12bc65418SDoug Thompson #include "amd64_edac.h" 223ac4ae8SAndreas Herrmann #include <asm/amd_nb.h> 32bc65418SDoug Thompson 4d1ea71cdSBorislav Petkov static struct edac_pci_ctl_info *pci_ctl; 52bc65418SDoug Thompson 62bc65418SDoug Thompson static int report_gart_errors; 72bc65418SDoug Thompson module_param(report_gart_errors, int, 0644); 82bc65418SDoug Thompson 92bc65418SDoug Thompson /* 102bc65418SDoug Thompson * Set by command line parameter. If BIOS has enabled the ECC, this override is 112bc65418SDoug Thompson * cleared to prevent re-enabling the hardware by this driver. 122bc65418SDoug Thompson */ 132bc65418SDoug Thompson static int ecc_enable_override; 142bc65418SDoug Thompson module_param(ecc_enable_override, int, 0644); 152bc65418SDoug Thompson 16a29d8b8eSTejun Heo static struct msr __percpu *msrs; 1750542251SBorislav Petkov 18360b7f3cSBorislav Petkov /* 19360b7f3cSBorislav Petkov * count successfully initialized driver instances for setup_pci_device() 20360b7f3cSBorislav Petkov */ 21360b7f3cSBorislav Petkov static atomic_t drv_instances = ATOMIC_INIT(0); 22360b7f3cSBorislav Petkov 23cc4d8860SBorislav Petkov /* Per-node driver instances */ 24cc4d8860SBorislav Petkov static struct mem_ctl_info **mcis; 25ae7bb7c6SBorislav Petkov static struct ecc_settings **ecc_stngs; 262bc65418SDoug Thompson 272bc65418SDoug Thompson /* 28b70ef010SBorislav Petkov * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing 29b70ef010SBorislav Petkov * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching- 30b70ef010SBorislav Petkov * or higher value'. 31b70ef010SBorislav Petkov * 32b70ef010SBorislav Petkov *FIXME: Produce a better mapping/linearisation. 33b70ef010SBorislav Petkov */ 34c7e5301aSDaniel J Blueman static const struct scrubrate { 3539094443SBorislav Petkov u32 scrubval; /* bit pattern for scrub rate */ 3639094443SBorislav Petkov u32 bandwidth; /* bandwidth consumed (bytes/sec) */ 3739094443SBorislav Petkov } scrubrates[] = { 38b70ef010SBorislav Petkov { 0x01, 1600000000UL}, 39b70ef010SBorislav Petkov { 0x02, 800000000UL}, 40b70ef010SBorislav Petkov { 0x03, 400000000UL}, 41b70ef010SBorislav Petkov { 0x04, 200000000UL}, 42b70ef010SBorislav Petkov { 0x05, 100000000UL}, 43b70ef010SBorislav Petkov { 0x06, 50000000UL}, 44b70ef010SBorislav Petkov { 0x07, 25000000UL}, 45b70ef010SBorislav Petkov { 0x08, 12284069UL}, 46b70ef010SBorislav Petkov { 0x09, 6274509UL}, 47b70ef010SBorislav Petkov { 0x0A, 3121951UL}, 48b70ef010SBorislav Petkov { 0x0B, 1560975UL}, 49b70ef010SBorislav Petkov { 0x0C, 781440UL}, 50b70ef010SBorislav Petkov { 0x0D, 390720UL}, 51b70ef010SBorislav Petkov { 0x0E, 195300UL}, 52b70ef010SBorislav Petkov { 0x0F, 97650UL}, 53b70ef010SBorislav Petkov { 0x10, 48854UL}, 54b70ef010SBorislav Petkov { 0x11, 24427UL}, 55b70ef010SBorislav Petkov { 0x12, 12213UL}, 56b70ef010SBorislav Petkov { 0x13, 6101UL}, 57b70ef010SBorislav Petkov { 0x14, 3051UL}, 58b70ef010SBorislav Petkov { 0x15, 1523UL}, 59b70ef010SBorislav Petkov { 0x16, 761UL}, 60b70ef010SBorislav Petkov { 0x00, 0UL}, /* scrubbing off */ 61b70ef010SBorislav Petkov }; 62b70ef010SBorislav Petkov 6366fed2d4SBorislav Petkov int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset, 64b2b0c605SBorislav Petkov u32 *val, const char *func) 65b2b0c605SBorislav Petkov { 66b2b0c605SBorislav Petkov int err = 0; 67b2b0c605SBorislav Petkov 68b2b0c605SBorislav Petkov err = pci_read_config_dword(pdev, offset, val); 69b2b0c605SBorislav Petkov if (err) 70b2b0c605SBorislav Petkov amd64_warn("%s: error reading F%dx%03x.\n", 71b2b0c605SBorislav Petkov func, PCI_FUNC(pdev->devfn), offset); 72b2b0c605SBorislav Petkov 73b2b0c605SBorislav Petkov return err; 74b2b0c605SBorislav Petkov } 75b2b0c605SBorislav Petkov 76b2b0c605SBorislav Petkov int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset, 77b2b0c605SBorislav Petkov u32 val, const char *func) 78b2b0c605SBorislav Petkov { 79b2b0c605SBorislav Petkov int err = 0; 80b2b0c605SBorislav Petkov 81b2b0c605SBorislav Petkov err = pci_write_config_dword(pdev, offset, val); 82b2b0c605SBorislav Petkov if (err) 83b2b0c605SBorislav Petkov amd64_warn("%s: error writing to F%dx%03x.\n", 84b2b0c605SBorislav Petkov func, PCI_FUNC(pdev->devfn), offset); 85b2b0c605SBorislav Petkov 86b2b0c605SBorislav Petkov return err; 87b2b0c605SBorislav Petkov } 88b2b0c605SBorislav Petkov 89b2b0c605SBorislav Petkov /* 9073ba8593SBorislav Petkov * Select DCT to which PCI cfg accesses are routed 9173ba8593SBorislav Petkov */ 9273ba8593SBorislav Petkov static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct) 9373ba8593SBorislav Petkov { 9473ba8593SBorislav Petkov u32 reg = 0; 9573ba8593SBorislav Petkov 9673ba8593SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, ®); 977981a28fSAravind Gopalakrishnan reg &= (pvt->model == 0x30) ? ~3 : ~1; 9873ba8593SBorislav Petkov reg |= dct; 9973ba8593SBorislav Petkov amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg); 10073ba8593SBorislav Petkov } 10173ba8593SBorislav Petkov 1027981a28fSAravind Gopalakrishnan /* 1037981a28fSAravind Gopalakrishnan * 1047981a28fSAravind Gopalakrishnan * Depending on the family, F2 DCT reads need special handling: 1057981a28fSAravind Gopalakrishnan * 1067981a28fSAravind Gopalakrishnan * K8: has a single DCT only and no address offsets >= 0x100 1077981a28fSAravind Gopalakrishnan * 1087981a28fSAravind Gopalakrishnan * F10h: each DCT has its own set of regs 1097981a28fSAravind Gopalakrishnan * DCT0 -> F2x040.. 1107981a28fSAravind Gopalakrishnan * DCT1 -> F2x140.. 1117981a28fSAravind Gopalakrishnan * 1127981a28fSAravind Gopalakrishnan * F16h: has only 1 DCT 1137981a28fSAravind Gopalakrishnan * 1147981a28fSAravind Gopalakrishnan * F15h: we select which DCT we access using F1x10C[DctCfgSel] 1157981a28fSAravind Gopalakrishnan */ 1167981a28fSAravind Gopalakrishnan static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct, 1177981a28fSAravind Gopalakrishnan int offset, u32 *val) 118b2b0c605SBorislav Petkov { 1197981a28fSAravind Gopalakrishnan switch (pvt->fam) { 1207981a28fSAravind Gopalakrishnan case 0xf: 1217981a28fSAravind Gopalakrishnan if (dct || offset >= 0x100) 1227981a28fSAravind Gopalakrishnan return -EINVAL; 1237981a28fSAravind Gopalakrishnan break; 124b2b0c605SBorislav Petkov 1257981a28fSAravind Gopalakrishnan case 0x10: 1267981a28fSAravind Gopalakrishnan if (dct) { 1277981a28fSAravind Gopalakrishnan /* 1287981a28fSAravind Gopalakrishnan * Note: If ganging is enabled, barring the regs 1297981a28fSAravind Gopalakrishnan * F2x[1,0]98 and F2x[1,0]9C; reads reads to F2x1xx 1307981a28fSAravind Gopalakrishnan * return 0. (cf. Section 2.8.1 F10h BKDG) 1317981a28fSAravind Gopalakrishnan */ 1327981a28fSAravind Gopalakrishnan if (dct_ganging_enabled(pvt)) 1337981a28fSAravind Gopalakrishnan return 0; 1347981a28fSAravind Gopalakrishnan 1357981a28fSAravind Gopalakrishnan offset += 0x100; 136b2b0c605SBorislav Petkov } 1377981a28fSAravind Gopalakrishnan break; 138b2b0c605SBorislav Petkov 1397981a28fSAravind Gopalakrishnan case 0x15: 1407981a28fSAravind Gopalakrishnan /* 1417981a28fSAravind Gopalakrishnan * F15h: F2x1xx addresses do not map explicitly to DCT1. 1427981a28fSAravind Gopalakrishnan * We should select which DCT we access using F1x10C[DctCfgSel] 1437981a28fSAravind Gopalakrishnan */ 1447981a28fSAravind Gopalakrishnan dct = (dct && pvt->model == 0x30) ? 3 : dct; 14573ba8593SBorislav Petkov f15h_select_dct(pvt, dct); 1467981a28fSAravind Gopalakrishnan break; 147b2b0c605SBorislav Petkov 1487981a28fSAravind Gopalakrishnan case 0x16: 1497981a28fSAravind Gopalakrishnan if (dct) 1507981a28fSAravind Gopalakrishnan return -EINVAL; 1517981a28fSAravind Gopalakrishnan break; 1527981a28fSAravind Gopalakrishnan 1537981a28fSAravind Gopalakrishnan default: 1547981a28fSAravind Gopalakrishnan break; 1557981a28fSAravind Gopalakrishnan } 1567981a28fSAravind Gopalakrishnan return amd64_read_pci_cfg(pvt->F2, offset, val); 157b2b0c605SBorislav Petkov } 158b2b0c605SBorislav Petkov 159b70ef010SBorislav Petkov /* 1602bc65418SDoug Thompson * Memory scrubber control interface. For K8, memory scrubbing is handled by 1612bc65418SDoug Thompson * hardware and can involve L2 cache, dcache as well as the main memory. With 1622bc65418SDoug Thompson * F10, this is extended to L3 cache scrubbing on CPU models sporting that 1632bc65418SDoug Thompson * functionality. 1642bc65418SDoug Thompson * 1652bc65418SDoug Thompson * This causes the "units" for the scrubbing speed to vary from 64 byte blocks 1662bc65418SDoug Thompson * (dram) over to cache lines. This is nasty, so we will use bandwidth in 1672bc65418SDoug Thompson * bytes/sec for the setting. 1682bc65418SDoug Thompson * 1692bc65418SDoug Thompson * Currently, we only do dram scrubbing. If the scrubbing is done in software on 1702bc65418SDoug Thompson * other archs, we might not have access to the caches directly. 1712bc65418SDoug Thompson */ 1722bc65418SDoug Thompson 1732bc65418SDoug Thompson /* 1742bc65418SDoug Thompson * scan the scrub rate mapping table for a close or matching bandwidth value to 1752bc65418SDoug Thompson * issue. If requested is too big, then use last maximum value found. 1762bc65418SDoug Thompson */ 177d1ea71cdSBorislav Petkov static int __set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate) 1782bc65418SDoug Thompson { 1792bc65418SDoug Thompson u32 scrubval; 1802bc65418SDoug Thompson int i; 1812bc65418SDoug Thompson 1822bc65418SDoug Thompson /* 1832bc65418SDoug Thompson * map the configured rate (new_bw) to a value specific to the AMD64 1842bc65418SDoug Thompson * memory controller and apply to register. Search for the first 1852bc65418SDoug Thompson * bandwidth entry that is greater or equal than the setting requested 1862bc65418SDoug Thompson * and program that. If at last entry, turn off DRAM scrubbing. 187168bfeefSAndrew Morton * 188168bfeefSAndrew Morton * If no suitable bandwidth is found, turn off DRAM scrubbing entirely 189168bfeefSAndrew Morton * by falling back to the last element in scrubrates[]. 1902bc65418SDoug Thompson */ 191168bfeefSAndrew Morton for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) { 1922bc65418SDoug Thompson /* 1932bc65418SDoug Thompson * skip scrub rates which aren't recommended 1942bc65418SDoug Thompson * (see F10 BKDG, F3x58) 1952bc65418SDoug Thompson */ 196395ae783SBorislav Petkov if (scrubrates[i].scrubval < min_rate) 1972bc65418SDoug Thompson continue; 1982bc65418SDoug Thompson 1992bc65418SDoug Thompson if (scrubrates[i].bandwidth <= new_bw) 2002bc65418SDoug Thompson break; 2012bc65418SDoug Thompson } 2022bc65418SDoug Thompson 2032bc65418SDoug Thompson scrubval = scrubrates[i].scrubval; 2042bc65418SDoug Thompson 2055980bb9cSBorislav Petkov pci_write_bits32(ctl, SCRCTRL, scrubval, 0x001F); 2062bc65418SDoug Thompson 20739094443SBorislav Petkov if (scrubval) 20839094443SBorislav Petkov return scrubrates[i].bandwidth; 20939094443SBorislav Petkov 2102bc65418SDoug Thompson return 0; 2112bc65418SDoug Thompson } 2122bc65418SDoug Thompson 213d1ea71cdSBorislav Petkov static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw) 2142bc65418SDoug Thompson { 2152bc65418SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 21687b3e0e6SBorislav Petkov u32 min_scrubrate = 0x5; 2172bc65418SDoug Thompson 218a4b4bedcSBorislav Petkov if (pvt->fam == 0xf) 21987b3e0e6SBorislav Petkov min_scrubrate = 0x0; 22087b3e0e6SBorislav Petkov 2213f0aba4fSBorislav Petkov /* Erratum #505 */ 2223f0aba4fSBorislav Petkov if (pvt->fam == 0x15 && pvt->model < 0x10) 22373ba8593SBorislav Petkov f15h_select_dct(pvt, 0); 22473ba8593SBorislav Petkov 225d1ea71cdSBorislav Petkov return __set_scrub_rate(pvt->F3, bw, min_scrubrate); 2262bc65418SDoug Thompson } 2272bc65418SDoug Thompson 228d1ea71cdSBorislav Petkov static int get_scrub_rate(struct mem_ctl_info *mci) 2292bc65418SDoug Thompson { 2302bc65418SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 2312bc65418SDoug Thompson u32 scrubval = 0; 23239094443SBorislav Petkov int i, retval = -EINVAL; 2332bc65418SDoug Thompson 2343f0aba4fSBorislav Petkov /* Erratum #505 */ 2353f0aba4fSBorislav Petkov if (pvt->fam == 0x15 && pvt->model < 0x10) 23673ba8593SBorislav Petkov f15h_select_dct(pvt, 0); 23773ba8593SBorislav Petkov 2385980bb9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval); 2392bc65418SDoug Thompson 2402bc65418SDoug Thompson scrubval = scrubval & 0x001F; 2412bc65418SDoug Thompson 242926311fdSRoel Kluin for (i = 0; i < ARRAY_SIZE(scrubrates); i++) { 2432bc65418SDoug Thompson if (scrubrates[i].scrubval == scrubval) { 24439094443SBorislav Petkov retval = scrubrates[i].bandwidth; 2452bc65418SDoug Thompson break; 2462bc65418SDoug Thompson } 2472bc65418SDoug Thompson } 24839094443SBorislav Petkov return retval; 2492bc65418SDoug Thompson } 2502bc65418SDoug Thompson 2516775763aSDoug Thompson /* 2527f19bf75SBorislav Petkov * returns true if the SysAddr given by sys_addr matches the 2537f19bf75SBorislav Petkov * DRAM base/limit associated with node_id 2546775763aSDoug Thompson */ 255d1ea71cdSBorislav Petkov static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid) 2566775763aSDoug Thompson { 2577f19bf75SBorislav Petkov u64 addr; 2586775763aSDoug Thompson 2596775763aSDoug Thompson /* The K8 treats this as a 40-bit value. However, bits 63-40 will be 2606775763aSDoug Thompson * all ones if the most significant implemented address bit is 1. 2616775763aSDoug Thompson * Here we discard bits 63-40. See section 3.4.2 of AMD publication 2626775763aSDoug Thompson * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1 2636775763aSDoug Thompson * Application Programming. 2646775763aSDoug Thompson */ 2656775763aSDoug Thompson addr = sys_addr & 0x000000ffffffffffull; 2666775763aSDoug Thompson 2677f19bf75SBorislav Petkov return ((addr >= get_dram_base(pvt, nid)) && 2687f19bf75SBorislav Petkov (addr <= get_dram_limit(pvt, nid))); 2696775763aSDoug Thompson } 2706775763aSDoug Thompson 2716775763aSDoug Thompson /* 2726775763aSDoug Thompson * Attempt to map a SysAddr to a node. On success, return a pointer to the 2736775763aSDoug Thompson * mem_ctl_info structure for the node that the SysAddr maps to. 2746775763aSDoug Thompson * 2756775763aSDoug Thompson * On failure, return NULL. 2766775763aSDoug Thompson */ 2776775763aSDoug Thompson static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci, 2786775763aSDoug Thompson u64 sys_addr) 2796775763aSDoug Thompson { 2806775763aSDoug Thompson struct amd64_pvt *pvt; 281c7e5301aSDaniel J Blueman u8 node_id; 2826775763aSDoug Thompson u32 intlv_en, bits; 2836775763aSDoug Thompson 2846775763aSDoug Thompson /* 2856775763aSDoug Thompson * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section 2866775763aSDoug Thompson * 3.4.4.2) registers to map the SysAddr to a node ID. 2876775763aSDoug Thompson */ 2886775763aSDoug Thompson pvt = mci->pvt_info; 2896775763aSDoug Thompson 2906775763aSDoug Thompson /* 2916775763aSDoug Thompson * The value of this field should be the same for all DRAM Base 2926775763aSDoug Thompson * registers. Therefore we arbitrarily choose to read it from the 2936775763aSDoug Thompson * register for node 0. 2946775763aSDoug Thompson */ 2957f19bf75SBorislav Petkov intlv_en = dram_intlv_en(pvt, 0); 2966775763aSDoug Thompson 2976775763aSDoug Thompson if (intlv_en == 0) { 2987f19bf75SBorislav Petkov for (node_id = 0; node_id < DRAM_RANGES; node_id++) { 299d1ea71cdSBorislav Petkov if (base_limit_match(pvt, sys_addr, node_id)) 3006775763aSDoug Thompson goto found; 3016775763aSDoug Thompson } 3028edc5445SBorislav Petkov goto err_no_match; 3038edc5445SBorislav Petkov } 3046775763aSDoug Thompson 30572f158feSBorislav Petkov if (unlikely((intlv_en != 0x01) && 30672f158feSBorislav Petkov (intlv_en != 0x03) && 30772f158feSBorislav Petkov (intlv_en != 0x07))) { 30824f9a7feSBorislav Petkov amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en); 3096775763aSDoug Thompson return NULL; 3106775763aSDoug Thompson } 3116775763aSDoug Thompson 3126775763aSDoug Thompson bits = (((u32) sys_addr) >> 12) & intlv_en; 3136775763aSDoug Thompson 3146775763aSDoug Thompson for (node_id = 0; ; ) { 3157f19bf75SBorislav Petkov if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits) 3166775763aSDoug Thompson break; /* intlv_sel field matches */ 3176775763aSDoug Thompson 3187f19bf75SBorislav Petkov if (++node_id >= DRAM_RANGES) 3196775763aSDoug Thompson goto err_no_match; 3206775763aSDoug Thompson } 3216775763aSDoug Thompson 3226775763aSDoug Thompson /* sanity test for sys_addr */ 323d1ea71cdSBorislav Petkov if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) { 32424f9a7feSBorislav Petkov amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address" 32524f9a7feSBorislav Petkov "range for node %d with node interleaving enabled.\n", 3268edc5445SBorislav Petkov __func__, sys_addr, node_id); 3276775763aSDoug Thompson return NULL; 3286775763aSDoug Thompson } 3296775763aSDoug Thompson 3306775763aSDoug Thompson found: 331b487c33eSBorislav Petkov return edac_mc_find((int)node_id); 3326775763aSDoug Thompson 3336775763aSDoug Thompson err_no_match: 334956b9ba1SJoe Perches edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n", 3356775763aSDoug Thompson (unsigned long)sys_addr); 3366775763aSDoug Thompson 3376775763aSDoug Thompson return NULL; 3386775763aSDoug Thompson } 339e2ce7255SDoug Thompson 340e2ce7255SDoug Thompson /* 34111c75eadSBorislav Petkov * compute the CS base address of the @csrow on the DRAM controller @dct. 34211c75eadSBorislav Petkov * For details see F2x[5C:40] in the processor's BKDG 343e2ce7255SDoug Thompson */ 34411c75eadSBorislav Petkov static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct, 34511c75eadSBorislav Petkov u64 *base, u64 *mask) 346e2ce7255SDoug Thompson { 34711c75eadSBorislav Petkov u64 csbase, csmask, base_bits, mask_bits; 34811c75eadSBorislav Petkov u8 addr_shift; 34911c75eadSBorislav Petkov 35018b94f66SAravind Gopalakrishnan if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) { 35111c75eadSBorislav Petkov csbase = pvt->csels[dct].csbases[csrow]; 35211c75eadSBorislav Petkov csmask = pvt->csels[dct].csmasks[csrow]; 35310ef6b0dSChen, Gong base_bits = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9); 35410ef6b0dSChen, Gong mask_bits = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9); 35511c75eadSBorislav Petkov addr_shift = 4; 35694c1acf2SAravind Gopalakrishnan 35794c1acf2SAravind Gopalakrishnan /* 35818b94f66SAravind Gopalakrishnan * F16h and F15h, models 30h and later need two addr_shift values: 35918b94f66SAravind Gopalakrishnan * 8 for high and 6 for low (cf. F16h BKDG). 36094c1acf2SAravind Gopalakrishnan */ 36118b94f66SAravind Gopalakrishnan } else if (pvt->fam == 0x16 || 36218b94f66SAravind Gopalakrishnan (pvt->fam == 0x15 && pvt->model >= 0x30)) { 36394c1acf2SAravind Gopalakrishnan csbase = pvt->csels[dct].csbases[csrow]; 36494c1acf2SAravind Gopalakrishnan csmask = pvt->csels[dct].csmasks[csrow >> 1]; 36594c1acf2SAravind Gopalakrishnan 36610ef6b0dSChen, Gong *base = (csbase & GENMASK_ULL(15, 5)) << 6; 36710ef6b0dSChen, Gong *base |= (csbase & GENMASK_ULL(30, 19)) << 8; 36894c1acf2SAravind Gopalakrishnan 36994c1acf2SAravind Gopalakrishnan *mask = ~0ULL; 37094c1acf2SAravind Gopalakrishnan /* poke holes for the csmask */ 37110ef6b0dSChen, Gong *mask &= ~((GENMASK_ULL(15, 5) << 6) | 37210ef6b0dSChen, Gong (GENMASK_ULL(30, 19) << 8)); 37394c1acf2SAravind Gopalakrishnan 37410ef6b0dSChen, Gong *mask |= (csmask & GENMASK_ULL(15, 5)) << 6; 37510ef6b0dSChen, Gong *mask |= (csmask & GENMASK_ULL(30, 19)) << 8; 37694c1acf2SAravind Gopalakrishnan 37794c1acf2SAravind Gopalakrishnan return; 37811c75eadSBorislav Petkov } else { 37911c75eadSBorislav Petkov csbase = pvt->csels[dct].csbases[csrow]; 38011c75eadSBorislav Petkov csmask = pvt->csels[dct].csmasks[csrow >> 1]; 38111c75eadSBorislav Petkov addr_shift = 8; 38211c75eadSBorislav Petkov 383a4b4bedcSBorislav Petkov if (pvt->fam == 0x15) 38410ef6b0dSChen, Gong base_bits = mask_bits = 38510ef6b0dSChen, Gong GENMASK_ULL(30,19) | GENMASK_ULL(13,5); 38611c75eadSBorislav Petkov else 38710ef6b0dSChen, Gong base_bits = mask_bits = 38810ef6b0dSChen, Gong GENMASK_ULL(28,19) | GENMASK_ULL(13,5); 389e2ce7255SDoug Thompson } 390e2ce7255SDoug Thompson 39111c75eadSBorislav Petkov *base = (csbase & base_bits) << addr_shift; 392e2ce7255SDoug Thompson 39311c75eadSBorislav Petkov *mask = ~0ULL; 39411c75eadSBorislav Petkov /* poke holes for the csmask */ 39511c75eadSBorislav Petkov *mask &= ~(mask_bits << addr_shift); 39611c75eadSBorislav Petkov /* OR them in */ 39711c75eadSBorislav Petkov *mask |= (csmask & mask_bits) << addr_shift; 398e2ce7255SDoug Thompson } 399e2ce7255SDoug Thompson 40011c75eadSBorislav Petkov #define for_each_chip_select(i, dct, pvt) \ 40111c75eadSBorislav Petkov for (i = 0; i < pvt->csels[dct].b_cnt; i++) 40211c75eadSBorislav Petkov 403614ec9d8SBorislav Petkov #define chip_select_base(i, dct, pvt) \ 404614ec9d8SBorislav Petkov pvt->csels[dct].csbases[i] 405614ec9d8SBorislav Petkov 40611c75eadSBorislav Petkov #define for_each_chip_select_mask(i, dct, pvt) \ 40711c75eadSBorislav Petkov for (i = 0; i < pvt->csels[dct].m_cnt; i++) 40811c75eadSBorislav Petkov 409e2ce7255SDoug Thompson /* 410e2ce7255SDoug Thompson * @input_addr is an InputAddr associated with the node given by mci. Return the 411e2ce7255SDoug Thompson * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr). 412e2ce7255SDoug Thompson */ 413e2ce7255SDoug Thompson static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr) 414e2ce7255SDoug Thompson { 415e2ce7255SDoug Thompson struct amd64_pvt *pvt; 416e2ce7255SDoug Thompson int csrow; 417e2ce7255SDoug Thompson u64 base, mask; 418e2ce7255SDoug Thompson 419e2ce7255SDoug Thompson pvt = mci->pvt_info; 420e2ce7255SDoug Thompson 42111c75eadSBorislav Petkov for_each_chip_select(csrow, 0, pvt) { 42211c75eadSBorislav Petkov if (!csrow_enabled(csrow, 0, pvt)) 423e2ce7255SDoug Thompson continue; 424e2ce7255SDoug Thompson 42511c75eadSBorislav Petkov get_cs_base_and_mask(pvt, csrow, 0, &base, &mask); 42611c75eadSBorislav Petkov 42711c75eadSBorislav Petkov mask = ~mask; 428e2ce7255SDoug Thompson 429e2ce7255SDoug Thompson if ((input_addr & mask) == (base & mask)) { 430956b9ba1SJoe Perches edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n", 431e2ce7255SDoug Thompson (unsigned long)input_addr, csrow, 432e2ce7255SDoug Thompson pvt->mc_node_id); 433e2ce7255SDoug Thompson 434e2ce7255SDoug Thompson return csrow; 435e2ce7255SDoug Thompson } 436e2ce7255SDoug Thompson } 437956b9ba1SJoe Perches edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n", 438e2ce7255SDoug Thompson (unsigned long)input_addr, pvt->mc_node_id); 439e2ce7255SDoug Thompson 440e2ce7255SDoug Thompson return -1; 441e2ce7255SDoug Thompson } 442e2ce7255SDoug Thompson 443e2ce7255SDoug Thompson /* 444e2ce7255SDoug Thompson * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094) 445e2ce7255SDoug Thompson * for the node represented by mci. Info is passed back in *hole_base, 446e2ce7255SDoug Thompson * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if 447e2ce7255SDoug Thompson * info is invalid. Info may be invalid for either of the following reasons: 448e2ce7255SDoug Thompson * 449e2ce7255SDoug Thompson * - The revision of the node is not E or greater. In this case, the DRAM Hole 450e2ce7255SDoug Thompson * Address Register does not exist. 451e2ce7255SDoug Thompson * 452e2ce7255SDoug Thompson * - The DramHoleValid bit is cleared in the DRAM Hole Address Register, 453e2ce7255SDoug Thompson * indicating that its contents are not valid. 454e2ce7255SDoug Thompson * 455e2ce7255SDoug Thompson * The values passed back in *hole_base, *hole_offset, and *hole_size are 456e2ce7255SDoug Thompson * complete 32-bit values despite the fact that the bitfields in the DHAR 457e2ce7255SDoug Thompson * only represent bits 31-24 of the base and offset values. 458e2ce7255SDoug Thompson */ 459e2ce7255SDoug Thompson int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base, 460e2ce7255SDoug Thompson u64 *hole_offset, u64 *hole_size) 461e2ce7255SDoug Thompson { 462e2ce7255SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 463e2ce7255SDoug Thompson 464e2ce7255SDoug Thompson /* only revE and later have the DRAM Hole Address Register */ 465a4b4bedcSBorislav Petkov if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) { 466956b9ba1SJoe Perches edac_dbg(1, " revision %d for node %d does not support DHAR\n", 467e2ce7255SDoug Thompson pvt->ext_model, pvt->mc_node_id); 468e2ce7255SDoug Thompson return 1; 469e2ce7255SDoug Thompson } 470e2ce7255SDoug Thompson 471bc21fa57SBorislav Petkov /* valid for Fam10h and above */ 472a4b4bedcSBorislav Petkov if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) { 473956b9ba1SJoe Perches edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n"); 474e2ce7255SDoug Thompson return 1; 475e2ce7255SDoug Thompson } 476e2ce7255SDoug Thompson 477c8e518d5SBorislav Petkov if (!dhar_valid(pvt)) { 478956b9ba1SJoe Perches edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n", 479e2ce7255SDoug Thompson pvt->mc_node_id); 480e2ce7255SDoug Thompson return 1; 481e2ce7255SDoug Thompson } 482e2ce7255SDoug Thompson 483e2ce7255SDoug Thompson /* This node has Memory Hoisting */ 484e2ce7255SDoug Thompson 485e2ce7255SDoug Thompson /* +------------------+--------------------+--------------------+----- 486e2ce7255SDoug Thompson * | memory | DRAM hole | relocated | 487e2ce7255SDoug Thompson * | [0, (x - 1)] | [x, 0xffffffff] | addresses from | 488e2ce7255SDoug Thompson * | | | DRAM hole | 489e2ce7255SDoug Thompson * | | | [0x100000000, | 490e2ce7255SDoug Thompson * | | | (0x100000000+ | 491e2ce7255SDoug Thompson * | | | (0xffffffff-x))] | 492e2ce7255SDoug Thompson * +------------------+--------------------+--------------------+----- 493e2ce7255SDoug Thompson * 494e2ce7255SDoug Thompson * Above is a diagram of physical memory showing the DRAM hole and the 495e2ce7255SDoug Thompson * relocated addresses from the DRAM hole. As shown, the DRAM hole 496e2ce7255SDoug Thompson * starts at address x (the base address) and extends through address 497e2ce7255SDoug Thompson * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the 498e2ce7255SDoug Thompson * addresses in the hole so that they start at 0x100000000. 499e2ce7255SDoug Thompson */ 500e2ce7255SDoug Thompson 5011f31677eSBorislav Petkov *hole_base = dhar_base(pvt); 5021f31677eSBorislav Petkov *hole_size = (1ULL << 32) - *hole_base; 503e2ce7255SDoug Thompson 504a4b4bedcSBorislav Petkov *hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt) 505a4b4bedcSBorislav Petkov : k8_dhar_offset(pvt); 506e2ce7255SDoug Thompson 507956b9ba1SJoe Perches edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n", 508e2ce7255SDoug Thompson pvt->mc_node_id, (unsigned long)*hole_base, 509e2ce7255SDoug Thompson (unsigned long)*hole_offset, (unsigned long)*hole_size); 510e2ce7255SDoug Thompson 511e2ce7255SDoug Thompson return 0; 512e2ce7255SDoug Thompson } 513e2ce7255SDoug Thompson EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info); 514e2ce7255SDoug Thompson 51593c2df58SDoug Thompson /* 51693c2df58SDoug Thompson * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is 51793c2df58SDoug Thompson * assumed that sys_addr maps to the node given by mci. 51893c2df58SDoug Thompson * 51993c2df58SDoug Thompson * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section 52093c2df58SDoug Thompson * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a 52193c2df58SDoug Thompson * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled, 52293c2df58SDoug Thompson * then it is also involved in translating a SysAddr to a DramAddr. Sections 52393c2df58SDoug Thompson * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting. 52493c2df58SDoug Thompson * These parts of the documentation are unclear. I interpret them as follows: 52593c2df58SDoug Thompson * 52693c2df58SDoug Thompson * When node n receives a SysAddr, it processes the SysAddr as follows: 52793c2df58SDoug Thompson * 52893c2df58SDoug Thompson * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM 52993c2df58SDoug Thompson * Limit registers for node n. If the SysAddr is not within the range 53093c2df58SDoug Thompson * specified by the base and limit values, then node n ignores the Sysaddr 53193c2df58SDoug Thompson * (since it does not map to node n). Otherwise continue to step 2 below. 53293c2df58SDoug Thompson * 53393c2df58SDoug Thompson * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is 53493c2df58SDoug Thompson * disabled so skip to step 3 below. Otherwise see if the SysAddr is within 53593c2df58SDoug Thompson * the range of relocated addresses (starting at 0x100000000) from the DRAM 53693c2df58SDoug Thompson * hole. If not, skip to step 3 below. Else get the value of the 53793c2df58SDoug Thompson * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the 53893c2df58SDoug Thompson * offset defined by this value from the SysAddr. 53993c2df58SDoug Thompson * 54093c2df58SDoug Thompson * 3. Obtain the base address for node n from the DRAMBase field of the DRAM 54193c2df58SDoug Thompson * Base register for node n. To obtain the DramAddr, subtract the base 54293c2df58SDoug Thompson * address from the SysAddr, as shown near the start of section 3.4.4 (p.70). 54393c2df58SDoug Thompson */ 54493c2df58SDoug Thompson static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr) 54593c2df58SDoug Thompson { 5467f19bf75SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 54793c2df58SDoug Thompson u64 dram_base, hole_base, hole_offset, hole_size, dram_addr; 5481f31677eSBorislav Petkov int ret; 54993c2df58SDoug Thompson 5507f19bf75SBorislav Petkov dram_base = get_dram_base(pvt, pvt->mc_node_id); 55193c2df58SDoug Thompson 55293c2df58SDoug Thompson ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset, 55393c2df58SDoug Thompson &hole_size); 55493c2df58SDoug Thompson if (!ret) { 5551f31677eSBorislav Petkov if ((sys_addr >= (1ULL << 32)) && 5561f31677eSBorislav Petkov (sys_addr < ((1ULL << 32) + hole_size))) { 55793c2df58SDoug Thompson /* use DHAR to translate SysAddr to DramAddr */ 55893c2df58SDoug Thompson dram_addr = sys_addr - hole_offset; 55993c2df58SDoug Thompson 560956b9ba1SJoe Perches edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n", 56193c2df58SDoug Thompson (unsigned long)sys_addr, 56293c2df58SDoug Thompson (unsigned long)dram_addr); 56393c2df58SDoug Thompson 56493c2df58SDoug Thompson return dram_addr; 56593c2df58SDoug Thompson } 56693c2df58SDoug Thompson } 56793c2df58SDoug Thompson 56893c2df58SDoug Thompson /* 56993c2df58SDoug Thompson * Translate the SysAddr to a DramAddr as shown near the start of 57093c2df58SDoug Thompson * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8 57193c2df58SDoug Thompson * only deals with 40-bit values. Therefore we discard bits 63-40 of 57293c2df58SDoug Thompson * sys_addr below. If bit 39 of sys_addr is 1 then the bits we 57393c2df58SDoug Thompson * discard are all 1s. Otherwise the bits we discard are all 0s. See 57493c2df58SDoug Thompson * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture 57593c2df58SDoug Thompson * Programmer's Manual Volume 1 Application Programming. 57693c2df58SDoug Thompson */ 57710ef6b0dSChen, Gong dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base; 57893c2df58SDoug Thompson 579956b9ba1SJoe Perches edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n", 580956b9ba1SJoe Perches (unsigned long)sys_addr, (unsigned long)dram_addr); 58193c2df58SDoug Thompson return dram_addr; 58293c2df58SDoug Thompson } 58393c2df58SDoug Thompson 58493c2df58SDoug Thompson /* 58593c2df58SDoug Thompson * @intlv_en is the value of the IntlvEn field from a DRAM Base register 58693c2df58SDoug Thompson * (section 3.4.4.1). Return the number of bits from a SysAddr that are used 58793c2df58SDoug Thompson * for node interleaving. 58893c2df58SDoug Thompson */ 58993c2df58SDoug Thompson static int num_node_interleave_bits(unsigned intlv_en) 59093c2df58SDoug Thompson { 59193c2df58SDoug Thompson static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 }; 59293c2df58SDoug Thompson int n; 59393c2df58SDoug Thompson 59493c2df58SDoug Thompson BUG_ON(intlv_en > 7); 59593c2df58SDoug Thompson n = intlv_shift_table[intlv_en]; 59693c2df58SDoug Thompson return n; 59793c2df58SDoug Thompson } 59893c2df58SDoug Thompson 59993c2df58SDoug Thompson /* Translate the DramAddr given by @dram_addr to an InputAddr. */ 60093c2df58SDoug Thompson static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr) 60193c2df58SDoug Thompson { 60293c2df58SDoug Thompson struct amd64_pvt *pvt; 60393c2df58SDoug Thompson int intlv_shift; 60493c2df58SDoug Thompson u64 input_addr; 60593c2df58SDoug Thompson 60693c2df58SDoug Thompson pvt = mci->pvt_info; 60793c2df58SDoug Thompson 60893c2df58SDoug Thompson /* 60993c2df58SDoug Thompson * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E) 61093c2df58SDoug Thompson * concerning translating a DramAddr to an InputAddr. 61193c2df58SDoug Thompson */ 6127f19bf75SBorislav Petkov intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0)); 61310ef6b0dSChen, Gong input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) + 61493c2df58SDoug Thompson (dram_addr & 0xfff); 61593c2df58SDoug Thompson 616956b9ba1SJoe Perches edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n", 61793c2df58SDoug Thompson intlv_shift, (unsigned long)dram_addr, 61893c2df58SDoug Thompson (unsigned long)input_addr); 61993c2df58SDoug Thompson 62093c2df58SDoug Thompson return input_addr; 62193c2df58SDoug Thompson } 62293c2df58SDoug Thompson 62393c2df58SDoug Thompson /* 62493c2df58SDoug Thompson * Translate the SysAddr represented by @sys_addr to an InputAddr. It is 62593c2df58SDoug Thompson * assumed that @sys_addr maps to the node given by mci. 62693c2df58SDoug Thompson */ 62793c2df58SDoug Thompson static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr) 62893c2df58SDoug Thompson { 62993c2df58SDoug Thompson u64 input_addr; 63093c2df58SDoug Thompson 63193c2df58SDoug Thompson input_addr = 63293c2df58SDoug Thompson dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr)); 63393c2df58SDoug Thompson 634956b9ba1SJoe Perches edac_dbg(2, "SysAdddr 0x%lx translates to InputAddr 0x%lx\n", 63593c2df58SDoug Thompson (unsigned long)sys_addr, (unsigned long)input_addr); 63693c2df58SDoug Thompson 63793c2df58SDoug Thompson return input_addr; 63893c2df58SDoug Thompson } 63993c2df58SDoug Thompson 64093c2df58SDoug Thompson /* Map the Error address to a PAGE and PAGE OFFSET. */ 64193c2df58SDoug Thompson static inline void error_address_to_page_and_offset(u64 error_address, 64233ca0643SBorislav Petkov struct err_info *err) 64393c2df58SDoug Thompson { 64433ca0643SBorislav Petkov err->page = (u32) (error_address >> PAGE_SHIFT); 64533ca0643SBorislav Petkov err->offset = ((u32) error_address) & ~PAGE_MASK; 64693c2df58SDoug Thompson } 64793c2df58SDoug Thompson 64893c2df58SDoug Thompson /* 64993c2df58SDoug Thompson * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address 65093c2df58SDoug Thompson * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers 65193c2df58SDoug Thompson * of a node that detected an ECC memory error. mci represents the node that 65293c2df58SDoug Thompson * the error address maps to (possibly different from the node that detected 65393c2df58SDoug Thompson * the error). Return the number of the csrow that sys_addr maps to, or -1 on 65493c2df58SDoug Thompson * error. 65593c2df58SDoug Thompson */ 65693c2df58SDoug Thompson static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr) 65793c2df58SDoug Thompson { 65893c2df58SDoug Thompson int csrow; 65993c2df58SDoug Thompson 66093c2df58SDoug Thompson csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr)); 66193c2df58SDoug Thompson 66293c2df58SDoug Thompson if (csrow == -1) 66324f9a7feSBorislav Petkov amd64_mc_err(mci, "Failed to translate InputAddr to csrow for " 66493c2df58SDoug Thompson "address 0x%lx\n", (unsigned long)sys_addr); 66593c2df58SDoug Thompson return csrow; 66693c2df58SDoug Thompson } 667e2ce7255SDoug Thompson 668bfc04aecSBorislav Petkov static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16); 6692da11654SDoug Thompson 6702da11654SDoug Thompson /* 6712da11654SDoug Thompson * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs 6722da11654SDoug Thompson * are ECC capable. 6732da11654SDoug Thompson */ 674d1ea71cdSBorislav Petkov static unsigned long determine_edac_cap(struct amd64_pvt *pvt) 6752da11654SDoug Thompson { 676cb328507SBorislav Petkov u8 bit; 6771f6189edSDan Carpenter unsigned long edac_cap = EDAC_FLAG_NONE; 6782da11654SDoug Thompson 679a4b4bedcSBorislav Petkov bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F) 6802da11654SDoug Thompson ? 19 6812da11654SDoug Thompson : 17; 6822da11654SDoug Thompson 683584fcff4SBorislav Petkov if (pvt->dclr0 & BIT(bit)) 6842da11654SDoug Thompson edac_cap = EDAC_FLAG_SECDED; 6852da11654SDoug Thompson 6862da11654SDoug Thompson return edac_cap; 6872da11654SDoug Thompson } 6882da11654SDoug Thompson 689d1ea71cdSBorislav Petkov static void debug_display_dimm_sizes(struct amd64_pvt *, u8); 6902da11654SDoug Thompson 691d1ea71cdSBorislav Petkov static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan) 69268798e17SBorislav Petkov { 693956b9ba1SJoe Perches edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr); 69468798e17SBorislav Petkov 695a597d2a5SAravind Gopalakrishnan if (pvt->dram_type == MEM_LRDDR3) { 696a597d2a5SAravind Gopalakrishnan u32 dcsm = pvt->csels[chan].csmasks[0]; 697a597d2a5SAravind Gopalakrishnan /* 698a597d2a5SAravind Gopalakrishnan * It's assumed all LRDIMMs in a DCT are going to be of 699a597d2a5SAravind Gopalakrishnan * same 'type' until proven otherwise. So, use a cs 700a597d2a5SAravind Gopalakrishnan * value of '0' here to get dcsm value. 701a597d2a5SAravind Gopalakrishnan */ 702a597d2a5SAravind Gopalakrishnan edac_dbg(1, " LRDIMM %dx rank multiply\n", (dcsm & 0x3)); 703a597d2a5SAravind Gopalakrishnan } 704a597d2a5SAravind Gopalakrishnan 705a597d2a5SAravind Gopalakrishnan edac_dbg(1, "All DIMMs support ECC:%s\n", 70668798e17SBorislav Petkov (dclr & BIT(19)) ? "yes" : "no"); 70768798e17SBorislav Petkov 708a597d2a5SAravind Gopalakrishnan 709956b9ba1SJoe Perches edac_dbg(1, " PAR/ERR parity: %s\n", 71068798e17SBorislav Petkov (dclr & BIT(8)) ? "enabled" : "disabled"); 71168798e17SBorislav Petkov 712a4b4bedcSBorislav Petkov if (pvt->fam == 0x10) 713956b9ba1SJoe Perches edac_dbg(1, " DCT 128bit mode width: %s\n", 71468798e17SBorislav Petkov (dclr & BIT(11)) ? "128b" : "64b"); 71568798e17SBorislav Petkov 716956b9ba1SJoe Perches edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n", 71768798e17SBorislav Petkov (dclr & BIT(12)) ? "yes" : "no", 71868798e17SBorislav Petkov (dclr & BIT(13)) ? "yes" : "no", 71968798e17SBorislav Petkov (dclr & BIT(14)) ? "yes" : "no", 72068798e17SBorislav Petkov (dclr & BIT(15)) ? "yes" : "no"); 72168798e17SBorislav Petkov } 72268798e17SBorislav Petkov 7232da11654SDoug Thompson /* Display and decode various NB registers for debug purposes. */ 724b2b0c605SBorislav Petkov static void dump_misc_regs(struct amd64_pvt *pvt) 7252da11654SDoug Thompson { 726956b9ba1SJoe Perches edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap); 7272da11654SDoug Thompson 728956b9ba1SJoe Perches edac_dbg(1, " NB two channel DRAM capable: %s\n", 7295980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no"); 73068798e17SBorislav Petkov 731956b9ba1SJoe Perches edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n", 7325980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no", 7335980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no"); 73468798e17SBorislav Petkov 735d1ea71cdSBorislav Petkov debug_dump_dramcfg_low(pvt, pvt->dclr0, 0); 7362da11654SDoug Thompson 737956b9ba1SJoe Perches edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare); 7382da11654SDoug Thompson 739956b9ba1SJoe Perches edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n", 740bc21fa57SBorislav Petkov pvt->dhar, dhar_base(pvt), 741a4b4bedcSBorislav Petkov (pvt->fam == 0xf) ? k8_dhar_offset(pvt) 742bc21fa57SBorislav Petkov : f10_dhar_offset(pvt)); 7432da11654SDoug Thompson 744956b9ba1SJoe Perches edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no"); 7452da11654SDoug Thompson 746d1ea71cdSBorislav Petkov debug_display_dimm_sizes(pvt, 0); 7474d796364SBorislav Petkov 7484d796364SBorislav Petkov /* everything below this point is Fam10h and above */ 749a4b4bedcSBorislav Petkov if (pvt->fam == 0xf) 7502da11654SDoug Thompson return; 7514d796364SBorislav Petkov 752d1ea71cdSBorislav Petkov debug_display_dimm_sizes(pvt, 1); 7532da11654SDoug Thompson 754a3b7db09SBorislav Petkov amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4")); 755ad6a32e9SBorislav Petkov 7568de1d91eSBorislav Petkov /* Only if NOT ganged does dclr1 have valid info */ 75768798e17SBorislav Petkov if (!dct_ganging_enabled(pvt)) 758d1ea71cdSBorislav Petkov debug_dump_dramcfg_low(pvt, pvt->dclr1, 1); 7592da11654SDoug Thompson } 7602da11654SDoug Thompson 76194be4bffSDoug Thompson /* 76218b94f66SAravind Gopalakrishnan * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60] 76394be4bffSDoug Thompson */ 76411c75eadSBorislav Petkov static void prep_chip_selects(struct amd64_pvt *pvt) 76594be4bffSDoug Thompson { 76618b94f66SAravind Gopalakrishnan if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) { 76711c75eadSBorislav Petkov pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8; 76811c75eadSBorislav Petkov pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8; 769a597d2a5SAravind Gopalakrishnan } else if (pvt->fam == 0x15 && pvt->model == 0x30) { 77018b94f66SAravind Gopalakrishnan pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4; 77118b94f66SAravind Gopalakrishnan pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2; 7729d858bb1SBorislav Petkov } else { 77311c75eadSBorislav Petkov pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8; 77411c75eadSBorislav Petkov pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4; 7759d858bb1SBorislav Petkov } 77694be4bffSDoug Thompson } 77794be4bffSDoug Thompson 77894be4bffSDoug Thompson /* 77911c75eadSBorislav Petkov * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers 78094be4bffSDoug Thompson */ 781b2b0c605SBorislav Petkov static void read_dct_base_mask(struct amd64_pvt *pvt) 78294be4bffSDoug Thompson { 78311c75eadSBorislav Petkov int cs; 78494be4bffSDoug Thompson 78511c75eadSBorislav Petkov prep_chip_selects(pvt); 78694be4bffSDoug Thompson 78711c75eadSBorislav Petkov for_each_chip_select(cs, 0, pvt) { 78871d2a32eSBorislav Petkov int reg0 = DCSB0 + (cs * 4); 78971d2a32eSBorislav Petkov int reg1 = DCSB1 + (cs * 4); 79011c75eadSBorislav Petkov u32 *base0 = &pvt->csels[0].csbases[cs]; 79111c75eadSBorislav Petkov u32 *base1 = &pvt->csels[1].csbases[cs]; 792b2b0c605SBorislav Petkov 7937981a28fSAravind Gopalakrishnan if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0)) 794956b9ba1SJoe Perches edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n", 79511c75eadSBorislav Petkov cs, *base0, reg0); 79694be4bffSDoug Thompson 7977981a28fSAravind Gopalakrishnan if (pvt->fam == 0xf) 79811c75eadSBorislav Petkov continue; 799b2b0c605SBorislav Petkov 8007981a28fSAravind Gopalakrishnan if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1)) 801956b9ba1SJoe Perches edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n", 8027981a28fSAravind Gopalakrishnan cs, *base1, (pvt->fam == 0x10) ? reg1 8037981a28fSAravind Gopalakrishnan : reg0); 80494be4bffSDoug Thompson } 80594be4bffSDoug Thompson 80611c75eadSBorislav Petkov for_each_chip_select_mask(cs, 0, pvt) { 80771d2a32eSBorislav Petkov int reg0 = DCSM0 + (cs * 4); 80871d2a32eSBorislav Petkov int reg1 = DCSM1 + (cs * 4); 80911c75eadSBorislav Petkov u32 *mask0 = &pvt->csels[0].csmasks[cs]; 81011c75eadSBorislav Petkov u32 *mask1 = &pvt->csels[1].csmasks[cs]; 811b2b0c605SBorislav Petkov 8127981a28fSAravind Gopalakrishnan if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0)) 813956b9ba1SJoe Perches edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n", 81411c75eadSBorislav Petkov cs, *mask0, reg0); 81594be4bffSDoug Thompson 8167981a28fSAravind Gopalakrishnan if (pvt->fam == 0xf) 81711c75eadSBorislav Petkov continue; 818b2b0c605SBorislav Petkov 8197981a28fSAravind Gopalakrishnan if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1)) 820956b9ba1SJoe Perches edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n", 8217981a28fSAravind Gopalakrishnan cs, *mask1, (pvt->fam == 0x10) ? reg1 8227981a28fSAravind Gopalakrishnan : reg0); 82394be4bffSDoug Thompson } 8246ba5dcdcSBorislav Petkov } 82594be4bffSDoug Thompson 826a597d2a5SAravind Gopalakrishnan static void determine_memory_type(struct amd64_pvt *pvt) 82794be4bffSDoug Thompson { 828a597d2a5SAravind Gopalakrishnan u32 dram_ctrl, dcsm; 82994be4bffSDoug Thompson 830a597d2a5SAravind Gopalakrishnan switch (pvt->fam) { 831a597d2a5SAravind Gopalakrishnan case 0xf: 832a597d2a5SAravind Gopalakrishnan if (pvt->ext_model >= K8_REV_F) 833a597d2a5SAravind Gopalakrishnan goto ddr3; 834a597d2a5SAravind Gopalakrishnan 835a597d2a5SAravind Gopalakrishnan pvt->dram_type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR; 836a597d2a5SAravind Gopalakrishnan return; 837a597d2a5SAravind Gopalakrishnan 838a597d2a5SAravind Gopalakrishnan case 0x10: 8396b4c0bdeSBorislav Petkov if (pvt->dchr0 & DDR3_MODE) 840a597d2a5SAravind Gopalakrishnan goto ddr3; 841a597d2a5SAravind Gopalakrishnan 842a597d2a5SAravind Gopalakrishnan pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2; 843a597d2a5SAravind Gopalakrishnan return; 844a597d2a5SAravind Gopalakrishnan 845a597d2a5SAravind Gopalakrishnan case 0x15: 846a597d2a5SAravind Gopalakrishnan if (pvt->model < 0x60) 847a597d2a5SAravind Gopalakrishnan goto ddr3; 848a597d2a5SAravind Gopalakrishnan 849a597d2a5SAravind Gopalakrishnan /* 850a597d2a5SAravind Gopalakrishnan * Model 0x60h needs special handling: 851a597d2a5SAravind Gopalakrishnan * 852a597d2a5SAravind Gopalakrishnan * We use a Chip Select value of '0' to obtain dcsm. 853a597d2a5SAravind Gopalakrishnan * Theoretically, it is possible to populate LRDIMMs of different 854a597d2a5SAravind Gopalakrishnan * 'Rank' value on a DCT. But this is not the common case. So, 855a597d2a5SAravind Gopalakrishnan * it's reasonable to assume all DIMMs are going to be of same 856a597d2a5SAravind Gopalakrishnan * 'type' until proven otherwise. 857a597d2a5SAravind Gopalakrishnan */ 858a597d2a5SAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 0, DRAM_CONTROL, &dram_ctrl); 859a597d2a5SAravind Gopalakrishnan dcsm = pvt->csels[0].csmasks[0]; 860a597d2a5SAravind Gopalakrishnan 861a597d2a5SAravind Gopalakrishnan if (((dram_ctrl >> 8) & 0x7) == 0x2) 862a597d2a5SAravind Gopalakrishnan pvt->dram_type = MEM_DDR4; 863a597d2a5SAravind Gopalakrishnan else if (pvt->dclr0 & BIT(16)) 864a597d2a5SAravind Gopalakrishnan pvt->dram_type = MEM_DDR3; 865a597d2a5SAravind Gopalakrishnan else if (dcsm & 0x3) 866a597d2a5SAravind Gopalakrishnan pvt->dram_type = MEM_LRDDR3; 8676b4c0bdeSBorislav Petkov else 868a597d2a5SAravind Gopalakrishnan pvt->dram_type = MEM_RDDR3; 869a597d2a5SAravind Gopalakrishnan 870a597d2a5SAravind Gopalakrishnan return; 871a597d2a5SAravind Gopalakrishnan 872a597d2a5SAravind Gopalakrishnan case 0x16: 873a597d2a5SAravind Gopalakrishnan goto ddr3; 874a597d2a5SAravind Gopalakrishnan 875a597d2a5SAravind Gopalakrishnan default: 876a597d2a5SAravind Gopalakrishnan WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam); 877a597d2a5SAravind Gopalakrishnan pvt->dram_type = MEM_EMPTY; 87894be4bffSDoug Thompson } 879a597d2a5SAravind Gopalakrishnan return; 88094be4bffSDoug Thompson 881a597d2a5SAravind Gopalakrishnan ddr3: 882a597d2a5SAravind Gopalakrishnan pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3; 88394be4bffSDoug Thompson } 88494be4bffSDoug Thompson 885cb328507SBorislav Petkov /* Get the number of DCT channels the memory controller is using. */ 886ddff876dSDoug Thompson static int k8_early_channel_count(struct amd64_pvt *pvt) 887ddff876dSDoug Thompson { 888cb328507SBorislav Petkov int flag; 889ddff876dSDoug Thompson 8909f56da0eSBorislav Petkov if (pvt->ext_model >= K8_REV_F) 891ddff876dSDoug Thompson /* RevF (NPT) and later */ 89241d8bfabSBorislav Petkov flag = pvt->dclr0 & WIDTH_128; 8939f56da0eSBorislav Petkov else 894ddff876dSDoug Thompson /* RevE and earlier */ 895ddff876dSDoug Thompson flag = pvt->dclr0 & REVE_WIDTH_128; 896ddff876dSDoug Thompson 897ddff876dSDoug Thompson /* not used */ 898ddff876dSDoug Thompson pvt->dclr1 = 0; 899ddff876dSDoug Thompson 900ddff876dSDoug Thompson return (flag) ? 2 : 1; 901ddff876dSDoug Thompson } 902ddff876dSDoug Thompson 90370046624SBorislav Petkov /* On F10h and later ErrAddr is MC4_ADDR[47:1] */ 904a4b4bedcSBorislav Petkov static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m) 905ddff876dSDoug Thompson { 906c1ae6830SBorislav Petkov u64 addr; 90770046624SBorislav Petkov u8 start_bit = 1; 90870046624SBorislav Petkov u8 end_bit = 47; 90970046624SBorislav Petkov 910a4b4bedcSBorislav Petkov if (pvt->fam == 0xf) { 91170046624SBorislav Petkov start_bit = 3; 91270046624SBorislav Petkov end_bit = 39; 91370046624SBorislav Petkov } 91470046624SBorislav Petkov 91510ef6b0dSChen, Gong addr = m->addr & GENMASK_ULL(end_bit, start_bit); 916c1ae6830SBorislav Petkov 917c1ae6830SBorislav Petkov /* 918c1ae6830SBorislav Petkov * Erratum 637 workaround 919c1ae6830SBorislav Petkov */ 920a4b4bedcSBorislav Petkov if (pvt->fam == 0x15) { 921c1ae6830SBorislav Petkov struct amd64_pvt *pvt; 922c1ae6830SBorislav Petkov u64 cc6_base, tmp_addr; 923c1ae6830SBorislav Petkov u32 tmp; 9248b84c8dfSDaniel J Blueman u16 mce_nid; 9258b84c8dfSDaniel J Blueman u8 intlv_en; 926c1ae6830SBorislav Petkov 92710ef6b0dSChen, Gong if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7) 928c1ae6830SBorislav Petkov return addr; 929c1ae6830SBorislav Petkov 930c1ae6830SBorislav Petkov mce_nid = amd_get_nb_id(m->extcpu); 931c1ae6830SBorislav Petkov pvt = mcis[mce_nid]->pvt_info; 932c1ae6830SBorislav Petkov 933c1ae6830SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp); 934c1ae6830SBorislav Petkov intlv_en = tmp >> 21 & 0x7; 935c1ae6830SBorislav Petkov 936c1ae6830SBorislav Petkov /* add [47:27] + 3 trailing bits */ 93710ef6b0dSChen, Gong cc6_base = (tmp & GENMASK_ULL(20, 0)) << 3; 938c1ae6830SBorislav Petkov 939c1ae6830SBorislav Petkov /* reverse and add DramIntlvEn */ 940c1ae6830SBorislav Petkov cc6_base |= intlv_en ^ 0x7; 941c1ae6830SBorislav Petkov 942c1ae6830SBorislav Petkov /* pin at [47:24] */ 943c1ae6830SBorislav Petkov cc6_base <<= 24; 944c1ae6830SBorislav Petkov 945c1ae6830SBorislav Petkov if (!intlv_en) 94610ef6b0dSChen, Gong return cc6_base | (addr & GENMASK_ULL(23, 0)); 947c1ae6830SBorislav Petkov 948c1ae6830SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp); 949c1ae6830SBorislav Petkov 950c1ae6830SBorislav Petkov /* faster log2 */ 95110ef6b0dSChen, Gong tmp_addr = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1); 952c1ae6830SBorislav Petkov 953c1ae6830SBorislav Petkov /* OR DramIntlvSel into bits [14:12] */ 95410ef6b0dSChen, Gong tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9; 955c1ae6830SBorislav Petkov 956c1ae6830SBorislav Petkov /* add remaining [11:0] bits from original MC4_ADDR */ 95710ef6b0dSChen, Gong tmp_addr |= addr & GENMASK_ULL(11, 0); 958c1ae6830SBorislav Petkov 959c1ae6830SBorislav Petkov return cc6_base | tmp_addr; 960c1ae6830SBorislav Petkov } 961c1ae6830SBorislav Petkov 962c1ae6830SBorislav Petkov return addr; 963ddff876dSDoug Thompson } 964ddff876dSDoug Thompson 965e2c0bffeSDaniel J Blueman static struct pci_dev *pci_get_related_function(unsigned int vendor, 966e2c0bffeSDaniel J Blueman unsigned int device, 967e2c0bffeSDaniel J Blueman struct pci_dev *related) 968e2c0bffeSDaniel J Blueman { 969e2c0bffeSDaniel J Blueman struct pci_dev *dev = NULL; 970e2c0bffeSDaniel J Blueman 971e2c0bffeSDaniel J Blueman while ((dev = pci_get_device(vendor, device, dev))) { 972e2c0bffeSDaniel J Blueman if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) && 973e2c0bffeSDaniel J Blueman (dev->bus->number == related->bus->number) && 974e2c0bffeSDaniel J Blueman (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn))) 975e2c0bffeSDaniel J Blueman break; 976e2c0bffeSDaniel J Blueman } 977e2c0bffeSDaniel J Blueman 978e2c0bffeSDaniel J Blueman return dev; 979e2c0bffeSDaniel J Blueman } 980e2c0bffeSDaniel J Blueman 9817f19bf75SBorislav Petkov static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range) 982ddff876dSDoug Thompson { 983e2c0bffeSDaniel J Blueman struct amd_northbridge *nb; 98418b94f66SAravind Gopalakrishnan struct pci_dev *f1 = NULL; 98518b94f66SAravind Gopalakrishnan unsigned int pci_func; 98671d2a32eSBorislav Petkov int off = range << 3; 987e2c0bffeSDaniel J Blueman u32 llim; 988ddff876dSDoug Thompson 9897f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo); 9907f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo); 991ddff876dSDoug Thompson 99218b94f66SAravind Gopalakrishnan if (pvt->fam == 0xf) 9937f19bf75SBorislav Petkov return; 994ddff876dSDoug Thompson 9957f19bf75SBorislav Petkov if (!dram_rw(pvt, range)) 9967f19bf75SBorislav Petkov return; 997ddff876dSDoug Thompson 9987f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi); 9997f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi); 1000f08e457cSBorislav Petkov 1001e2c0bffeSDaniel J Blueman /* F15h: factor in CC6 save area by reading dst node's limit reg */ 100218b94f66SAravind Gopalakrishnan if (pvt->fam != 0x15) 1003e2c0bffeSDaniel J Blueman return; 1004f08e457cSBorislav Petkov 1005e2c0bffeSDaniel J Blueman nb = node_to_amd_nb(dram_dst_node(pvt, range)); 1006e2c0bffeSDaniel J Blueman if (WARN_ON(!nb)) 1007e2c0bffeSDaniel J Blueman return; 1008e2c0bffeSDaniel J Blueman 1009a597d2a5SAravind Gopalakrishnan if (pvt->model == 0x60) 1010a597d2a5SAravind Gopalakrishnan pci_func = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1; 1011a597d2a5SAravind Gopalakrishnan else if (pvt->model == 0x30) 1012a597d2a5SAravind Gopalakrishnan pci_func = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1; 1013a597d2a5SAravind Gopalakrishnan else 1014a597d2a5SAravind Gopalakrishnan pci_func = PCI_DEVICE_ID_AMD_15H_NB_F1; 101518b94f66SAravind Gopalakrishnan 101618b94f66SAravind Gopalakrishnan f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc); 1017f08e457cSBorislav Petkov if (WARN_ON(!f1)) 1018f08e457cSBorislav Petkov return; 1019f08e457cSBorislav Petkov 1020f08e457cSBorislav Petkov amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim); 1021f08e457cSBorislav Petkov 102210ef6b0dSChen, Gong pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0); 1023f08e457cSBorislav Petkov 1024f08e457cSBorislav Petkov /* {[39:27],111b} */ 1025f08e457cSBorislav Petkov pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16; 1026f08e457cSBorislav Petkov 102710ef6b0dSChen, Gong pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0); 1028f08e457cSBorislav Petkov 1029f08e457cSBorislav Petkov /* [47:40] */ 1030f08e457cSBorislav Petkov pvt->ranges[range].lim.hi |= llim >> 13; 1031f08e457cSBorislav Petkov 1032f08e457cSBorislav Petkov pci_dev_put(f1); 1033f08e457cSBorislav Petkov } 1034ddff876dSDoug Thompson 1035f192c7b1SBorislav Petkov static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr, 103633ca0643SBorislav Petkov struct err_info *err) 1037ddff876dSDoug Thompson { 1038f192c7b1SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 1039ddff876dSDoug Thompson 104033ca0643SBorislav Petkov error_address_to_page_and_offset(sys_addr, err); 1041ab5a503cSMauro Carvalho Chehab 1042ab5a503cSMauro Carvalho Chehab /* 1043ab5a503cSMauro Carvalho Chehab * Find out which node the error address belongs to. This may be 1044ab5a503cSMauro Carvalho Chehab * different from the node that detected the error. 1045ab5a503cSMauro Carvalho Chehab */ 104633ca0643SBorislav Petkov err->src_mci = find_mc_by_sys_addr(mci, sys_addr); 104733ca0643SBorislav Petkov if (!err->src_mci) { 1048ab5a503cSMauro Carvalho Chehab amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n", 1049ab5a503cSMauro Carvalho Chehab (unsigned long)sys_addr); 105033ca0643SBorislav Petkov err->err_code = ERR_NODE; 1051ab5a503cSMauro Carvalho Chehab return; 1052ab5a503cSMauro Carvalho Chehab } 1053ab5a503cSMauro Carvalho Chehab 1054ab5a503cSMauro Carvalho Chehab /* Now map the sys_addr to a CSROW */ 105533ca0643SBorislav Petkov err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr); 105633ca0643SBorislav Petkov if (err->csrow < 0) { 105733ca0643SBorislav Petkov err->err_code = ERR_CSROW; 1058ab5a503cSMauro Carvalho Chehab return; 1059ab5a503cSMauro Carvalho Chehab } 1060ab5a503cSMauro Carvalho Chehab 1061ddff876dSDoug Thompson /* CHIPKILL enabled */ 1062f192c7b1SBorislav Petkov if (pvt->nbcfg & NBCFG_CHIPKILL) { 106333ca0643SBorislav Petkov err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome); 106433ca0643SBorislav Petkov if (err->channel < 0) { 1065ddff876dSDoug Thompson /* 1066ddff876dSDoug Thompson * Syndrome didn't map, so we don't know which of the 1067ddff876dSDoug Thompson * 2 DIMMs is in error. So we need to ID 'both' of them 1068ddff876dSDoug Thompson * as suspect. 1069ddff876dSDoug Thompson */ 107033ca0643SBorislav Petkov amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - " 1071ab5a503cSMauro Carvalho Chehab "possible error reporting race\n", 107233ca0643SBorislav Petkov err->syndrome); 107333ca0643SBorislav Petkov err->err_code = ERR_CHANNEL; 1074ddff876dSDoug Thompson return; 1075ddff876dSDoug Thompson } 1076ddff876dSDoug Thompson } else { 1077ddff876dSDoug Thompson /* 1078ddff876dSDoug Thompson * non-chipkill ecc mode 1079ddff876dSDoug Thompson * 1080ddff876dSDoug Thompson * The k8 documentation is unclear about how to determine the 1081ddff876dSDoug Thompson * channel number when using non-chipkill memory. This method 1082ddff876dSDoug Thompson * was obtained from email communication with someone at AMD. 1083ddff876dSDoug Thompson * (Wish the email was placed in this comment - norsk) 1084ddff876dSDoug Thompson */ 108533ca0643SBorislav Petkov err->channel = ((sys_addr & BIT(3)) != 0); 1086ddff876dSDoug Thompson } 1087ddff876dSDoug Thompson } 1088ddff876dSDoug Thompson 108941d8bfabSBorislav Petkov static int ddr2_cs_size(unsigned i, bool dct_width) 1090ddff876dSDoug Thompson { 109141d8bfabSBorislav Petkov unsigned shift = 0; 1092ddff876dSDoug Thompson 109341d8bfabSBorislav Petkov if (i <= 2) 109441d8bfabSBorislav Petkov shift = i; 109541d8bfabSBorislav Petkov else if (!(i & 0x1)) 109641d8bfabSBorislav Petkov shift = i >> 1; 10971433eb99SBorislav Petkov else 109841d8bfabSBorislav Petkov shift = (i + 1) >> 1; 1099ddff876dSDoug Thompson 110041d8bfabSBorislav Petkov return 128 << (shift + !!dct_width); 110141d8bfabSBorislav Petkov } 110241d8bfabSBorislav Petkov 110341d8bfabSBorislav Petkov static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 1104a597d2a5SAravind Gopalakrishnan unsigned cs_mode, int cs_mask_nr) 110541d8bfabSBorislav Petkov { 110641d8bfabSBorislav Petkov u32 dclr = dct ? pvt->dclr1 : pvt->dclr0; 110741d8bfabSBorislav Petkov 110841d8bfabSBorislav Petkov if (pvt->ext_model >= K8_REV_F) { 110941d8bfabSBorislav Petkov WARN_ON(cs_mode > 11); 111041d8bfabSBorislav Petkov return ddr2_cs_size(cs_mode, dclr & WIDTH_128); 111141d8bfabSBorislav Petkov } 111241d8bfabSBorislav Petkov else if (pvt->ext_model >= K8_REV_D) { 111311b0a314SBorislav Petkov unsigned diff; 111441d8bfabSBorislav Petkov WARN_ON(cs_mode > 10); 111541d8bfabSBorislav Petkov 111611b0a314SBorislav Petkov /* 111711b0a314SBorislav Petkov * the below calculation, besides trying to win an obfuscated C 111811b0a314SBorislav Petkov * contest, maps cs_mode values to DIMM chip select sizes. The 111911b0a314SBorislav Petkov * mappings are: 112011b0a314SBorislav Petkov * 112111b0a314SBorislav Petkov * cs_mode CS size (mb) 112211b0a314SBorislav Petkov * ======= ============ 112311b0a314SBorislav Petkov * 0 32 112411b0a314SBorislav Petkov * 1 64 112511b0a314SBorislav Petkov * 2 128 112611b0a314SBorislav Petkov * 3 128 112711b0a314SBorislav Petkov * 4 256 112811b0a314SBorislav Petkov * 5 512 112911b0a314SBorislav Petkov * 6 256 113011b0a314SBorislav Petkov * 7 512 113111b0a314SBorislav Petkov * 8 1024 113211b0a314SBorislav Petkov * 9 1024 113311b0a314SBorislav Petkov * 10 2048 113411b0a314SBorislav Petkov * 113511b0a314SBorislav Petkov * Basically, it calculates a value with which to shift the 113611b0a314SBorislav Petkov * smallest CS size of 32MB. 113711b0a314SBorislav Petkov * 113811b0a314SBorislav Petkov * ddr[23]_cs_size have a similar purpose. 113911b0a314SBorislav Petkov */ 114011b0a314SBorislav Petkov diff = cs_mode/3 + (unsigned)(cs_mode > 5); 114111b0a314SBorislav Petkov 114211b0a314SBorislav Petkov return 32 << (cs_mode - diff); 114341d8bfabSBorislav Petkov } 114441d8bfabSBorislav Petkov else { 114541d8bfabSBorislav Petkov WARN_ON(cs_mode > 6); 114641d8bfabSBorislav Petkov return 32 << cs_mode; 114741d8bfabSBorislav Petkov } 1148ddff876dSDoug Thompson } 1149ddff876dSDoug Thompson 11501afd3c98SDoug Thompson /* 11511afd3c98SDoug Thompson * Get the number of DCT channels in use. 11521afd3c98SDoug Thompson * 11531afd3c98SDoug Thompson * Return: 11541afd3c98SDoug Thompson * number of Memory Channels in operation 11551afd3c98SDoug Thompson * Pass back: 11561afd3c98SDoug Thompson * contents of the DCL0_LOW register 11571afd3c98SDoug Thompson */ 11587d20d14dSBorislav Petkov static int f1x_early_channel_count(struct amd64_pvt *pvt) 11591afd3c98SDoug Thompson { 11606ba5dcdcSBorislav Petkov int i, j, channels = 0; 1161ddff876dSDoug Thompson 11627d20d14dSBorislav Petkov /* On F10h, if we are in 128 bit mode, then we are using 2 channels */ 1163a4b4bedcSBorislav Petkov if (pvt->fam == 0x10 && (pvt->dclr0 & WIDTH_128)) 11647d20d14dSBorislav Petkov return 2; 11651afd3c98SDoug Thompson 11661afd3c98SDoug Thompson /* 1167d16149e8SBorislav Petkov * Need to check if in unganged mode: In such, there are 2 channels, 1168d16149e8SBorislav Petkov * but they are not in 128 bit mode and thus the above 'dclr0' status 1169d16149e8SBorislav Petkov * bit will be OFF. 11701afd3c98SDoug Thompson * 11711afd3c98SDoug Thompson * Need to check DCT0[0] and DCT1[0] to see if only one of them has 11721afd3c98SDoug Thompson * their CSEnable bit on. If so, then SINGLE DIMM case. 11731afd3c98SDoug Thompson */ 1174956b9ba1SJoe Perches edac_dbg(0, "Data width is not 128 bits - need more decoding\n"); 11751afd3c98SDoug Thompson 11761afd3c98SDoug Thompson /* 11771afd3c98SDoug Thompson * Check DRAM Bank Address Mapping values for each DIMM to see if there 11781afd3c98SDoug Thompson * is more than just one DIMM present in unganged mode. Need to check 11791afd3c98SDoug Thompson * both controllers since DIMMs can be placed in either one. 11801afd3c98SDoug Thompson */ 1181525a1b20SBorislav Petkov for (i = 0; i < 2; i++) { 1182525a1b20SBorislav Petkov u32 dbam = (i ? pvt->dbam1 : pvt->dbam0); 11831afd3c98SDoug Thompson 118457a30854SWan Wei for (j = 0; j < 4; j++) { 118557a30854SWan Wei if (DBAM_DIMM(j, dbam) > 0) { 11861afd3c98SDoug Thompson channels++; 118757a30854SWan Wei break; 11881afd3c98SDoug Thompson } 118957a30854SWan Wei } 119057a30854SWan Wei } 11911afd3c98SDoug Thompson 1192d16149e8SBorislav Petkov if (channels > 2) 1193d16149e8SBorislav Petkov channels = 2; 1194d16149e8SBorislav Petkov 119524f9a7feSBorislav Petkov amd64_info("MCT channel count: %d\n", channels); 11961afd3c98SDoug Thompson 11971afd3c98SDoug Thompson return channels; 11981afd3c98SDoug Thompson } 11991afd3c98SDoug Thompson 120041d8bfabSBorislav Petkov static int ddr3_cs_size(unsigned i, bool dct_width) 12011afd3c98SDoug Thompson { 120241d8bfabSBorislav Petkov unsigned shift = 0; 120341d8bfabSBorislav Petkov int cs_size = 0; 120441d8bfabSBorislav Petkov 120541d8bfabSBorislav Petkov if (i == 0 || i == 3 || i == 4) 120641d8bfabSBorislav Petkov cs_size = -1; 120741d8bfabSBorislav Petkov else if (i <= 2) 120841d8bfabSBorislav Petkov shift = i; 120941d8bfabSBorislav Petkov else if (i == 12) 121041d8bfabSBorislav Petkov shift = 7; 121141d8bfabSBorislav Petkov else if (!(i & 0x1)) 121241d8bfabSBorislav Petkov shift = i >> 1; 121341d8bfabSBorislav Petkov else 121441d8bfabSBorislav Petkov shift = (i + 1) >> 1; 121541d8bfabSBorislav Petkov 121641d8bfabSBorislav Petkov if (cs_size != -1) 121741d8bfabSBorislav Petkov cs_size = (128 * (1 << !!dct_width)) << shift; 121841d8bfabSBorislav Petkov 121941d8bfabSBorislav Petkov return cs_size; 122041d8bfabSBorislav Petkov } 122141d8bfabSBorislav Petkov 1222a597d2a5SAravind Gopalakrishnan static int ddr3_lrdimm_cs_size(unsigned i, unsigned rank_multiply) 1223a597d2a5SAravind Gopalakrishnan { 1224a597d2a5SAravind Gopalakrishnan unsigned shift = 0; 1225a597d2a5SAravind Gopalakrishnan int cs_size = 0; 1226a597d2a5SAravind Gopalakrishnan 1227a597d2a5SAravind Gopalakrishnan if (i < 4 || i == 6) 1228a597d2a5SAravind Gopalakrishnan cs_size = -1; 1229a597d2a5SAravind Gopalakrishnan else if (i == 12) 1230a597d2a5SAravind Gopalakrishnan shift = 7; 1231a597d2a5SAravind Gopalakrishnan else if (!(i & 0x1)) 1232a597d2a5SAravind Gopalakrishnan shift = i >> 1; 1233a597d2a5SAravind Gopalakrishnan else 1234a597d2a5SAravind Gopalakrishnan shift = (i + 1) >> 1; 1235a597d2a5SAravind Gopalakrishnan 1236a597d2a5SAravind Gopalakrishnan if (cs_size != -1) 1237a597d2a5SAravind Gopalakrishnan cs_size = rank_multiply * (128 << shift); 1238a597d2a5SAravind Gopalakrishnan 1239a597d2a5SAravind Gopalakrishnan return cs_size; 1240a597d2a5SAravind Gopalakrishnan } 1241a597d2a5SAravind Gopalakrishnan 1242a597d2a5SAravind Gopalakrishnan static int ddr4_cs_size(unsigned i) 1243a597d2a5SAravind Gopalakrishnan { 1244a597d2a5SAravind Gopalakrishnan int cs_size = 0; 1245a597d2a5SAravind Gopalakrishnan 1246a597d2a5SAravind Gopalakrishnan if (i == 0) 1247a597d2a5SAravind Gopalakrishnan cs_size = -1; 1248a597d2a5SAravind Gopalakrishnan else if (i == 1) 1249a597d2a5SAravind Gopalakrishnan cs_size = 1024; 1250a597d2a5SAravind Gopalakrishnan else 1251a597d2a5SAravind Gopalakrishnan /* Min cs_size = 1G */ 1252a597d2a5SAravind Gopalakrishnan cs_size = 1024 * (1 << (i >> 1)); 1253a597d2a5SAravind Gopalakrishnan 1254a597d2a5SAravind Gopalakrishnan return cs_size; 1255a597d2a5SAravind Gopalakrishnan } 1256a597d2a5SAravind Gopalakrishnan 125741d8bfabSBorislav Petkov static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 1258a597d2a5SAravind Gopalakrishnan unsigned cs_mode, int cs_mask_nr) 125941d8bfabSBorislav Petkov { 126041d8bfabSBorislav Petkov u32 dclr = dct ? pvt->dclr1 : pvt->dclr0; 126141d8bfabSBorislav Petkov 126241d8bfabSBorislav Petkov WARN_ON(cs_mode > 11); 12631433eb99SBorislav Petkov 12641433eb99SBorislav Petkov if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE) 126541d8bfabSBorislav Petkov return ddr3_cs_size(cs_mode, dclr & WIDTH_128); 12661433eb99SBorislav Petkov else 126741d8bfabSBorislav Petkov return ddr2_cs_size(cs_mode, dclr & WIDTH_128); 126841d8bfabSBorislav Petkov } 12691433eb99SBorislav Petkov 127041d8bfabSBorislav Petkov /* 127141d8bfabSBorislav Petkov * F15h supports only 64bit DCT interfaces 127241d8bfabSBorislav Petkov */ 127341d8bfabSBorislav Petkov static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 1274a597d2a5SAravind Gopalakrishnan unsigned cs_mode, int cs_mask_nr) 127541d8bfabSBorislav Petkov { 127641d8bfabSBorislav Petkov WARN_ON(cs_mode > 12); 127741d8bfabSBorislav Petkov 127841d8bfabSBorislav Petkov return ddr3_cs_size(cs_mode, false); 12791afd3c98SDoug Thompson } 12801afd3c98SDoug Thompson 1281a597d2a5SAravind Gopalakrishnan /* F15h M60h supports DDR4 mapping as well.. */ 1282a597d2a5SAravind Gopalakrishnan static int f15_m60h_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 1283a597d2a5SAravind Gopalakrishnan unsigned cs_mode, int cs_mask_nr) 1284a597d2a5SAravind Gopalakrishnan { 1285a597d2a5SAravind Gopalakrishnan int cs_size; 1286a597d2a5SAravind Gopalakrishnan u32 dcsm = pvt->csels[dct].csmasks[cs_mask_nr]; 1287a597d2a5SAravind Gopalakrishnan 1288a597d2a5SAravind Gopalakrishnan WARN_ON(cs_mode > 12); 1289a597d2a5SAravind Gopalakrishnan 1290a597d2a5SAravind Gopalakrishnan if (pvt->dram_type == MEM_DDR4) { 1291a597d2a5SAravind Gopalakrishnan if (cs_mode > 9) 1292a597d2a5SAravind Gopalakrishnan return -1; 1293a597d2a5SAravind Gopalakrishnan 1294a597d2a5SAravind Gopalakrishnan cs_size = ddr4_cs_size(cs_mode); 1295a597d2a5SAravind Gopalakrishnan } else if (pvt->dram_type == MEM_LRDDR3) { 1296a597d2a5SAravind Gopalakrishnan unsigned rank_multiply = dcsm & 0xf; 1297a597d2a5SAravind Gopalakrishnan 1298a597d2a5SAravind Gopalakrishnan if (rank_multiply == 3) 1299a597d2a5SAravind Gopalakrishnan rank_multiply = 4; 1300a597d2a5SAravind Gopalakrishnan cs_size = ddr3_lrdimm_cs_size(cs_mode, rank_multiply); 1301a597d2a5SAravind Gopalakrishnan } else { 1302a597d2a5SAravind Gopalakrishnan /* Minimum cs size is 512mb for F15hM60h*/ 1303a597d2a5SAravind Gopalakrishnan if (cs_mode == 0x1) 1304a597d2a5SAravind Gopalakrishnan return -1; 1305a597d2a5SAravind Gopalakrishnan 1306a597d2a5SAravind Gopalakrishnan cs_size = ddr3_cs_size(cs_mode, false); 1307a597d2a5SAravind Gopalakrishnan } 1308a597d2a5SAravind Gopalakrishnan 1309a597d2a5SAravind Gopalakrishnan return cs_size; 1310a597d2a5SAravind Gopalakrishnan } 1311a597d2a5SAravind Gopalakrishnan 131294c1acf2SAravind Gopalakrishnan /* 131318b94f66SAravind Gopalakrishnan * F16h and F15h model 30h have only limited cs_modes. 131494c1acf2SAravind Gopalakrishnan */ 131594c1acf2SAravind Gopalakrishnan static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 1316a597d2a5SAravind Gopalakrishnan unsigned cs_mode, int cs_mask_nr) 131794c1acf2SAravind Gopalakrishnan { 131894c1acf2SAravind Gopalakrishnan WARN_ON(cs_mode > 12); 131994c1acf2SAravind Gopalakrishnan 132094c1acf2SAravind Gopalakrishnan if (cs_mode == 6 || cs_mode == 8 || 132194c1acf2SAravind Gopalakrishnan cs_mode == 9 || cs_mode == 12) 132294c1acf2SAravind Gopalakrishnan return -1; 132394c1acf2SAravind Gopalakrishnan else 132494c1acf2SAravind Gopalakrishnan return ddr3_cs_size(cs_mode, false); 132594c1acf2SAravind Gopalakrishnan } 132694c1acf2SAravind Gopalakrishnan 13275a5d2371SBorislav Petkov static void read_dram_ctl_register(struct amd64_pvt *pvt) 13286163b5d4SDoug Thompson { 13296163b5d4SDoug Thompson 1330a4b4bedcSBorislav Petkov if (pvt->fam == 0xf) 13315a5d2371SBorislav Petkov return; 13325a5d2371SBorislav Petkov 13337981a28fSAravind Gopalakrishnan if (!amd64_read_pci_cfg(pvt->F2, DCT_SEL_LO, &pvt->dct_sel_lo)) { 1334956b9ba1SJoe Perches edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n", 133578da121eSBorislav Petkov pvt->dct_sel_lo, dct_sel_baseaddr(pvt)); 13366163b5d4SDoug Thompson 1337956b9ba1SJoe Perches edac_dbg(0, " DCTs operate in %s mode\n", 13385a5d2371SBorislav Petkov (dct_ganging_enabled(pvt) ? "ganged" : "unganged")); 13396163b5d4SDoug Thompson 134072381bd5SBorislav Petkov if (!dct_ganging_enabled(pvt)) 1341956b9ba1SJoe Perches edac_dbg(0, " Address range split per DCT: %s\n", 134272381bd5SBorislav Petkov (dct_high_range_enabled(pvt) ? "yes" : "no")); 134372381bd5SBorislav Petkov 1344956b9ba1SJoe Perches edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n", 134572381bd5SBorislav Petkov (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"), 134672381bd5SBorislav Petkov (dct_memory_cleared(pvt) ? "yes" : "no")); 134772381bd5SBorislav Petkov 1348956b9ba1SJoe Perches edac_dbg(0, " channel interleave: %s, " 134978da121eSBorislav Petkov "interleave bits selector: 0x%x\n", 135072381bd5SBorislav Petkov (dct_interleave_enabled(pvt) ? "enabled" : "disabled"), 13516163b5d4SDoug Thompson dct_sel_interleave_addr(pvt)); 13526163b5d4SDoug Thompson } 13536163b5d4SDoug Thompson 13547981a28fSAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F2, DCT_SEL_HI, &pvt->dct_sel_hi); 13556163b5d4SDoug Thompson } 13566163b5d4SDoug Thompson 1357f71d0a05SDoug Thompson /* 135818b94f66SAravind Gopalakrishnan * Determine channel (DCT) based on the interleaving mode (see F15h M30h BKDG, 135918b94f66SAravind Gopalakrishnan * 2.10.12 Memory Interleaving Modes). 136018b94f66SAravind Gopalakrishnan */ 136118b94f66SAravind Gopalakrishnan static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr, 136218b94f66SAravind Gopalakrishnan u8 intlv_en, int num_dcts_intlv, 136318b94f66SAravind Gopalakrishnan u32 dct_sel) 136418b94f66SAravind Gopalakrishnan { 136518b94f66SAravind Gopalakrishnan u8 channel = 0; 136618b94f66SAravind Gopalakrishnan u8 select; 136718b94f66SAravind Gopalakrishnan 136818b94f66SAravind Gopalakrishnan if (!(intlv_en)) 136918b94f66SAravind Gopalakrishnan return (u8)(dct_sel); 137018b94f66SAravind Gopalakrishnan 137118b94f66SAravind Gopalakrishnan if (num_dcts_intlv == 2) { 137218b94f66SAravind Gopalakrishnan select = (sys_addr >> 8) & 0x3; 137318b94f66SAravind Gopalakrishnan channel = select ? 0x3 : 0; 13749d0e8d83SAravind Gopalakrishnan } else if (num_dcts_intlv == 4) { 13759d0e8d83SAravind Gopalakrishnan u8 intlv_addr = dct_sel_interleave_addr(pvt); 13769d0e8d83SAravind Gopalakrishnan switch (intlv_addr) { 13779d0e8d83SAravind Gopalakrishnan case 0x4: 13789d0e8d83SAravind Gopalakrishnan channel = (sys_addr >> 8) & 0x3; 13799d0e8d83SAravind Gopalakrishnan break; 13809d0e8d83SAravind Gopalakrishnan case 0x5: 13819d0e8d83SAravind Gopalakrishnan channel = (sys_addr >> 9) & 0x3; 13829d0e8d83SAravind Gopalakrishnan break; 13839d0e8d83SAravind Gopalakrishnan } 13849d0e8d83SAravind Gopalakrishnan } 138518b94f66SAravind Gopalakrishnan return channel; 138618b94f66SAravind Gopalakrishnan } 138718b94f66SAravind Gopalakrishnan 138818b94f66SAravind Gopalakrishnan /* 1389229a7a11SBorislav Petkov * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory 1390f71d0a05SDoug Thompson * Interleaving Modes. 1391f71d0a05SDoug Thompson */ 1392b15f0fcaSBorislav Petkov static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr, 1393229a7a11SBorislav Petkov bool hi_range_sel, u8 intlv_en) 13946163b5d4SDoug Thompson { 1395151fa71cSBorislav Petkov u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1; 13966163b5d4SDoug Thompson 13976163b5d4SDoug Thompson if (dct_ganging_enabled(pvt)) 1398229a7a11SBorislav Petkov return 0; 1399229a7a11SBorislav Petkov 1400229a7a11SBorislav Petkov if (hi_range_sel) 1401229a7a11SBorislav Petkov return dct_sel_high; 1402229a7a11SBorislav Petkov 1403f71d0a05SDoug Thompson /* 1404f71d0a05SDoug Thompson * see F2x110[DctSelIntLvAddr] - channel interleave mode 1405f71d0a05SDoug Thompson */ 1406229a7a11SBorislav Petkov if (dct_interleave_enabled(pvt)) { 1407229a7a11SBorislav Petkov u8 intlv_addr = dct_sel_interleave_addr(pvt); 14086163b5d4SDoug Thompson 1409229a7a11SBorislav Petkov /* return DCT select function: 0=DCT0, 1=DCT1 */ 1410229a7a11SBorislav Petkov if (!intlv_addr) 1411229a7a11SBorislav Petkov return sys_addr >> 6 & 1; 14126163b5d4SDoug Thompson 1413229a7a11SBorislav Petkov if (intlv_addr & 0x2) { 1414229a7a11SBorislav Petkov u8 shift = intlv_addr & 0x1 ? 9 : 6; 1415229a7a11SBorislav Petkov u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2; 1416229a7a11SBorislav Petkov 1417229a7a11SBorislav Petkov return ((sys_addr >> shift) & 1) ^ temp; 14186163b5d4SDoug Thompson } 14196163b5d4SDoug Thompson 1420229a7a11SBorislav Petkov return (sys_addr >> (12 + hweight8(intlv_en))) & 1; 1421229a7a11SBorislav Petkov } 1422229a7a11SBorislav Petkov 1423229a7a11SBorislav Petkov if (dct_high_range_enabled(pvt)) 1424229a7a11SBorislav Petkov return ~dct_sel_high & 1; 14256163b5d4SDoug Thompson 14266163b5d4SDoug Thompson return 0; 14276163b5d4SDoug Thompson } 14286163b5d4SDoug Thompson 1429c8e518d5SBorislav Petkov /* Convert the sys_addr to the normalized DCT address */ 1430c7e5301aSDaniel J Blueman static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range, 1431c8e518d5SBorislav Petkov u64 sys_addr, bool hi_rng, 1432c8e518d5SBorislav Petkov u32 dct_sel_base_addr) 14336163b5d4SDoug Thompson { 14346163b5d4SDoug Thompson u64 chan_off; 1435c8e518d5SBorislav Petkov u64 dram_base = get_dram_base(pvt, range); 1436c8e518d5SBorislav Petkov u64 hole_off = f10_dhar_offset(pvt); 1437c8e518d5SBorislav Petkov u64 dct_sel_base_off = (pvt->dct_sel_hi & 0xFFFFFC00) << 16; 14386163b5d4SDoug Thompson 1439c8e518d5SBorislav Petkov if (hi_rng) { 1440c8e518d5SBorislav Petkov /* 1441c8e518d5SBorislav Petkov * if 1442c8e518d5SBorislav Petkov * base address of high range is below 4Gb 1443c8e518d5SBorislav Petkov * (bits [47:27] at [31:11]) 1444c8e518d5SBorislav Petkov * DRAM address space on this DCT is hoisted above 4Gb && 1445c8e518d5SBorislav Petkov * sys_addr > 4Gb 1446c8e518d5SBorislav Petkov * 1447c8e518d5SBorislav Petkov * remove hole offset from sys_addr 1448c8e518d5SBorislav Petkov * else 1449c8e518d5SBorislav Petkov * remove high range offset from sys_addr 1450c8e518d5SBorislav Petkov */ 1451c8e518d5SBorislav Petkov if ((!(dct_sel_base_addr >> 16) || 1452c8e518d5SBorislav Petkov dct_sel_base_addr < dhar_base(pvt)) && 1453972ea17aSBorislav Petkov dhar_valid(pvt) && 1454c8e518d5SBorislav Petkov (sys_addr >= BIT_64(32))) 1455bc21fa57SBorislav Petkov chan_off = hole_off; 14566163b5d4SDoug Thompson else 14576163b5d4SDoug Thompson chan_off = dct_sel_base_off; 14586163b5d4SDoug Thompson } else { 1459c8e518d5SBorislav Petkov /* 1460c8e518d5SBorislav Petkov * if 1461c8e518d5SBorislav Petkov * we have a valid hole && 1462c8e518d5SBorislav Petkov * sys_addr > 4Gb 1463c8e518d5SBorislav Petkov * 1464c8e518d5SBorislav Petkov * remove hole 1465c8e518d5SBorislav Petkov * else 1466c8e518d5SBorislav Petkov * remove dram base to normalize to DCT address 1467c8e518d5SBorislav Petkov */ 1468972ea17aSBorislav Petkov if (dhar_valid(pvt) && (sys_addr >= BIT_64(32))) 1469bc21fa57SBorislav Petkov chan_off = hole_off; 14706163b5d4SDoug Thompson else 1471c8e518d5SBorislav Petkov chan_off = dram_base; 14726163b5d4SDoug Thompson } 14736163b5d4SDoug Thompson 147410ef6b0dSChen, Gong return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23)); 14756163b5d4SDoug Thompson } 14766163b5d4SDoug Thompson 14776163b5d4SDoug Thompson /* 14786163b5d4SDoug Thompson * checks if the csrow passed in is marked as SPARED, if so returns the new 14796163b5d4SDoug Thompson * spare row 14806163b5d4SDoug Thompson */ 148111c75eadSBorislav Petkov static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow) 14826163b5d4SDoug Thompson { 1483614ec9d8SBorislav Petkov int tmp_cs; 14846163b5d4SDoug Thompson 1485614ec9d8SBorislav Petkov if (online_spare_swap_done(pvt, dct) && 1486614ec9d8SBorislav Petkov csrow == online_spare_bad_dramcs(pvt, dct)) { 1487614ec9d8SBorislav Petkov 1488614ec9d8SBorislav Petkov for_each_chip_select(tmp_cs, dct, pvt) { 1489614ec9d8SBorislav Petkov if (chip_select_base(tmp_cs, dct, pvt) & 0x2) { 1490614ec9d8SBorislav Petkov csrow = tmp_cs; 1491614ec9d8SBorislav Petkov break; 1492614ec9d8SBorislav Petkov } 1493614ec9d8SBorislav Petkov } 14946163b5d4SDoug Thompson } 14956163b5d4SDoug Thompson return csrow; 14966163b5d4SDoug Thompson } 14976163b5d4SDoug Thompson 14986163b5d4SDoug Thompson /* 14996163b5d4SDoug Thompson * Iterate over the DRAM DCT "base" and "mask" registers looking for a 15006163b5d4SDoug Thompson * SystemAddr match on the specified 'ChannelSelect' and 'NodeID' 15016163b5d4SDoug Thompson * 15026163b5d4SDoug Thompson * Return: 15036163b5d4SDoug Thompson * -EINVAL: NOT FOUND 15046163b5d4SDoug Thompson * 0..csrow = Chip-Select Row 15056163b5d4SDoug Thompson */ 1506c7e5301aSDaniel J Blueman static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct) 15076163b5d4SDoug Thompson { 15086163b5d4SDoug Thompson struct mem_ctl_info *mci; 15096163b5d4SDoug Thompson struct amd64_pvt *pvt; 151011c75eadSBorislav Petkov u64 cs_base, cs_mask; 15116163b5d4SDoug Thompson int cs_found = -EINVAL; 15126163b5d4SDoug Thompson int csrow; 15136163b5d4SDoug Thompson 1514cc4d8860SBorislav Petkov mci = mcis[nid]; 15156163b5d4SDoug Thompson if (!mci) 15166163b5d4SDoug Thompson return cs_found; 15176163b5d4SDoug Thompson 15186163b5d4SDoug Thompson pvt = mci->pvt_info; 15196163b5d4SDoug Thompson 1520956b9ba1SJoe Perches edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct); 15216163b5d4SDoug Thompson 152211c75eadSBorislav Petkov for_each_chip_select(csrow, dct, pvt) { 152311c75eadSBorislav Petkov if (!csrow_enabled(csrow, dct, pvt)) 15246163b5d4SDoug Thompson continue; 15256163b5d4SDoug Thompson 152611c75eadSBorislav Petkov get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask); 15276163b5d4SDoug Thompson 1528956b9ba1SJoe Perches edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n", 15296163b5d4SDoug Thompson csrow, cs_base, cs_mask); 15306163b5d4SDoug Thompson 153111c75eadSBorislav Petkov cs_mask = ~cs_mask; 15326163b5d4SDoug Thompson 1533956b9ba1SJoe Perches edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n", 153411c75eadSBorislav Petkov (in_addr & cs_mask), (cs_base & cs_mask)); 15356163b5d4SDoug Thompson 153611c75eadSBorislav Petkov if ((in_addr & cs_mask) == (cs_base & cs_mask)) { 153718b94f66SAravind Gopalakrishnan if (pvt->fam == 0x15 && pvt->model >= 0x30) { 153818b94f66SAravind Gopalakrishnan cs_found = csrow; 153918b94f66SAravind Gopalakrishnan break; 154018b94f66SAravind Gopalakrishnan } 154111c75eadSBorislav Petkov cs_found = f10_process_possible_spare(pvt, dct, csrow); 15426163b5d4SDoug Thompson 1543956b9ba1SJoe Perches edac_dbg(1, " MATCH csrow=%d\n", cs_found); 15446163b5d4SDoug Thompson break; 15456163b5d4SDoug Thompson } 15466163b5d4SDoug Thompson } 15476163b5d4SDoug Thompson return cs_found; 15486163b5d4SDoug Thompson } 15496163b5d4SDoug Thompson 155095b0ef55SBorislav Petkov /* 155195b0ef55SBorislav Petkov * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is 155295b0ef55SBorislav Petkov * swapped with a region located at the bottom of memory so that the GPU can use 155395b0ef55SBorislav Petkov * the interleaved region and thus two channels. 155495b0ef55SBorislav Petkov */ 1555b15f0fcaSBorislav Petkov static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr) 155695b0ef55SBorislav Petkov { 155795b0ef55SBorislav Petkov u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr; 155895b0ef55SBorislav Petkov 1559a4b4bedcSBorislav Petkov if (pvt->fam == 0x10) { 156095b0ef55SBorislav Petkov /* only revC3 and revE have that feature */ 1561a4b4bedcSBorislav Petkov if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3)) 156295b0ef55SBorislav Petkov return sys_addr; 156395b0ef55SBorislav Petkov } 156495b0ef55SBorislav Petkov 15657981a28fSAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F2, SWAP_INTLV_REG, &swap_reg); 156695b0ef55SBorislav Petkov 156795b0ef55SBorislav Petkov if (!(swap_reg & 0x1)) 156895b0ef55SBorislav Petkov return sys_addr; 156995b0ef55SBorislav Petkov 157095b0ef55SBorislav Petkov swap_base = (swap_reg >> 3) & 0x7f; 157195b0ef55SBorislav Petkov swap_limit = (swap_reg >> 11) & 0x7f; 157295b0ef55SBorislav Petkov rgn_size = (swap_reg >> 20) & 0x7f; 157395b0ef55SBorislav Petkov tmp_addr = sys_addr >> 27; 157495b0ef55SBorislav Petkov 157595b0ef55SBorislav Petkov if (!(sys_addr >> 34) && 157695b0ef55SBorislav Petkov (((tmp_addr >= swap_base) && 157795b0ef55SBorislav Petkov (tmp_addr <= swap_limit)) || 157895b0ef55SBorislav Petkov (tmp_addr < rgn_size))) 157995b0ef55SBorislav Petkov return sys_addr ^ (u64)swap_base << 27; 158095b0ef55SBorislav Petkov 158195b0ef55SBorislav Petkov return sys_addr; 158295b0ef55SBorislav Petkov } 158395b0ef55SBorislav Petkov 1584f71d0a05SDoug Thompson /* For a given @dram_range, check if @sys_addr falls within it. */ 1585e761359aSBorislav Petkov static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range, 158633ca0643SBorislav Petkov u64 sys_addr, int *chan_sel) 1587f71d0a05SDoug Thompson { 1588229a7a11SBorislav Petkov int cs_found = -EINVAL; 1589c8e518d5SBorislav Petkov u64 chan_addr; 15905d4b58e8SBorislav Petkov u32 dct_sel_base; 159111c75eadSBorislav Petkov u8 channel; 1592229a7a11SBorislav Petkov bool high_range = false; 1593f71d0a05SDoug Thompson 15947f19bf75SBorislav Petkov u8 node_id = dram_dst_node(pvt, range); 1595229a7a11SBorislav Petkov u8 intlv_en = dram_intlv_en(pvt, range); 15967f19bf75SBorislav Petkov u32 intlv_sel = dram_intlv_sel(pvt, range); 1597f71d0a05SDoug Thompson 1598956b9ba1SJoe Perches edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n", 1599c8e518d5SBorislav Petkov range, sys_addr, get_dram_limit(pvt, range)); 1600f71d0a05SDoug Thompson 1601355fba60SBorislav Petkov if (dhar_valid(pvt) && 1602355fba60SBorislav Petkov dhar_base(pvt) <= sys_addr && 1603355fba60SBorislav Petkov sys_addr < BIT_64(32)) { 1604355fba60SBorislav Petkov amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n", 1605355fba60SBorislav Petkov sys_addr); 1606f71d0a05SDoug Thompson return -EINVAL; 1607355fba60SBorislav Petkov } 1608355fba60SBorislav Petkov 1609f030ddfbSBorislav Petkov if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en))) 1610355fba60SBorislav Petkov return -EINVAL; 1611f71d0a05SDoug Thompson 1612b15f0fcaSBorislav Petkov sys_addr = f1x_swap_interleaved_region(pvt, sys_addr); 161395b0ef55SBorislav Petkov 1614f71d0a05SDoug Thompson dct_sel_base = dct_sel_baseaddr(pvt); 1615f71d0a05SDoug Thompson 1616f71d0a05SDoug Thompson /* 1617f71d0a05SDoug Thompson * check whether addresses >= DctSelBaseAddr[47:27] are to be used to 1618f71d0a05SDoug Thompson * select between DCT0 and DCT1. 1619f71d0a05SDoug Thompson */ 1620f71d0a05SDoug Thompson if (dct_high_range_enabled(pvt) && 1621f71d0a05SDoug Thompson !dct_ganging_enabled(pvt) && 1622f71d0a05SDoug Thompson ((sys_addr >> 27) >= (dct_sel_base >> 11))) 1623229a7a11SBorislav Petkov high_range = true; 1624f71d0a05SDoug Thompson 1625b15f0fcaSBorislav Petkov channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en); 1626f71d0a05SDoug Thompson 1627b15f0fcaSBorislav Petkov chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr, 1628c8e518d5SBorislav Petkov high_range, dct_sel_base); 1629f71d0a05SDoug Thompson 1630e2f79dbdSBorislav Petkov /* Remove node interleaving, see F1x120 */ 1631e2f79dbdSBorislav Petkov if (intlv_en) 1632e2f79dbdSBorislav Petkov chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) | 1633e2f79dbdSBorislav Petkov (chan_addr & 0xfff); 1634f71d0a05SDoug Thompson 16355d4b58e8SBorislav Petkov /* remove channel interleave */ 1636f71d0a05SDoug Thompson if (dct_interleave_enabled(pvt) && 1637f71d0a05SDoug Thompson !dct_high_range_enabled(pvt) && 1638f71d0a05SDoug Thompson !dct_ganging_enabled(pvt)) { 16395d4b58e8SBorislav Petkov 16405d4b58e8SBorislav Petkov if (dct_sel_interleave_addr(pvt) != 1) { 16415d4b58e8SBorislav Petkov if (dct_sel_interleave_addr(pvt) == 0x3) 16425d4b58e8SBorislav Petkov /* hash 9 */ 16435d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 10) << 9) | 16445d4b58e8SBorislav Petkov (chan_addr & 0x1ff); 16455d4b58e8SBorislav Petkov else 16465d4b58e8SBorislav Petkov /* A[6] or hash 6 */ 16475d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 7) << 6) | 16485d4b58e8SBorislav Petkov (chan_addr & 0x3f); 16495d4b58e8SBorislav Petkov } else 16505d4b58e8SBorislav Petkov /* A[12] */ 16515d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 13) << 12) | 16525d4b58e8SBorislav Petkov (chan_addr & 0xfff); 1653f71d0a05SDoug Thompson } 1654f71d0a05SDoug Thompson 1655956b9ba1SJoe Perches edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr); 1656f71d0a05SDoug Thompson 1657b15f0fcaSBorislav Petkov cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel); 1658f71d0a05SDoug Thompson 165933ca0643SBorislav Petkov if (cs_found >= 0) 1660f71d0a05SDoug Thompson *chan_sel = channel; 166133ca0643SBorislav Petkov 1662f71d0a05SDoug Thompson return cs_found; 1663f71d0a05SDoug Thompson } 1664f71d0a05SDoug Thompson 166518b94f66SAravind Gopalakrishnan static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range, 166618b94f66SAravind Gopalakrishnan u64 sys_addr, int *chan_sel) 166718b94f66SAravind Gopalakrishnan { 166818b94f66SAravind Gopalakrishnan int cs_found = -EINVAL; 166918b94f66SAravind Gopalakrishnan int num_dcts_intlv = 0; 167018b94f66SAravind Gopalakrishnan u64 chan_addr, chan_offset; 167118b94f66SAravind Gopalakrishnan u64 dct_base, dct_limit; 167218b94f66SAravind Gopalakrishnan u32 dct_cont_base_reg, dct_cont_limit_reg, tmp; 167318b94f66SAravind Gopalakrishnan u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en; 167418b94f66SAravind Gopalakrishnan 167518b94f66SAravind Gopalakrishnan u64 dhar_offset = f10_dhar_offset(pvt); 167618b94f66SAravind Gopalakrishnan u8 intlv_addr = dct_sel_interleave_addr(pvt); 167718b94f66SAravind Gopalakrishnan u8 node_id = dram_dst_node(pvt, range); 167818b94f66SAravind Gopalakrishnan u8 intlv_en = dram_intlv_en(pvt, range); 167918b94f66SAravind Gopalakrishnan 168018b94f66SAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg); 168118b94f66SAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg); 168218b94f66SAravind Gopalakrishnan 168318b94f66SAravind Gopalakrishnan dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0)); 168418b94f66SAravind Gopalakrishnan dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7); 168518b94f66SAravind Gopalakrishnan 168618b94f66SAravind Gopalakrishnan edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n", 168718b94f66SAravind Gopalakrishnan range, sys_addr, get_dram_limit(pvt, range)); 168818b94f66SAravind Gopalakrishnan 168918b94f66SAravind Gopalakrishnan if (!(get_dram_base(pvt, range) <= sys_addr) && 169018b94f66SAravind Gopalakrishnan !(get_dram_limit(pvt, range) >= sys_addr)) 169118b94f66SAravind Gopalakrishnan return -EINVAL; 169218b94f66SAravind Gopalakrishnan 169318b94f66SAravind Gopalakrishnan if (dhar_valid(pvt) && 169418b94f66SAravind Gopalakrishnan dhar_base(pvt) <= sys_addr && 169518b94f66SAravind Gopalakrishnan sys_addr < BIT_64(32)) { 169618b94f66SAravind Gopalakrishnan amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n", 169718b94f66SAravind Gopalakrishnan sys_addr); 169818b94f66SAravind Gopalakrishnan return -EINVAL; 169918b94f66SAravind Gopalakrishnan } 170018b94f66SAravind Gopalakrishnan 170118b94f66SAravind Gopalakrishnan /* Verify sys_addr is within DCT Range. */ 17024fc06b31SAravind Gopalakrishnan dct_base = (u64) dct_sel_baseaddr(pvt); 17034fc06b31SAravind Gopalakrishnan dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF; 170418b94f66SAravind Gopalakrishnan 170518b94f66SAravind Gopalakrishnan if (!(dct_cont_base_reg & BIT(0)) && 17064fc06b31SAravind Gopalakrishnan !(dct_base <= (sys_addr >> 27) && 17074fc06b31SAravind Gopalakrishnan dct_limit >= (sys_addr >> 27))) 170818b94f66SAravind Gopalakrishnan return -EINVAL; 170918b94f66SAravind Gopalakrishnan 171018b94f66SAravind Gopalakrishnan /* Verify number of dct's that participate in channel interleaving. */ 171118b94f66SAravind Gopalakrishnan num_dcts_intlv = (int) hweight8(intlv_en); 171218b94f66SAravind Gopalakrishnan 171318b94f66SAravind Gopalakrishnan if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4)) 171418b94f66SAravind Gopalakrishnan return -EINVAL; 171518b94f66SAravind Gopalakrishnan 171618b94f66SAravind Gopalakrishnan channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en, 171718b94f66SAravind Gopalakrishnan num_dcts_intlv, dct_sel); 171818b94f66SAravind Gopalakrishnan 171918b94f66SAravind Gopalakrishnan /* Verify we stay within the MAX number of channels allowed */ 17207f3f5240SAravind Gopalakrishnan if (channel > 3) 172118b94f66SAravind Gopalakrishnan return -EINVAL; 172218b94f66SAravind Gopalakrishnan 172318b94f66SAravind Gopalakrishnan leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0)); 172418b94f66SAravind Gopalakrishnan 172518b94f66SAravind Gopalakrishnan /* Get normalized DCT addr */ 172618b94f66SAravind Gopalakrishnan if (leg_mmio_hole && (sys_addr >= BIT_64(32))) 172718b94f66SAravind Gopalakrishnan chan_offset = dhar_offset; 172818b94f66SAravind Gopalakrishnan else 17294fc06b31SAravind Gopalakrishnan chan_offset = dct_base << 27; 173018b94f66SAravind Gopalakrishnan 173118b94f66SAravind Gopalakrishnan chan_addr = sys_addr - chan_offset; 173218b94f66SAravind Gopalakrishnan 173318b94f66SAravind Gopalakrishnan /* remove channel interleave */ 173418b94f66SAravind Gopalakrishnan if (num_dcts_intlv == 2) { 173518b94f66SAravind Gopalakrishnan if (intlv_addr == 0x4) 173618b94f66SAravind Gopalakrishnan chan_addr = ((chan_addr >> 9) << 8) | 173718b94f66SAravind Gopalakrishnan (chan_addr & 0xff); 173818b94f66SAravind Gopalakrishnan else if (intlv_addr == 0x5) 173918b94f66SAravind Gopalakrishnan chan_addr = ((chan_addr >> 10) << 9) | 174018b94f66SAravind Gopalakrishnan (chan_addr & 0x1ff); 174118b94f66SAravind Gopalakrishnan else 174218b94f66SAravind Gopalakrishnan return -EINVAL; 174318b94f66SAravind Gopalakrishnan 174418b94f66SAravind Gopalakrishnan } else if (num_dcts_intlv == 4) { 174518b94f66SAravind Gopalakrishnan if (intlv_addr == 0x4) 174618b94f66SAravind Gopalakrishnan chan_addr = ((chan_addr >> 10) << 8) | 174718b94f66SAravind Gopalakrishnan (chan_addr & 0xff); 174818b94f66SAravind Gopalakrishnan else if (intlv_addr == 0x5) 174918b94f66SAravind Gopalakrishnan chan_addr = ((chan_addr >> 11) << 9) | 175018b94f66SAravind Gopalakrishnan (chan_addr & 0x1ff); 175118b94f66SAravind Gopalakrishnan else 175218b94f66SAravind Gopalakrishnan return -EINVAL; 175318b94f66SAravind Gopalakrishnan } 175418b94f66SAravind Gopalakrishnan 175518b94f66SAravind Gopalakrishnan if (dct_offset_en) { 175618b94f66SAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F1, 175718b94f66SAravind Gopalakrishnan DRAM_CONT_HIGH_OFF + (int) channel * 4, 175818b94f66SAravind Gopalakrishnan &tmp); 17594fc06b31SAravind Gopalakrishnan chan_addr += (u64) ((tmp >> 11) & 0xfff) << 27; 176018b94f66SAravind Gopalakrishnan } 176118b94f66SAravind Gopalakrishnan 176218b94f66SAravind Gopalakrishnan f15h_select_dct(pvt, channel); 176318b94f66SAravind Gopalakrishnan 176418b94f66SAravind Gopalakrishnan edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr); 176518b94f66SAravind Gopalakrishnan 176618b94f66SAravind Gopalakrishnan /* 176718b94f66SAravind Gopalakrishnan * Find Chip select: 176818b94f66SAravind Gopalakrishnan * if channel = 3, then alias it to 1. This is because, in F15 M30h, 176918b94f66SAravind Gopalakrishnan * there is support for 4 DCT's, but only 2 are currently functional. 177018b94f66SAravind Gopalakrishnan * They are DCT0 and DCT3. But we have read all registers of DCT3 into 177118b94f66SAravind Gopalakrishnan * pvt->csels[1]. So we need to use '1' here to get correct info. 177218b94f66SAravind Gopalakrishnan * Refer F15 M30h BKDG Section 2.10 and 2.10.3 for clarifications. 177318b94f66SAravind Gopalakrishnan */ 177418b94f66SAravind Gopalakrishnan alias_channel = (channel == 3) ? 1 : channel; 177518b94f66SAravind Gopalakrishnan 177618b94f66SAravind Gopalakrishnan cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel); 177718b94f66SAravind Gopalakrishnan 177818b94f66SAravind Gopalakrishnan if (cs_found >= 0) 177918b94f66SAravind Gopalakrishnan *chan_sel = alias_channel; 178018b94f66SAravind Gopalakrishnan 178118b94f66SAravind Gopalakrishnan return cs_found; 178218b94f66SAravind Gopalakrishnan } 178318b94f66SAravind Gopalakrishnan 178418b94f66SAravind Gopalakrishnan static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt, 178518b94f66SAravind Gopalakrishnan u64 sys_addr, 178633ca0643SBorislav Petkov int *chan_sel) 1787f71d0a05SDoug Thompson { 1788e761359aSBorislav Petkov int cs_found = -EINVAL; 1789e761359aSBorislav Petkov unsigned range; 1790f71d0a05SDoug Thompson 17917f19bf75SBorislav Petkov for (range = 0; range < DRAM_RANGES; range++) { 17927f19bf75SBorislav Petkov if (!dram_rw(pvt, range)) 1793f71d0a05SDoug Thompson continue; 1794f71d0a05SDoug Thompson 179518b94f66SAravind Gopalakrishnan if (pvt->fam == 0x15 && pvt->model >= 0x30) 179618b94f66SAravind Gopalakrishnan cs_found = f15_m30h_match_to_this_node(pvt, range, 179718b94f66SAravind Gopalakrishnan sys_addr, 179818b94f66SAravind Gopalakrishnan chan_sel); 1799f71d0a05SDoug Thompson 180018b94f66SAravind Gopalakrishnan else if ((get_dram_base(pvt, range) <= sys_addr) && 180118b94f66SAravind Gopalakrishnan (get_dram_limit(pvt, range) >= sys_addr)) { 1802b15f0fcaSBorislav Petkov cs_found = f1x_match_to_this_node(pvt, range, 180333ca0643SBorislav Petkov sys_addr, chan_sel); 1804f71d0a05SDoug Thompson if (cs_found >= 0) 1805f71d0a05SDoug Thompson break; 1806f71d0a05SDoug Thompson } 1807f71d0a05SDoug Thompson } 1808f71d0a05SDoug Thompson return cs_found; 1809f71d0a05SDoug Thompson } 1810f71d0a05SDoug Thompson 1811f71d0a05SDoug Thompson /* 1812bdc30a0cSBorislav Petkov * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps 1813bdc30a0cSBorislav Petkov * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW). 1814f71d0a05SDoug Thompson * 1815bdc30a0cSBorislav Petkov * The @sys_addr is usually an error address received from the hardware 1816bdc30a0cSBorislav Petkov * (MCX_ADDR). 1817f71d0a05SDoug Thompson */ 1818b15f0fcaSBorislav Petkov static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr, 181933ca0643SBorislav Petkov struct err_info *err) 1820f71d0a05SDoug Thompson { 1821f71d0a05SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 1822f71d0a05SDoug Thompson 182333ca0643SBorislav Petkov error_address_to_page_and_offset(sys_addr, err); 1824ab5a503cSMauro Carvalho Chehab 182533ca0643SBorislav Petkov err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel); 182633ca0643SBorislav Petkov if (err->csrow < 0) { 182733ca0643SBorislav Petkov err->err_code = ERR_CSROW; 1828bdc30a0cSBorislav Petkov return; 1829bdc30a0cSBorislav Petkov } 1830bdc30a0cSBorislav Petkov 1831f71d0a05SDoug Thompson /* 1832bdc30a0cSBorislav Petkov * We need the syndromes for channel detection only when we're 1833bdc30a0cSBorislav Petkov * ganged. Otherwise @chan should already contain the channel at 1834bdc30a0cSBorislav Petkov * this point. 1835f71d0a05SDoug Thompson */ 1836a97fa68eSBorislav Petkov if (dct_ganging_enabled(pvt)) 183733ca0643SBorislav Petkov err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome); 1838f71d0a05SDoug Thompson } 1839f71d0a05SDoug Thompson 1840f71d0a05SDoug Thompson /* 18418566c4dfSBorislav Petkov * debug routine to display the memory sizes of all logical DIMMs and its 1842cb328507SBorislav Petkov * CSROWs 1843f71d0a05SDoug Thompson */ 1844d1ea71cdSBorislav Petkov static void debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl) 1845f71d0a05SDoug Thompson { 1846bb89f5a0SBorislav Petkov int dimm, size0, size1; 1847525a1b20SBorislav Petkov u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases; 1848525a1b20SBorislav Petkov u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0; 1849f71d0a05SDoug Thompson 1850a4b4bedcSBorislav Petkov if (pvt->fam == 0xf) { 18518566c4dfSBorislav Petkov /* K8 families < revF not supported yet */ 18521433eb99SBorislav Petkov if (pvt->ext_model < K8_REV_F) 18538566c4dfSBorislav Petkov return; 18548566c4dfSBorislav Petkov else 18558566c4dfSBorislav Petkov WARN_ON(ctrl != 0); 18568566c4dfSBorislav Petkov } 18578566c4dfSBorislav Petkov 18587981a28fSAravind Gopalakrishnan if (pvt->fam == 0x10) { 18597981a28fSAravind Gopalakrishnan dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 18607981a28fSAravind Gopalakrishnan : pvt->dbam0; 18617981a28fSAravind Gopalakrishnan dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? 18627981a28fSAravind Gopalakrishnan pvt->csels[1].csbases : 18637981a28fSAravind Gopalakrishnan pvt->csels[0].csbases; 18647981a28fSAravind Gopalakrishnan } else if (ctrl) { 18657981a28fSAravind Gopalakrishnan dbam = pvt->dbam0; 18667981a28fSAravind Gopalakrishnan dcsb = pvt->csels[1].csbases; 18677981a28fSAravind Gopalakrishnan } 1868956b9ba1SJoe Perches edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", 1869956b9ba1SJoe Perches ctrl, dbam); 1870f71d0a05SDoug Thompson 18718566c4dfSBorislav Petkov edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl); 18728566c4dfSBorislav Petkov 1873f71d0a05SDoug Thompson /* Dump memory sizes for DIMM and its CSROWs */ 1874f71d0a05SDoug Thompson for (dimm = 0; dimm < 4; dimm++) { 1875f71d0a05SDoug Thompson 1876f71d0a05SDoug Thompson size0 = 0; 187711c75eadSBorislav Petkov if (dcsb[dimm*2] & DCSB_CS_ENABLE) 1878a597d2a5SAravind Gopalakrishnan /* For f15m60h, need multiplier for LRDIMM cs_size 1879a597d2a5SAravind Gopalakrishnan * calculation. We pass 'dimm' value to the dbam_to_cs 1880a597d2a5SAravind Gopalakrishnan * mapper so we can find the multiplier from the 1881a597d2a5SAravind Gopalakrishnan * corresponding DCSM. 1882a597d2a5SAravind Gopalakrishnan */ 188341d8bfabSBorislav Petkov size0 = pvt->ops->dbam_to_cs(pvt, ctrl, 1884a597d2a5SAravind Gopalakrishnan DBAM_DIMM(dimm, dbam), 1885a597d2a5SAravind Gopalakrishnan dimm); 1886f71d0a05SDoug Thompson 1887f71d0a05SDoug Thompson size1 = 0; 188811c75eadSBorislav Petkov if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE) 188941d8bfabSBorislav Petkov size1 = pvt->ops->dbam_to_cs(pvt, ctrl, 1890a597d2a5SAravind Gopalakrishnan DBAM_DIMM(dimm, dbam), 1891a597d2a5SAravind Gopalakrishnan dimm); 1892f71d0a05SDoug Thompson 189324f9a7feSBorislav Petkov amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n", 1894bb89f5a0SBorislav Petkov dimm * 2, size0, 1895bb89f5a0SBorislav Petkov dimm * 2 + 1, size1); 1896f71d0a05SDoug Thompson } 1897f71d0a05SDoug Thompson } 1898f71d0a05SDoug Thompson 1899d1ea71cdSBorislav Petkov static struct amd64_family_type family_types[] = { 19004d37607aSDoug Thompson [K8_CPUS] = { 19010092b20dSBorislav Petkov .ctl_name = "K8", 19028d5b5d9cSBorislav Petkov .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP, 19038d5b5d9cSBorislav Petkov .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC, 19044d37607aSDoug Thompson .ops = { 19054d37607aSDoug Thompson .early_channel_count = k8_early_channel_count, 19064d37607aSDoug Thompson .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow, 19071433eb99SBorislav Petkov .dbam_to_cs = k8_dbam_to_chip_select, 19084d37607aSDoug Thompson } 19094d37607aSDoug Thompson }, 19104d37607aSDoug Thompson [F10_CPUS] = { 19110092b20dSBorislav Petkov .ctl_name = "F10h", 19128d5b5d9cSBorislav Petkov .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP, 19138d5b5d9cSBorislav Petkov .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC, 19144d37607aSDoug Thompson .ops = { 19157d20d14dSBorislav Petkov .early_channel_count = f1x_early_channel_count, 1916b15f0fcaSBorislav Petkov .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, 19171433eb99SBorislav Petkov .dbam_to_cs = f10_dbam_to_chip_select, 1918b2b0c605SBorislav Petkov } 1919b2b0c605SBorislav Petkov }, 1920b2b0c605SBorislav Petkov [F15_CPUS] = { 1921b2b0c605SBorislav Petkov .ctl_name = "F15h", 1922df71a053SBorislav Petkov .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1, 1923df71a053SBorislav Petkov .f3_id = PCI_DEVICE_ID_AMD_15H_NB_F3, 1924b2b0c605SBorislav Petkov .ops = { 19257d20d14dSBorislav Petkov .early_channel_count = f1x_early_channel_count, 1926b15f0fcaSBorislav Petkov .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, 192741d8bfabSBorislav Petkov .dbam_to_cs = f15_dbam_to_chip_select, 19284d37607aSDoug Thompson } 19294d37607aSDoug Thompson }, 193018b94f66SAravind Gopalakrishnan [F15_M30H_CPUS] = { 193118b94f66SAravind Gopalakrishnan .ctl_name = "F15h_M30h", 193218b94f66SAravind Gopalakrishnan .f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1, 193318b94f66SAravind Gopalakrishnan .f3_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F3, 193418b94f66SAravind Gopalakrishnan .ops = { 193518b94f66SAravind Gopalakrishnan .early_channel_count = f1x_early_channel_count, 193618b94f66SAravind Gopalakrishnan .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, 193718b94f66SAravind Gopalakrishnan .dbam_to_cs = f16_dbam_to_chip_select, 193818b94f66SAravind Gopalakrishnan } 193918b94f66SAravind Gopalakrishnan }, 1940a597d2a5SAravind Gopalakrishnan [F15_M60H_CPUS] = { 1941a597d2a5SAravind Gopalakrishnan .ctl_name = "F15h_M60h", 1942a597d2a5SAravind Gopalakrishnan .f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1, 1943a597d2a5SAravind Gopalakrishnan .f3_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F3, 1944a597d2a5SAravind Gopalakrishnan .ops = { 1945a597d2a5SAravind Gopalakrishnan .early_channel_count = f1x_early_channel_count, 1946a597d2a5SAravind Gopalakrishnan .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, 1947a597d2a5SAravind Gopalakrishnan .dbam_to_cs = f15_m60h_dbam_to_chip_select, 1948a597d2a5SAravind Gopalakrishnan } 1949a597d2a5SAravind Gopalakrishnan }, 195094c1acf2SAravind Gopalakrishnan [F16_CPUS] = { 195194c1acf2SAravind Gopalakrishnan .ctl_name = "F16h", 195294c1acf2SAravind Gopalakrishnan .f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1, 195394c1acf2SAravind Gopalakrishnan .f3_id = PCI_DEVICE_ID_AMD_16H_NB_F3, 195494c1acf2SAravind Gopalakrishnan .ops = { 195594c1acf2SAravind Gopalakrishnan .early_channel_count = f1x_early_channel_count, 195694c1acf2SAravind Gopalakrishnan .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, 195794c1acf2SAravind Gopalakrishnan .dbam_to_cs = f16_dbam_to_chip_select, 195894c1acf2SAravind Gopalakrishnan } 195994c1acf2SAravind Gopalakrishnan }, 196085a8885bSAravind Gopalakrishnan [F16_M30H_CPUS] = { 196185a8885bSAravind Gopalakrishnan .ctl_name = "F16h_M30h", 196285a8885bSAravind Gopalakrishnan .f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1, 196385a8885bSAravind Gopalakrishnan .f3_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F3, 196485a8885bSAravind Gopalakrishnan .ops = { 196585a8885bSAravind Gopalakrishnan .early_channel_count = f1x_early_channel_count, 196685a8885bSAravind Gopalakrishnan .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, 196785a8885bSAravind Gopalakrishnan .dbam_to_cs = f16_dbam_to_chip_select, 196885a8885bSAravind Gopalakrishnan } 196985a8885bSAravind Gopalakrishnan }, 19704d37607aSDoug Thompson }; 19714d37607aSDoug Thompson 1972b1289d6fSDoug Thompson /* 1973bfc04aecSBorislav Petkov * These are tables of eigenvectors (one per line) which can be used for the 1974bfc04aecSBorislav Petkov * construction of the syndrome tables. The modified syndrome search algorithm 1975bfc04aecSBorislav Petkov * uses those to find the symbol in error and thus the DIMM. 1976b1289d6fSDoug Thompson * 1977bfc04aecSBorislav Petkov * Algorithm courtesy of Ross LaFetra from AMD. 1978b1289d6fSDoug Thompson */ 1979c7e5301aSDaniel J Blueman static const u16 x4_vectors[] = { 1980bfc04aecSBorislav Petkov 0x2f57, 0x1afe, 0x66cc, 0xdd88, 1981bfc04aecSBorislav Petkov 0x11eb, 0x3396, 0x7f4c, 0xeac8, 1982bfc04aecSBorislav Petkov 0x0001, 0x0002, 0x0004, 0x0008, 1983bfc04aecSBorislav Petkov 0x1013, 0x3032, 0x4044, 0x8088, 1984bfc04aecSBorislav Petkov 0x106b, 0x30d6, 0x70fc, 0xe0a8, 1985bfc04aecSBorislav Petkov 0x4857, 0xc4fe, 0x13cc, 0x3288, 1986bfc04aecSBorislav Petkov 0x1ac5, 0x2f4a, 0x5394, 0xa1e8, 1987bfc04aecSBorislav Petkov 0x1f39, 0x251e, 0xbd6c, 0x6bd8, 1988bfc04aecSBorislav Petkov 0x15c1, 0x2a42, 0x89ac, 0x4758, 1989bfc04aecSBorislav Petkov 0x2b03, 0x1602, 0x4f0c, 0xca08, 1990bfc04aecSBorislav Petkov 0x1f07, 0x3a0e, 0x6b04, 0xbd08, 1991bfc04aecSBorislav Petkov 0x8ba7, 0x465e, 0x244c, 0x1cc8, 1992bfc04aecSBorislav Petkov 0x2b87, 0x164e, 0x642c, 0xdc18, 1993bfc04aecSBorislav Petkov 0x40b9, 0x80de, 0x1094, 0x20e8, 1994bfc04aecSBorislav Petkov 0x27db, 0x1eb6, 0x9dac, 0x7b58, 1995bfc04aecSBorislav Petkov 0x11c1, 0x2242, 0x84ac, 0x4c58, 1996bfc04aecSBorislav Petkov 0x1be5, 0x2d7a, 0x5e34, 0xa718, 1997bfc04aecSBorislav Petkov 0x4b39, 0x8d1e, 0x14b4, 0x28d8, 1998bfc04aecSBorislav Petkov 0x4c97, 0xc87e, 0x11fc, 0x33a8, 1999bfc04aecSBorislav Petkov 0x8e97, 0x497e, 0x2ffc, 0x1aa8, 2000bfc04aecSBorislav Petkov 0x16b3, 0x3d62, 0x4f34, 0x8518, 2001bfc04aecSBorislav Petkov 0x1e2f, 0x391a, 0x5cac, 0xf858, 2002bfc04aecSBorislav Petkov 0x1d9f, 0x3b7a, 0x572c, 0xfe18, 2003bfc04aecSBorislav Petkov 0x15f5, 0x2a5a, 0x5264, 0xa3b8, 2004bfc04aecSBorislav Petkov 0x1dbb, 0x3b66, 0x715c, 0xe3f8, 2005bfc04aecSBorislav Petkov 0x4397, 0xc27e, 0x17fc, 0x3ea8, 2006bfc04aecSBorislav Petkov 0x1617, 0x3d3e, 0x6464, 0xb8b8, 2007bfc04aecSBorislav Petkov 0x23ff, 0x12aa, 0xab6c, 0x56d8, 2008bfc04aecSBorislav Petkov 0x2dfb, 0x1ba6, 0x913c, 0x7328, 2009bfc04aecSBorislav Petkov 0x185d, 0x2ca6, 0x7914, 0x9e28, 2010bfc04aecSBorislav Petkov 0x171b, 0x3e36, 0x7d7c, 0xebe8, 2011bfc04aecSBorislav Petkov 0x4199, 0x82ee, 0x19f4, 0x2e58, 2012bfc04aecSBorislav Petkov 0x4807, 0xc40e, 0x130c, 0x3208, 2013bfc04aecSBorislav Petkov 0x1905, 0x2e0a, 0x5804, 0xac08, 2014bfc04aecSBorislav Petkov 0x213f, 0x132a, 0xadfc, 0x5ba8, 2015bfc04aecSBorislav Petkov 0x19a9, 0x2efe, 0xb5cc, 0x6f88, 2016b1289d6fSDoug Thompson }; 2017b1289d6fSDoug Thompson 2018c7e5301aSDaniel J Blueman static const u16 x8_vectors[] = { 2019bfc04aecSBorislav Petkov 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480, 2020bfc04aecSBorislav Petkov 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80, 2021bfc04aecSBorislav Petkov 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80, 2022bfc04aecSBorislav Petkov 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80, 2023bfc04aecSBorislav Petkov 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780, 2024bfc04aecSBorislav Petkov 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080, 2025bfc04aecSBorislav Petkov 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080, 2026bfc04aecSBorislav Petkov 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080, 2027bfc04aecSBorislav Petkov 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80, 2028bfc04aecSBorislav Petkov 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580, 2029bfc04aecSBorislav Petkov 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880, 2030bfc04aecSBorislav Petkov 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280, 2031bfc04aecSBorislav Petkov 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180, 2032bfc04aecSBorislav Petkov 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580, 2033bfc04aecSBorislav Petkov 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280, 2034bfc04aecSBorislav Petkov 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180, 2035bfc04aecSBorislav Petkov 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080, 2036bfc04aecSBorislav Petkov 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 2037bfc04aecSBorislav Petkov 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000, 2038bfc04aecSBorislav Petkov }; 2039bfc04aecSBorislav Petkov 2040c7e5301aSDaniel J Blueman static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs, 2041d34a6ecdSBorislav Petkov unsigned v_dim) 2042b1289d6fSDoug Thompson { 2043bfc04aecSBorislav Petkov unsigned int i, err_sym; 2044b1289d6fSDoug Thompson 2045bfc04aecSBorislav Petkov for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) { 2046bfc04aecSBorislav Petkov u16 s = syndrome; 2047d34a6ecdSBorislav Petkov unsigned v_idx = err_sym * v_dim; 2048d34a6ecdSBorislav Petkov unsigned v_end = (err_sym + 1) * v_dim; 2049b1289d6fSDoug Thompson 2050bfc04aecSBorislav Petkov /* walk over all 16 bits of the syndrome */ 2051bfc04aecSBorislav Petkov for (i = 1; i < (1U << 16); i <<= 1) { 2052bfc04aecSBorislav Petkov 2053bfc04aecSBorislav Petkov /* if bit is set in that eigenvector... */ 2054bfc04aecSBorislav Petkov if (v_idx < v_end && vectors[v_idx] & i) { 2055bfc04aecSBorislav Petkov u16 ev_comp = vectors[v_idx++]; 2056bfc04aecSBorislav Petkov 2057bfc04aecSBorislav Petkov /* ... and bit set in the modified syndrome, */ 2058bfc04aecSBorislav Petkov if (s & i) { 2059bfc04aecSBorislav Petkov /* remove it. */ 2060bfc04aecSBorislav Petkov s ^= ev_comp; 2061bfc04aecSBorislav Petkov 2062bfc04aecSBorislav Petkov if (!s) 2063bfc04aecSBorislav Petkov return err_sym; 2064bfc04aecSBorislav Petkov } 2065bfc04aecSBorislav Petkov 2066bfc04aecSBorislav Petkov } else if (s & i) 2067bfc04aecSBorislav Petkov /* can't get to zero, move to next symbol */ 2068bfc04aecSBorislav Petkov break; 2069bfc04aecSBorislav Petkov } 2070b1289d6fSDoug Thompson } 2071b1289d6fSDoug Thompson 2072956b9ba1SJoe Perches edac_dbg(0, "syndrome(%x) not found\n", syndrome); 2073b1289d6fSDoug Thompson return -1; 2074b1289d6fSDoug Thompson } 2075d27bf6faSDoug Thompson 2076bfc04aecSBorislav Petkov static int map_err_sym_to_channel(int err_sym, int sym_size) 2077bfc04aecSBorislav Petkov { 2078bfc04aecSBorislav Petkov if (sym_size == 4) 2079bfc04aecSBorislav Petkov switch (err_sym) { 2080bfc04aecSBorislav Petkov case 0x20: 2081bfc04aecSBorislav Petkov case 0x21: 2082bfc04aecSBorislav Petkov return 0; 2083bfc04aecSBorislav Petkov break; 2084bfc04aecSBorislav Petkov case 0x22: 2085bfc04aecSBorislav Petkov case 0x23: 2086bfc04aecSBorislav Petkov return 1; 2087bfc04aecSBorislav Petkov break; 2088bfc04aecSBorislav Petkov default: 2089bfc04aecSBorislav Petkov return err_sym >> 4; 2090bfc04aecSBorislav Petkov break; 2091bfc04aecSBorislav Petkov } 2092bfc04aecSBorislav Petkov /* x8 symbols */ 2093bfc04aecSBorislav Petkov else 2094bfc04aecSBorislav Petkov switch (err_sym) { 2095bfc04aecSBorislav Petkov /* imaginary bits not in a DIMM */ 2096bfc04aecSBorislav Petkov case 0x10: 2097bfc04aecSBorislav Petkov WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n", 2098bfc04aecSBorislav Petkov err_sym); 2099bfc04aecSBorislav Petkov return -1; 2100bfc04aecSBorislav Petkov break; 2101bfc04aecSBorislav Petkov 2102bfc04aecSBorislav Petkov case 0x11: 2103bfc04aecSBorislav Petkov return 0; 2104bfc04aecSBorislav Petkov break; 2105bfc04aecSBorislav Petkov case 0x12: 2106bfc04aecSBorislav Petkov return 1; 2107bfc04aecSBorislav Petkov break; 2108bfc04aecSBorislav Petkov default: 2109bfc04aecSBorislav Petkov return err_sym >> 3; 2110bfc04aecSBorislav Petkov break; 2111bfc04aecSBorislav Petkov } 2112bfc04aecSBorislav Petkov return -1; 2113bfc04aecSBorislav Petkov } 2114bfc04aecSBorislav Petkov 2115bfc04aecSBorislav Petkov static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome) 2116bfc04aecSBorislav Petkov { 2117bfc04aecSBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 2118ad6a32e9SBorislav Petkov int err_sym = -1; 2119bfc04aecSBorislav Petkov 2120a3b7db09SBorislav Petkov if (pvt->ecc_sym_sz == 8) 2121bfc04aecSBorislav Petkov err_sym = decode_syndrome(syndrome, x8_vectors, 2122ad6a32e9SBorislav Petkov ARRAY_SIZE(x8_vectors), 2123a3b7db09SBorislav Petkov pvt->ecc_sym_sz); 2124a3b7db09SBorislav Petkov else if (pvt->ecc_sym_sz == 4) 2125ad6a32e9SBorislav Petkov err_sym = decode_syndrome(syndrome, x4_vectors, 2126ad6a32e9SBorislav Petkov ARRAY_SIZE(x4_vectors), 2127a3b7db09SBorislav Petkov pvt->ecc_sym_sz); 2128ad6a32e9SBorislav Petkov else { 2129a3b7db09SBorislav Petkov amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz); 2130ad6a32e9SBorislav Petkov return err_sym; 2131bfc04aecSBorislav Petkov } 2132ad6a32e9SBorislav Petkov 2133a3b7db09SBorislav Petkov return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz); 213441c31044SBorislav Petkov } 2135bfc04aecSBorislav Petkov 213633ca0643SBorislav Petkov static void __log_bus_error(struct mem_ctl_info *mci, struct err_info *err, 213733ca0643SBorislav Petkov u8 ecc_type) 2138d27bf6faSDoug Thompson { 213933ca0643SBorislav Petkov enum hw_event_mc_err_type err_type; 214033ca0643SBorislav Petkov const char *string; 2141d27bf6faSDoug Thompson 214233ca0643SBorislav Petkov if (ecc_type == 2) 214333ca0643SBorislav Petkov err_type = HW_EVENT_ERR_CORRECTED; 214433ca0643SBorislav Petkov else if (ecc_type == 1) 214533ca0643SBorislav Petkov err_type = HW_EVENT_ERR_UNCORRECTED; 214633ca0643SBorislav Petkov else { 214733ca0643SBorislav Petkov WARN(1, "Something is rotten in the state of Denmark.\n"); 2148d27bf6faSDoug Thompson return; 2149d27bf6faSDoug Thompson } 2150d27bf6faSDoug Thompson 215133ca0643SBorislav Petkov switch (err->err_code) { 215233ca0643SBorislav Petkov case DECODE_OK: 215333ca0643SBorislav Petkov string = ""; 215433ca0643SBorislav Petkov break; 215533ca0643SBorislav Petkov case ERR_NODE: 215633ca0643SBorislav Petkov string = "Failed to map error addr to a node"; 215733ca0643SBorislav Petkov break; 215833ca0643SBorislav Petkov case ERR_CSROW: 215933ca0643SBorislav Petkov string = "Failed to map error addr to a csrow"; 216033ca0643SBorislav Petkov break; 216133ca0643SBorislav Petkov case ERR_CHANNEL: 216233ca0643SBorislav Petkov string = "unknown syndrome - possible error reporting race"; 216333ca0643SBorislav Petkov break; 216433ca0643SBorislav Petkov default: 216533ca0643SBorislav Petkov string = "WTF error"; 216633ca0643SBorislav Petkov break; 2167d27bf6faSDoug Thompson } 216833ca0643SBorislav Petkov 216933ca0643SBorislav Petkov edac_mc_handle_error(err_type, mci, 1, 217033ca0643SBorislav Petkov err->page, err->offset, err->syndrome, 217133ca0643SBorislav Petkov err->csrow, err->channel, -1, 217233ca0643SBorislav Petkov string, ""); 2173d27bf6faSDoug Thompson } 2174d27bf6faSDoug Thompson 2175df781d03SBorislav Petkov static inline void decode_bus_error(int node_id, struct mce *m) 2176d27bf6faSDoug Thompson { 2177df781d03SBorislav Petkov struct mem_ctl_info *mci = mcis[node_id]; 217833ca0643SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 2179f192c7b1SBorislav Petkov u8 ecc_type = (m->status >> 45) & 0x3; 218066fed2d4SBorislav Petkov u8 xec = XEC(m->status, 0x1f); 218166fed2d4SBorislav Petkov u16 ec = EC(m->status); 218233ca0643SBorislav Petkov u64 sys_addr; 218333ca0643SBorislav Petkov struct err_info err; 2184d27bf6faSDoug Thompson 218566fed2d4SBorislav Petkov /* Bail out early if this was an 'observed' error */ 21865980bb9cSBorislav Petkov if (PP(ec) == NBSL_PP_OBS) 2187b70ef010SBorislav Petkov return; 2188d27bf6faSDoug Thompson 2189ecaf5606SBorislav Petkov /* Do only ECC errors */ 2190ecaf5606SBorislav Petkov if (xec && xec != F10_NBSL_EXT_ERR_ECC) 2191d27bf6faSDoug Thompson return; 2192d27bf6faSDoug Thompson 219333ca0643SBorislav Petkov memset(&err, 0, sizeof(err)); 219433ca0643SBorislav Petkov 2195a4b4bedcSBorislav Petkov sys_addr = get_error_address(pvt, m); 219633ca0643SBorislav Petkov 2197ecaf5606SBorislav Petkov if (ecc_type == 2) 219833ca0643SBorislav Petkov err.syndrome = extract_syndrome(m->status); 219933ca0643SBorislav Petkov 220033ca0643SBorislav Petkov pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err); 220133ca0643SBorislav Petkov 220233ca0643SBorislav Petkov __log_bus_error(mci, &err, ecc_type); 2203d27bf6faSDoug Thompson } 2204d27bf6faSDoug Thompson 22050ec449eeSDoug Thompson /* 22068d5b5d9cSBorislav Petkov * Use pvt->F2 which contains the F2 CPU PCI device to get the related 2207bbd0c1f6SBorislav Petkov * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error. 22080ec449eeSDoug Thompson */ 2209360b7f3cSBorislav Petkov static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id) 22100ec449eeSDoug Thompson { 22110ec449eeSDoug Thompson /* Reserve the ADDRESS MAP Device */ 22128d5b5d9cSBorislav Petkov pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2); 22138d5b5d9cSBorislav Petkov if (!pvt->F1) { 221424f9a7feSBorislav Petkov amd64_err("error address map device not found: " 22150ec449eeSDoug Thompson "vendor %x device 0x%x (broken BIOS?)\n", 2216bbd0c1f6SBorislav Petkov PCI_VENDOR_ID_AMD, f1_id); 2217bbd0c1f6SBorislav Petkov return -ENODEV; 22180ec449eeSDoug Thompson } 22190ec449eeSDoug Thompson 22200ec449eeSDoug Thompson /* Reserve the MISC Device */ 22218d5b5d9cSBorislav Petkov pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2); 22228d5b5d9cSBorislav Petkov if (!pvt->F3) { 22238d5b5d9cSBorislav Petkov pci_dev_put(pvt->F1); 22248d5b5d9cSBorislav Petkov pvt->F1 = NULL; 22250ec449eeSDoug Thompson 222624f9a7feSBorislav Petkov amd64_err("error F3 device not found: " 22270ec449eeSDoug Thompson "vendor %x device 0x%x (broken BIOS?)\n", 2228bbd0c1f6SBorislav Petkov PCI_VENDOR_ID_AMD, f3_id); 22298d5b5d9cSBorislav Petkov 2230bbd0c1f6SBorislav Petkov return -ENODEV; 22310ec449eeSDoug Thompson } 2232956b9ba1SJoe Perches edac_dbg(1, "F1: %s\n", pci_name(pvt->F1)); 2233956b9ba1SJoe Perches edac_dbg(1, "F2: %s\n", pci_name(pvt->F2)); 2234956b9ba1SJoe Perches edac_dbg(1, "F3: %s\n", pci_name(pvt->F3)); 22350ec449eeSDoug Thompson 22360ec449eeSDoug Thompson return 0; 22370ec449eeSDoug Thompson } 22380ec449eeSDoug Thompson 2239360b7f3cSBorislav Petkov static void free_mc_sibling_devs(struct amd64_pvt *pvt) 22400ec449eeSDoug Thompson { 22418d5b5d9cSBorislav Petkov pci_dev_put(pvt->F1); 22428d5b5d9cSBorislav Petkov pci_dev_put(pvt->F3); 22430ec449eeSDoug Thompson } 22440ec449eeSDoug Thompson 22450ec449eeSDoug Thompson /* 22460ec449eeSDoug Thompson * Retrieve the hardware registers of the memory controller (this includes the 22470ec449eeSDoug Thompson * 'Address Map' and 'Misc' device regs) 22480ec449eeSDoug Thompson */ 2249360b7f3cSBorislav Petkov static void read_mc_regs(struct amd64_pvt *pvt) 22500ec449eeSDoug Thompson { 2251a4b4bedcSBorislav Petkov unsigned range; 22520ec449eeSDoug Thompson u64 msr_val; 2253ad6a32e9SBorislav Petkov u32 tmp; 22540ec449eeSDoug Thompson 22550ec449eeSDoug Thompson /* 22560ec449eeSDoug Thompson * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since 22570ec449eeSDoug Thompson * those are Read-As-Zero 22580ec449eeSDoug Thompson */ 2259e97f8bb8SBorislav Petkov rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem); 2260956b9ba1SJoe Perches edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem); 22610ec449eeSDoug Thompson 22620ec449eeSDoug Thompson /* check first whether TOP_MEM2 is enabled */ 22630ec449eeSDoug Thompson rdmsrl(MSR_K8_SYSCFG, msr_val); 22640ec449eeSDoug Thompson if (msr_val & (1U << 21)) { 2265e97f8bb8SBorislav Petkov rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2); 2266956b9ba1SJoe Perches edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2); 22670ec449eeSDoug Thompson } else 2268956b9ba1SJoe Perches edac_dbg(0, " TOP_MEM2 disabled\n"); 22690ec449eeSDoug Thompson 22705980bb9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap); 22710ec449eeSDoug Thompson 22725a5d2371SBorislav Petkov read_dram_ctl_register(pvt); 22730ec449eeSDoug Thompson 22747f19bf75SBorislav Petkov for (range = 0; range < DRAM_RANGES; range++) { 22757f19bf75SBorislav Petkov u8 rw; 22760ec449eeSDoug Thompson 22777f19bf75SBorislav Petkov /* read settings for this DRAM range */ 22787f19bf75SBorislav Petkov read_dram_base_limit_regs(pvt, range); 2279e97f8bb8SBorislav Petkov 22807f19bf75SBorislav Petkov rw = dram_rw(pvt, range); 22817f19bf75SBorislav Petkov if (!rw) 22827f19bf75SBorislav Petkov continue; 22837f19bf75SBorislav Petkov 2284956b9ba1SJoe Perches edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n", 22857f19bf75SBorislav Petkov range, 22867f19bf75SBorislav Petkov get_dram_base(pvt, range), 22877f19bf75SBorislav Petkov get_dram_limit(pvt, range)); 22887f19bf75SBorislav Petkov 2289956b9ba1SJoe Perches edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n", 22907f19bf75SBorislav Petkov dram_intlv_en(pvt, range) ? "Enabled" : "Disabled", 22917f19bf75SBorislav Petkov (rw & 0x1) ? "R" : "-", 22927f19bf75SBorislav Petkov (rw & 0x2) ? "W" : "-", 22937f19bf75SBorislav Petkov dram_intlv_sel(pvt, range), 22947f19bf75SBorislav Petkov dram_dst_node(pvt, range)); 22950ec449eeSDoug Thompson } 22960ec449eeSDoug Thompson 2297b2b0c605SBorislav Petkov read_dct_base_mask(pvt); 22980ec449eeSDoug Thompson 2299bc21fa57SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar); 23007981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 0, DBAM0, &pvt->dbam0); 23010ec449eeSDoug Thompson 23028d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare); 23030ec449eeSDoug Thompson 23047981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 0, DCLR0, &pvt->dclr0); 23057981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 0, DCHR0, &pvt->dchr0); 23060ec449eeSDoug Thompson 230778da121eSBorislav Petkov if (!dct_ganging_enabled(pvt)) { 23087981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 1, DCLR0, &pvt->dclr1); 23097981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1); 23100ec449eeSDoug Thompson } 2311b2b0c605SBorislav Petkov 2312a3b7db09SBorislav Petkov pvt->ecc_sym_sz = 4; 2313a597d2a5SAravind Gopalakrishnan determine_memory_type(pvt); 2314a597d2a5SAravind Gopalakrishnan edac_dbg(1, " DIMM type: %s\n", edac_mem_types[pvt->dram_type]); 2315a3b7db09SBorislav Petkov 2316a4b4bedcSBorislav Petkov if (pvt->fam >= 0x10) { 23178d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp); 23187981a28fSAravind Gopalakrishnan /* F16h has only DCT0, so no need to read dbam1 */ 2319a4b4bedcSBorislav Petkov if (pvt->fam != 0x16) 23207981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1); 2321a3b7db09SBorislav Petkov 2322a3b7db09SBorislav Petkov /* F10h, revD and later can do x8 ECC too */ 2323a4b4bedcSBorislav Petkov if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25)) 2324a3b7db09SBorislav Petkov pvt->ecc_sym_sz = 8; 2325525a1b20SBorislav Petkov } 2326b2b0c605SBorislav Petkov dump_misc_regs(pvt); 23270ec449eeSDoug Thompson } 23280ec449eeSDoug Thompson 23290ec449eeSDoug Thompson /* 23300ec449eeSDoug Thompson * NOTE: CPU Revision Dependent code 23310ec449eeSDoug Thompson * 23320ec449eeSDoug Thompson * Input: 233311c75eadSBorislav Petkov * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1) 23340ec449eeSDoug Thompson * k8 private pointer to --> 23350ec449eeSDoug Thompson * DRAM Bank Address mapping register 23360ec449eeSDoug Thompson * node_id 23370ec449eeSDoug Thompson * DCL register where dual_channel_active is 23380ec449eeSDoug Thompson * 23390ec449eeSDoug Thompson * The DBAM register consists of 4 sets of 4 bits each definitions: 23400ec449eeSDoug Thompson * 23410ec449eeSDoug Thompson * Bits: CSROWs 23420ec449eeSDoug Thompson * 0-3 CSROWs 0 and 1 23430ec449eeSDoug Thompson * 4-7 CSROWs 2 and 3 23440ec449eeSDoug Thompson * 8-11 CSROWs 4 and 5 23450ec449eeSDoug Thompson * 12-15 CSROWs 6 and 7 23460ec449eeSDoug Thompson * 23470ec449eeSDoug Thompson * Values range from: 0 to 15 23480ec449eeSDoug Thompson * The meaning of the values depends on CPU revision and dual-channel state, 23490ec449eeSDoug Thompson * see relevant BKDG more info. 23500ec449eeSDoug Thompson * 23510ec449eeSDoug Thompson * The memory controller provides for total of only 8 CSROWs in its current 23520ec449eeSDoug Thompson * architecture. Each "pair" of CSROWs normally represents just one DIMM in 23530ec449eeSDoug Thompson * single channel or two (2) DIMMs in dual channel mode. 23540ec449eeSDoug Thompson * 23550ec449eeSDoug Thompson * The following code logic collapses the various tables for CSROW based on CPU 23560ec449eeSDoug Thompson * revision. 23570ec449eeSDoug Thompson * 23580ec449eeSDoug Thompson * Returns: 23590ec449eeSDoug Thompson * The number of PAGE_SIZE pages on the specified CSROW number it 23600ec449eeSDoug Thompson * encompasses 23610ec449eeSDoug Thompson * 23620ec449eeSDoug Thompson */ 2363d1ea71cdSBorislav Petkov static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr) 23640ec449eeSDoug Thompson { 23651433eb99SBorislav Petkov u32 cs_mode, nr_pages; 2366f92cae45SAshish Shenoy u32 dbam = dct ? pvt->dbam1 : pvt->dbam0; 23670ec449eeSDoug Thompson 236810de6497SBorislav Petkov 23690ec449eeSDoug Thompson /* 23700ec449eeSDoug Thompson * The math on this doesn't look right on the surface because x/2*4 can 23710ec449eeSDoug Thompson * be simplified to x*2 but this expression makes use of the fact that 23720ec449eeSDoug Thompson * it is integral math where 1/2=0. This intermediate value becomes the 23730ec449eeSDoug Thompson * number of bits to shift the DBAM register to extract the proper CSROW 23740ec449eeSDoug Thompson * field. 23750ec449eeSDoug Thompson */ 23760a5dfc31SBorislav Petkov cs_mode = DBAM_DIMM(csrow_nr / 2, dbam); 23770ec449eeSDoug Thompson 2378a597d2a5SAravind Gopalakrishnan nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, (csrow_nr / 2)) 2379a597d2a5SAravind Gopalakrishnan << (20 - PAGE_SHIFT); 23800ec449eeSDoug Thompson 238110de6497SBorislav Petkov edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n", 238210de6497SBorislav Petkov csrow_nr, dct, cs_mode); 238310de6497SBorislav Petkov edac_dbg(0, "nr_pages/channel: %u\n", nr_pages); 23840ec449eeSDoug Thompson 23850ec449eeSDoug Thompson return nr_pages; 23860ec449eeSDoug Thompson } 23870ec449eeSDoug Thompson 23880ec449eeSDoug Thompson /* 23890ec449eeSDoug Thompson * Initialize the array of csrow attribute instances, based on the values 23900ec449eeSDoug Thompson * from pci config hardware registers. 23910ec449eeSDoug Thompson */ 2392360b7f3cSBorislav Petkov static int init_csrows(struct mem_ctl_info *mci) 23930ec449eeSDoug Thompson { 239410de6497SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 23950ec449eeSDoug Thompson struct csrow_info *csrow; 2396de3910ebSMauro Carvalho Chehab struct dimm_info *dimm; 2397084a4fccSMauro Carvalho Chehab enum edac_type edac_mode; 239810de6497SBorislav Petkov int i, j, empty = 1; 2399a895bf8bSMauro Carvalho Chehab int nr_pages = 0; 240010de6497SBorislav Petkov u32 val; 24010ec449eeSDoug Thompson 2402a97fa68eSBorislav Petkov amd64_read_pci_cfg(pvt->F3, NBCFG, &val); 24030ec449eeSDoug Thompson 24042299ef71SBorislav Petkov pvt->nbcfg = val; 24050ec449eeSDoug Thompson 2406956b9ba1SJoe Perches edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n", 24072299ef71SBorislav Petkov pvt->mc_node_id, val, 2408a97fa68eSBorislav Petkov !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE)); 24090ec449eeSDoug Thompson 241010de6497SBorislav Petkov /* 241110de6497SBorislav Petkov * We iterate over DCT0 here but we look at DCT1 in parallel, if needed. 241210de6497SBorislav Petkov */ 241311c75eadSBorislav Petkov for_each_chip_select(i, 0, pvt) { 241410de6497SBorislav Petkov bool row_dct0 = !!csrow_enabled(i, 0, pvt); 241510de6497SBorislav Petkov bool row_dct1 = false; 24160ec449eeSDoug Thompson 2417a4b4bedcSBorislav Petkov if (pvt->fam != 0xf) 241810de6497SBorislav Petkov row_dct1 = !!csrow_enabled(i, 1, pvt); 241910de6497SBorislav Petkov 242010de6497SBorislav Petkov if (!row_dct0 && !row_dct1) 24210ec449eeSDoug Thompson continue; 24220ec449eeSDoug Thompson 242310de6497SBorislav Petkov csrow = mci->csrows[i]; 24240ec449eeSDoug Thompson empty = 0; 242511c75eadSBorislav Petkov 242610de6497SBorislav Petkov edac_dbg(1, "MC node: %d, csrow: %d\n", 242710de6497SBorislav Petkov pvt->mc_node_id, i); 242810de6497SBorislav Petkov 24291eef1282SMauro Carvalho Chehab if (row_dct0) { 2430d1ea71cdSBorislav Petkov nr_pages = get_csrow_nr_pages(pvt, 0, i); 24311eef1282SMauro Carvalho Chehab csrow->channels[0]->dimm->nr_pages = nr_pages; 24321eef1282SMauro Carvalho Chehab } 243310de6497SBorislav Petkov 243410de6497SBorislav Petkov /* K8 has only one DCT */ 2435a4b4bedcSBorislav Petkov if (pvt->fam != 0xf && row_dct1) { 2436d1ea71cdSBorislav Petkov int row_dct1_pages = get_csrow_nr_pages(pvt, 1, i); 24371eef1282SMauro Carvalho Chehab 24381eef1282SMauro Carvalho Chehab csrow->channels[1]->dimm->nr_pages = row_dct1_pages; 24391eef1282SMauro Carvalho Chehab nr_pages += row_dct1_pages; 24401eef1282SMauro Carvalho Chehab } 24410ec449eeSDoug Thompson 244210de6497SBorislav Petkov edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages); 24430ec449eeSDoug Thompson 24440ec449eeSDoug Thompson /* 24450ec449eeSDoug Thompson * determine whether CHIPKILL or JUST ECC or NO ECC is operating 24460ec449eeSDoug Thompson */ 2447a97fa68eSBorislav Petkov if (pvt->nbcfg & NBCFG_ECC_ENABLE) 2448084a4fccSMauro Carvalho Chehab edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL) ? 24490ec449eeSDoug Thompson EDAC_S4ECD4ED : EDAC_SECDED; 24500ec449eeSDoug Thompson else 2451084a4fccSMauro Carvalho Chehab edac_mode = EDAC_NONE; 2452084a4fccSMauro Carvalho Chehab 2453084a4fccSMauro Carvalho Chehab for (j = 0; j < pvt->channel_count; j++) { 2454de3910ebSMauro Carvalho Chehab dimm = csrow->channels[j]->dimm; 2455a597d2a5SAravind Gopalakrishnan dimm->mtype = pvt->dram_type; 2456de3910ebSMauro Carvalho Chehab dimm->edac_mode = edac_mode; 2457084a4fccSMauro Carvalho Chehab } 24580ec449eeSDoug Thompson } 24590ec449eeSDoug Thompson 24600ec449eeSDoug Thompson return empty; 24610ec449eeSDoug Thompson } 2462d27bf6faSDoug Thompson 246306724535SBorislav Petkov /* get all cores on this DCT */ 24648b84c8dfSDaniel J Blueman static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid) 2465f9431992SDoug Thompson { 246606724535SBorislav Petkov int cpu; 2467f9431992SDoug Thompson 246806724535SBorislav Petkov for_each_online_cpu(cpu) 246906724535SBorislav Petkov if (amd_get_nb_id(cpu) == nid) 247006724535SBorislav Petkov cpumask_set_cpu(cpu, mask); 2471f9431992SDoug Thompson } 2472f9431992SDoug Thompson 2473f9431992SDoug Thompson /* check MCG_CTL on all the cpus on this node */ 2474d1ea71cdSBorislav Petkov static bool nb_mce_bank_enabled_on_node(u16 nid) 2475f9431992SDoug Thompson { 2476ba578cb3SRusty Russell cpumask_var_t mask; 247750542251SBorislav Petkov int cpu, nbe; 247806724535SBorislav Petkov bool ret = false; 2479f9431992SDoug Thompson 2480ba578cb3SRusty Russell if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { 248124f9a7feSBorislav Petkov amd64_warn("%s: Error allocating mask\n", __func__); 248206724535SBorislav Petkov return false; 248306724535SBorislav Petkov } 248406724535SBorislav Petkov 2485ba578cb3SRusty Russell get_cpus_on_this_dct_cpumask(mask, nid); 248606724535SBorislav Petkov 2487ba578cb3SRusty Russell rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs); 2488ba578cb3SRusty Russell 2489ba578cb3SRusty Russell for_each_cpu(cpu, mask) { 249050542251SBorislav Petkov struct msr *reg = per_cpu_ptr(msrs, cpu); 24915980bb9cSBorislav Petkov nbe = reg->l & MSR_MCGCTL_NBE; 249206724535SBorislav Petkov 2493956b9ba1SJoe Perches edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n", 249450542251SBorislav Petkov cpu, reg->q, 249506724535SBorislav Petkov (nbe ? "enabled" : "disabled")); 249606724535SBorislav Petkov 249706724535SBorislav Petkov if (!nbe) 249806724535SBorislav Petkov goto out; 249906724535SBorislav Petkov } 250006724535SBorislav Petkov ret = true; 250106724535SBorislav Petkov 250206724535SBorislav Petkov out: 2503ba578cb3SRusty Russell free_cpumask_var(mask); 2504f9431992SDoug Thompson return ret; 2505f9431992SDoug Thompson } 2506f9431992SDoug Thompson 2507c7e5301aSDaniel J Blueman static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on) 2508f6d6ae96SBorislav Petkov { 2509f6d6ae96SBorislav Petkov cpumask_var_t cmask; 251050542251SBorislav Petkov int cpu; 2511f6d6ae96SBorislav Petkov 2512f6d6ae96SBorislav Petkov if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) { 251324f9a7feSBorislav Petkov amd64_warn("%s: error allocating mask\n", __func__); 2514f6d6ae96SBorislav Petkov return false; 2515f6d6ae96SBorislav Petkov } 2516f6d6ae96SBorislav Petkov 2517ae7bb7c6SBorislav Petkov get_cpus_on_this_dct_cpumask(cmask, nid); 2518f6d6ae96SBorislav Petkov 2519f6d6ae96SBorislav Petkov rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); 2520f6d6ae96SBorislav Petkov 2521f6d6ae96SBorislav Petkov for_each_cpu(cpu, cmask) { 2522f6d6ae96SBorislav Petkov 252350542251SBorislav Petkov struct msr *reg = per_cpu_ptr(msrs, cpu); 252450542251SBorislav Petkov 2525f6d6ae96SBorislav Petkov if (on) { 25265980bb9cSBorislav Petkov if (reg->l & MSR_MCGCTL_NBE) 2527ae7bb7c6SBorislav Petkov s->flags.nb_mce_enable = 1; 2528f6d6ae96SBorislav Petkov 25295980bb9cSBorislav Petkov reg->l |= MSR_MCGCTL_NBE; 2530f6d6ae96SBorislav Petkov } else { 2531f6d6ae96SBorislav Petkov /* 2532d95cf4deSBorislav Petkov * Turn off NB MCE reporting only when it was off before 2533f6d6ae96SBorislav Petkov */ 2534ae7bb7c6SBorislav Petkov if (!s->flags.nb_mce_enable) 25355980bb9cSBorislav Petkov reg->l &= ~MSR_MCGCTL_NBE; 2536f6d6ae96SBorislav Petkov } 2537f6d6ae96SBorislav Petkov } 2538f6d6ae96SBorislav Petkov wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); 2539f6d6ae96SBorislav Petkov 2540f6d6ae96SBorislav Petkov free_cpumask_var(cmask); 2541f6d6ae96SBorislav Petkov 2542f6d6ae96SBorislav Petkov return 0; 2543f6d6ae96SBorislav Petkov } 2544f6d6ae96SBorislav Petkov 2545c7e5301aSDaniel J Blueman static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid, 25462299ef71SBorislav Petkov struct pci_dev *F3) 2547f6d6ae96SBorislav Petkov { 25482299ef71SBorislav Petkov bool ret = true; 2549c9f4f26eSBorislav Petkov u32 value, mask = 0x3; /* UECC/CECC enable */ 2550f6d6ae96SBorislav Petkov 25512299ef71SBorislav Petkov if (toggle_ecc_err_reporting(s, nid, ON)) { 25522299ef71SBorislav Petkov amd64_warn("Error enabling ECC reporting over MCGCTL!\n"); 25532299ef71SBorislav Petkov return false; 25542299ef71SBorislav Petkov } 25552299ef71SBorislav Petkov 2556c9f4f26eSBorislav Petkov amd64_read_pci_cfg(F3, NBCTL, &value); 2557f6d6ae96SBorislav Petkov 2558ae7bb7c6SBorislav Petkov s->old_nbctl = value & mask; 2559ae7bb7c6SBorislav Petkov s->nbctl_valid = true; 2560f6d6ae96SBorislav Petkov 2561f6d6ae96SBorislav Petkov value |= mask; 2562c9f4f26eSBorislav Petkov amd64_write_pci_cfg(F3, NBCTL, value); 2563f6d6ae96SBorislav Petkov 2564a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 2565f6d6ae96SBorislav Petkov 2566956b9ba1SJoe Perches edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n", 2567a97fa68eSBorislav Petkov nid, value, !!(value & NBCFG_ECC_ENABLE)); 2568f6d6ae96SBorislav Petkov 2569a97fa68eSBorislav Petkov if (!(value & NBCFG_ECC_ENABLE)) { 257024f9a7feSBorislav Petkov amd64_warn("DRAM ECC disabled on this node, enabling...\n"); 2571f6d6ae96SBorislav Petkov 2572ae7bb7c6SBorislav Petkov s->flags.nb_ecc_prev = 0; 2573d95cf4deSBorislav Petkov 2574f6d6ae96SBorislav Petkov /* Attempt to turn on DRAM ECC Enable */ 2575a97fa68eSBorislav Petkov value |= NBCFG_ECC_ENABLE; 2576a97fa68eSBorislav Petkov amd64_write_pci_cfg(F3, NBCFG, value); 2577f6d6ae96SBorislav Petkov 2578a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 2579f6d6ae96SBorislav Petkov 2580a97fa68eSBorislav Petkov if (!(value & NBCFG_ECC_ENABLE)) { 258124f9a7feSBorislav Petkov amd64_warn("Hardware rejected DRAM ECC enable," 258224f9a7feSBorislav Petkov "check memory DIMM configuration.\n"); 25832299ef71SBorislav Petkov ret = false; 2584f6d6ae96SBorislav Petkov } else { 258524f9a7feSBorislav Petkov amd64_info("Hardware accepted DRAM ECC Enable\n"); 2586f6d6ae96SBorislav Petkov } 2587d95cf4deSBorislav Petkov } else { 2588ae7bb7c6SBorislav Petkov s->flags.nb_ecc_prev = 1; 2589f6d6ae96SBorislav Petkov } 2590d95cf4deSBorislav Petkov 2591956b9ba1SJoe Perches edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n", 2592a97fa68eSBorislav Petkov nid, value, !!(value & NBCFG_ECC_ENABLE)); 2593f6d6ae96SBorislav Petkov 25942299ef71SBorislav Petkov return ret; 2595f6d6ae96SBorislav Petkov } 2596f6d6ae96SBorislav Petkov 2597c7e5301aSDaniel J Blueman static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid, 2598360b7f3cSBorislav Petkov struct pci_dev *F3) 2599f6d6ae96SBorislav Petkov { 2600c9f4f26eSBorislav Petkov u32 value, mask = 0x3; /* UECC/CECC enable */ 2601c9f4f26eSBorislav Petkov 2602f6d6ae96SBorislav Petkov 2603ae7bb7c6SBorislav Petkov if (!s->nbctl_valid) 2604f6d6ae96SBorislav Petkov return; 2605f6d6ae96SBorislav Petkov 2606c9f4f26eSBorislav Petkov amd64_read_pci_cfg(F3, NBCTL, &value); 2607f6d6ae96SBorislav Petkov value &= ~mask; 2608ae7bb7c6SBorislav Petkov value |= s->old_nbctl; 2609f6d6ae96SBorislav Petkov 2610c9f4f26eSBorislav Petkov amd64_write_pci_cfg(F3, NBCTL, value); 2611f6d6ae96SBorislav Petkov 2612ae7bb7c6SBorislav Petkov /* restore previous BIOS DRAM ECC "off" setting we force-enabled */ 2613ae7bb7c6SBorislav Petkov if (!s->flags.nb_ecc_prev) { 2614a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 2615a97fa68eSBorislav Petkov value &= ~NBCFG_ECC_ENABLE; 2616a97fa68eSBorislav Petkov amd64_write_pci_cfg(F3, NBCFG, value); 2617d95cf4deSBorislav Petkov } 2618d95cf4deSBorislav Petkov 2619d95cf4deSBorislav Petkov /* restore the NB Enable MCGCTL bit */ 26202299ef71SBorislav Petkov if (toggle_ecc_err_reporting(s, nid, OFF)) 262124f9a7feSBorislav Petkov amd64_warn("Error restoring NB MCGCTL settings!\n"); 2622f6d6ae96SBorislav Petkov } 2623f6d6ae96SBorislav Petkov 2624f9431992SDoug Thompson /* 26252299ef71SBorislav Petkov * EDAC requires that the BIOS have ECC enabled before 26262299ef71SBorislav Petkov * taking over the processing of ECC errors. A command line 26272299ef71SBorislav Petkov * option allows to force-enable hardware ECC later in 26282299ef71SBorislav Petkov * enable_ecc_error_reporting(). 2629f9431992SDoug Thompson */ 2630cab4d277SBorislav Petkov static const char *ecc_msg = 2631cab4d277SBorislav Petkov "ECC disabled in the BIOS or no ECC capability, module will not load.\n" 2632cab4d277SBorislav Petkov " Either enable ECC checking or force module loading by setting " 2633cab4d277SBorislav Petkov "'ecc_enable_override'.\n" 2634cab4d277SBorislav Petkov " (Note that use of the override may cause unknown side effects.)\n"; 2635be3468e8SBorislav Petkov 2636c7e5301aSDaniel J Blueman static bool ecc_enabled(struct pci_dev *F3, u16 nid) 2637f9431992SDoug Thompson { 2638f9431992SDoug Thompson u32 value; 26392299ef71SBorislav Petkov u8 ecc_en = 0; 264006724535SBorislav Petkov bool nb_mce_en = false; 2641f9431992SDoug Thompson 2642a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 2643f9431992SDoug Thompson 2644a97fa68eSBorislav Petkov ecc_en = !!(value & NBCFG_ECC_ENABLE); 26452299ef71SBorislav Petkov amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled")); 2646be3468e8SBorislav Petkov 2647d1ea71cdSBorislav Petkov nb_mce_en = nb_mce_bank_enabled_on_node(nid); 264806724535SBorislav Petkov if (!nb_mce_en) 26492299ef71SBorislav Petkov amd64_notice("NB MCE bank disabled, set MSR " 26502299ef71SBorislav Petkov "0x%08x[4] on node %d to enable.\n", 26512299ef71SBorislav Petkov MSR_IA32_MCG_CTL, nid); 2652be3468e8SBorislav Petkov 26532299ef71SBorislav Petkov if (!ecc_en || !nb_mce_en) { 265424f9a7feSBorislav Petkov amd64_notice("%s", ecc_msg); 26552299ef71SBorislav Petkov return false; 2656be3468e8SBorislav Petkov } 26572299ef71SBorislav Petkov return true; 2658f9431992SDoug Thompson } 2659f9431992SDoug Thompson 2660c5608759SMauro Carvalho Chehab static int set_mc_sysfs_attrs(struct mem_ctl_info *mci) 26617d6034d3SDoug Thompson { 2662a4b4bedcSBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 2663c5608759SMauro Carvalho Chehab int rc; 26647d6034d3SDoug Thompson 2665c5608759SMauro Carvalho Chehab rc = amd64_create_sysfs_dbg_files(mci); 2666c5608759SMauro Carvalho Chehab if (rc < 0) 2667c5608759SMauro Carvalho Chehab return rc; 2668c5608759SMauro Carvalho Chehab 2669a4b4bedcSBorislav Petkov if (pvt->fam >= 0x10) { 2670c5608759SMauro Carvalho Chehab rc = amd64_create_sysfs_inject_files(mci); 2671c5608759SMauro Carvalho Chehab if (rc < 0) 2672c5608759SMauro Carvalho Chehab return rc; 2673c5608759SMauro Carvalho Chehab } 2674c5608759SMauro Carvalho Chehab 2675c5608759SMauro Carvalho Chehab return 0; 2676c5608759SMauro Carvalho Chehab } 2677c5608759SMauro Carvalho Chehab 2678c5608759SMauro Carvalho Chehab static void del_mc_sysfs_attrs(struct mem_ctl_info *mci) 2679c5608759SMauro Carvalho Chehab { 2680a4b4bedcSBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 2681a4b4bedcSBorislav Petkov 2682c5608759SMauro Carvalho Chehab amd64_remove_sysfs_dbg_files(mci); 26837d6034d3SDoug Thompson 2684a4b4bedcSBorislav Petkov if (pvt->fam >= 0x10) 2685c5608759SMauro Carvalho Chehab amd64_remove_sysfs_inject_files(mci); 26867d6034d3SDoug Thompson } 26877d6034d3SDoug Thompson 2688df71a053SBorislav Petkov static void setup_mci_misc_attrs(struct mem_ctl_info *mci, 2689df71a053SBorislav Petkov struct amd64_family_type *fam) 26907d6034d3SDoug Thompson { 26917d6034d3SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 26927d6034d3SDoug Thompson 26937d6034d3SDoug Thompson mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2; 26947d6034d3SDoug Thompson mci->edac_ctl_cap = EDAC_FLAG_NONE; 26957d6034d3SDoug Thompson 26965980bb9cSBorislav Petkov if (pvt->nbcap & NBCAP_SECDED) 26977d6034d3SDoug Thompson mci->edac_ctl_cap |= EDAC_FLAG_SECDED; 26987d6034d3SDoug Thompson 26995980bb9cSBorislav Petkov if (pvt->nbcap & NBCAP_CHIPKILL) 27007d6034d3SDoug Thompson mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED; 27017d6034d3SDoug Thompson 2702d1ea71cdSBorislav Petkov mci->edac_cap = determine_edac_cap(pvt); 27037d6034d3SDoug Thompson mci->mod_name = EDAC_MOD_STR; 27047d6034d3SDoug Thompson mci->mod_ver = EDAC_AMD64_VERSION; 2705df71a053SBorislav Petkov mci->ctl_name = fam->ctl_name; 27068d5b5d9cSBorislav Petkov mci->dev_name = pci_name(pvt->F2); 27077d6034d3SDoug Thompson mci->ctl_page_to_phys = NULL; 27087d6034d3SDoug Thompson 27097d6034d3SDoug Thompson /* memory scrubber interface */ 2710d1ea71cdSBorislav Petkov mci->set_sdram_scrub_rate = set_scrub_rate; 2711d1ea71cdSBorislav Petkov mci->get_sdram_scrub_rate = get_scrub_rate; 27127d6034d3SDoug Thompson } 27137d6034d3SDoug Thompson 27140092b20dSBorislav Petkov /* 27150092b20dSBorislav Petkov * returns a pointer to the family descriptor on success, NULL otherwise. 27160092b20dSBorislav Petkov */ 2717d1ea71cdSBorislav Petkov static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt) 2718395ae783SBorislav Petkov { 27190092b20dSBorislav Petkov struct amd64_family_type *fam_type = NULL; 27200092b20dSBorislav Petkov 272118b94f66SAravind Gopalakrishnan pvt->ext_model = boot_cpu_data.x86_model >> 4; 2722a4b4bedcSBorislav Petkov pvt->stepping = boot_cpu_data.x86_mask; 272318b94f66SAravind Gopalakrishnan pvt->model = boot_cpu_data.x86_model; 272418b94f66SAravind Gopalakrishnan pvt->fam = boot_cpu_data.x86; 272518b94f66SAravind Gopalakrishnan 272618b94f66SAravind Gopalakrishnan switch (pvt->fam) { 2727395ae783SBorislav Petkov case 0xf: 2728d1ea71cdSBorislav Petkov fam_type = &family_types[K8_CPUS]; 2729d1ea71cdSBorislav Petkov pvt->ops = &family_types[K8_CPUS].ops; 2730395ae783SBorislav Petkov break; 2731df71a053SBorislav Petkov 2732395ae783SBorislav Petkov case 0x10: 2733d1ea71cdSBorislav Petkov fam_type = &family_types[F10_CPUS]; 2734d1ea71cdSBorislav Petkov pvt->ops = &family_types[F10_CPUS].ops; 2735df71a053SBorislav Petkov break; 2736df71a053SBorislav Petkov 2737df71a053SBorislav Petkov case 0x15: 273818b94f66SAravind Gopalakrishnan if (pvt->model == 0x30) { 2739d1ea71cdSBorislav Petkov fam_type = &family_types[F15_M30H_CPUS]; 2740d1ea71cdSBorislav Petkov pvt->ops = &family_types[F15_M30H_CPUS].ops; 274118b94f66SAravind Gopalakrishnan break; 2742a597d2a5SAravind Gopalakrishnan } else if (pvt->model == 0x60) { 2743a597d2a5SAravind Gopalakrishnan fam_type = &family_types[F15_M60H_CPUS]; 2744a597d2a5SAravind Gopalakrishnan pvt->ops = &family_types[F15_M60H_CPUS].ops; 2745a597d2a5SAravind Gopalakrishnan break; 274618b94f66SAravind Gopalakrishnan } 274718b94f66SAravind Gopalakrishnan 2748d1ea71cdSBorislav Petkov fam_type = &family_types[F15_CPUS]; 2749d1ea71cdSBorislav Petkov pvt->ops = &family_types[F15_CPUS].ops; 2750395ae783SBorislav Petkov break; 2751395ae783SBorislav Petkov 275294c1acf2SAravind Gopalakrishnan case 0x16: 275385a8885bSAravind Gopalakrishnan if (pvt->model == 0x30) { 275485a8885bSAravind Gopalakrishnan fam_type = &family_types[F16_M30H_CPUS]; 275585a8885bSAravind Gopalakrishnan pvt->ops = &family_types[F16_M30H_CPUS].ops; 275685a8885bSAravind Gopalakrishnan break; 275785a8885bSAravind Gopalakrishnan } 2758d1ea71cdSBorislav Petkov fam_type = &family_types[F16_CPUS]; 2759d1ea71cdSBorislav Petkov pvt->ops = &family_types[F16_CPUS].ops; 276094c1acf2SAravind Gopalakrishnan break; 276194c1acf2SAravind Gopalakrishnan 2762395ae783SBorislav Petkov default: 276324f9a7feSBorislav Petkov amd64_err("Unsupported family!\n"); 27640092b20dSBorislav Petkov return NULL; 2765395ae783SBorislav Petkov } 27660092b20dSBorislav Petkov 2767df71a053SBorislav Petkov amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name, 276818b94f66SAravind Gopalakrishnan (pvt->fam == 0xf ? 27690092b20dSBorislav Petkov (pvt->ext_model >= K8_REV_F ? "revF or later " 27700092b20dSBorislav Petkov : "revE or earlier ") 277124f9a7feSBorislav Petkov : ""), pvt->mc_node_id); 27720092b20dSBorislav Petkov return fam_type; 2773395ae783SBorislav Petkov } 2774395ae783SBorislav Petkov 2775d1ea71cdSBorislav Petkov static int init_one_instance(struct pci_dev *F2) 27767d6034d3SDoug Thompson { 27777d6034d3SDoug Thompson struct amd64_pvt *pvt = NULL; 27780092b20dSBorislav Petkov struct amd64_family_type *fam_type = NULL; 2779360b7f3cSBorislav Petkov struct mem_ctl_info *mci = NULL; 2780ab5a503cSMauro Carvalho Chehab struct edac_mc_layer layers[2]; 27817d6034d3SDoug Thompson int err = 0, ret; 2782772c3ff3SDaniel J Blueman u16 nid = amd_get_node_id(F2); 27837d6034d3SDoug Thompson 27847d6034d3SDoug Thompson ret = -ENOMEM; 27857d6034d3SDoug Thompson pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL); 27867d6034d3SDoug Thompson if (!pvt) 2787360b7f3cSBorislav Petkov goto err_ret; 27887d6034d3SDoug Thompson 2789360b7f3cSBorislav Petkov pvt->mc_node_id = nid; 27908d5b5d9cSBorislav Petkov pvt->F2 = F2; 27917d6034d3SDoug Thompson 2792395ae783SBorislav Petkov ret = -EINVAL; 2793d1ea71cdSBorislav Petkov fam_type = per_family_init(pvt); 27940092b20dSBorislav Petkov if (!fam_type) 2795395ae783SBorislav Petkov goto err_free; 2796395ae783SBorislav Petkov 27977d6034d3SDoug Thompson ret = -ENODEV; 2798360b7f3cSBorislav Petkov err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id); 27997d6034d3SDoug Thompson if (err) 28007d6034d3SDoug Thompson goto err_free; 28017d6034d3SDoug Thompson 2802360b7f3cSBorislav Petkov read_mc_regs(pvt); 28037d6034d3SDoug Thompson 28047d6034d3SDoug Thompson /* 28057d6034d3SDoug Thompson * We need to determine how many memory channels there are. Then use 28067d6034d3SDoug Thompson * that information for calculating the size of the dynamic instance 2807360b7f3cSBorislav Petkov * tables in the 'mci' structure. 28087d6034d3SDoug Thompson */ 2809360b7f3cSBorislav Petkov ret = -EINVAL; 28107d6034d3SDoug Thompson pvt->channel_count = pvt->ops->early_channel_count(pvt); 28117d6034d3SDoug Thompson if (pvt->channel_count < 0) 2812360b7f3cSBorislav Petkov goto err_siblings; 28137d6034d3SDoug Thompson 28147d6034d3SDoug Thompson ret = -ENOMEM; 2815ab5a503cSMauro Carvalho Chehab layers[0].type = EDAC_MC_LAYER_CHIP_SELECT; 2816ab5a503cSMauro Carvalho Chehab layers[0].size = pvt->csels[0].b_cnt; 2817ab5a503cSMauro Carvalho Chehab layers[0].is_virt_csrow = true; 2818ab5a503cSMauro Carvalho Chehab layers[1].type = EDAC_MC_LAYER_CHANNEL; 2819f0a56c48SBorislav Petkov 2820f0a56c48SBorislav Petkov /* 2821f0a56c48SBorislav Petkov * Always allocate two channels since we can have setups with DIMMs on 2822f0a56c48SBorislav Petkov * only one channel. Also, this simplifies handling later for the price 2823f0a56c48SBorislav Petkov * of a couple of KBs tops. 2824f0a56c48SBorislav Petkov */ 2825f0a56c48SBorislav Petkov layers[1].size = 2; 2826ab5a503cSMauro Carvalho Chehab layers[1].is_virt_csrow = false; 2827f0a56c48SBorislav Petkov 2828ca0907b9SMauro Carvalho Chehab mci = edac_mc_alloc(nid, ARRAY_SIZE(layers), layers, 0); 28297d6034d3SDoug Thompson if (!mci) 2830360b7f3cSBorislav Petkov goto err_siblings; 28317d6034d3SDoug Thompson 28327d6034d3SDoug Thompson mci->pvt_info = pvt; 2833fd687502SMauro Carvalho Chehab mci->pdev = &pvt->F2->dev; 28347d6034d3SDoug Thompson 2835df71a053SBorislav Petkov setup_mci_misc_attrs(mci, fam_type); 2836360b7f3cSBorislav Petkov 2837360b7f3cSBorislav Petkov if (init_csrows(mci)) 28387d6034d3SDoug Thompson mci->edac_cap = EDAC_FLAG_NONE; 28397d6034d3SDoug Thompson 28407d6034d3SDoug Thompson ret = -ENODEV; 28417d6034d3SDoug Thompson if (edac_mc_add_mc(mci)) { 2842956b9ba1SJoe Perches edac_dbg(1, "failed edac_mc_add_mc()\n"); 28437d6034d3SDoug Thompson goto err_add_mc; 28447d6034d3SDoug Thompson } 2845c5608759SMauro Carvalho Chehab if (set_mc_sysfs_attrs(mci)) { 2846956b9ba1SJoe Perches edac_dbg(1, "failed edac_mc_add_mc()\n"); 2847c5608759SMauro Carvalho Chehab goto err_add_sysfs; 2848c5608759SMauro Carvalho Chehab } 28497d6034d3SDoug Thompson 2850549d042dSBorislav Petkov /* register stuff with EDAC MCE */ 2851549d042dSBorislav Petkov if (report_gart_errors) 2852549d042dSBorislav Petkov amd_report_gart_errors(true); 2853549d042dSBorislav Petkov 2854df781d03SBorislav Petkov amd_register_ecc_decoder(decode_bus_error); 2855549d042dSBorislav Petkov 2856360b7f3cSBorislav Petkov mcis[nid] = mci; 2857360b7f3cSBorislav Petkov 2858360b7f3cSBorislav Petkov atomic_inc(&drv_instances); 2859360b7f3cSBorislav Petkov 28607d6034d3SDoug Thompson return 0; 28617d6034d3SDoug Thompson 2862c5608759SMauro Carvalho Chehab err_add_sysfs: 2863c5608759SMauro Carvalho Chehab edac_mc_del_mc(mci->pdev); 28647d6034d3SDoug Thompson err_add_mc: 28657d6034d3SDoug Thompson edac_mc_free(mci); 28667d6034d3SDoug Thompson 2867360b7f3cSBorislav Petkov err_siblings: 2868360b7f3cSBorislav Petkov free_mc_sibling_devs(pvt); 28697d6034d3SDoug Thompson 2870360b7f3cSBorislav Petkov err_free: 2871360b7f3cSBorislav Petkov kfree(pvt); 28727d6034d3SDoug Thompson 2873360b7f3cSBorislav Petkov err_ret: 28747d6034d3SDoug Thompson return ret; 28757d6034d3SDoug Thompson } 28767d6034d3SDoug Thompson 2877d1ea71cdSBorislav Petkov static int probe_one_instance(struct pci_dev *pdev, 28787d6034d3SDoug Thompson const struct pci_device_id *mc_type) 28797d6034d3SDoug Thompson { 2880772c3ff3SDaniel J Blueman u16 nid = amd_get_node_id(pdev); 28812299ef71SBorislav Petkov struct pci_dev *F3 = node_to_amd_nb(nid)->misc; 2882ae7bb7c6SBorislav Petkov struct ecc_settings *s; 28832299ef71SBorislav Petkov int ret = 0; 28847d6034d3SDoug Thompson 28857d6034d3SDoug Thompson ret = pci_enable_device(pdev); 2886b8cfa02fSBorislav Petkov if (ret < 0) { 2887956b9ba1SJoe Perches edac_dbg(0, "ret=%d\n", ret); 2888b8cfa02fSBorislav Petkov return -EIO; 2889b8cfa02fSBorislav Petkov } 2890b8cfa02fSBorislav Petkov 2891ae7bb7c6SBorislav Petkov ret = -ENOMEM; 2892ae7bb7c6SBorislav Petkov s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL); 2893ae7bb7c6SBorislav Petkov if (!s) 28942299ef71SBorislav Petkov goto err_out; 2895ae7bb7c6SBorislav Petkov 2896ae7bb7c6SBorislav Petkov ecc_stngs[nid] = s; 2897ae7bb7c6SBorislav Petkov 28982299ef71SBorislav Petkov if (!ecc_enabled(F3, nid)) { 28992299ef71SBorislav Petkov ret = -ENODEV; 29002299ef71SBorislav Petkov 29012299ef71SBorislav Petkov if (!ecc_enable_override) 29022299ef71SBorislav Petkov goto err_enable; 29032299ef71SBorislav Petkov 29042299ef71SBorislav Petkov amd64_warn("Forcing ECC on!\n"); 29052299ef71SBorislav Petkov 29062299ef71SBorislav Petkov if (!enable_ecc_error_reporting(s, nid, F3)) 29072299ef71SBorislav Petkov goto err_enable; 29082299ef71SBorislav Petkov } 29092299ef71SBorislav Petkov 2910d1ea71cdSBorislav Petkov ret = init_one_instance(pdev); 2911360b7f3cSBorislav Petkov if (ret < 0) { 2912ae7bb7c6SBorislav Petkov amd64_err("Error probing instance: %d\n", nid); 2913360b7f3cSBorislav Petkov restore_ecc_error_reporting(s, nid, F3); 2914360b7f3cSBorislav Petkov } 29157d6034d3SDoug Thompson 29167d6034d3SDoug Thompson return ret; 29172299ef71SBorislav Petkov 29182299ef71SBorislav Petkov err_enable: 29192299ef71SBorislav Petkov kfree(s); 29202299ef71SBorislav Petkov ecc_stngs[nid] = NULL; 29212299ef71SBorislav Petkov 29222299ef71SBorislav Petkov err_out: 29232299ef71SBorislav Petkov return ret; 29247d6034d3SDoug Thompson } 29257d6034d3SDoug Thompson 2926d1ea71cdSBorislav Petkov static void remove_one_instance(struct pci_dev *pdev) 29277d6034d3SDoug Thompson { 29287d6034d3SDoug Thompson struct mem_ctl_info *mci; 29297d6034d3SDoug Thompson struct amd64_pvt *pvt; 2930772c3ff3SDaniel J Blueman u16 nid = amd_get_node_id(pdev); 2931360b7f3cSBorislav Petkov struct pci_dev *F3 = node_to_amd_nb(nid)->misc; 2932360b7f3cSBorislav Petkov struct ecc_settings *s = ecc_stngs[nid]; 29337d6034d3SDoug Thompson 2934c5608759SMauro Carvalho Chehab mci = find_mci_by_dev(&pdev->dev); 2935a4b4bedcSBorislav Petkov WARN_ON(!mci); 2936a4b4bedcSBorislav Petkov 2937c5608759SMauro Carvalho Chehab del_mc_sysfs_attrs(mci); 29387d6034d3SDoug Thompson /* Remove from EDAC CORE tracking list */ 29397d6034d3SDoug Thompson mci = edac_mc_del_mc(&pdev->dev); 29407d6034d3SDoug Thompson if (!mci) 29417d6034d3SDoug Thompson return; 29427d6034d3SDoug Thompson 29437d6034d3SDoug Thompson pvt = mci->pvt_info; 29447d6034d3SDoug Thompson 2945360b7f3cSBorislav Petkov restore_ecc_error_reporting(s, nid, F3); 29467d6034d3SDoug Thompson 2947360b7f3cSBorislav Petkov free_mc_sibling_devs(pvt); 29487d6034d3SDoug Thompson 2949549d042dSBorislav Petkov /* unregister from EDAC MCE */ 2950549d042dSBorislav Petkov amd_report_gart_errors(false); 2951df781d03SBorislav Petkov amd_unregister_ecc_decoder(decode_bus_error); 2952549d042dSBorislav Petkov 2953360b7f3cSBorislav Petkov kfree(ecc_stngs[nid]); 2954360b7f3cSBorislav Petkov ecc_stngs[nid] = NULL; 2955ae7bb7c6SBorislav Petkov 29567d6034d3SDoug Thompson /* Free the EDAC CORE resources */ 29578f68ed97SBorislav Petkov mci->pvt_info = NULL; 2958360b7f3cSBorislav Petkov mcis[nid] = NULL; 29598f68ed97SBorislav Petkov 29608f68ed97SBorislav Petkov kfree(pvt); 29617d6034d3SDoug Thompson edac_mc_free(mci); 29627d6034d3SDoug Thompson } 29637d6034d3SDoug Thompson 29647d6034d3SDoug Thompson /* 29657d6034d3SDoug Thompson * This table is part of the interface for loading drivers for PCI devices. The 29667d6034d3SDoug Thompson * PCI core identifies what devices are on a system during boot, and then 29677d6034d3SDoug Thompson * inquiry this table to see if this driver is for a given device found. 29687d6034d3SDoug Thompson */ 2969ba935f40SJingoo Han static const struct pci_device_id amd64_pci_table[] = { 2970a597d2a5SAravind Gopalakrishnan { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_K8_NB_MEMCTL) }, 2971a597d2a5SAravind Gopalakrishnan { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_10H_NB_DRAM) }, 2972a597d2a5SAravind Gopalakrishnan { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_15H_NB_F2) }, 2973a597d2a5SAravind Gopalakrishnan { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_15H_M30H_NB_F2) }, 2974a597d2a5SAravind Gopalakrishnan { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_15H_M60H_NB_F2) }, 2975a597d2a5SAravind Gopalakrishnan { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_16H_NB_F2) }, 2976a597d2a5SAravind Gopalakrishnan { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_16H_M30H_NB_F2) }, 29777d6034d3SDoug Thompson {0, } 29787d6034d3SDoug Thompson }; 29797d6034d3SDoug Thompson MODULE_DEVICE_TABLE(pci, amd64_pci_table); 29807d6034d3SDoug Thompson 29817d6034d3SDoug Thompson static struct pci_driver amd64_pci_driver = { 29827d6034d3SDoug Thompson .name = EDAC_MOD_STR, 2983d1ea71cdSBorislav Petkov .probe = probe_one_instance, 2984d1ea71cdSBorislav Petkov .remove = remove_one_instance, 29857d6034d3SDoug Thompson .id_table = amd64_pci_table, 29867d6034d3SDoug Thompson }; 29877d6034d3SDoug Thompson 2988360b7f3cSBorislav Petkov static void setup_pci_device(void) 29897d6034d3SDoug Thompson { 29907d6034d3SDoug Thompson struct mem_ctl_info *mci; 29917d6034d3SDoug Thompson struct amd64_pvt *pvt; 29927d6034d3SDoug Thompson 2993d1ea71cdSBorislav Petkov if (pci_ctl) 29947d6034d3SDoug Thompson return; 29957d6034d3SDoug Thompson 2996cc4d8860SBorislav Petkov mci = mcis[0]; 2997d1ea71cdSBorislav Petkov if (!mci) 2998d1ea71cdSBorislav Petkov return; 29997d6034d3SDoug Thompson 30007d6034d3SDoug Thompson pvt = mci->pvt_info; 3001d1ea71cdSBorislav Petkov pci_ctl = edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR); 3002d1ea71cdSBorislav Petkov if (!pci_ctl) { 3003d1ea71cdSBorislav Petkov pr_warn("%s(): Unable to create PCI control\n", __func__); 3004d1ea71cdSBorislav Petkov pr_warn("%s(): PCI error report via EDAC not set\n", __func__); 30057d6034d3SDoug Thompson } 30067d6034d3SDoug Thompson } 30077d6034d3SDoug Thompson 30087d6034d3SDoug Thompson static int __init amd64_edac_init(void) 30097d6034d3SDoug Thompson { 3010360b7f3cSBorislav Petkov int err = -ENODEV; 30117d6034d3SDoug Thompson 3012df71a053SBorislav Petkov printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION); 30137d6034d3SDoug Thompson 30147d6034d3SDoug Thompson opstate_init(); 30157d6034d3SDoug Thompson 30169653a5c7SHans Rosenfeld if (amd_cache_northbridges() < 0) 301756b34b91SBorislav Petkov goto err_ret; 30187d6034d3SDoug Thompson 3019cc4d8860SBorislav Petkov err = -ENOMEM; 3020cc4d8860SBorislav Petkov mcis = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL); 3021ae7bb7c6SBorislav Petkov ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL); 3022360b7f3cSBorislav Petkov if (!(mcis && ecc_stngs)) 3023a9f0fbe2SBorislav Petkov goto err_free; 3024cc4d8860SBorislav Petkov 302550542251SBorislav Petkov msrs = msrs_alloc(); 302656b34b91SBorislav Petkov if (!msrs) 3027360b7f3cSBorislav Petkov goto err_free; 302850542251SBorislav Petkov 30297d6034d3SDoug Thompson err = pci_register_driver(&amd64_pci_driver); 30307d6034d3SDoug Thompson if (err) 303156b34b91SBorislav Petkov goto err_pci; 30327d6034d3SDoug Thompson 303356b34b91SBorislav Petkov err = -ENODEV; 3034360b7f3cSBorislav Petkov if (!atomic_read(&drv_instances)) 3035360b7f3cSBorislav Petkov goto err_no_instances; 30367d6034d3SDoug Thompson 3037360b7f3cSBorislav Petkov setup_pci_device(); 30387d6034d3SDoug Thompson return 0; 30397d6034d3SDoug Thompson 3040360b7f3cSBorislav Petkov err_no_instances: 30417d6034d3SDoug Thompson pci_unregister_driver(&amd64_pci_driver); 3042cc4d8860SBorislav Petkov 304356b34b91SBorislav Petkov err_pci: 304456b34b91SBorislav Petkov msrs_free(msrs); 304556b34b91SBorislav Petkov msrs = NULL; 3046cc4d8860SBorislav Petkov 3047360b7f3cSBorislav Petkov err_free: 3048360b7f3cSBorislav Petkov kfree(mcis); 3049360b7f3cSBorislav Petkov mcis = NULL; 3050360b7f3cSBorislav Petkov 3051360b7f3cSBorislav Petkov kfree(ecc_stngs); 3052360b7f3cSBorislav Petkov ecc_stngs = NULL; 3053360b7f3cSBorislav Petkov 305456b34b91SBorislav Petkov err_ret: 30557d6034d3SDoug Thompson return err; 30567d6034d3SDoug Thompson } 30577d6034d3SDoug Thompson 30587d6034d3SDoug Thompson static void __exit amd64_edac_exit(void) 30597d6034d3SDoug Thompson { 3060d1ea71cdSBorislav Petkov if (pci_ctl) 3061d1ea71cdSBorislav Petkov edac_pci_release_generic_ctl(pci_ctl); 30627d6034d3SDoug Thompson 30637d6034d3SDoug Thompson pci_unregister_driver(&amd64_pci_driver); 306450542251SBorislav Petkov 3065ae7bb7c6SBorislav Petkov kfree(ecc_stngs); 3066ae7bb7c6SBorislav Petkov ecc_stngs = NULL; 3067ae7bb7c6SBorislav Petkov 3068cc4d8860SBorislav Petkov kfree(mcis); 3069cc4d8860SBorislav Petkov mcis = NULL; 3070cc4d8860SBorislav Petkov 307150542251SBorislav Petkov msrs_free(msrs); 307250542251SBorislav Petkov msrs = NULL; 30737d6034d3SDoug Thompson } 30747d6034d3SDoug Thompson 30757d6034d3SDoug Thompson module_init(amd64_edac_init); 30767d6034d3SDoug Thompson module_exit(amd64_edac_exit); 30777d6034d3SDoug Thompson 30787d6034d3SDoug Thompson MODULE_LICENSE("GPL"); 30797d6034d3SDoug Thompson MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, " 30807d6034d3SDoug Thompson "Dave Peterson, Thayne Harbaugh"); 30817d6034d3SDoug Thompson MODULE_DESCRIPTION("MC support for AMD64 memory controllers - " 30827d6034d3SDoug Thompson EDAC_AMD64_VERSION); 30837d6034d3SDoug Thompson 30847d6034d3SDoug Thompson module_param(edac_op_state, int, 0444); 30857d6034d3SDoug Thompson MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI"); 3086