109c434b8SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 22bc65418SDoug Thompson #include "amd64_edac.h" 323ac4ae8SAndreas Herrmann #include <asm/amd_nb.h> 42bc65418SDoug Thompson 5d1ea71cdSBorislav Petkov static struct edac_pci_ctl_info *pci_ctl; 62bc65418SDoug Thompson 72bc65418SDoug Thompson /* 82bc65418SDoug Thompson * Set by command line parameter. If BIOS has enabled the ECC, this override is 92bc65418SDoug Thompson * cleared to prevent re-enabling the hardware by this driver. 102bc65418SDoug Thompson */ 112bc65418SDoug Thompson static int ecc_enable_override; 122bc65418SDoug Thompson module_param(ecc_enable_override, int, 0644); 132bc65418SDoug Thompson 14a29d8b8eSTejun Heo static struct msr __percpu *msrs; 1550542251SBorislav Petkov 16ed623d55SMuralidhara M K static inline u32 get_umc_reg(struct amd64_pvt *pvt, u32 reg) 172151c84eSYazen Ghannam { 18ed623d55SMuralidhara M K if (!pvt->flags.zn_regs_v2) 192151c84eSYazen Ghannam return reg; 202151c84eSYazen Ghannam 212151c84eSYazen Ghannam switch (reg) { 222151c84eSYazen Ghannam case UMCCH_ADDR_CFG: return UMCCH_ADDR_CFG_DDR5; 232151c84eSYazen Ghannam case UMCCH_ADDR_MASK_SEC: return UMCCH_ADDR_MASK_SEC_DDR5; 242151c84eSYazen Ghannam case UMCCH_DIMM_CFG: return UMCCH_DIMM_CFG_DDR5; 252151c84eSYazen Ghannam } 262151c84eSYazen Ghannam 272151c84eSYazen Ghannam WARN_ONCE(1, "%s: unknown register 0x%x", __func__, reg); 282151c84eSYazen Ghannam return 0; 292151c84eSYazen Ghannam } 302151c84eSYazen Ghannam 312ec591acSBorislav Petkov /* Per-node stuff */ 32ae7bb7c6SBorislav Petkov static struct ecc_settings **ecc_stngs; 332bc65418SDoug Thompson 34706657b1SBorislav Petkov /* Device for the PCI component */ 35706657b1SBorislav Petkov static struct device *pci_ctl_dev; 36706657b1SBorislav Petkov 372bc65418SDoug Thompson /* 38b70ef010SBorislav Petkov * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing 39b70ef010SBorislav Petkov * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching- 40b70ef010SBorislav Petkov * or higher value'. 41b70ef010SBorislav Petkov * 42b70ef010SBorislav Petkov *FIXME: Produce a better mapping/linearisation. 43b70ef010SBorislav Petkov */ 44c7e5301aSDaniel J Blueman static const struct scrubrate { 4539094443SBorislav Petkov u32 scrubval; /* bit pattern for scrub rate */ 4639094443SBorislav Petkov u32 bandwidth; /* bandwidth consumed (bytes/sec) */ 4739094443SBorislav Petkov } scrubrates[] = { 48b70ef010SBorislav Petkov { 0x01, 1600000000UL}, 49b70ef010SBorislav Petkov { 0x02, 800000000UL}, 50b70ef010SBorislav Petkov { 0x03, 400000000UL}, 51b70ef010SBorislav Petkov { 0x04, 200000000UL}, 52b70ef010SBorislav Petkov { 0x05, 100000000UL}, 53b70ef010SBorislav Petkov { 0x06, 50000000UL}, 54b70ef010SBorislav Petkov { 0x07, 25000000UL}, 55b70ef010SBorislav Petkov { 0x08, 12284069UL}, 56b70ef010SBorislav Petkov { 0x09, 6274509UL}, 57b70ef010SBorislav Petkov { 0x0A, 3121951UL}, 58b70ef010SBorislav Petkov { 0x0B, 1560975UL}, 59b70ef010SBorislav Petkov { 0x0C, 781440UL}, 60b70ef010SBorislav Petkov { 0x0D, 390720UL}, 61b70ef010SBorislav Petkov { 0x0E, 195300UL}, 62b70ef010SBorislav Petkov { 0x0F, 97650UL}, 63b70ef010SBorislav Petkov { 0x10, 48854UL}, 64b70ef010SBorislav Petkov { 0x11, 24427UL}, 65b70ef010SBorislav Petkov { 0x12, 12213UL}, 66b70ef010SBorislav Petkov { 0x13, 6101UL}, 67b70ef010SBorislav Petkov { 0x14, 3051UL}, 68b70ef010SBorislav Petkov { 0x15, 1523UL}, 69b70ef010SBorislav Petkov { 0x16, 761UL}, 70b70ef010SBorislav Petkov { 0x00, 0UL}, /* scrubbing off */ 71b70ef010SBorislav Petkov }; 72b70ef010SBorislav Petkov 7366fed2d4SBorislav Petkov int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset, 74b2b0c605SBorislav Petkov u32 *val, const char *func) 75b2b0c605SBorislav Petkov { 76b2b0c605SBorislav Petkov int err = 0; 77b2b0c605SBorislav Petkov 78b2b0c605SBorislav Petkov err = pci_read_config_dword(pdev, offset, val); 79b2b0c605SBorislav Petkov if (err) 80b2b0c605SBorislav Petkov amd64_warn("%s: error reading F%dx%03x.\n", 81b2b0c605SBorislav Petkov func, PCI_FUNC(pdev->devfn), offset); 82b2b0c605SBorislav Petkov 83b2b0c605SBorislav Petkov return err; 84b2b0c605SBorislav Petkov } 85b2b0c605SBorislav Petkov 86b2b0c605SBorislav Petkov int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset, 87b2b0c605SBorislav Petkov u32 val, const char *func) 88b2b0c605SBorislav Petkov { 89b2b0c605SBorislav Petkov int err = 0; 90b2b0c605SBorislav Petkov 91b2b0c605SBorislav Petkov err = pci_write_config_dword(pdev, offset, val); 92b2b0c605SBorislav Petkov if (err) 93b2b0c605SBorislav Petkov amd64_warn("%s: error writing to F%dx%03x.\n", 94b2b0c605SBorislav Petkov func, PCI_FUNC(pdev->devfn), offset); 95b2b0c605SBorislav Petkov 96b2b0c605SBorislav Petkov return err; 97b2b0c605SBorislav Petkov } 98b2b0c605SBorislav Petkov 99b2b0c605SBorislav Petkov /* 10073ba8593SBorislav Petkov * Select DCT to which PCI cfg accesses are routed 10173ba8593SBorislav Petkov */ 10273ba8593SBorislav Petkov static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct) 10373ba8593SBorislav Petkov { 10473ba8593SBorislav Petkov u32 reg = 0; 10573ba8593SBorislav Petkov 10673ba8593SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, ®); 1077981a28fSAravind Gopalakrishnan reg &= (pvt->model == 0x30) ? ~3 : ~1; 10873ba8593SBorislav Petkov reg |= dct; 10973ba8593SBorislav Petkov amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg); 11073ba8593SBorislav Petkov } 11173ba8593SBorislav Petkov 1127981a28fSAravind Gopalakrishnan /* 1137981a28fSAravind Gopalakrishnan * 1147981a28fSAravind Gopalakrishnan * Depending on the family, F2 DCT reads need special handling: 1157981a28fSAravind Gopalakrishnan * 1167981a28fSAravind Gopalakrishnan * K8: has a single DCT only and no address offsets >= 0x100 1177981a28fSAravind Gopalakrishnan * 1187981a28fSAravind Gopalakrishnan * F10h: each DCT has its own set of regs 1197981a28fSAravind Gopalakrishnan * DCT0 -> F2x040.. 1207981a28fSAravind Gopalakrishnan * DCT1 -> F2x140.. 1217981a28fSAravind Gopalakrishnan * 1227981a28fSAravind Gopalakrishnan * F16h: has only 1 DCT 1237981a28fSAravind Gopalakrishnan * 1247981a28fSAravind Gopalakrishnan * F15h: we select which DCT we access using F1x10C[DctCfgSel] 1257981a28fSAravind Gopalakrishnan */ 1267981a28fSAravind Gopalakrishnan static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct, 1277981a28fSAravind Gopalakrishnan int offset, u32 *val) 128b2b0c605SBorislav Petkov { 1297981a28fSAravind Gopalakrishnan switch (pvt->fam) { 1307981a28fSAravind Gopalakrishnan case 0xf: 1317981a28fSAravind Gopalakrishnan if (dct || offset >= 0x100) 1327981a28fSAravind Gopalakrishnan return -EINVAL; 1337981a28fSAravind Gopalakrishnan break; 134b2b0c605SBorislav Petkov 1357981a28fSAravind Gopalakrishnan case 0x10: 1367981a28fSAravind Gopalakrishnan if (dct) { 1377981a28fSAravind Gopalakrishnan /* 1387981a28fSAravind Gopalakrishnan * Note: If ganging is enabled, barring the regs 1397981a28fSAravind Gopalakrishnan * F2x[1,0]98 and F2x[1,0]9C; reads reads to F2x1xx 1407981a28fSAravind Gopalakrishnan * return 0. (cf. Section 2.8.1 F10h BKDG) 1417981a28fSAravind Gopalakrishnan */ 1427981a28fSAravind Gopalakrishnan if (dct_ganging_enabled(pvt)) 1437981a28fSAravind Gopalakrishnan return 0; 1447981a28fSAravind Gopalakrishnan 1457981a28fSAravind Gopalakrishnan offset += 0x100; 146b2b0c605SBorislav Petkov } 1477981a28fSAravind Gopalakrishnan break; 148b2b0c605SBorislav Petkov 1497981a28fSAravind Gopalakrishnan case 0x15: 1507981a28fSAravind Gopalakrishnan /* 1517981a28fSAravind Gopalakrishnan * F15h: F2x1xx addresses do not map explicitly to DCT1. 1527981a28fSAravind Gopalakrishnan * We should select which DCT we access using F1x10C[DctCfgSel] 1537981a28fSAravind Gopalakrishnan */ 1547981a28fSAravind Gopalakrishnan dct = (dct && pvt->model == 0x30) ? 3 : dct; 15573ba8593SBorislav Petkov f15h_select_dct(pvt, dct); 1567981a28fSAravind Gopalakrishnan break; 157b2b0c605SBorislav Petkov 1587981a28fSAravind Gopalakrishnan case 0x16: 1597981a28fSAravind Gopalakrishnan if (dct) 1607981a28fSAravind Gopalakrishnan return -EINVAL; 1617981a28fSAravind Gopalakrishnan break; 1627981a28fSAravind Gopalakrishnan 1637981a28fSAravind Gopalakrishnan default: 1647981a28fSAravind Gopalakrishnan break; 1657981a28fSAravind Gopalakrishnan } 1667981a28fSAravind Gopalakrishnan return amd64_read_pci_cfg(pvt->F2, offset, val); 167b2b0c605SBorislav Petkov } 168b2b0c605SBorislav Petkov 169b70ef010SBorislav Petkov /* 1702bc65418SDoug Thompson * Memory scrubber control interface. For K8, memory scrubbing is handled by 1712bc65418SDoug Thompson * hardware and can involve L2 cache, dcache as well as the main memory. With 1722bc65418SDoug Thompson * F10, this is extended to L3 cache scrubbing on CPU models sporting that 1732bc65418SDoug Thompson * functionality. 1742bc65418SDoug Thompson * 1752bc65418SDoug Thompson * This causes the "units" for the scrubbing speed to vary from 64 byte blocks 1762bc65418SDoug Thompson * (dram) over to cache lines. This is nasty, so we will use bandwidth in 1772bc65418SDoug Thompson * bytes/sec for the setting. 1782bc65418SDoug Thompson * 1792bc65418SDoug Thompson * Currently, we only do dram scrubbing. If the scrubbing is done in software on 1802bc65418SDoug Thompson * other archs, we might not have access to the caches directly. 1812bc65418SDoug Thompson */ 1822bc65418SDoug Thompson 1838051c0afSYazen Ghannam /* 1848051c0afSYazen Ghannam * Scan the scrub rate mapping table for a close or matching bandwidth value to 1852bc65418SDoug Thompson * issue. If requested is too big, then use last maximum value found. 1862bc65418SDoug Thompson */ 187da92110dSAravind Gopalakrishnan static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate) 1882bc65418SDoug Thompson { 1892bc65418SDoug Thompson u32 scrubval; 1902bc65418SDoug Thompson int i; 1912bc65418SDoug Thompson 1922bc65418SDoug Thompson /* 1932bc65418SDoug Thompson * map the configured rate (new_bw) to a value specific to the AMD64 1942bc65418SDoug Thompson * memory controller and apply to register. Search for the first 1952bc65418SDoug Thompson * bandwidth entry that is greater or equal than the setting requested 1962bc65418SDoug Thompson * and program that. If at last entry, turn off DRAM scrubbing. 197168bfeefSAndrew Morton * 198168bfeefSAndrew Morton * If no suitable bandwidth is found, turn off DRAM scrubbing entirely 199168bfeefSAndrew Morton * by falling back to the last element in scrubrates[]. 2002bc65418SDoug Thompson */ 201168bfeefSAndrew Morton for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) { 2022bc65418SDoug Thompson /* 2032bc65418SDoug Thompson * skip scrub rates which aren't recommended 2042bc65418SDoug Thompson * (see F10 BKDG, F3x58) 2052bc65418SDoug Thompson */ 206395ae783SBorislav Petkov if (scrubrates[i].scrubval < min_rate) 2072bc65418SDoug Thompson continue; 2082bc65418SDoug Thompson 2092bc65418SDoug Thompson if (scrubrates[i].bandwidth <= new_bw) 2102bc65418SDoug Thompson break; 2112bc65418SDoug Thompson } 2122bc65418SDoug Thompson 2132bc65418SDoug Thompson scrubval = scrubrates[i].scrubval; 2142bc65418SDoug Thompson 2156e241bc9SYazen Ghannam if (pvt->fam == 0x15 && pvt->model == 0x60) { 216da92110dSAravind Gopalakrishnan f15h_select_dct(pvt, 0); 217da92110dSAravind Gopalakrishnan pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F); 218da92110dSAravind Gopalakrishnan f15h_select_dct(pvt, 1); 219da92110dSAravind Gopalakrishnan pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F); 220da92110dSAravind Gopalakrishnan } else { 221da92110dSAravind Gopalakrishnan pci_write_bits32(pvt->F3, SCRCTRL, scrubval, 0x001F); 222da92110dSAravind Gopalakrishnan } 2232bc65418SDoug Thompson 22439094443SBorislav Petkov if (scrubval) 22539094443SBorislav Petkov return scrubrates[i].bandwidth; 22639094443SBorislav Petkov 2272bc65418SDoug Thompson return 0; 2282bc65418SDoug Thompson } 2292bc65418SDoug Thompson 230d1ea71cdSBorislav Petkov static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw) 2312bc65418SDoug Thompson { 2322bc65418SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 23387b3e0e6SBorislav Petkov u32 min_scrubrate = 0x5; 2342bc65418SDoug Thompson 235a4b4bedcSBorislav Petkov if (pvt->fam == 0xf) 23687b3e0e6SBorislav Petkov min_scrubrate = 0x0; 23787b3e0e6SBorislav Petkov 238da92110dSAravind Gopalakrishnan if (pvt->fam == 0x15) { 2393f0aba4fSBorislav Petkov /* Erratum #505 */ 240da92110dSAravind Gopalakrishnan if (pvt->model < 0x10) 24173ba8593SBorislav Petkov f15h_select_dct(pvt, 0); 24273ba8593SBorislav Petkov 243da92110dSAravind Gopalakrishnan if (pvt->model == 0x60) 244da92110dSAravind Gopalakrishnan min_scrubrate = 0x6; 245da92110dSAravind Gopalakrishnan } 246da92110dSAravind Gopalakrishnan return __set_scrub_rate(pvt, bw, min_scrubrate); 2472bc65418SDoug Thompson } 2482bc65418SDoug Thompson 249d1ea71cdSBorislav Petkov static int get_scrub_rate(struct mem_ctl_info *mci) 2502bc65418SDoug Thompson { 2512bc65418SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 25239094443SBorislav Petkov int i, retval = -EINVAL; 2538051c0afSYazen Ghannam u32 scrubval = 0; 2542bc65418SDoug Thompson 2556e241bc9SYazen Ghannam if (pvt->fam == 0x15) { 256dcd01394SYazen Ghannam /* Erratum #505 */ 257dcd01394SYazen Ghannam if (pvt->model < 0x10) 258dcd01394SYazen Ghannam f15h_select_dct(pvt, 0); 2598051c0afSYazen Ghannam 260dcd01394SYazen Ghannam if (pvt->model == 0x60) 261dcd01394SYazen Ghannam amd64_read_pci_cfg(pvt->F2, F15H_M60H_SCRCTRL, &scrubval); 262ee470bb2SBorislav Petkov else 263ee470bb2SBorislav Petkov amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval); 264dcd01394SYazen Ghannam } else { 2655980bb9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval); 2668051c0afSYazen Ghannam } 2672bc65418SDoug Thompson 2682bc65418SDoug Thompson scrubval = scrubval & 0x001F; 2692bc65418SDoug Thompson 270926311fdSRoel Kluin for (i = 0; i < ARRAY_SIZE(scrubrates); i++) { 2712bc65418SDoug Thompson if (scrubrates[i].scrubval == scrubval) { 27239094443SBorislav Petkov retval = scrubrates[i].bandwidth; 2732bc65418SDoug Thompson break; 2742bc65418SDoug Thompson } 2752bc65418SDoug Thompson } 27639094443SBorislav Petkov return retval; 2772bc65418SDoug Thompson } 2782bc65418SDoug Thompson 2796775763aSDoug Thompson /* 2807f19bf75SBorislav Petkov * returns true if the SysAddr given by sys_addr matches the 2817f19bf75SBorislav Petkov * DRAM base/limit associated with node_id 2826775763aSDoug Thompson */ 283d1ea71cdSBorislav Petkov static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid) 2846775763aSDoug Thompson { 2857f19bf75SBorislav Petkov u64 addr; 2866775763aSDoug Thompson 2876775763aSDoug Thompson /* The K8 treats this as a 40-bit value. However, bits 63-40 will be 2886775763aSDoug Thompson * all ones if the most significant implemented address bit is 1. 2896775763aSDoug Thompson * Here we discard bits 63-40. See section 3.4.2 of AMD publication 2906775763aSDoug Thompson * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1 2916775763aSDoug Thompson * Application Programming. 2926775763aSDoug Thompson */ 2936775763aSDoug Thompson addr = sys_addr & 0x000000ffffffffffull; 2946775763aSDoug Thompson 2957f19bf75SBorislav Petkov return ((addr >= get_dram_base(pvt, nid)) && 2967f19bf75SBorislav Petkov (addr <= get_dram_limit(pvt, nid))); 2976775763aSDoug Thompson } 2986775763aSDoug Thompson 2996775763aSDoug Thompson /* 3006775763aSDoug Thompson * Attempt to map a SysAddr to a node. On success, return a pointer to the 3016775763aSDoug Thompson * mem_ctl_info structure for the node that the SysAddr maps to. 3026775763aSDoug Thompson * 3036775763aSDoug Thompson * On failure, return NULL. 3046775763aSDoug Thompson */ 3056775763aSDoug Thompson static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci, 3066775763aSDoug Thompson u64 sys_addr) 3076775763aSDoug Thompson { 3086775763aSDoug Thompson struct amd64_pvt *pvt; 309c7e5301aSDaniel J Blueman u8 node_id; 3106775763aSDoug Thompson u32 intlv_en, bits; 3116775763aSDoug Thompson 3126775763aSDoug Thompson /* 3136775763aSDoug Thompson * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section 3146775763aSDoug Thompson * 3.4.4.2) registers to map the SysAddr to a node ID. 3156775763aSDoug Thompson */ 3166775763aSDoug Thompson pvt = mci->pvt_info; 3176775763aSDoug Thompson 3186775763aSDoug Thompson /* 3196775763aSDoug Thompson * The value of this field should be the same for all DRAM Base 3206775763aSDoug Thompson * registers. Therefore we arbitrarily choose to read it from the 3216775763aSDoug Thompson * register for node 0. 3226775763aSDoug Thompson */ 3237f19bf75SBorislav Petkov intlv_en = dram_intlv_en(pvt, 0); 3246775763aSDoug Thompson 3256775763aSDoug Thompson if (intlv_en == 0) { 3267f19bf75SBorislav Petkov for (node_id = 0; node_id < DRAM_RANGES; node_id++) { 327d1ea71cdSBorislav Petkov if (base_limit_match(pvt, sys_addr, node_id)) 3286775763aSDoug Thompson goto found; 3296775763aSDoug Thompson } 3308edc5445SBorislav Petkov goto err_no_match; 3318edc5445SBorislav Petkov } 3326775763aSDoug Thompson 33372f158feSBorislav Petkov if (unlikely((intlv_en != 0x01) && 33472f158feSBorislav Petkov (intlv_en != 0x03) && 33572f158feSBorislav Petkov (intlv_en != 0x07))) { 33624f9a7feSBorislav Petkov amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en); 3376775763aSDoug Thompson return NULL; 3386775763aSDoug Thompson } 3396775763aSDoug Thompson 3406775763aSDoug Thompson bits = (((u32) sys_addr) >> 12) & intlv_en; 3416775763aSDoug Thompson 3426775763aSDoug Thompson for (node_id = 0; ; ) { 3437f19bf75SBorislav Petkov if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits) 3446775763aSDoug Thompson break; /* intlv_sel field matches */ 3456775763aSDoug Thompson 3467f19bf75SBorislav Petkov if (++node_id >= DRAM_RANGES) 3476775763aSDoug Thompson goto err_no_match; 3486775763aSDoug Thompson } 3496775763aSDoug Thompson 3506775763aSDoug Thompson /* sanity test for sys_addr */ 351d1ea71cdSBorislav Petkov if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) { 35224f9a7feSBorislav Petkov amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address" 35324f9a7feSBorislav Petkov "range for node %d with node interleaving enabled.\n", 3548edc5445SBorislav Petkov __func__, sys_addr, node_id); 3556775763aSDoug Thompson return NULL; 3566775763aSDoug Thompson } 3576775763aSDoug Thompson 3586775763aSDoug Thompson found: 359b487c33eSBorislav Petkov return edac_mc_find((int)node_id); 3606775763aSDoug Thompson 3616775763aSDoug Thompson err_no_match: 362956b9ba1SJoe Perches edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n", 3636775763aSDoug Thompson (unsigned long)sys_addr); 3646775763aSDoug Thompson 3656775763aSDoug Thompson return NULL; 3666775763aSDoug Thompson } 367e2ce7255SDoug Thompson 368e2ce7255SDoug Thompson /* 36911c75eadSBorislav Petkov * compute the CS base address of the @csrow on the DRAM controller @dct. 37011c75eadSBorislav Petkov * For details see F2x[5C:40] in the processor's BKDG 371e2ce7255SDoug Thompson */ 37211c75eadSBorislav Petkov static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct, 37311c75eadSBorislav Petkov u64 *base, u64 *mask) 374e2ce7255SDoug Thompson { 37511c75eadSBorislav Petkov u64 csbase, csmask, base_bits, mask_bits; 37611c75eadSBorislav Petkov u8 addr_shift; 37711c75eadSBorislav Petkov 37818b94f66SAravind Gopalakrishnan if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) { 37911c75eadSBorislav Petkov csbase = pvt->csels[dct].csbases[csrow]; 38011c75eadSBorislav Petkov csmask = pvt->csels[dct].csmasks[csrow]; 38110ef6b0dSChen, Gong base_bits = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9); 38210ef6b0dSChen, Gong mask_bits = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9); 38311c75eadSBorislav Petkov addr_shift = 4; 38494c1acf2SAravind Gopalakrishnan 38594c1acf2SAravind Gopalakrishnan /* 38618b94f66SAravind Gopalakrishnan * F16h and F15h, models 30h and later need two addr_shift values: 38718b94f66SAravind Gopalakrishnan * 8 for high and 6 for low (cf. F16h BKDG). 38894c1acf2SAravind Gopalakrishnan */ 38918b94f66SAravind Gopalakrishnan } else if (pvt->fam == 0x16 || 39018b94f66SAravind Gopalakrishnan (pvt->fam == 0x15 && pvt->model >= 0x30)) { 39194c1acf2SAravind Gopalakrishnan csbase = pvt->csels[dct].csbases[csrow]; 39294c1acf2SAravind Gopalakrishnan csmask = pvt->csels[dct].csmasks[csrow >> 1]; 39394c1acf2SAravind Gopalakrishnan 39410ef6b0dSChen, Gong *base = (csbase & GENMASK_ULL(15, 5)) << 6; 39510ef6b0dSChen, Gong *base |= (csbase & GENMASK_ULL(30, 19)) << 8; 39694c1acf2SAravind Gopalakrishnan 39794c1acf2SAravind Gopalakrishnan *mask = ~0ULL; 39894c1acf2SAravind Gopalakrishnan /* poke holes for the csmask */ 39910ef6b0dSChen, Gong *mask &= ~((GENMASK_ULL(15, 5) << 6) | 40010ef6b0dSChen, Gong (GENMASK_ULL(30, 19) << 8)); 40194c1acf2SAravind Gopalakrishnan 40210ef6b0dSChen, Gong *mask |= (csmask & GENMASK_ULL(15, 5)) << 6; 40310ef6b0dSChen, Gong *mask |= (csmask & GENMASK_ULL(30, 19)) << 8; 40494c1acf2SAravind Gopalakrishnan 40594c1acf2SAravind Gopalakrishnan return; 40611c75eadSBorislav Petkov } else { 40711c75eadSBorislav Petkov csbase = pvt->csels[dct].csbases[csrow]; 40811c75eadSBorislav Petkov csmask = pvt->csels[dct].csmasks[csrow >> 1]; 40911c75eadSBorislav Petkov addr_shift = 8; 41011c75eadSBorislav Petkov 411a4b4bedcSBorislav Petkov if (pvt->fam == 0x15) 41210ef6b0dSChen, Gong base_bits = mask_bits = 41310ef6b0dSChen, Gong GENMASK_ULL(30,19) | GENMASK_ULL(13,5); 41411c75eadSBorislav Petkov else 41510ef6b0dSChen, Gong base_bits = mask_bits = 41610ef6b0dSChen, Gong GENMASK_ULL(28,19) | GENMASK_ULL(13,5); 417e2ce7255SDoug Thompson } 418e2ce7255SDoug Thompson 41911c75eadSBorislav Petkov *base = (csbase & base_bits) << addr_shift; 420e2ce7255SDoug Thompson 42111c75eadSBorislav Petkov *mask = ~0ULL; 42211c75eadSBorislav Petkov /* poke holes for the csmask */ 42311c75eadSBorislav Petkov *mask &= ~(mask_bits << addr_shift); 42411c75eadSBorislav Petkov /* OR them in */ 42511c75eadSBorislav Petkov *mask |= (csmask & mask_bits) << addr_shift; 426e2ce7255SDoug Thompson } 427e2ce7255SDoug Thompson 42811c75eadSBorislav Petkov #define for_each_chip_select(i, dct, pvt) \ 42911c75eadSBorislav Petkov for (i = 0; i < pvt->csels[dct].b_cnt; i++) 43011c75eadSBorislav Petkov 431614ec9d8SBorislav Petkov #define chip_select_base(i, dct, pvt) \ 432614ec9d8SBorislav Petkov pvt->csels[dct].csbases[i] 433614ec9d8SBorislav Petkov 43411c75eadSBorislav Petkov #define for_each_chip_select_mask(i, dct, pvt) \ 43511c75eadSBorislav Petkov for (i = 0; i < pvt->csels[dct].m_cnt; i++) 43611c75eadSBorislav Petkov 4374d30d2bcSYazen Ghannam #define for_each_umc(i) \ 438ed623d55SMuralidhara M K for (i = 0; i < pvt->max_mcs; i++) 4394d30d2bcSYazen Ghannam 440e2ce7255SDoug Thompson /* 441e2ce7255SDoug Thompson * @input_addr is an InputAddr associated with the node given by mci. Return the 442e2ce7255SDoug Thompson * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr). 443e2ce7255SDoug Thompson */ 444e2ce7255SDoug Thompson static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr) 445e2ce7255SDoug Thompson { 446e2ce7255SDoug Thompson struct amd64_pvt *pvt; 447e2ce7255SDoug Thompson int csrow; 448e2ce7255SDoug Thompson u64 base, mask; 449e2ce7255SDoug Thompson 450e2ce7255SDoug Thompson pvt = mci->pvt_info; 451e2ce7255SDoug Thompson 45211c75eadSBorislav Petkov for_each_chip_select(csrow, 0, pvt) { 45311c75eadSBorislav Petkov if (!csrow_enabled(csrow, 0, pvt)) 454e2ce7255SDoug Thompson continue; 455e2ce7255SDoug Thompson 45611c75eadSBorislav Petkov get_cs_base_and_mask(pvt, csrow, 0, &base, &mask); 45711c75eadSBorislav Petkov 45811c75eadSBorislav Petkov mask = ~mask; 459e2ce7255SDoug Thompson 460e2ce7255SDoug Thompson if ((input_addr & mask) == (base & mask)) { 461956b9ba1SJoe Perches edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n", 462e2ce7255SDoug Thompson (unsigned long)input_addr, csrow, 463e2ce7255SDoug Thompson pvt->mc_node_id); 464e2ce7255SDoug Thompson 465e2ce7255SDoug Thompson return csrow; 466e2ce7255SDoug Thompson } 467e2ce7255SDoug Thompson } 468956b9ba1SJoe Perches edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n", 469e2ce7255SDoug Thompson (unsigned long)input_addr, pvt->mc_node_id); 470e2ce7255SDoug Thompson 471e2ce7255SDoug Thompson return -1; 472e2ce7255SDoug Thompson } 473e2ce7255SDoug Thompson 474e2ce7255SDoug Thompson /* 475e2ce7255SDoug Thompson * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094) 476e2ce7255SDoug Thompson * for the node represented by mci. Info is passed back in *hole_base, 477e2ce7255SDoug Thompson * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if 478e2ce7255SDoug Thompson * info is invalid. Info may be invalid for either of the following reasons: 479e2ce7255SDoug Thompson * 480e2ce7255SDoug Thompson * - The revision of the node is not E or greater. In this case, the DRAM Hole 481e2ce7255SDoug Thompson * Address Register does not exist. 482e2ce7255SDoug Thompson * 483e2ce7255SDoug Thompson * - The DramHoleValid bit is cleared in the DRAM Hole Address Register, 484e2ce7255SDoug Thompson * indicating that its contents are not valid. 485e2ce7255SDoug Thompson * 486e2ce7255SDoug Thompson * The values passed back in *hole_base, *hole_offset, and *hole_size are 487e2ce7255SDoug Thompson * complete 32-bit values despite the fact that the bitfields in the DHAR 488e2ce7255SDoug Thompson * only represent bits 31-24 of the base and offset values. 489e2ce7255SDoug Thompson */ 4902a28ceefSBorislav Petkov static int get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base, 491e2ce7255SDoug Thompson u64 *hole_offset, u64 *hole_size) 492e2ce7255SDoug Thompson { 493e2ce7255SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 494e2ce7255SDoug Thompson 495e2ce7255SDoug Thompson /* only revE and later have the DRAM Hole Address Register */ 496a4b4bedcSBorislav Petkov if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) { 497956b9ba1SJoe Perches edac_dbg(1, " revision %d for node %d does not support DHAR\n", 498e2ce7255SDoug Thompson pvt->ext_model, pvt->mc_node_id); 499e2ce7255SDoug Thompson return 1; 500e2ce7255SDoug Thompson } 501e2ce7255SDoug Thompson 502bc21fa57SBorislav Petkov /* valid for Fam10h and above */ 503a4b4bedcSBorislav Petkov if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) { 504956b9ba1SJoe Perches edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n"); 505e2ce7255SDoug Thompson return 1; 506e2ce7255SDoug Thompson } 507e2ce7255SDoug Thompson 508c8e518d5SBorislav Petkov if (!dhar_valid(pvt)) { 509956b9ba1SJoe Perches edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n", 510e2ce7255SDoug Thompson pvt->mc_node_id); 511e2ce7255SDoug Thompson return 1; 512e2ce7255SDoug Thompson } 513e2ce7255SDoug Thompson 514e2ce7255SDoug Thompson /* This node has Memory Hoisting */ 515e2ce7255SDoug Thompson 516e2ce7255SDoug Thompson /* +------------------+--------------------+--------------------+----- 517e2ce7255SDoug Thompson * | memory | DRAM hole | relocated | 518e2ce7255SDoug Thompson * | [0, (x - 1)] | [x, 0xffffffff] | addresses from | 519e2ce7255SDoug Thompson * | | | DRAM hole | 520e2ce7255SDoug Thompson * | | | [0x100000000, | 521e2ce7255SDoug Thompson * | | | (0x100000000+ | 522e2ce7255SDoug Thompson * | | | (0xffffffff-x))] | 523e2ce7255SDoug Thompson * +------------------+--------------------+--------------------+----- 524e2ce7255SDoug Thompson * 525e2ce7255SDoug Thompson * Above is a diagram of physical memory showing the DRAM hole and the 526e2ce7255SDoug Thompson * relocated addresses from the DRAM hole. As shown, the DRAM hole 527e2ce7255SDoug Thompson * starts at address x (the base address) and extends through address 528e2ce7255SDoug Thompson * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the 529e2ce7255SDoug Thompson * addresses in the hole so that they start at 0x100000000. 530e2ce7255SDoug Thompson */ 531e2ce7255SDoug Thompson 5321f31677eSBorislav Petkov *hole_base = dhar_base(pvt); 5331f31677eSBorislav Petkov *hole_size = (1ULL << 32) - *hole_base; 534e2ce7255SDoug Thompson 535a4b4bedcSBorislav Petkov *hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt) 536a4b4bedcSBorislav Petkov : k8_dhar_offset(pvt); 537e2ce7255SDoug Thompson 538956b9ba1SJoe Perches edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n", 539e2ce7255SDoug Thompson pvt->mc_node_id, (unsigned long)*hole_base, 540e2ce7255SDoug Thompson (unsigned long)*hole_offset, (unsigned long)*hole_size); 541e2ce7255SDoug Thompson 542e2ce7255SDoug Thompson return 0; 543e2ce7255SDoug Thompson } 5442a28ceefSBorislav Petkov 5452a28ceefSBorislav Petkov #ifdef CONFIG_EDAC_DEBUG 5462a28ceefSBorislav Petkov #define EDAC_DCT_ATTR_SHOW(reg) \ 5472a28ceefSBorislav Petkov static ssize_t reg##_show(struct device *dev, \ 5482a28ceefSBorislav Petkov struct device_attribute *mattr, char *data) \ 5492a28ceefSBorislav Petkov { \ 5502a28ceefSBorislav Petkov struct mem_ctl_info *mci = to_mci(dev); \ 5512a28ceefSBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; \ 5522a28ceefSBorislav Petkov \ 5532a28ceefSBorislav Petkov return sprintf(data, "0x%016llx\n", (u64)pvt->reg); \ 5542a28ceefSBorislav Petkov } 5552a28ceefSBorislav Petkov 5562a28ceefSBorislav Petkov EDAC_DCT_ATTR_SHOW(dhar); 5572a28ceefSBorislav Petkov EDAC_DCT_ATTR_SHOW(dbam0); 5582a28ceefSBorislav Petkov EDAC_DCT_ATTR_SHOW(top_mem); 5592a28ceefSBorislav Petkov EDAC_DCT_ATTR_SHOW(top_mem2); 5602a28ceefSBorislav Petkov 561d19faf0eSDwaipayan Ray static ssize_t dram_hole_show(struct device *dev, struct device_attribute *mattr, 5622a28ceefSBorislav Petkov char *data) 5632a28ceefSBorislav Petkov { 5642a28ceefSBorislav Petkov struct mem_ctl_info *mci = to_mci(dev); 5652a28ceefSBorislav Petkov 5662a28ceefSBorislav Petkov u64 hole_base = 0; 5672a28ceefSBorislav Petkov u64 hole_offset = 0; 5682a28ceefSBorislav Petkov u64 hole_size = 0; 5692a28ceefSBorislav Petkov 5702a28ceefSBorislav Petkov get_dram_hole_info(mci, &hole_base, &hole_offset, &hole_size); 5712a28ceefSBorislav Petkov 5722a28ceefSBorislav Petkov return sprintf(data, "%llx %llx %llx\n", hole_base, hole_offset, 5732a28ceefSBorislav Petkov hole_size); 5742a28ceefSBorislav Petkov } 5752a28ceefSBorislav Petkov 5762a28ceefSBorislav Petkov /* 5772a28ceefSBorislav Petkov * update NUM_DBG_ATTRS in case you add new members 5782a28ceefSBorislav Petkov */ 5792a28ceefSBorislav Petkov static DEVICE_ATTR(dhar, S_IRUGO, dhar_show, NULL); 5802a28ceefSBorislav Petkov static DEVICE_ATTR(dbam, S_IRUGO, dbam0_show, NULL); 5812a28ceefSBorislav Petkov static DEVICE_ATTR(topmem, S_IRUGO, top_mem_show, NULL); 5822a28ceefSBorislav Petkov static DEVICE_ATTR(topmem2, S_IRUGO, top_mem2_show, NULL); 583d19faf0eSDwaipayan Ray static DEVICE_ATTR_RO(dram_hole); 5842a28ceefSBorislav Petkov 5852a28ceefSBorislav Petkov static struct attribute *dbg_attrs[] = { 5862a28ceefSBorislav Petkov &dev_attr_dhar.attr, 5872a28ceefSBorislav Petkov &dev_attr_dbam.attr, 5882a28ceefSBorislav Petkov &dev_attr_topmem.attr, 5892a28ceefSBorislav Petkov &dev_attr_topmem2.attr, 5902a28ceefSBorislav Petkov &dev_attr_dram_hole.attr, 5912a28ceefSBorislav Petkov NULL 5922a28ceefSBorislav Petkov }; 5932a28ceefSBorislav Petkov 5942a28ceefSBorislav Petkov static const struct attribute_group dbg_group = { 5952a28ceefSBorislav Petkov .attrs = dbg_attrs, 5962a28ceefSBorislav Petkov }; 5972a28ceefSBorislav Petkov 59861810096SBorislav Petkov static ssize_t inject_section_show(struct device *dev, 59961810096SBorislav Petkov struct device_attribute *mattr, char *buf) 60061810096SBorislav Petkov { 60161810096SBorislav Petkov struct mem_ctl_info *mci = to_mci(dev); 60261810096SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 60361810096SBorislav Petkov return sprintf(buf, "0x%x\n", pvt->injection.section); 60461810096SBorislav Petkov } 60561810096SBorislav Petkov 60661810096SBorislav Petkov /* 60761810096SBorislav Petkov * store error injection section value which refers to one of 4 16-byte sections 60861810096SBorislav Petkov * within a 64-byte cacheline 60961810096SBorislav Petkov * 61061810096SBorislav Petkov * range: 0..3 61161810096SBorislav Petkov */ 61261810096SBorislav Petkov static ssize_t inject_section_store(struct device *dev, 61361810096SBorislav Petkov struct device_attribute *mattr, 61461810096SBorislav Petkov const char *data, size_t count) 61561810096SBorislav Petkov { 61661810096SBorislav Petkov struct mem_ctl_info *mci = to_mci(dev); 61761810096SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 61861810096SBorislav Petkov unsigned long value; 61961810096SBorislav Petkov int ret; 62061810096SBorislav Petkov 62161810096SBorislav Petkov ret = kstrtoul(data, 10, &value); 62261810096SBorislav Petkov if (ret < 0) 62361810096SBorislav Petkov return ret; 62461810096SBorislav Petkov 62561810096SBorislav Petkov if (value > 3) { 62661810096SBorislav Petkov amd64_warn("%s: invalid section 0x%lx\n", __func__, value); 62761810096SBorislav Petkov return -EINVAL; 62861810096SBorislav Petkov } 62961810096SBorislav Petkov 63061810096SBorislav Petkov pvt->injection.section = (u32) value; 63161810096SBorislav Petkov return count; 63261810096SBorislav Petkov } 63361810096SBorislav Petkov 63461810096SBorislav Petkov static ssize_t inject_word_show(struct device *dev, 63561810096SBorislav Petkov struct device_attribute *mattr, char *buf) 63661810096SBorislav Petkov { 63761810096SBorislav Petkov struct mem_ctl_info *mci = to_mci(dev); 63861810096SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 63961810096SBorislav Petkov return sprintf(buf, "0x%x\n", pvt->injection.word); 64061810096SBorislav Petkov } 64161810096SBorislav Petkov 64261810096SBorislav Petkov /* 64361810096SBorislav Petkov * store error injection word value which refers to one of 9 16-bit word of the 64461810096SBorislav Petkov * 16-byte (128-bit + ECC bits) section 64561810096SBorislav Petkov * 64661810096SBorislav Petkov * range: 0..8 64761810096SBorislav Petkov */ 64861810096SBorislav Petkov static ssize_t inject_word_store(struct device *dev, 64961810096SBorislav Petkov struct device_attribute *mattr, 65061810096SBorislav Petkov const char *data, size_t count) 65161810096SBorislav Petkov { 65261810096SBorislav Petkov struct mem_ctl_info *mci = to_mci(dev); 65361810096SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 65461810096SBorislav Petkov unsigned long value; 65561810096SBorislav Petkov int ret; 65661810096SBorislav Petkov 65761810096SBorislav Petkov ret = kstrtoul(data, 10, &value); 65861810096SBorislav Petkov if (ret < 0) 65961810096SBorislav Petkov return ret; 66061810096SBorislav Petkov 66161810096SBorislav Petkov if (value > 8) { 66261810096SBorislav Petkov amd64_warn("%s: invalid word 0x%lx\n", __func__, value); 66361810096SBorislav Petkov return -EINVAL; 66461810096SBorislav Petkov } 66561810096SBorislav Petkov 66661810096SBorislav Petkov pvt->injection.word = (u32) value; 66761810096SBorislav Petkov return count; 66861810096SBorislav Petkov } 66961810096SBorislav Petkov 67061810096SBorislav Petkov static ssize_t inject_ecc_vector_show(struct device *dev, 67161810096SBorislav Petkov struct device_attribute *mattr, 67261810096SBorislav Petkov char *buf) 67361810096SBorislav Petkov { 67461810096SBorislav Petkov struct mem_ctl_info *mci = to_mci(dev); 67561810096SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 67661810096SBorislav Petkov return sprintf(buf, "0x%x\n", pvt->injection.bit_map); 67761810096SBorislav Petkov } 67861810096SBorislav Petkov 67961810096SBorislav Petkov /* 68061810096SBorislav Petkov * store 16 bit error injection vector which enables injecting errors to the 68161810096SBorislav Petkov * corresponding bit within the error injection word above. When used during a 68261810096SBorislav Petkov * DRAM ECC read, it holds the contents of the of the DRAM ECC bits. 68361810096SBorislav Petkov */ 68461810096SBorislav Petkov static ssize_t inject_ecc_vector_store(struct device *dev, 68561810096SBorislav Petkov struct device_attribute *mattr, 68661810096SBorislav Petkov const char *data, size_t count) 68761810096SBorislav Petkov { 68861810096SBorislav Petkov struct mem_ctl_info *mci = to_mci(dev); 68961810096SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 69061810096SBorislav Petkov unsigned long value; 69161810096SBorislav Petkov int ret; 69261810096SBorislav Petkov 69361810096SBorislav Petkov ret = kstrtoul(data, 16, &value); 69461810096SBorislav Petkov if (ret < 0) 69561810096SBorislav Petkov return ret; 69661810096SBorislav Petkov 69761810096SBorislav Petkov if (value & 0xFFFF0000) { 69861810096SBorislav Petkov amd64_warn("%s: invalid EccVector: 0x%lx\n", __func__, value); 69961810096SBorislav Petkov return -EINVAL; 70061810096SBorislav Petkov } 70161810096SBorislav Petkov 70261810096SBorislav Petkov pvt->injection.bit_map = (u32) value; 70361810096SBorislav Petkov return count; 70461810096SBorislav Petkov } 70561810096SBorislav Petkov 70661810096SBorislav Petkov /* 70761810096SBorislav Petkov * Do a DRAM ECC read. Assemble staged values in the pvt area, format into 70861810096SBorislav Petkov * fields needed by the injection registers and read the NB Array Data Port. 70961810096SBorislav Petkov */ 71061810096SBorislav Petkov static ssize_t inject_read_store(struct device *dev, 71161810096SBorislav Petkov struct device_attribute *mattr, 71261810096SBorislav Petkov const char *data, size_t count) 71361810096SBorislav Petkov { 71461810096SBorislav Petkov struct mem_ctl_info *mci = to_mci(dev); 71561810096SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 71661810096SBorislav Petkov unsigned long value; 71761810096SBorislav Petkov u32 section, word_bits; 71861810096SBorislav Petkov int ret; 71961810096SBorislav Petkov 72061810096SBorislav Petkov ret = kstrtoul(data, 10, &value); 72161810096SBorislav Petkov if (ret < 0) 72261810096SBorislav Petkov return ret; 72361810096SBorislav Petkov 72461810096SBorislav Petkov /* Form value to choose 16-byte section of cacheline */ 72561810096SBorislav Petkov section = F10_NB_ARRAY_DRAM | SET_NB_ARRAY_ADDR(pvt->injection.section); 72661810096SBorislav Petkov 72761810096SBorislav Petkov amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_ADDR, section); 72861810096SBorislav Petkov 72961810096SBorislav Petkov word_bits = SET_NB_DRAM_INJECTION_READ(pvt->injection); 73061810096SBorislav Petkov 73161810096SBorislav Petkov /* Issue 'word' and 'bit' along with the READ request */ 73261810096SBorislav Petkov amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, word_bits); 73361810096SBorislav Petkov 73461810096SBorislav Petkov edac_dbg(0, "section=0x%x word_bits=0x%x\n", section, word_bits); 73561810096SBorislav Petkov 73661810096SBorislav Petkov return count; 73761810096SBorislav Petkov } 73861810096SBorislav Petkov 73961810096SBorislav Petkov /* 74061810096SBorislav Petkov * Do a DRAM ECC write. Assemble staged values in the pvt area and format into 74161810096SBorislav Petkov * fields needed by the injection registers. 74261810096SBorislav Petkov */ 74361810096SBorislav Petkov static ssize_t inject_write_store(struct device *dev, 74461810096SBorislav Petkov struct device_attribute *mattr, 74561810096SBorislav Petkov const char *data, size_t count) 74661810096SBorislav Petkov { 74761810096SBorislav Petkov struct mem_ctl_info *mci = to_mci(dev); 74861810096SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 74961810096SBorislav Petkov u32 section, word_bits, tmp; 75061810096SBorislav Petkov unsigned long value; 75161810096SBorislav Petkov int ret; 75261810096SBorislav Petkov 75361810096SBorislav Petkov ret = kstrtoul(data, 10, &value); 75461810096SBorislav Petkov if (ret < 0) 75561810096SBorislav Petkov return ret; 75661810096SBorislav Petkov 75761810096SBorislav Petkov /* Form value to choose 16-byte section of cacheline */ 75861810096SBorislav Petkov section = F10_NB_ARRAY_DRAM | SET_NB_ARRAY_ADDR(pvt->injection.section); 75961810096SBorislav Petkov 76061810096SBorislav Petkov amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_ADDR, section); 76161810096SBorislav Petkov 76261810096SBorislav Petkov word_bits = SET_NB_DRAM_INJECTION_WRITE(pvt->injection); 76361810096SBorislav Petkov 76461810096SBorislav Petkov pr_notice_once("Don't forget to decrease MCE polling interval in\n" 76561810096SBorislav Petkov "/sys/bus/machinecheck/devices/machinecheck<CPUNUM>/check_interval\n" 76661810096SBorislav Petkov "so that you can get the error report faster.\n"); 76761810096SBorislav Petkov 76861810096SBorislav Petkov on_each_cpu(disable_caches, NULL, 1); 76961810096SBorislav Petkov 77061810096SBorislav Petkov /* Issue 'word' and 'bit' along with the READ request */ 77161810096SBorislav Petkov amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, word_bits); 77261810096SBorislav Petkov 77361810096SBorislav Petkov retry: 77461810096SBorislav Petkov /* wait until injection happens */ 77561810096SBorislav Petkov amd64_read_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, &tmp); 77661810096SBorislav Petkov if (tmp & F10_NB_ARR_ECC_WR_REQ) { 77761810096SBorislav Petkov cpu_relax(); 77861810096SBorislav Petkov goto retry; 77961810096SBorislav Petkov } 78061810096SBorislav Petkov 78161810096SBorislav Petkov on_each_cpu(enable_caches, NULL, 1); 78261810096SBorislav Petkov 78361810096SBorislav Petkov edac_dbg(0, "section=0x%x word_bits=0x%x\n", section, word_bits); 78461810096SBorislav Petkov 78561810096SBorislav Petkov return count; 78661810096SBorislav Petkov } 78761810096SBorislav Petkov 78861810096SBorislav Petkov /* 78961810096SBorislav Petkov * update NUM_INJ_ATTRS in case you add new members 79061810096SBorislav Petkov */ 79161810096SBorislav Petkov 792d19faf0eSDwaipayan Ray static DEVICE_ATTR_RW(inject_section); 793d19faf0eSDwaipayan Ray static DEVICE_ATTR_RW(inject_word); 794d19faf0eSDwaipayan Ray static DEVICE_ATTR_RW(inject_ecc_vector); 795d19faf0eSDwaipayan Ray static DEVICE_ATTR_WO(inject_write); 796d19faf0eSDwaipayan Ray static DEVICE_ATTR_WO(inject_read); 79761810096SBorislav Petkov 79861810096SBorislav Petkov static struct attribute *inj_attrs[] = { 79961810096SBorislav Petkov &dev_attr_inject_section.attr, 80061810096SBorislav Petkov &dev_attr_inject_word.attr, 80161810096SBorislav Petkov &dev_attr_inject_ecc_vector.attr, 80261810096SBorislav Petkov &dev_attr_inject_write.attr, 80361810096SBorislav Petkov &dev_attr_inject_read.attr, 80461810096SBorislav Petkov NULL 80561810096SBorislav Petkov }; 80661810096SBorislav Petkov 80761810096SBorislav Petkov static umode_t inj_is_visible(struct kobject *kobj, struct attribute *attr, int idx) 80861810096SBorislav Petkov { 80961810096SBorislav Petkov struct device *dev = kobj_to_dev(kobj); 81061810096SBorislav Petkov struct mem_ctl_info *mci = container_of(dev, struct mem_ctl_info, dev); 81161810096SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 81261810096SBorislav Petkov 8131865bc71SBorislav Petkov /* Families which have that injection hw */ 8141865bc71SBorislav Petkov if (pvt->fam >= 0x10 && pvt->fam <= 0x16) 81561810096SBorislav Petkov return attr->mode; 8161865bc71SBorislav Petkov 8171865bc71SBorislav Petkov return 0; 81861810096SBorislav Petkov } 81961810096SBorislav Petkov 82061810096SBorislav Petkov static const struct attribute_group inj_group = { 82161810096SBorislav Petkov .attrs = inj_attrs, 82261810096SBorislav Petkov .is_visible = inj_is_visible, 82361810096SBorislav Petkov }; 82461810096SBorislav Petkov #endif /* CONFIG_EDAC_DEBUG */ 825e2ce7255SDoug Thompson 82693c2df58SDoug Thompson /* 82793c2df58SDoug Thompson * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is 82893c2df58SDoug Thompson * assumed that sys_addr maps to the node given by mci. 82993c2df58SDoug Thompson * 83093c2df58SDoug Thompson * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section 83193c2df58SDoug Thompson * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a 83293c2df58SDoug Thompson * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled, 83393c2df58SDoug Thompson * then it is also involved in translating a SysAddr to a DramAddr. Sections 83493c2df58SDoug Thompson * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting. 83593c2df58SDoug Thompson * These parts of the documentation are unclear. I interpret them as follows: 83693c2df58SDoug Thompson * 83793c2df58SDoug Thompson * When node n receives a SysAddr, it processes the SysAddr as follows: 83893c2df58SDoug Thompson * 83993c2df58SDoug Thompson * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM 84093c2df58SDoug Thompson * Limit registers for node n. If the SysAddr is not within the range 84193c2df58SDoug Thompson * specified by the base and limit values, then node n ignores the Sysaddr 84293c2df58SDoug Thompson * (since it does not map to node n). Otherwise continue to step 2 below. 84393c2df58SDoug Thompson * 84493c2df58SDoug Thompson * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is 84593c2df58SDoug Thompson * disabled so skip to step 3 below. Otherwise see if the SysAddr is within 84693c2df58SDoug Thompson * the range of relocated addresses (starting at 0x100000000) from the DRAM 84793c2df58SDoug Thompson * hole. If not, skip to step 3 below. Else get the value of the 84893c2df58SDoug Thompson * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the 84993c2df58SDoug Thompson * offset defined by this value from the SysAddr. 85093c2df58SDoug Thompson * 85193c2df58SDoug Thompson * 3. Obtain the base address for node n from the DRAMBase field of the DRAM 85293c2df58SDoug Thompson * Base register for node n. To obtain the DramAddr, subtract the base 85393c2df58SDoug Thompson * address from the SysAddr, as shown near the start of section 3.4.4 (p.70). 85493c2df58SDoug Thompson */ 85593c2df58SDoug Thompson static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr) 85693c2df58SDoug Thompson { 8577f19bf75SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 85893c2df58SDoug Thompson u64 dram_base, hole_base, hole_offset, hole_size, dram_addr; 8591f31677eSBorislav Petkov int ret; 86093c2df58SDoug Thompson 8617f19bf75SBorislav Petkov dram_base = get_dram_base(pvt, pvt->mc_node_id); 86293c2df58SDoug Thompson 8632a28ceefSBorislav Petkov ret = get_dram_hole_info(mci, &hole_base, &hole_offset, &hole_size); 86493c2df58SDoug Thompson if (!ret) { 8651f31677eSBorislav Petkov if ((sys_addr >= (1ULL << 32)) && 8661f31677eSBorislav Petkov (sys_addr < ((1ULL << 32) + hole_size))) { 86793c2df58SDoug Thompson /* use DHAR to translate SysAddr to DramAddr */ 86893c2df58SDoug Thompson dram_addr = sys_addr - hole_offset; 86993c2df58SDoug Thompson 870956b9ba1SJoe Perches edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n", 87193c2df58SDoug Thompson (unsigned long)sys_addr, 87293c2df58SDoug Thompson (unsigned long)dram_addr); 87393c2df58SDoug Thompson 87493c2df58SDoug Thompson return dram_addr; 87593c2df58SDoug Thompson } 87693c2df58SDoug Thompson } 87793c2df58SDoug Thompson 87893c2df58SDoug Thompson /* 87993c2df58SDoug Thompson * Translate the SysAddr to a DramAddr as shown near the start of 88093c2df58SDoug Thompson * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8 88193c2df58SDoug Thompson * only deals with 40-bit values. Therefore we discard bits 63-40 of 88293c2df58SDoug Thompson * sys_addr below. If bit 39 of sys_addr is 1 then the bits we 88393c2df58SDoug Thompson * discard are all 1s. Otherwise the bits we discard are all 0s. See 88493c2df58SDoug Thompson * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture 88593c2df58SDoug Thompson * Programmer's Manual Volume 1 Application Programming. 88693c2df58SDoug Thompson */ 88710ef6b0dSChen, Gong dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base; 88893c2df58SDoug Thompson 889956b9ba1SJoe Perches edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n", 890956b9ba1SJoe Perches (unsigned long)sys_addr, (unsigned long)dram_addr); 89193c2df58SDoug Thompson return dram_addr; 89293c2df58SDoug Thompson } 89393c2df58SDoug Thompson 89493c2df58SDoug Thompson /* 89593c2df58SDoug Thompson * @intlv_en is the value of the IntlvEn field from a DRAM Base register 89693c2df58SDoug Thompson * (section 3.4.4.1). Return the number of bits from a SysAddr that are used 89793c2df58SDoug Thompson * for node interleaving. 89893c2df58SDoug Thompson */ 89993c2df58SDoug Thompson static int num_node_interleave_bits(unsigned intlv_en) 90093c2df58SDoug Thompson { 90193c2df58SDoug Thompson static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 }; 90293c2df58SDoug Thompson int n; 90393c2df58SDoug Thompson 90493c2df58SDoug Thompson BUG_ON(intlv_en > 7); 90593c2df58SDoug Thompson n = intlv_shift_table[intlv_en]; 90693c2df58SDoug Thompson return n; 90793c2df58SDoug Thompson } 90893c2df58SDoug Thompson 90993c2df58SDoug Thompson /* Translate the DramAddr given by @dram_addr to an InputAddr. */ 91093c2df58SDoug Thompson static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr) 91193c2df58SDoug Thompson { 91293c2df58SDoug Thompson struct amd64_pvt *pvt; 91393c2df58SDoug Thompson int intlv_shift; 91493c2df58SDoug Thompson u64 input_addr; 91593c2df58SDoug Thompson 91693c2df58SDoug Thompson pvt = mci->pvt_info; 91793c2df58SDoug Thompson 91893c2df58SDoug Thompson /* 91993c2df58SDoug Thompson * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E) 92093c2df58SDoug Thompson * concerning translating a DramAddr to an InputAddr. 92193c2df58SDoug Thompson */ 9227f19bf75SBorislav Petkov intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0)); 92310ef6b0dSChen, Gong input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) + 92493c2df58SDoug Thompson (dram_addr & 0xfff); 92593c2df58SDoug Thompson 926956b9ba1SJoe Perches edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n", 92793c2df58SDoug Thompson intlv_shift, (unsigned long)dram_addr, 92893c2df58SDoug Thompson (unsigned long)input_addr); 92993c2df58SDoug Thompson 93093c2df58SDoug Thompson return input_addr; 93193c2df58SDoug Thompson } 93293c2df58SDoug Thompson 93393c2df58SDoug Thompson /* 93493c2df58SDoug Thompson * Translate the SysAddr represented by @sys_addr to an InputAddr. It is 93593c2df58SDoug Thompson * assumed that @sys_addr maps to the node given by mci. 93693c2df58SDoug Thompson */ 93793c2df58SDoug Thompson static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr) 93893c2df58SDoug Thompson { 93993c2df58SDoug Thompson u64 input_addr; 94093c2df58SDoug Thompson 94193c2df58SDoug Thompson input_addr = 94293c2df58SDoug Thompson dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr)); 94393c2df58SDoug Thompson 944c19ca6cbSMasanari Iida edac_dbg(2, "SysAddr 0x%lx translates to InputAddr 0x%lx\n", 94593c2df58SDoug Thompson (unsigned long)sys_addr, (unsigned long)input_addr); 94693c2df58SDoug Thompson 94793c2df58SDoug Thompson return input_addr; 94893c2df58SDoug Thompson } 94993c2df58SDoug Thompson 95093c2df58SDoug Thompson /* Map the Error address to a PAGE and PAGE OFFSET. */ 95193c2df58SDoug Thompson static inline void error_address_to_page_and_offset(u64 error_address, 95233ca0643SBorislav Petkov struct err_info *err) 95393c2df58SDoug Thompson { 95433ca0643SBorislav Petkov err->page = (u32) (error_address >> PAGE_SHIFT); 95533ca0643SBorislav Petkov err->offset = ((u32) error_address) & ~PAGE_MASK; 95693c2df58SDoug Thompson } 95793c2df58SDoug Thompson 95893c2df58SDoug Thompson /* 95993c2df58SDoug Thompson * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address 96093c2df58SDoug Thompson * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers 96193c2df58SDoug Thompson * of a node that detected an ECC memory error. mci represents the node that 96293c2df58SDoug Thompson * the error address maps to (possibly different from the node that detected 96393c2df58SDoug Thompson * the error). Return the number of the csrow that sys_addr maps to, or -1 on 96493c2df58SDoug Thompson * error. 96593c2df58SDoug Thompson */ 96693c2df58SDoug Thompson static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr) 96793c2df58SDoug Thompson { 96893c2df58SDoug Thompson int csrow; 96993c2df58SDoug Thompson 97093c2df58SDoug Thompson csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr)); 97193c2df58SDoug Thompson 97293c2df58SDoug Thompson if (csrow == -1) 97324f9a7feSBorislav Petkov amd64_mc_err(mci, "Failed to translate InputAddr to csrow for " 97493c2df58SDoug Thompson "address 0x%lx\n", (unsigned long)sys_addr); 97593c2df58SDoug Thompson return csrow; 97693c2df58SDoug Thompson } 977e2ce7255SDoug Thompson 978b3218ae4SYazen Ghannam /* Protect the PCI config register pairs used for DF indirect access. */ 979b3218ae4SYazen Ghannam static DEFINE_MUTEX(df_indirect_mutex); 980b3218ae4SYazen Ghannam 981b3218ae4SYazen Ghannam /* 982b3218ae4SYazen Ghannam * Data Fabric Indirect Access uses FICAA/FICAD. 983b3218ae4SYazen Ghannam * 984b3218ae4SYazen Ghannam * Fabric Indirect Configuration Access Address (FICAA): Constructed based 985b3218ae4SYazen Ghannam * on the device's Instance Id and the PCI function and register offset of 986b3218ae4SYazen Ghannam * the desired register. 987b3218ae4SYazen Ghannam * 988b3218ae4SYazen Ghannam * Fabric Indirect Configuration Access Data (FICAD): There are FICAD LO 989b3218ae4SYazen Ghannam * and FICAD HI registers but so far we only need the LO register. 990448c3d60SYazen Ghannam * 991448c3d60SYazen Ghannam * Use Instance Id 0xFF to indicate a broadcast read. 992b3218ae4SYazen Ghannam */ 993448c3d60SYazen Ghannam #define DF_BROADCAST 0xFF 994448c3d60SYazen Ghannam static int __df_indirect_read(u16 node, u8 func, u16 reg, u8 instance_id, u32 *lo) 995b3218ae4SYazen Ghannam { 996b3218ae4SYazen Ghannam struct pci_dev *F4; 997b3218ae4SYazen Ghannam u32 ficaa; 998b3218ae4SYazen Ghannam int err = -ENODEV; 999b3218ae4SYazen Ghannam 1000b3218ae4SYazen Ghannam if (node >= amd_nb_num()) 1001b3218ae4SYazen Ghannam goto out; 1002b3218ae4SYazen Ghannam 1003b3218ae4SYazen Ghannam F4 = node_to_amd_nb(node)->link; 1004b3218ae4SYazen Ghannam if (!F4) 1005b3218ae4SYazen Ghannam goto out; 1006b3218ae4SYazen Ghannam 1007448c3d60SYazen Ghannam ficaa = (instance_id == DF_BROADCAST) ? 0 : 1; 1008b3218ae4SYazen Ghannam ficaa |= reg & 0x3FC; 1009b3218ae4SYazen Ghannam ficaa |= (func & 0x7) << 11; 1010b3218ae4SYazen Ghannam ficaa |= instance_id << 16; 1011b3218ae4SYazen Ghannam 1012b3218ae4SYazen Ghannam mutex_lock(&df_indirect_mutex); 1013b3218ae4SYazen Ghannam 1014b3218ae4SYazen Ghannam err = pci_write_config_dword(F4, 0x5C, ficaa); 1015b3218ae4SYazen Ghannam if (err) { 1016b3218ae4SYazen Ghannam pr_warn("Error writing DF Indirect FICAA, FICAA=0x%x\n", ficaa); 1017b3218ae4SYazen Ghannam goto out_unlock; 1018b3218ae4SYazen Ghannam } 1019b3218ae4SYazen Ghannam 1020b3218ae4SYazen Ghannam err = pci_read_config_dword(F4, 0x98, lo); 1021b3218ae4SYazen Ghannam if (err) 1022b3218ae4SYazen Ghannam pr_warn("Error reading DF Indirect FICAD LO, FICAA=0x%x.\n", ficaa); 1023b3218ae4SYazen Ghannam 1024b3218ae4SYazen Ghannam out_unlock: 1025b3218ae4SYazen Ghannam mutex_unlock(&df_indirect_mutex); 1026b3218ae4SYazen Ghannam 1027b3218ae4SYazen Ghannam out: 1028b3218ae4SYazen Ghannam return err; 1029b3218ae4SYazen Ghannam } 1030b3218ae4SYazen Ghannam 1031448c3d60SYazen Ghannam static int df_indirect_read_instance(u16 node, u8 func, u16 reg, u8 instance_id, u32 *lo) 1032448c3d60SYazen Ghannam { 1033448c3d60SYazen Ghannam return __df_indirect_read(node, func, reg, instance_id, lo); 1034448c3d60SYazen Ghannam } 1035448c3d60SYazen Ghannam 1036448c3d60SYazen Ghannam static int df_indirect_read_broadcast(u16 node, u8 func, u16 reg, u32 *lo) 1037448c3d60SYazen Ghannam { 1038448c3d60SYazen Ghannam return __df_indirect_read(node, func, reg, DF_BROADCAST, lo); 1039448c3d60SYazen Ghannam } 1040448c3d60SYazen Ghannam 104170aeb807SYazen Ghannam struct addr_ctx { 104270aeb807SYazen Ghannam u64 ret_addr; 104370aeb807SYazen Ghannam u32 tmp; 104470aeb807SYazen Ghannam u16 nid; 104570aeb807SYazen Ghannam u8 inst_id; 104670aeb807SYazen Ghannam }; 104770aeb807SYazen Ghannam 10480b746e8cSYazen Ghannam static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr) 10490b746e8cSYazen Ghannam { 10500b746e8cSYazen Ghannam u64 dram_base_addr, dram_limit_addr, dram_hole_base; 10510b746e8cSYazen Ghannam 10520b746e8cSYazen Ghannam u8 die_id_shift, die_id_mask, socket_id_shift, socket_id_mask; 10530b746e8cSYazen Ghannam u8 intlv_num_dies, intlv_num_chan, intlv_num_sockets; 10540b746e8cSYazen Ghannam u8 intlv_addr_sel, intlv_addr_bit; 10550b746e8cSYazen Ghannam u8 num_intlv_bits, hashed_bit; 10560b746e8cSYazen Ghannam u8 lgcy_mmio_hole_en, base = 0; 10570b746e8cSYazen Ghannam u8 cs_mask, cs_id = 0; 10580b746e8cSYazen Ghannam bool hash_enabled = false; 10590b746e8cSYazen Ghannam 106070aeb807SYazen Ghannam struct addr_ctx ctx; 106170aeb807SYazen Ghannam 106270aeb807SYazen Ghannam memset(&ctx, 0, sizeof(ctx)); 106370aeb807SYazen Ghannam 106470aeb807SYazen Ghannam /* Start from the normalized address */ 106570aeb807SYazen Ghannam ctx.ret_addr = norm_addr; 106670aeb807SYazen Ghannam 106770aeb807SYazen Ghannam ctx.nid = nid; 106870aeb807SYazen Ghannam ctx.inst_id = umc; 106970aeb807SYazen Ghannam 10700b746e8cSYazen Ghannam /* Read D18F0x1B4 (DramOffset), check if base 1 is used. */ 107170aeb807SYazen Ghannam if (df_indirect_read_instance(nid, 0, 0x1B4, umc, &ctx.tmp)) 10720b746e8cSYazen Ghannam goto out_err; 10730b746e8cSYazen Ghannam 10740b746e8cSYazen Ghannam /* Remove HiAddrOffset from normalized address, if enabled: */ 107570aeb807SYazen Ghannam if (ctx.tmp & BIT(0)) { 107670aeb807SYazen Ghannam u64 hi_addr_offset = (ctx.tmp & GENMASK_ULL(31, 20)) << 8; 10770b746e8cSYazen Ghannam 10780b746e8cSYazen Ghannam if (norm_addr >= hi_addr_offset) { 107970aeb807SYazen Ghannam ctx.ret_addr -= hi_addr_offset; 10800b746e8cSYazen Ghannam base = 1; 10810b746e8cSYazen Ghannam } 10820b746e8cSYazen Ghannam } 10830b746e8cSYazen Ghannam 10840b746e8cSYazen Ghannam /* Read D18F0x110 (DramBaseAddress). */ 108570aeb807SYazen Ghannam if (df_indirect_read_instance(nid, 0, 0x110 + (8 * base), umc, &ctx.tmp)) 10860b746e8cSYazen Ghannam goto out_err; 10870b746e8cSYazen Ghannam 10880b746e8cSYazen Ghannam /* Check if address range is valid. */ 108970aeb807SYazen Ghannam if (!(ctx.tmp & BIT(0))) { 10900b746e8cSYazen Ghannam pr_err("%s: Invalid DramBaseAddress range: 0x%x.\n", 109170aeb807SYazen Ghannam __func__, ctx.tmp); 10920b746e8cSYazen Ghannam goto out_err; 10930b746e8cSYazen Ghannam } 10940b746e8cSYazen Ghannam 109570aeb807SYazen Ghannam lgcy_mmio_hole_en = ctx.tmp & BIT(1); 109670aeb807SYazen Ghannam intlv_num_chan = (ctx.tmp >> 4) & 0xF; 109770aeb807SYazen Ghannam intlv_addr_sel = (ctx.tmp >> 8) & 0x7; 109870aeb807SYazen Ghannam dram_base_addr = (ctx.tmp & GENMASK_ULL(31, 12)) << 16; 10990b746e8cSYazen Ghannam 11000b746e8cSYazen Ghannam /* {0, 1, 2, 3} map to address bits {8, 9, 10, 11} respectively */ 11010b746e8cSYazen Ghannam if (intlv_addr_sel > 3) { 11020b746e8cSYazen Ghannam pr_err("%s: Invalid interleave address select %d.\n", 11030b746e8cSYazen Ghannam __func__, intlv_addr_sel); 11040b746e8cSYazen Ghannam goto out_err; 11050b746e8cSYazen Ghannam } 11060b746e8cSYazen Ghannam 11070b746e8cSYazen Ghannam /* Read D18F0x114 (DramLimitAddress). */ 110870aeb807SYazen Ghannam if (df_indirect_read_instance(nid, 0, 0x114 + (8 * base), umc, &ctx.tmp)) 11090b746e8cSYazen Ghannam goto out_err; 11100b746e8cSYazen Ghannam 111170aeb807SYazen Ghannam intlv_num_sockets = (ctx.tmp >> 8) & 0x1; 111270aeb807SYazen Ghannam intlv_num_dies = (ctx.tmp >> 10) & 0x3; 111370aeb807SYazen Ghannam dram_limit_addr = ((ctx.tmp & GENMASK_ULL(31, 12)) << 16) | GENMASK_ULL(27, 0); 11140b746e8cSYazen Ghannam 11150b746e8cSYazen Ghannam intlv_addr_bit = intlv_addr_sel + 8; 11160b746e8cSYazen Ghannam 11170b746e8cSYazen Ghannam /* Re-use intlv_num_chan by setting it equal to log2(#channels) */ 11180b746e8cSYazen Ghannam switch (intlv_num_chan) { 11190b746e8cSYazen Ghannam case 0: intlv_num_chan = 0; break; 11200b746e8cSYazen Ghannam case 1: intlv_num_chan = 1; break; 11210b746e8cSYazen Ghannam case 3: intlv_num_chan = 2; break; 11220b746e8cSYazen Ghannam case 5: intlv_num_chan = 3; break; 11230b746e8cSYazen Ghannam case 7: intlv_num_chan = 4; break; 11240b746e8cSYazen Ghannam 11250b746e8cSYazen Ghannam case 8: intlv_num_chan = 1; 11260b746e8cSYazen Ghannam hash_enabled = true; 11270b746e8cSYazen Ghannam break; 11280b746e8cSYazen Ghannam default: 11290b746e8cSYazen Ghannam pr_err("%s: Invalid number of interleaved channels %d.\n", 11300b746e8cSYazen Ghannam __func__, intlv_num_chan); 11310b746e8cSYazen Ghannam goto out_err; 11320b746e8cSYazen Ghannam } 11330b746e8cSYazen Ghannam 11340b746e8cSYazen Ghannam num_intlv_bits = intlv_num_chan; 11350b746e8cSYazen Ghannam 11360b746e8cSYazen Ghannam if (intlv_num_dies > 2) { 11370b746e8cSYazen Ghannam pr_err("%s: Invalid number of interleaved nodes/dies %d.\n", 11380b746e8cSYazen Ghannam __func__, intlv_num_dies); 11390b746e8cSYazen Ghannam goto out_err; 11400b746e8cSYazen Ghannam } 11410b746e8cSYazen Ghannam 11420b746e8cSYazen Ghannam num_intlv_bits += intlv_num_dies; 11430b746e8cSYazen Ghannam 11440b746e8cSYazen Ghannam /* Add a bit if sockets are interleaved. */ 11450b746e8cSYazen Ghannam num_intlv_bits += intlv_num_sockets; 11460b746e8cSYazen Ghannam 11470b746e8cSYazen Ghannam /* Assert num_intlv_bits <= 4 */ 11480b746e8cSYazen Ghannam if (num_intlv_bits > 4) { 11490b746e8cSYazen Ghannam pr_err("%s: Invalid interleave bits %d.\n", 11500b746e8cSYazen Ghannam __func__, num_intlv_bits); 11510b746e8cSYazen Ghannam goto out_err; 11520b746e8cSYazen Ghannam } 11530b746e8cSYazen Ghannam 11540b746e8cSYazen Ghannam if (num_intlv_bits > 0) { 11550b746e8cSYazen Ghannam u64 temp_addr_x, temp_addr_i, temp_addr_y; 11560b746e8cSYazen Ghannam u8 die_id_bit, sock_id_bit, cs_fabric_id; 11570b746e8cSYazen Ghannam 11580b746e8cSYazen Ghannam /* 11590b746e8cSYazen Ghannam * Read FabricBlockInstanceInformation3_CS[BlockFabricID]. 11600b746e8cSYazen Ghannam * This is the fabric id for this coherent slave. Use 11610b746e8cSYazen Ghannam * umc/channel# as instance id of the coherent slave 11620b746e8cSYazen Ghannam * for FICAA. 11630b746e8cSYazen Ghannam */ 116470aeb807SYazen Ghannam if (df_indirect_read_instance(nid, 0, 0x50, umc, &ctx.tmp)) 11650b746e8cSYazen Ghannam goto out_err; 11660b746e8cSYazen Ghannam 116770aeb807SYazen Ghannam cs_fabric_id = (ctx.tmp >> 8) & 0xFF; 11680b746e8cSYazen Ghannam die_id_bit = 0; 11690b746e8cSYazen Ghannam 11700b746e8cSYazen Ghannam /* If interleaved over more than 1 channel: */ 11710b746e8cSYazen Ghannam if (intlv_num_chan) { 11720b746e8cSYazen Ghannam die_id_bit = intlv_num_chan; 11730b746e8cSYazen Ghannam cs_mask = (1 << die_id_bit) - 1; 11740b746e8cSYazen Ghannam cs_id = cs_fabric_id & cs_mask; 11750b746e8cSYazen Ghannam } 11760b746e8cSYazen Ghannam 11770b746e8cSYazen Ghannam sock_id_bit = die_id_bit; 11780b746e8cSYazen Ghannam 11790b746e8cSYazen Ghannam /* Read D18F1x208 (SystemFabricIdMask). */ 11800b746e8cSYazen Ghannam if (intlv_num_dies || intlv_num_sockets) 118170aeb807SYazen Ghannam if (df_indirect_read_broadcast(nid, 1, 0x208, &ctx.tmp)) 11820b746e8cSYazen Ghannam goto out_err; 11830b746e8cSYazen Ghannam 11840b746e8cSYazen Ghannam /* If interleaved over more than 1 die. */ 11850b746e8cSYazen Ghannam if (intlv_num_dies) { 11860b746e8cSYazen Ghannam sock_id_bit = die_id_bit + intlv_num_dies; 118770aeb807SYazen Ghannam die_id_shift = (ctx.tmp >> 24) & 0xF; 118870aeb807SYazen Ghannam die_id_mask = (ctx.tmp >> 8) & 0xFF; 11890b746e8cSYazen Ghannam 11900b746e8cSYazen Ghannam cs_id |= ((cs_fabric_id & die_id_mask) >> die_id_shift) << die_id_bit; 11910b746e8cSYazen Ghannam } 11920b746e8cSYazen Ghannam 11930b746e8cSYazen Ghannam /* If interleaved over more than 1 socket. */ 11940b746e8cSYazen Ghannam if (intlv_num_sockets) { 119570aeb807SYazen Ghannam socket_id_shift = (ctx.tmp >> 28) & 0xF; 119670aeb807SYazen Ghannam socket_id_mask = (ctx.tmp >> 16) & 0xFF; 11970b746e8cSYazen Ghannam 11980b746e8cSYazen Ghannam cs_id |= ((cs_fabric_id & socket_id_mask) >> socket_id_shift) << sock_id_bit; 11990b746e8cSYazen Ghannam } 12000b746e8cSYazen Ghannam 12010b746e8cSYazen Ghannam /* 12020b746e8cSYazen Ghannam * The pre-interleaved address consists of XXXXXXIIIYYYYY 12030b746e8cSYazen Ghannam * where III is the ID for this CS, and XXXXXXYYYYY are the 12040b746e8cSYazen Ghannam * address bits from the post-interleaved address. 12050b746e8cSYazen Ghannam * "num_intlv_bits" has been calculated to tell us how many "I" 12060b746e8cSYazen Ghannam * bits there are. "intlv_addr_bit" tells us how many "Y" bits 12070b746e8cSYazen Ghannam * there are (where "I" starts). 12080b746e8cSYazen Ghannam */ 120970aeb807SYazen Ghannam temp_addr_y = ctx.ret_addr & GENMASK_ULL(intlv_addr_bit - 1, 0); 12100b746e8cSYazen Ghannam temp_addr_i = (cs_id << intlv_addr_bit); 121170aeb807SYazen Ghannam temp_addr_x = (ctx.ret_addr & GENMASK_ULL(63, intlv_addr_bit)) << num_intlv_bits; 121270aeb807SYazen Ghannam ctx.ret_addr = temp_addr_x | temp_addr_i | temp_addr_y; 12130b746e8cSYazen Ghannam } 12140b746e8cSYazen Ghannam 12150b746e8cSYazen Ghannam /* Add dram base address */ 121670aeb807SYazen Ghannam ctx.ret_addr += dram_base_addr; 12170b746e8cSYazen Ghannam 12180b746e8cSYazen Ghannam /* If legacy MMIO hole enabled */ 12190b746e8cSYazen Ghannam if (lgcy_mmio_hole_en) { 122070aeb807SYazen Ghannam if (df_indirect_read_broadcast(nid, 0, 0x104, &ctx.tmp)) 12210b746e8cSYazen Ghannam goto out_err; 12220b746e8cSYazen Ghannam 122370aeb807SYazen Ghannam dram_hole_base = ctx.tmp & GENMASK(31, 24); 122470aeb807SYazen Ghannam if (ctx.ret_addr >= dram_hole_base) 122570aeb807SYazen Ghannam ctx.ret_addr += (BIT_ULL(32) - dram_hole_base); 12260b746e8cSYazen Ghannam } 12270b746e8cSYazen Ghannam 12280b746e8cSYazen Ghannam if (hash_enabled) { 12290b746e8cSYazen Ghannam /* Save some parentheses and grab ls-bit at the end. */ 123070aeb807SYazen Ghannam hashed_bit = (ctx.ret_addr >> 12) ^ 123170aeb807SYazen Ghannam (ctx.ret_addr >> 18) ^ 123270aeb807SYazen Ghannam (ctx.ret_addr >> 21) ^ 123370aeb807SYazen Ghannam (ctx.ret_addr >> 30) ^ 12340b746e8cSYazen Ghannam cs_id; 12350b746e8cSYazen Ghannam 12360b746e8cSYazen Ghannam hashed_bit &= BIT(0); 12370b746e8cSYazen Ghannam 123870aeb807SYazen Ghannam if (hashed_bit != ((ctx.ret_addr >> intlv_addr_bit) & BIT(0))) 123970aeb807SYazen Ghannam ctx.ret_addr ^= BIT(intlv_addr_bit); 12400b746e8cSYazen Ghannam } 12410b746e8cSYazen Ghannam 12420b746e8cSYazen Ghannam /* Is calculated system address is above DRAM limit address? */ 124370aeb807SYazen Ghannam if (ctx.ret_addr > dram_limit_addr) 12440b746e8cSYazen Ghannam goto out_err; 12450b746e8cSYazen Ghannam 124670aeb807SYazen Ghannam *sys_addr = ctx.ret_addr; 12470b746e8cSYazen Ghannam return 0; 12480b746e8cSYazen Ghannam 12490b746e8cSYazen Ghannam out_err: 12500b746e8cSYazen Ghannam return -EINVAL; 12510b746e8cSYazen Ghannam } 12520b746e8cSYazen Ghannam 1253bfc04aecSBorislav Petkov static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16); 12542da11654SDoug Thompson 12552da11654SDoug Thompson /* 12562da11654SDoug Thompson * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs 12572da11654SDoug Thompson * are ECC capable. 12582da11654SDoug Thompson */ 1259*f6a4b4a1SMuralidhara M K static unsigned long dct_determine_edac_cap(struct amd64_pvt *pvt) 12602da11654SDoug Thompson { 12611f6189edSDan Carpenter unsigned long edac_cap = EDAC_FLAG_NONE; 1262d27f3a34SYazen Ghannam u8 bit; 12632da11654SDoug Thompson 1264*f6a4b4a1SMuralidhara M K bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F) 1265*f6a4b4a1SMuralidhara M K ? 19 1266*f6a4b4a1SMuralidhara M K : 17; 1267*f6a4b4a1SMuralidhara M K 1268*f6a4b4a1SMuralidhara M K if (pvt->dclr0 & BIT(bit)) 1269*f6a4b4a1SMuralidhara M K edac_cap = EDAC_FLAG_SECDED; 1270*f6a4b4a1SMuralidhara M K 1271*f6a4b4a1SMuralidhara M K return edac_cap; 1272*f6a4b4a1SMuralidhara M K } 1273*f6a4b4a1SMuralidhara M K 1274*f6a4b4a1SMuralidhara M K static unsigned long umc_determine_edac_cap(struct amd64_pvt *pvt) 1275*f6a4b4a1SMuralidhara M K { 1276d27f3a34SYazen Ghannam u8 i, umc_en_mask = 0, dimm_ecc_en_mask = 0; 1277*f6a4b4a1SMuralidhara M K unsigned long edac_cap = EDAC_FLAG_NONE; 1278d27f3a34SYazen Ghannam 12794d30d2bcSYazen Ghannam for_each_umc(i) { 1280d27f3a34SYazen Ghannam if (!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT)) 1281d27f3a34SYazen Ghannam continue; 1282d27f3a34SYazen Ghannam 1283d27f3a34SYazen Ghannam umc_en_mask |= BIT(i); 1284d27f3a34SYazen Ghannam 1285d27f3a34SYazen Ghannam /* UMC Configuration bit 12 (DimmEccEn) */ 1286d27f3a34SYazen Ghannam if (pvt->umc[i].umc_cfg & BIT(12)) 1287d27f3a34SYazen Ghannam dimm_ecc_en_mask |= BIT(i); 1288d27f3a34SYazen Ghannam } 1289d27f3a34SYazen Ghannam 1290d27f3a34SYazen Ghannam if (umc_en_mask == dimm_ecc_en_mask) 1291d27f3a34SYazen Ghannam edac_cap = EDAC_FLAG_SECDED; 12922da11654SDoug Thompson 12932da11654SDoug Thompson return edac_cap; 12942da11654SDoug Thompson } 12952da11654SDoug Thompson 129600e4feb8SYazen Ghannam /* 129700e4feb8SYazen Ghannam * debug routine to display the memory sizes of all logical DIMMs and its 129800e4feb8SYazen Ghannam * CSROWs 129900e4feb8SYazen Ghannam */ 130000e4feb8SYazen Ghannam static void dct_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl) 130100e4feb8SYazen Ghannam { 130200e4feb8SYazen Ghannam u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases; 130300e4feb8SYazen Ghannam u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0; 130400e4feb8SYazen Ghannam int dimm, size0, size1; 130500e4feb8SYazen Ghannam 130600e4feb8SYazen Ghannam if (pvt->fam == 0xf) { 130700e4feb8SYazen Ghannam /* K8 families < revF not supported yet */ 130800e4feb8SYazen Ghannam if (pvt->ext_model < K8_REV_F) 130900e4feb8SYazen Ghannam return; 131000e4feb8SYazen Ghannam 131100e4feb8SYazen Ghannam WARN_ON(ctrl != 0); 131200e4feb8SYazen Ghannam } 131300e4feb8SYazen Ghannam 131400e4feb8SYazen Ghannam if (pvt->fam == 0x10) { 131500e4feb8SYazen Ghannam dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 131600e4feb8SYazen Ghannam : pvt->dbam0; 131700e4feb8SYazen Ghannam dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? 131800e4feb8SYazen Ghannam pvt->csels[1].csbases : 131900e4feb8SYazen Ghannam pvt->csels[0].csbases; 132000e4feb8SYazen Ghannam } else if (ctrl) { 132100e4feb8SYazen Ghannam dbam = pvt->dbam0; 132200e4feb8SYazen Ghannam dcsb = pvt->csels[1].csbases; 132300e4feb8SYazen Ghannam } 132400e4feb8SYazen Ghannam edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", 132500e4feb8SYazen Ghannam ctrl, dbam); 132600e4feb8SYazen Ghannam 132700e4feb8SYazen Ghannam edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl); 132800e4feb8SYazen Ghannam 132900e4feb8SYazen Ghannam /* Dump memory sizes for DIMM and its CSROWs */ 133000e4feb8SYazen Ghannam for (dimm = 0; dimm < 4; dimm++) { 133100e4feb8SYazen Ghannam size0 = 0; 133200e4feb8SYazen Ghannam if (dcsb[dimm * 2] & DCSB_CS_ENABLE) 133300e4feb8SYazen Ghannam /* 133400e4feb8SYazen Ghannam * For F15m60h, we need multiplier for LRDIMM cs_size 133500e4feb8SYazen Ghannam * calculation. We pass dimm value to the dbam_to_cs 133600e4feb8SYazen Ghannam * mapper so we can find the multiplier from the 133700e4feb8SYazen Ghannam * corresponding DCSM. 133800e4feb8SYazen Ghannam */ 133900e4feb8SYazen Ghannam size0 = pvt->ops->dbam_to_cs(pvt, ctrl, 134000e4feb8SYazen Ghannam DBAM_DIMM(dimm, dbam), 134100e4feb8SYazen Ghannam dimm); 134200e4feb8SYazen Ghannam 134300e4feb8SYazen Ghannam size1 = 0; 134400e4feb8SYazen Ghannam if (dcsb[dimm * 2 + 1] & DCSB_CS_ENABLE) 134500e4feb8SYazen Ghannam size1 = pvt->ops->dbam_to_cs(pvt, ctrl, 134600e4feb8SYazen Ghannam DBAM_DIMM(dimm, dbam), 134700e4feb8SYazen Ghannam dimm); 134800e4feb8SYazen Ghannam 134900e4feb8SYazen Ghannam amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n", 135000e4feb8SYazen Ghannam dimm * 2, size0, 135100e4feb8SYazen Ghannam dimm * 2 + 1, size1); 135200e4feb8SYazen Ghannam } 135300e4feb8SYazen Ghannam } 135400e4feb8SYazen Ghannam 13552da11654SDoug Thompson 1356d1ea71cdSBorislav Petkov static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan) 135768798e17SBorislav Petkov { 1358956b9ba1SJoe Perches edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr); 135968798e17SBorislav Petkov 1360a597d2a5SAravind Gopalakrishnan if (pvt->dram_type == MEM_LRDDR3) { 1361a597d2a5SAravind Gopalakrishnan u32 dcsm = pvt->csels[chan].csmasks[0]; 1362a597d2a5SAravind Gopalakrishnan /* 1363a597d2a5SAravind Gopalakrishnan * It's assumed all LRDIMMs in a DCT are going to be of 1364a597d2a5SAravind Gopalakrishnan * same 'type' until proven otherwise. So, use a cs 1365a597d2a5SAravind Gopalakrishnan * value of '0' here to get dcsm value. 1366a597d2a5SAravind Gopalakrishnan */ 1367a597d2a5SAravind Gopalakrishnan edac_dbg(1, " LRDIMM %dx rank multiply\n", (dcsm & 0x3)); 1368a597d2a5SAravind Gopalakrishnan } 1369a597d2a5SAravind Gopalakrishnan 1370a597d2a5SAravind Gopalakrishnan edac_dbg(1, "All DIMMs support ECC:%s\n", 137168798e17SBorislav Petkov (dclr & BIT(19)) ? "yes" : "no"); 137268798e17SBorislav Petkov 1373a597d2a5SAravind Gopalakrishnan 1374956b9ba1SJoe Perches edac_dbg(1, " PAR/ERR parity: %s\n", 137568798e17SBorislav Petkov (dclr & BIT(8)) ? "enabled" : "disabled"); 137668798e17SBorislav Petkov 1377a4b4bedcSBorislav Petkov if (pvt->fam == 0x10) 1378956b9ba1SJoe Perches edac_dbg(1, " DCT 128bit mode width: %s\n", 137968798e17SBorislav Petkov (dclr & BIT(11)) ? "128b" : "64b"); 138068798e17SBorislav Petkov 1381956b9ba1SJoe Perches edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n", 138268798e17SBorislav Petkov (dclr & BIT(12)) ? "yes" : "no", 138368798e17SBorislav Petkov (dclr & BIT(13)) ? "yes" : "no", 138468798e17SBorislav Petkov (dclr & BIT(14)) ? "yes" : "no", 138568798e17SBorislav Petkov (dclr & BIT(15)) ? "yes" : "no"); 138668798e17SBorislav Petkov } 138768798e17SBorislav Petkov 1388e53a3b26SYazen Ghannam #define CS_EVEN_PRIMARY BIT(0) 1389e53a3b26SYazen Ghannam #define CS_ODD_PRIMARY BIT(1) 139081f5090dSYazen Ghannam #define CS_EVEN_SECONDARY BIT(2) 139181f5090dSYazen Ghannam #define CS_ODD_SECONDARY BIT(3) 13929f4873fbSYazen Ghannam #define CS_3R_INTERLEAVE BIT(4) 1393e53a3b26SYazen Ghannam 139481f5090dSYazen Ghannam #define CS_EVEN (CS_EVEN_PRIMARY | CS_EVEN_SECONDARY) 139581f5090dSYazen Ghannam #define CS_ODD (CS_ODD_PRIMARY | CS_ODD_SECONDARY) 1396e53a3b26SYazen Ghannam 1397c0984666SYazen Ghannam static int umc_get_cs_mode(int dimm, u8 ctrl, struct amd64_pvt *pvt) 1398fc00c6a4SYazen Ghannam { 13999f4873fbSYazen Ghannam u8 base, count = 0; 1400e53a3b26SYazen Ghannam int cs_mode = 0; 1401fc00c6a4SYazen Ghannam 1402e53a3b26SYazen Ghannam if (csrow_enabled(2 * dimm, ctrl, pvt)) 1403e53a3b26SYazen Ghannam cs_mode |= CS_EVEN_PRIMARY; 1404fc00c6a4SYazen Ghannam 1405e53a3b26SYazen Ghannam if (csrow_enabled(2 * dimm + 1, ctrl, pvt)) 1406e53a3b26SYazen Ghannam cs_mode |= CS_ODD_PRIMARY; 1407e53a3b26SYazen Ghannam 140881f5090dSYazen Ghannam /* Asymmetric dual-rank DIMM support. */ 140981f5090dSYazen Ghannam if (csrow_sec_enabled(2 * dimm + 1, ctrl, pvt)) 141081f5090dSYazen Ghannam cs_mode |= CS_ODD_SECONDARY; 141181f5090dSYazen Ghannam 14129f4873fbSYazen Ghannam /* 14139f4873fbSYazen Ghannam * 3 Rank inteleaving support. 14149f4873fbSYazen Ghannam * There should be only three bases enabled and their two masks should 14159f4873fbSYazen Ghannam * be equal. 14169f4873fbSYazen Ghannam */ 14179f4873fbSYazen Ghannam for_each_chip_select(base, ctrl, pvt) 14189f4873fbSYazen Ghannam count += csrow_enabled(base, ctrl, pvt); 14199f4873fbSYazen Ghannam 14209f4873fbSYazen Ghannam if (count == 3 && 14219f4873fbSYazen Ghannam pvt->csels[ctrl].csmasks[0] == pvt->csels[ctrl].csmasks[1]) { 14229f4873fbSYazen Ghannam edac_dbg(1, "3R interleaving in use.\n"); 14239f4873fbSYazen Ghannam cs_mode |= CS_3R_INTERLEAVE; 14249f4873fbSYazen Ghannam } 14259f4873fbSYazen Ghannam 1426e53a3b26SYazen Ghannam return cs_mode; 1427fc00c6a4SYazen Ghannam } 1428fc00c6a4SYazen Ghannam 1429a2e59ab8SYazen Ghannam static int umc_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc, 1430a2e59ab8SYazen Ghannam unsigned int cs_mode, int csrow_nr) 1431a2e59ab8SYazen Ghannam { 1432a2e59ab8SYazen Ghannam u32 addr_mask_orig, addr_mask_deinterleaved; 1433a2e59ab8SYazen Ghannam u32 msb, weight, num_zero_bits; 1434a2e59ab8SYazen Ghannam int cs_mask_nr = csrow_nr; 1435a2e59ab8SYazen Ghannam int dimm, size = 0; 1436a2e59ab8SYazen Ghannam 1437a2e59ab8SYazen Ghannam /* No Chip Selects are enabled. */ 1438a2e59ab8SYazen Ghannam if (!cs_mode) 1439a2e59ab8SYazen Ghannam return size; 1440a2e59ab8SYazen Ghannam 1441a2e59ab8SYazen Ghannam /* Requested size of an even CS but none are enabled. */ 1442a2e59ab8SYazen Ghannam if (!(cs_mode & CS_EVEN) && !(csrow_nr & 1)) 1443a2e59ab8SYazen Ghannam return size; 1444a2e59ab8SYazen Ghannam 1445a2e59ab8SYazen Ghannam /* Requested size of an odd CS but none are enabled. */ 1446a2e59ab8SYazen Ghannam if (!(cs_mode & CS_ODD) && (csrow_nr & 1)) 1447a2e59ab8SYazen Ghannam return size; 1448a2e59ab8SYazen Ghannam 1449a2e59ab8SYazen Ghannam /* 1450a2e59ab8SYazen Ghannam * Family 17h introduced systems with one mask per DIMM, 1451a2e59ab8SYazen Ghannam * and two Chip Selects per DIMM. 1452a2e59ab8SYazen Ghannam * 1453a2e59ab8SYazen Ghannam * CS0 and CS1 -> MASK0 / DIMM0 1454a2e59ab8SYazen Ghannam * CS2 and CS3 -> MASK1 / DIMM1 1455a2e59ab8SYazen Ghannam * 1456a2e59ab8SYazen Ghannam * Family 19h Model 10h introduced systems with one mask per Chip Select, 1457a2e59ab8SYazen Ghannam * and two Chip Selects per DIMM. 1458a2e59ab8SYazen Ghannam * 1459a2e59ab8SYazen Ghannam * CS0 -> MASK0 -> DIMM0 1460a2e59ab8SYazen Ghannam * CS1 -> MASK1 -> DIMM0 1461a2e59ab8SYazen Ghannam * CS2 -> MASK2 -> DIMM1 1462a2e59ab8SYazen Ghannam * CS3 -> MASK3 -> DIMM1 1463a2e59ab8SYazen Ghannam * 1464a2e59ab8SYazen Ghannam * Keep the mask number equal to the Chip Select number for newer systems, 1465a2e59ab8SYazen Ghannam * and shift the mask number for older systems. 1466a2e59ab8SYazen Ghannam */ 1467a2e59ab8SYazen Ghannam dimm = csrow_nr >> 1; 1468a2e59ab8SYazen Ghannam 1469ed623d55SMuralidhara M K if (!pvt->flags.zn_regs_v2) 1470a2e59ab8SYazen Ghannam cs_mask_nr >>= 1; 1471a2e59ab8SYazen Ghannam 1472a2e59ab8SYazen Ghannam /* Asymmetric dual-rank DIMM support. */ 1473a2e59ab8SYazen Ghannam if ((csrow_nr & 1) && (cs_mode & CS_ODD_SECONDARY)) 1474a2e59ab8SYazen Ghannam addr_mask_orig = pvt->csels[umc].csmasks_sec[cs_mask_nr]; 1475a2e59ab8SYazen Ghannam else 1476a2e59ab8SYazen Ghannam addr_mask_orig = pvt->csels[umc].csmasks[cs_mask_nr]; 1477a2e59ab8SYazen Ghannam 1478a2e59ab8SYazen Ghannam /* 1479a2e59ab8SYazen Ghannam * The number of zero bits in the mask is equal to the number of bits 1480a2e59ab8SYazen Ghannam * in a full mask minus the number of bits in the current mask. 1481a2e59ab8SYazen Ghannam * 1482a2e59ab8SYazen Ghannam * The MSB is the number of bits in the full mask because BIT[0] is 1483a2e59ab8SYazen Ghannam * always 0. 1484a2e59ab8SYazen Ghannam * 1485a2e59ab8SYazen Ghannam * In the special 3 Rank interleaving case, a single bit is flipped 1486a2e59ab8SYazen Ghannam * without swapping with the most significant bit. This can be handled 1487a2e59ab8SYazen Ghannam * by keeping the MSB where it is and ignoring the single zero bit. 1488a2e59ab8SYazen Ghannam */ 1489a2e59ab8SYazen Ghannam msb = fls(addr_mask_orig) - 1; 1490a2e59ab8SYazen Ghannam weight = hweight_long(addr_mask_orig); 1491a2e59ab8SYazen Ghannam num_zero_bits = msb - weight - !!(cs_mode & CS_3R_INTERLEAVE); 1492a2e59ab8SYazen Ghannam 1493a2e59ab8SYazen Ghannam /* Take the number of zero bits off from the top of the mask. */ 1494a2e59ab8SYazen Ghannam addr_mask_deinterleaved = GENMASK_ULL(msb - num_zero_bits, 1); 1495a2e59ab8SYazen Ghannam 1496a2e59ab8SYazen Ghannam edac_dbg(1, "CS%d DIMM%d AddrMasks:\n", csrow_nr, dimm); 1497a2e59ab8SYazen Ghannam edac_dbg(1, " Original AddrMask: 0x%x\n", addr_mask_orig); 1498a2e59ab8SYazen Ghannam edac_dbg(1, " Deinterleaved AddrMask: 0x%x\n", addr_mask_deinterleaved); 1499a2e59ab8SYazen Ghannam 1500a2e59ab8SYazen Ghannam /* Register [31:1] = Address [39:9]. Size is in kBs here. */ 1501a2e59ab8SYazen Ghannam size = (addr_mask_deinterleaved >> 2) + 1; 1502a2e59ab8SYazen Ghannam 1503a2e59ab8SYazen Ghannam /* Return size in MBs. */ 1504a2e59ab8SYazen Ghannam return size >> 10; 1505a2e59ab8SYazen Ghannam } 1506a2e59ab8SYazen Ghannam 150700e4feb8SYazen Ghannam static void umc_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl) 150807ed82efSYazen Ghannam { 1509e53a3b26SYazen Ghannam int dimm, size0, size1, cs0, cs1, cs_mode; 151007ed82efSYazen Ghannam 151107ed82efSYazen Ghannam edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:\n", ctrl); 151207ed82efSYazen Ghannam 1513d971e28eSYazen Ghannam for (dimm = 0; dimm < 2; dimm++) { 1514eb77e6b8SYazen Ghannam cs0 = dimm * 2; 1515eb77e6b8SYazen Ghannam cs1 = dimm * 2 + 1; 1516eb77e6b8SYazen Ghannam 1517c0984666SYazen Ghannam cs_mode = umc_get_cs_mode(dimm, ctrl, pvt); 1518e53a3b26SYazen Ghannam 1519a2e59ab8SYazen Ghannam size0 = umc_addr_mask_to_cs_size(pvt, ctrl, cs_mode, cs0); 1520a2e59ab8SYazen Ghannam size1 = umc_addr_mask_to_cs_size(pvt, ctrl, cs_mode, cs1); 152107ed82efSYazen Ghannam 152207ed82efSYazen Ghannam amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n", 1523eb77e6b8SYazen Ghannam cs0, size0, 1524eb77e6b8SYazen Ghannam cs1, size1); 152507ed82efSYazen Ghannam } 152607ed82efSYazen Ghannam } 152707ed82efSYazen Ghannam 152807ed82efSYazen Ghannam static void __dump_misc_regs_df(struct amd64_pvt *pvt) 152907ed82efSYazen Ghannam { 153007ed82efSYazen Ghannam struct amd64_umc *umc; 153107ed82efSYazen Ghannam u32 i, tmp, umc_base; 153207ed82efSYazen Ghannam 15334d30d2bcSYazen Ghannam for_each_umc(i) { 153407ed82efSYazen Ghannam umc_base = get_umc_base(i); 153507ed82efSYazen Ghannam umc = &pvt->umc[i]; 153607ed82efSYazen Ghannam 153707ed82efSYazen Ghannam edac_dbg(1, "UMC%d DIMM cfg: 0x%x\n", i, umc->dimm_cfg); 153807ed82efSYazen Ghannam edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i, umc->umc_cfg); 153907ed82efSYazen Ghannam edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i, umc->sdp_ctrl); 154007ed82efSYazen Ghannam edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i, umc->ecc_ctrl); 154107ed82efSYazen Ghannam 154207ed82efSYazen Ghannam amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ECC_BAD_SYMBOL, &tmp); 154307ed82efSYazen Ghannam edac_dbg(1, "UMC%d ECC bad symbol: 0x%x\n", i, tmp); 154407ed82efSYazen Ghannam 154507ed82efSYazen Ghannam amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_UMC_CAP, &tmp); 154607ed82efSYazen Ghannam edac_dbg(1, "UMC%d UMC cap: 0x%x\n", i, tmp); 154707ed82efSYazen Ghannam edac_dbg(1, "UMC%d UMC cap high: 0x%x\n", i, umc->umc_cap_hi); 154807ed82efSYazen Ghannam 154907ed82efSYazen Ghannam edac_dbg(1, "UMC%d ECC capable: %s, ChipKill ECC capable: %s\n", 155007ed82efSYazen Ghannam i, (umc->umc_cap_hi & BIT(30)) ? "yes" : "no", 155107ed82efSYazen Ghannam (umc->umc_cap_hi & BIT(31)) ? "yes" : "no"); 155207ed82efSYazen Ghannam edac_dbg(1, "UMC%d All DIMMs support ECC: %s\n", 155307ed82efSYazen Ghannam i, (umc->umc_cfg & BIT(12)) ? "yes" : "no"); 155407ed82efSYazen Ghannam edac_dbg(1, "UMC%d x4 DIMMs present: %s\n", 155507ed82efSYazen Ghannam i, (umc->dimm_cfg & BIT(6)) ? "yes" : "no"); 155607ed82efSYazen Ghannam edac_dbg(1, "UMC%d x16 DIMMs present: %s\n", 155707ed82efSYazen Ghannam i, (umc->dimm_cfg & BIT(7)) ? "yes" : "no"); 155807ed82efSYazen Ghannam 15592151c84eSYazen Ghannam if (umc->dram_type == MEM_LRDDR4 || umc->dram_type == MEM_LRDDR5) { 15602151c84eSYazen Ghannam amd_smn_read(pvt->mc_node_id, 1561ed623d55SMuralidhara M K umc_base + get_umc_reg(pvt, UMCCH_ADDR_CFG), 15622151c84eSYazen Ghannam &tmp); 156307ed82efSYazen Ghannam edac_dbg(1, "UMC%d LRDIMM %dx rank multiply\n", 156407ed82efSYazen Ghannam i, 1 << ((tmp >> 4) & 0x3)); 156507ed82efSYazen Ghannam } 156607ed82efSYazen Ghannam 156700e4feb8SYazen Ghannam umc_debug_display_dimm_sizes(pvt, i); 156807ed82efSYazen Ghannam } 156907ed82efSYazen Ghannam } 157007ed82efSYazen Ghannam 15712da11654SDoug Thompson /* Display and decode various NB registers for debug purposes. */ 157207ed82efSYazen Ghannam static void __dump_misc_regs(struct amd64_pvt *pvt) 15732da11654SDoug Thompson { 1574956b9ba1SJoe Perches edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap); 15752da11654SDoug Thompson 1576956b9ba1SJoe Perches edac_dbg(1, " NB two channel DRAM capable: %s\n", 15775980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no"); 157868798e17SBorislav Petkov 1579956b9ba1SJoe Perches edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n", 15805980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no", 15815980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no"); 158268798e17SBorislav Petkov 1583d1ea71cdSBorislav Petkov debug_dump_dramcfg_low(pvt, pvt->dclr0, 0); 15842da11654SDoug Thompson 1585956b9ba1SJoe Perches edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare); 15862da11654SDoug Thompson 1587956b9ba1SJoe Perches edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n", 1588bc21fa57SBorislav Petkov pvt->dhar, dhar_base(pvt), 1589a4b4bedcSBorislav Petkov (pvt->fam == 0xf) ? k8_dhar_offset(pvt) 1590bc21fa57SBorislav Petkov : f10_dhar_offset(pvt)); 15912da11654SDoug Thompson 159200e4feb8SYazen Ghannam dct_debug_display_dimm_sizes(pvt, 0); 15934d796364SBorislav Petkov 15944d796364SBorislav Petkov /* everything below this point is Fam10h and above */ 1595a4b4bedcSBorislav Petkov if (pvt->fam == 0xf) 15962da11654SDoug Thompson return; 15974d796364SBorislav Petkov 159800e4feb8SYazen Ghannam dct_debug_display_dimm_sizes(pvt, 1); 15992da11654SDoug Thompson 16008de1d91eSBorislav Petkov /* Only if NOT ganged does dclr1 have valid info */ 160168798e17SBorislav Petkov if (!dct_ganging_enabled(pvt)) 1602d1ea71cdSBorislav Petkov debug_dump_dramcfg_low(pvt, pvt->dclr1, 1); 1603cf981562SYazen Ghannam 1604cf981562SYazen Ghannam edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no"); 16055a1adb37SYazen Ghannam 16065a1adb37SYazen Ghannam amd64_info("using x%u syndromes.\n", pvt->ecc_sym_sz); 16072da11654SDoug Thompson } 16082da11654SDoug Thompson 160907ed82efSYazen Ghannam /* Display and decode various NB registers for debug purposes. */ 161007ed82efSYazen Ghannam static void dump_misc_regs(struct amd64_pvt *pvt) 161107ed82efSYazen Ghannam { 161207ed82efSYazen Ghannam if (pvt->umc) 161307ed82efSYazen Ghannam __dump_misc_regs_df(pvt); 161407ed82efSYazen Ghannam else 161507ed82efSYazen Ghannam __dump_misc_regs(pvt); 161607ed82efSYazen Ghannam } 161707ed82efSYazen Ghannam 161894be4bffSDoug Thompson /* 161918b94f66SAravind Gopalakrishnan * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60] 162094be4bffSDoug Thompson */ 1621637f60efSMuralidhara M K static void dct_prep_chip_selects(struct amd64_pvt *pvt) 162294be4bffSDoug Thompson { 162318b94f66SAravind Gopalakrishnan if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) { 162411c75eadSBorislav Petkov pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8; 162511c75eadSBorislav Petkov pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8; 1626a597d2a5SAravind Gopalakrishnan } else if (pvt->fam == 0x15 && pvt->model == 0x30) { 162718b94f66SAravind Gopalakrishnan pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4; 162818b94f66SAravind Gopalakrishnan pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2; 1629637f60efSMuralidhara M K } else { 1630637f60efSMuralidhara M K pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8; 1631637f60efSMuralidhara M K pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4; 1632637f60efSMuralidhara M K } 1633637f60efSMuralidhara M K } 1634637f60efSMuralidhara M K 1635637f60efSMuralidhara M K static void umc_prep_chip_selects(struct amd64_pvt *pvt) 1636637f60efSMuralidhara M K { 1637d971e28eSYazen Ghannam int umc; 1638d971e28eSYazen Ghannam 1639d971e28eSYazen Ghannam for_each_umc(umc) { 1640d971e28eSYazen Ghannam pvt->csels[umc].b_cnt = 4; 1641ed623d55SMuralidhara M K pvt->csels[umc].m_cnt = pvt->flags.zn_regs_v2 ? 4 : 2; 1642d971e28eSYazen Ghannam } 164394be4bffSDoug Thompson } 164494be4bffSDoug Thompson 1645b29dad9bSMuralidhara M K static void umc_read_base_mask(struct amd64_pvt *pvt) 1646d971e28eSYazen Ghannam { 16477574729eSYazen Ghannam u32 umc_base_reg, umc_base_reg_sec; 16487574729eSYazen Ghannam u32 umc_mask_reg, umc_mask_reg_sec; 16497574729eSYazen Ghannam u32 base_reg, base_reg_sec; 16507574729eSYazen Ghannam u32 mask_reg, mask_reg_sec; 16517574729eSYazen Ghannam u32 *base, *base_sec; 16527574729eSYazen Ghannam u32 *mask, *mask_sec; 1653d971e28eSYazen Ghannam int cs, umc; 1654d971e28eSYazen Ghannam 1655d971e28eSYazen Ghannam for_each_umc(umc) { 1656d971e28eSYazen Ghannam umc_base_reg = get_umc_base(umc) + UMCCH_BASE_ADDR; 16577574729eSYazen Ghannam umc_base_reg_sec = get_umc_base(umc) + UMCCH_BASE_ADDR_SEC; 1658d971e28eSYazen Ghannam 1659d971e28eSYazen Ghannam for_each_chip_select(cs, umc, pvt) { 1660d971e28eSYazen Ghannam base = &pvt->csels[umc].csbases[cs]; 16617574729eSYazen Ghannam base_sec = &pvt->csels[umc].csbases_sec[cs]; 1662d971e28eSYazen Ghannam 1663d971e28eSYazen Ghannam base_reg = umc_base_reg + (cs * 4); 16647574729eSYazen Ghannam base_reg_sec = umc_base_reg_sec + (cs * 4); 1665d971e28eSYazen Ghannam 1666d971e28eSYazen Ghannam if (!amd_smn_read(pvt->mc_node_id, base_reg, base)) 1667d971e28eSYazen Ghannam edac_dbg(0, " DCSB%d[%d]=0x%08x reg: 0x%x\n", 1668d971e28eSYazen Ghannam umc, cs, *base, base_reg); 16697574729eSYazen Ghannam 16707574729eSYazen Ghannam if (!amd_smn_read(pvt->mc_node_id, base_reg_sec, base_sec)) 16717574729eSYazen Ghannam edac_dbg(0, " DCSB_SEC%d[%d]=0x%08x reg: 0x%x\n", 16727574729eSYazen Ghannam umc, cs, *base_sec, base_reg_sec); 1673d971e28eSYazen Ghannam } 1674d971e28eSYazen Ghannam 1675d971e28eSYazen Ghannam umc_mask_reg = get_umc_base(umc) + UMCCH_ADDR_MASK; 1676ed623d55SMuralidhara M K umc_mask_reg_sec = get_umc_base(umc) + get_umc_reg(pvt, UMCCH_ADDR_MASK_SEC); 1677d971e28eSYazen Ghannam 1678d971e28eSYazen Ghannam for_each_chip_select_mask(cs, umc, pvt) { 1679d971e28eSYazen Ghannam mask = &pvt->csels[umc].csmasks[cs]; 16807574729eSYazen Ghannam mask_sec = &pvt->csels[umc].csmasks_sec[cs]; 1681d971e28eSYazen Ghannam 1682d971e28eSYazen Ghannam mask_reg = umc_mask_reg + (cs * 4); 16837574729eSYazen Ghannam mask_reg_sec = umc_mask_reg_sec + (cs * 4); 1684d971e28eSYazen Ghannam 1685d971e28eSYazen Ghannam if (!amd_smn_read(pvt->mc_node_id, mask_reg, mask)) 1686d971e28eSYazen Ghannam edac_dbg(0, " DCSM%d[%d]=0x%08x reg: 0x%x\n", 1687d971e28eSYazen Ghannam umc, cs, *mask, mask_reg); 16887574729eSYazen Ghannam 16897574729eSYazen Ghannam if (!amd_smn_read(pvt->mc_node_id, mask_reg_sec, mask_sec)) 16907574729eSYazen Ghannam edac_dbg(0, " DCSM_SEC%d[%d]=0x%08x reg: 0x%x\n", 16917574729eSYazen Ghannam umc, cs, *mask_sec, mask_reg_sec); 1692d971e28eSYazen Ghannam } 1693d971e28eSYazen Ghannam } 1694d971e28eSYazen Ghannam } 1695d971e28eSYazen Ghannam 169694be4bffSDoug Thompson /* 169711c75eadSBorislav Petkov * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers 169894be4bffSDoug Thompson */ 1699b29dad9bSMuralidhara M K static void dct_read_base_mask(struct amd64_pvt *pvt) 170094be4bffSDoug Thompson { 1701d971e28eSYazen Ghannam int cs; 170294be4bffSDoug Thompson 170311c75eadSBorislav Petkov for_each_chip_select(cs, 0, pvt) { 1704d971e28eSYazen Ghannam int reg0 = DCSB0 + (cs * 4); 1705d971e28eSYazen Ghannam int reg1 = DCSB1 + (cs * 4); 170611c75eadSBorislav Petkov u32 *base0 = &pvt->csels[0].csbases[cs]; 170711c75eadSBorislav Petkov u32 *base1 = &pvt->csels[1].csbases[cs]; 1708b2b0c605SBorislav Petkov 17097981a28fSAravind Gopalakrishnan if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0)) 1710956b9ba1SJoe Perches edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n", 171111c75eadSBorislav Petkov cs, *base0, reg0); 171294be4bffSDoug Thompson 17137981a28fSAravind Gopalakrishnan if (pvt->fam == 0xf) 171411c75eadSBorislav Petkov continue; 1715b2b0c605SBorislav Petkov 17167981a28fSAravind Gopalakrishnan if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1)) 1717956b9ba1SJoe Perches edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n", 17187981a28fSAravind Gopalakrishnan cs, *base1, (pvt->fam == 0x10) ? reg1 17197981a28fSAravind Gopalakrishnan : reg0); 172094be4bffSDoug Thompson } 172194be4bffSDoug Thompson 172211c75eadSBorislav Petkov for_each_chip_select_mask(cs, 0, pvt) { 1723d971e28eSYazen Ghannam int reg0 = DCSM0 + (cs * 4); 1724d971e28eSYazen Ghannam int reg1 = DCSM1 + (cs * 4); 172511c75eadSBorislav Petkov u32 *mask0 = &pvt->csels[0].csmasks[cs]; 172611c75eadSBorislav Petkov u32 *mask1 = &pvt->csels[1].csmasks[cs]; 1727b2b0c605SBorislav Petkov 17287981a28fSAravind Gopalakrishnan if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0)) 1729956b9ba1SJoe Perches edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n", 173011c75eadSBorislav Petkov cs, *mask0, reg0); 173194be4bffSDoug Thompson 17327981a28fSAravind Gopalakrishnan if (pvt->fam == 0xf) 173311c75eadSBorislav Petkov continue; 1734b2b0c605SBorislav Petkov 17357981a28fSAravind Gopalakrishnan if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1)) 1736956b9ba1SJoe Perches edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n", 17377981a28fSAravind Gopalakrishnan cs, *mask1, (pvt->fam == 0x10) ? reg1 17387981a28fSAravind Gopalakrishnan : reg0); 173994be4bffSDoug Thompson } 17406ba5dcdcSBorislav Petkov } 174194be4bffSDoug Thompson 174278ec161aSMuralidhara M K static void umc_determine_memory_type(struct amd64_pvt *pvt) 174375aeaaf2SYazen Ghannam { 174475aeaaf2SYazen Ghannam struct amd64_umc *umc; 174575aeaaf2SYazen Ghannam u32 i; 174675aeaaf2SYazen Ghannam 174775aeaaf2SYazen Ghannam for_each_umc(i) { 174875aeaaf2SYazen Ghannam umc = &pvt->umc[i]; 174975aeaaf2SYazen Ghannam 175075aeaaf2SYazen Ghannam if (!(umc->sdp_ctrl & UMC_SDP_INIT)) { 175175aeaaf2SYazen Ghannam umc->dram_type = MEM_EMPTY; 175275aeaaf2SYazen Ghannam continue; 175375aeaaf2SYazen Ghannam } 175475aeaaf2SYazen Ghannam 17552151c84eSYazen Ghannam /* 17562151c84eSYazen Ghannam * Check if the system supports the "DDR Type" field in UMC Config 17572151c84eSYazen Ghannam * and has DDR5 DIMMs in use. 17582151c84eSYazen Ghannam */ 1759ed623d55SMuralidhara M K if (pvt->flags.zn_regs_v2 && ((umc->umc_cfg & GENMASK(2, 0)) == 0x1)) { 17602151c84eSYazen Ghannam if (umc->dimm_cfg & BIT(5)) 17612151c84eSYazen Ghannam umc->dram_type = MEM_LRDDR5; 17622151c84eSYazen Ghannam else if (umc->dimm_cfg & BIT(4)) 17632151c84eSYazen Ghannam umc->dram_type = MEM_RDDR5; 17642151c84eSYazen Ghannam else 17652151c84eSYazen Ghannam umc->dram_type = MEM_DDR5; 17662151c84eSYazen Ghannam } else { 176775aeaaf2SYazen Ghannam if (umc->dimm_cfg & BIT(5)) 176875aeaaf2SYazen Ghannam umc->dram_type = MEM_LRDDR4; 176975aeaaf2SYazen Ghannam else if (umc->dimm_cfg & BIT(4)) 177075aeaaf2SYazen Ghannam umc->dram_type = MEM_RDDR4; 177175aeaaf2SYazen Ghannam else 177275aeaaf2SYazen Ghannam umc->dram_type = MEM_DDR4; 17732151c84eSYazen Ghannam } 177475aeaaf2SYazen Ghannam 177575aeaaf2SYazen Ghannam edac_dbg(1, " UMC%d DIMM type: %s\n", i, edac_mem_types[umc->dram_type]); 177675aeaaf2SYazen Ghannam } 177775aeaaf2SYazen Ghannam } 177875aeaaf2SYazen Ghannam 177978ec161aSMuralidhara M K static void dct_determine_memory_type(struct amd64_pvt *pvt) 178094be4bffSDoug Thompson { 1781a597d2a5SAravind Gopalakrishnan u32 dram_ctrl, dcsm; 178294be4bffSDoug Thompson 1783a597d2a5SAravind Gopalakrishnan switch (pvt->fam) { 1784a597d2a5SAravind Gopalakrishnan case 0xf: 1785a597d2a5SAravind Gopalakrishnan if (pvt->ext_model >= K8_REV_F) 1786a597d2a5SAravind Gopalakrishnan goto ddr3; 1787a597d2a5SAravind Gopalakrishnan 1788a597d2a5SAravind Gopalakrishnan pvt->dram_type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR; 1789a597d2a5SAravind Gopalakrishnan return; 1790a597d2a5SAravind Gopalakrishnan 1791a597d2a5SAravind Gopalakrishnan case 0x10: 17926b4c0bdeSBorislav Petkov if (pvt->dchr0 & DDR3_MODE) 1793a597d2a5SAravind Gopalakrishnan goto ddr3; 1794a597d2a5SAravind Gopalakrishnan 1795a597d2a5SAravind Gopalakrishnan pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2; 1796a597d2a5SAravind Gopalakrishnan return; 1797a597d2a5SAravind Gopalakrishnan 1798a597d2a5SAravind Gopalakrishnan case 0x15: 1799a597d2a5SAravind Gopalakrishnan if (pvt->model < 0x60) 1800a597d2a5SAravind Gopalakrishnan goto ddr3; 1801a597d2a5SAravind Gopalakrishnan 1802a597d2a5SAravind Gopalakrishnan /* 1803a597d2a5SAravind Gopalakrishnan * Model 0x60h needs special handling: 1804a597d2a5SAravind Gopalakrishnan * 1805a597d2a5SAravind Gopalakrishnan * We use a Chip Select value of '0' to obtain dcsm. 1806a597d2a5SAravind Gopalakrishnan * Theoretically, it is possible to populate LRDIMMs of different 1807a597d2a5SAravind Gopalakrishnan * 'Rank' value on a DCT. But this is not the common case. So, 1808a597d2a5SAravind Gopalakrishnan * it's reasonable to assume all DIMMs are going to be of same 1809a597d2a5SAravind Gopalakrishnan * 'type' until proven otherwise. 1810a597d2a5SAravind Gopalakrishnan */ 1811a597d2a5SAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 0, DRAM_CONTROL, &dram_ctrl); 1812a597d2a5SAravind Gopalakrishnan dcsm = pvt->csels[0].csmasks[0]; 1813a597d2a5SAravind Gopalakrishnan 1814a597d2a5SAravind Gopalakrishnan if (((dram_ctrl >> 8) & 0x7) == 0x2) 1815a597d2a5SAravind Gopalakrishnan pvt->dram_type = MEM_DDR4; 1816a597d2a5SAravind Gopalakrishnan else if (pvt->dclr0 & BIT(16)) 1817a597d2a5SAravind Gopalakrishnan pvt->dram_type = MEM_DDR3; 1818a597d2a5SAravind Gopalakrishnan else if (dcsm & 0x3) 1819a597d2a5SAravind Gopalakrishnan pvt->dram_type = MEM_LRDDR3; 18206b4c0bdeSBorislav Petkov else 1821a597d2a5SAravind Gopalakrishnan pvt->dram_type = MEM_RDDR3; 1822a597d2a5SAravind Gopalakrishnan 1823a597d2a5SAravind Gopalakrishnan return; 1824a597d2a5SAravind Gopalakrishnan 1825a597d2a5SAravind Gopalakrishnan case 0x16: 1826a597d2a5SAravind Gopalakrishnan goto ddr3; 1827a597d2a5SAravind Gopalakrishnan 1828a597d2a5SAravind Gopalakrishnan default: 1829a597d2a5SAravind Gopalakrishnan WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam); 1830a597d2a5SAravind Gopalakrishnan pvt->dram_type = MEM_EMPTY; 183194be4bffSDoug Thompson } 183278ec161aSMuralidhara M K 183378ec161aSMuralidhara M K edac_dbg(1, " DIMM type: %s\n", edac_mem_types[pvt->dram_type]); 1834a597d2a5SAravind Gopalakrishnan return; 183594be4bffSDoug Thompson 1836a597d2a5SAravind Gopalakrishnan ddr3: 1837a597d2a5SAravind Gopalakrishnan pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3; 183894be4bffSDoug Thompson } 183994be4bffSDoug Thompson 184070046624SBorislav Petkov /* On F10h and later ErrAddr is MC4_ADDR[47:1] */ 1841a4b4bedcSBorislav Petkov static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m) 1842ddff876dSDoug Thompson { 1843db970bd2SYazen Ghannam u16 mce_nid = topology_die_id(m->extcpu); 18442ec591acSBorislav Petkov struct mem_ctl_info *mci; 184570046624SBorislav Petkov u8 start_bit = 1; 184670046624SBorislav Petkov u8 end_bit = 47; 18472ec591acSBorislav Petkov u64 addr; 18482ec591acSBorislav Petkov 18492ec591acSBorislav Petkov mci = edac_mc_find(mce_nid); 18502ec591acSBorislav Petkov if (!mci) 18512ec591acSBorislav Petkov return 0; 18522ec591acSBorislav Petkov 18532ec591acSBorislav Petkov pvt = mci->pvt_info; 185470046624SBorislav Petkov 1855a4b4bedcSBorislav Petkov if (pvt->fam == 0xf) { 185670046624SBorislav Petkov start_bit = 3; 185770046624SBorislav Petkov end_bit = 39; 185870046624SBorislav Petkov } 185970046624SBorislav Petkov 186010ef6b0dSChen, Gong addr = m->addr & GENMASK_ULL(end_bit, start_bit); 1861c1ae6830SBorislav Petkov 1862c1ae6830SBorislav Petkov /* 1863c1ae6830SBorislav Petkov * Erratum 637 workaround 1864c1ae6830SBorislav Petkov */ 1865a4b4bedcSBorislav Petkov if (pvt->fam == 0x15) { 1866c1ae6830SBorislav Petkov u64 cc6_base, tmp_addr; 1867c1ae6830SBorislav Petkov u32 tmp; 18688b84c8dfSDaniel J Blueman u8 intlv_en; 1869c1ae6830SBorislav Petkov 187010ef6b0dSChen, Gong if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7) 1871c1ae6830SBorislav Petkov return addr; 1872c1ae6830SBorislav Petkov 1873c1ae6830SBorislav Petkov 1874c1ae6830SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp); 1875c1ae6830SBorislav Petkov intlv_en = tmp >> 21 & 0x7; 1876c1ae6830SBorislav Petkov 1877c1ae6830SBorislav Petkov /* add [47:27] + 3 trailing bits */ 187810ef6b0dSChen, Gong cc6_base = (tmp & GENMASK_ULL(20, 0)) << 3; 1879c1ae6830SBorislav Petkov 1880c1ae6830SBorislav Petkov /* reverse and add DramIntlvEn */ 1881c1ae6830SBorislav Petkov cc6_base |= intlv_en ^ 0x7; 1882c1ae6830SBorislav Petkov 1883c1ae6830SBorislav Petkov /* pin at [47:24] */ 1884c1ae6830SBorislav Petkov cc6_base <<= 24; 1885c1ae6830SBorislav Petkov 1886c1ae6830SBorislav Petkov if (!intlv_en) 188710ef6b0dSChen, Gong return cc6_base | (addr & GENMASK_ULL(23, 0)); 1888c1ae6830SBorislav Petkov 1889c1ae6830SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp); 1890c1ae6830SBorislav Petkov 1891c1ae6830SBorislav Petkov /* faster log2 */ 189210ef6b0dSChen, Gong tmp_addr = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1); 1893c1ae6830SBorislav Petkov 1894c1ae6830SBorislav Petkov /* OR DramIntlvSel into bits [14:12] */ 189510ef6b0dSChen, Gong tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9; 1896c1ae6830SBorislav Petkov 1897c1ae6830SBorislav Petkov /* add remaining [11:0] bits from original MC4_ADDR */ 189810ef6b0dSChen, Gong tmp_addr |= addr & GENMASK_ULL(11, 0); 1899c1ae6830SBorislav Petkov 1900c1ae6830SBorislav Petkov return cc6_base | tmp_addr; 1901c1ae6830SBorislav Petkov } 1902c1ae6830SBorislav Petkov 1903c1ae6830SBorislav Petkov return addr; 1904ddff876dSDoug Thompson } 1905ddff876dSDoug Thompson 1906e2c0bffeSDaniel J Blueman static struct pci_dev *pci_get_related_function(unsigned int vendor, 1907e2c0bffeSDaniel J Blueman unsigned int device, 1908e2c0bffeSDaniel J Blueman struct pci_dev *related) 1909e2c0bffeSDaniel J Blueman { 1910e2c0bffeSDaniel J Blueman struct pci_dev *dev = NULL; 1911e2c0bffeSDaniel J Blueman 1912e2c0bffeSDaniel J Blueman while ((dev = pci_get_device(vendor, device, dev))) { 1913e2c0bffeSDaniel J Blueman if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) && 1914e2c0bffeSDaniel J Blueman (dev->bus->number == related->bus->number) && 1915e2c0bffeSDaniel J Blueman (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn))) 1916e2c0bffeSDaniel J Blueman break; 1917e2c0bffeSDaniel J Blueman } 1918e2c0bffeSDaniel J Blueman 1919e2c0bffeSDaniel J Blueman return dev; 1920e2c0bffeSDaniel J Blueman } 1921e2c0bffeSDaniel J Blueman 19227f19bf75SBorislav Petkov static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range) 1923ddff876dSDoug Thompson { 1924e2c0bffeSDaniel J Blueman struct amd_northbridge *nb; 192518b94f66SAravind Gopalakrishnan struct pci_dev *f1 = NULL; 192618b94f66SAravind Gopalakrishnan unsigned int pci_func; 192771d2a32eSBorislav Petkov int off = range << 3; 1928e2c0bffeSDaniel J Blueman u32 llim; 1929ddff876dSDoug Thompson 19307f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo); 19317f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo); 1932ddff876dSDoug Thompson 193318b94f66SAravind Gopalakrishnan if (pvt->fam == 0xf) 19347f19bf75SBorislav Petkov return; 1935ddff876dSDoug Thompson 19367f19bf75SBorislav Petkov if (!dram_rw(pvt, range)) 19377f19bf75SBorislav Petkov return; 1938ddff876dSDoug Thompson 19397f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi); 19407f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi); 1941f08e457cSBorislav Petkov 1942e2c0bffeSDaniel J Blueman /* F15h: factor in CC6 save area by reading dst node's limit reg */ 194318b94f66SAravind Gopalakrishnan if (pvt->fam != 0x15) 1944e2c0bffeSDaniel J Blueman return; 1945f08e457cSBorislav Petkov 1946e2c0bffeSDaniel J Blueman nb = node_to_amd_nb(dram_dst_node(pvt, range)); 1947e2c0bffeSDaniel J Blueman if (WARN_ON(!nb)) 1948e2c0bffeSDaniel J Blueman return; 1949e2c0bffeSDaniel J Blueman 1950a597d2a5SAravind Gopalakrishnan if (pvt->model == 0x60) 1951a597d2a5SAravind Gopalakrishnan pci_func = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1; 1952a597d2a5SAravind Gopalakrishnan else if (pvt->model == 0x30) 1953a597d2a5SAravind Gopalakrishnan pci_func = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1; 1954a597d2a5SAravind Gopalakrishnan else 1955a597d2a5SAravind Gopalakrishnan pci_func = PCI_DEVICE_ID_AMD_15H_NB_F1; 195618b94f66SAravind Gopalakrishnan 195718b94f66SAravind Gopalakrishnan f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc); 1958f08e457cSBorislav Petkov if (WARN_ON(!f1)) 1959f08e457cSBorislav Petkov return; 1960f08e457cSBorislav Petkov 1961f08e457cSBorislav Petkov amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim); 1962f08e457cSBorislav Petkov 196310ef6b0dSChen, Gong pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0); 1964f08e457cSBorislav Petkov 1965f08e457cSBorislav Petkov /* {[39:27],111b} */ 1966f08e457cSBorislav Petkov pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16; 1967f08e457cSBorislav Petkov 196810ef6b0dSChen, Gong pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0); 1969f08e457cSBorislav Petkov 1970f08e457cSBorislav Petkov /* [47:40] */ 1971f08e457cSBorislav Petkov pvt->ranges[range].lim.hi |= llim >> 13; 1972f08e457cSBorislav Petkov 1973f08e457cSBorislav Petkov pci_dev_put(f1); 1974f08e457cSBorislav Petkov } 1975ddff876dSDoug Thompson 1976f192c7b1SBorislav Petkov static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr, 197733ca0643SBorislav Petkov struct err_info *err) 1978ddff876dSDoug Thompson { 1979f192c7b1SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 1980ddff876dSDoug Thompson 198133ca0643SBorislav Petkov error_address_to_page_and_offset(sys_addr, err); 1982ab5a503cSMauro Carvalho Chehab 1983ab5a503cSMauro Carvalho Chehab /* 1984ab5a503cSMauro Carvalho Chehab * Find out which node the error address belongs to. This may be 1985ab5a503cSMauro Carvalho Chehab * different from the node that detected the error. 1986ab5a503cSMauro Carvalho Chehab */ 198733ca0643SBorislav Petkov err->src_mci = find_mc_by_sys_addr(mci, sys_addr); 198833ca0643SBorislav Petkov if (!err->src_mci) { 1989ab5a503cSMauro Carvalho Chehab amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n", 1990ab5a503cSMauro Carvalho Chehab (unsigned long)sys_addr); 199133ca0643SBorislav Petkov err->err_code = ERR_NODE; 1992ab5a503cSMauro Carvalho Chehab return; 1993ab5a503cSMauro Carvalho Chehab } 1994ab5a503cSMauro Carvalho Chehab 1995ab5a503cSMauro Carvalho Chehab /* Now map the sys_addr to a CSROW */ 199633ca0643SBorislav Petkov err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr); 199733ca0643SBorislav Petkov if (err->csrow < 0) { 199833ca0643SBorislav Petkov err->err_code = ERR_CSROW; 1999ab5a503cSMauro Carvalho Chehab return; 2000ab5a503cSMauro Carvalho Chehab } 2001ab5a503cSMauro Carvalho Chehab 2002ddff876dSDoug Thompson /* CHIPKILL enabled */ 2003f192c7b1SBorislav Petkov if (pvt->nbcfg & NBCFG_CHIPKILL) { 200433ca0643SBorislav Petkov err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome); 200533ca0643SBorislav Petkov if (err->channel < 0) { 2006ddff876dSDoug Thompson /* 2007ddff876dSDoug Thompson * Syndrome didn't map, so we don't know which of the 2008ddff876dSDoug Thompson * 2 DIMMs is in error. So we need to ID 'both' of them 2009ddff876dSDoug Thompson * as suspect. 2010ddff876dSDoug Thompson */ 201133ca0643SBorislav Petkov amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - " 2012ab5a503cSMauro Carvalho Chehab "possible error reporting race\n", 201333ca0643SBorislav Petkov err->syndrome); 201433ca0643SBorislav Petkov err->err_code = ERR_CHANNEL; 2015ddff876dSDoug Thompson return; 2016ddff876dSDoug Thompson } 2017ddff876dSDoug Thompson } else { 2018ddff876dSDoug Thompson /* 2019ddff876dSDoug Thompson * non-chipkill ecc mode 2020ddff876dSDoug Thompson * 2021ddff876dSDoug Thompson * The k8 documentation is unclear about how to determine the 2022ddff876dSDoug Thompson * channel number when using non-chipkill memory. This method 2023ddff876dSDoug Thompson * was obtained from email communication with someone at AMD. 2024ddff876dSDoug Thompson * (Wish the email was placed in this comment - norsk) 2025ddff876dSDoug Thompson */ 202633ca0643SBorislav Petkov err->channel = ((sys_addr & BIT(3)) != 0); 2027ddff876dSDoug Thompson } 2028ddff876dSDoug Thompson } 2029ddff876dSDoug Thompson 203041d8bfabSBorislav Petkov static int ddr2_cs_size(unsigned i, bool dct_width) 2031ddff876dSDoug Thompson { 203241d8bfabSBorislav Petkov unsigned shift = 0; 2033ddff876dSDoug Thompson 203441d8bfabSBorislav Petkov if (i <= 2) 203541d8bfabSBorislav Petkov shift = i; 203641d8bfabSBorislav Petkov else if (!(i & 0x1)) 203741d8bfabSBorislav Petkov shift = i >> 1; 20381433eb99SBorislav Petkov else 203941d8bfabSBorislav Petkov shift = (i + 1) >> 1; 2040ddff876dSDoug Thompson 204141d8bfabSBorislav Petkov return 128 << (shift + !!dct_width); 204241d8bfabSBorislav Petkov } 204341d8bfabSBorislav Petkov 204441d8bfabSBorislav Petkov static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 2045a597d2a5SAravind Gopalakrishnan unsigned cs_mode, int cs_mask_nr) 204641d8bfabSBorislav Petkov { 204741d8bfabSBorislav Petkov u32 dclr = dct ? pvt->dclr1 : pvt->dclr0; 204841d8bfabSBorislav Petkov 204941d8bfabSBorislav Petkov if (pvt->ext_model >= K8_REV_F) { 205041d8bfabSBorislav Petkov WARN_ON(cs_mode > 11); 205141d8bfabSBorislav Petkov return ddr2_cs_size(cs_mode, dclr & WIDTH_128); 205241d8bfabSBorislav Petkov } 205341d8bfabSBorislav Petkov else if (pvt->ext_model >= K8_REV_D) { 205411b0a314SBorislav Petkov unsigned diff; 205541d8bfabSBorislav Petkov WARN_ON(cs_mode > 10); 205641d8bfabSBorislav Petkov 205711b0a314SBorislav Petkov /* 205811b0a314SBorislav Petkov * the below calculation, besides trying to win an obfuscated C 205911b0a314SBorislav Petkov * contest, maps cs_mode values to DIMM chip select sizes. The 206011b0a314SBorislav Petkov * mappings are: 206111b0a314SBorislav Petkov * 206211b0a314SBorislav Petkov * cs_mode CS size (mb) 206311b0a314SBorislav Petkov * ======= ============ 206411b0a314SBorislav Petkov * 0 32 206511b0a314SBorislav Petkov * 1 64 206611b0a314SBorislav Petkov * 2 128 206711b0a314SBorislav Petkov * 3 128 206811b0a314SBorislav Petkov * 4 256 206911b0a314SBorislav Petkov * 5 512 207011b0a314SBorislav Petkov * 6 256 207111b0a314SBorislav Petkov * 7 512 207211b0a314SBorislav Petkov * 8 1024 207311b0a314SBorislav Petkov * 9 1024 207411b0a314SBorislav Petkov * 10 2048 207511b0a314SBorislav Petkov * 207611b0a314SBorislav Petkov * Basically, it calculates a value with which to shift the 207711b0a314SBorislav Petkov * smallest CS size of 32MB. 207811b0a314SBorislav Petkov * 207911b0a314SBorislav Petkov * ddr[23]_cs_size have a similar purpose. 208011b0a314SBorislav Petkov */ 208111b0a314SBorislav Petkov diff = cs_mode/3 + (unsigned)(cs_mode > 5); 208211b0a314SBorislav Petkov 208311b0a314SBorislav Petkov return 32 << (cs_mode - diff); 208441d8bfabSBorislav Petkov } 208541d8bfabSBorislav Petkov else { 208641d8bfabSBorislav Petkov WARN_ON(cs_mode > 6); 208741d8bfabSBorislav Petkov return 32 << cs_mode; 208841d8bfabSBorislav Petkov } 2089ddff876dSDoug Thompson } 2090ddff876dSDoug Thompson 209141d8bfabSBorislav Petkov static int ddr3_cs_size(unsigned i, bool dct_width) 20921afd3c98SDoug Thompson { 209341d8bfabSBorislav Petkov unsigned shift = 0; 209441d8bfabSBorislav Petkov int cs_size = 0; 209541d8bfabSBorislav Petkov 209641d8bfabSBorislav Petkov if (i == 0 || i == 3 || i == 4) 209741d8bfabSBorislav Petkov cs_size = -1; 209841d8bfabSBorislav Petkov else if (i <= 2) 209941d8bfabSBorislav Petkov shift = i; 210041d8bfabSBorislav Petkov else if (i == 12) 210141d8bfabSBorislav Petkov shift = 7; 210241d8bfabSBorislav Petkov else if (!(i & 0x1)) 210341d8bfabSBorislav Petkov shift = i >> 1; 210441d8bfabSBorislav Petkov else 210541d8bfabSBorislav Petkov shift = (i + 1) >> 1; 210641d8bfabSBorislav Petkov 210741d8bfabSBorislav Petkov if (cs_size != -1) 210841d8bfabSBorislav Petkov cs_size = (128 * (1 << !!dct_width)) << shift; 210941d8bfabSBorislav Petkov 211041d8bfabSBorislav Petkov return cs_size; 211141d8bfabSBorislav Petkov } 211241d8bfabSBorislav Petkov 2113a597d2a5SAravind Gopalakrishnan static int ddr3_lrdimm_cs_size(unsigned i, unsigned rank_multiply) 2114a597d2a5SAravind Gopalakrishnan { 2115a597d2a5SAravind Gopalakrishnan unsigned shift = 0; 2116a597d2a5SAravind Gopalakrishnan int cs_size = 0; 2117a597d2a5SAravind Gopalakrishnan 2118a597d2a5SAravind Gopalakrishnan if (i < 4 || i == 6) 2119a597d2a5SAravind Gopalakrishnan cs_size = -1; 2120a597d2a5SAravind Gopalakrishnan else if (i == 12) 2121a597d2a5SAravind Gopalakrishnan shift = 7; 2122a597d2a5SAravind Gopalakrishnan else if (!(i & 0x1)) 2123a597d2a5SAravind Gopalakrishnan shift = i >> 1; 2124a597d2a5SAravind Gopalakrishnan else 2125a597d2a5SAravind Gopalakrishnan shift = (i + 1) >> 1; 2126a597d2a5SAravind Gopalakrishnan 2127a597d2a5SAravind Gopalakrishnan if (cs_size != -1) 2128a597d2a5SAravind Gopalakrishnan cs_size = rank_multiply * (128 << shift); 2129a597d2a5SAravind Gopalakrishnan 2130a597d2a5SAravind Gopalakrishnan return cs_size; 2131a597d2a5SAravind Gopalakrishnan } 2132a597d2a5SAravind Gopalakrishnan 2133a597d2a5SAravind Gopalakrishnan static int ddr4_cs_size(unsigned i) 2134a597d2a5SAravind Gopalakrishnan { 2135a597d2a5SAravind Gopalakrishnan int cs_size = 0; 2136a597d2a5SAravind Gopalakrishnan 2137a597d2a5SAravind Gopalakrishnan if (i == 0) 2138a597d2a5SAravind Gopalakrishnan cs_size = -1; 2139a597d2a5SAravind Gopalakrishnan else if (i == 1) 2140a597d2a5SAravind Gopalakrishnan cs_size = 1024; 2141a597d2a5SAravind Gopalakrishnan else 2142a597d2a5SAravind Gopalakrishnan /* Min cs_size = 1G */ 2143a597d2a5SAravind Gopalakrishnan cs_size = 1024 * (1 << (i >> 1)); 2144a597d2a5SAravind Gopalakrishnan 2145a597d2a5SAravind Gopalakrishnan return cs_size; 2146a597d2a5SAravind Gopalakrishnan } 2147a597d2a5SAravind Gopalakrishnan 214841d8bfabSBorislav Petkov static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 2149a597d2a5SAravind Gopalakrishnan unsigned cs_mode, int cs_mask_nr) 215041d8bfabSBorislav Petkov { 215141d8bfabSBorislav Petkov u32 dclr = dct ? pvt->dclr1 : pvt->dclr0; 215241d8bfabSBorislav Petkov 215341d8bfabSBorislav Petkov WARN_ON(cs_mode > 11); 21541433eb99SBorislav Petkov 21551433eb99SBorislav Petkov if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE) 215641d8bfabSBorislav Petkov return ddr3_cs_size(cs_mode, dclr & WIDTH_128); 21571433eb99SBorislav Petkov else 215841d8bfabSBorislav Petkov return ddr2_cs_size(cs_mode, dclr & WIDTH_128); 215941d8bfabSBorislav Petkov } 21601433eb99SBorislav Petkov 216141d8bfabSBorislav Petkov /* 216241d8bfabSBorislav Petkov * F15h supports only 64bit DCT interfaces 216341d8bfabSBorislav Petkov */ 216441d8bfabSBorislav Petkov static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 2165a597d2a5SAravind Gopalakrishnan unsigned cs_mode, int cs_mask_nr) 216641d8bfabSBorislav Petkov { 216741d8bfabSBorislav Petkov WARN_ON(cs_mode > 12); 216841d8bfabSBorislav Petkov 216941d8bfabSBorislav Petkov return ddr3_cs_size(cs_mode, false); 21701afd3c98SDoug Thompson } 21711afd3c98SDoug Thompson 2172a597d2a5SAravind Gopalakrishnan /* F15h M60h supports DDR4 mapping as well.. */ 2173a597d2a5SAravind Gopalakrishnan static int f15_m60h_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 2174a597d2a5SAravind Gopalakrishnan unsigned cs_mode, int cs_mask_nr) 2175a597d2a5SAravind Gopalakrishnan { 2176a597d2a5SAravind Gopalakrishnan int cs_size; 2177a597d2a5SAravind Gopalakrishnan u32 dcsm = pvt->csels[dct].csmasks[cs_mask_nr]; 2178a597d2a5SAravind Gopalakrishnan 2179a597d2a5SAravind Gopalakrishnan WARN_ON(cs_mode > 12); 2180a597d2a5SAravind Gopalakrishnan 2181a597d2a5SAravind Gopalakrishnan if (pvt->dram_type == MEM_DDR4) { 2182a597d2a5SAravind Gopalakrishnan if (cs_mode > 9) 2183a597d2a5SAravind Gopalakrishnan return -1; 2184a597d2a5SAravind Gopalakrishnan 2185a597d2a5SAravind Gopalakrishnan cs_size = ddr4_cs_size(cs_mode); 2186a597d2a5SAravind Gopalakrishnan } else if (pvt->dram_type == MEM_LRDDR3) { 2187a597d2a5SAravind Gopalakrishnan unsigned rank_multiply = dcsm & 0xf; 2188a597d2a5SAravind Gopalakrishnan 2189a597d2a5SAravind Gopalakrishnan if (rank_multiply == 3) 2190a597d2a5SAravind Gopalakrishnan rank_multiply = 4; 2191a597d2a5SAravind Gopalakrishnan cs_size = ddr3_lrdimm_cs_size(cs_mode, rank_multiply); 2192a597d2a5SAravind Gopalakrishnan } else { 2193a597d2a5SAravind Gopalakrishnan /* Minimum cs size is 512mb for F15hM60h*/ 2194a597d2a5SAravind Gopalakrishnan if (cs_mode == 0x1) 2195a597d2a5SAravind Gopalakrishnan return -1; 2196a597d2a5SAravind Gopalakrishnan 2197a597d2a5SAravind Gopalakrishnan cs_size = ddr3_cs_size(cs_mode, false); 2198a597d2a5SAravind Gopalakrishnan } 2199a597d2a5SAravind Gopalakrishnan 2200a597d2a5SAravind Gopalakrishnan return cs_size; 2201a597d2a5SAravind Gopalakrishnan } 2202a597d2a5SAravind Gopalakrishnan 220394c1acf2SAravind Gopalakrishnan /* 220418b94f66SAravind Gopalakrishnan * F16h and F15h model 30h have only limited cs_modes. 220594c1acf2SAravind Gopalakrishnan */ 220694c1acf2SAravind Gopalakrishnan static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 2207a597d2a5SAravind Gopalakrishnan unsigned cs_mode, int cs_mask_nr) 220894c1acf2SAravind Gopalakrishnan { 220994c1acf2SAravind Gopalakrishnan WARN_ON(cs_mode > 12); 221094c1acf2SAravind Gopalakrishnan 221194c1acf2SAravind Gopalakrishnan if (cs_mode == 6 || cs_mode == 8 || 221294c1acf2SAravind Gopalakrishnan cs_mode == 9 || cs_mode == 12) 221394c1acf2SAravind Gopalakrishnan return -1; 221494c1acf2SAravind Gopalakrishnan else 221594c1acf2SAravind Gopalakrishnan return ddr3_cs_size(cs_mode, false); 221694c1acf2SAravind Gopalakrishnan } 221794c1acf2SAravind Gopalakrishnan 22185a5d2371SBorislav Petkov static void read_dram_ctl_register(struct amd64_pvt *pvt) 22196163b5d4SDoug Thompson { 22206163b5d4SDoug Thompson 2221a4b4bedcSBorislav Petkov if (pvt->fam == 0xf) 22225a5d2371SBorislav Petkov return; 22235a5d2371SBorislav Petkov 22247981a28fSAravind Gopalakrishnan if (!amd64_read_pci_cfg(pvt->F2, DCT_SEL_LO, &pvt->dct_sel_lo)) { 2225956b9ba1SJoe Perches edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n", 222678da121eSBorislav Petkov pvt->dct_sel_lo, dct_sel_baseaddr(pvt)); 22276163b5d4SDoug Thompson 2228956b9ba1SJoe Perches edac_dbg(0, " DCTs operate in %s mode\n", 22295a5d2371SBorislav Petkov (dct_ganging_enabled(pvt) ? "ganged" : "unganged")); 22306163b5d4SDoug Thompson 223172381bd5SBorislav Petkov if (!dct_ganging_enabled(pvt)) 2232956b9ba1SJoe Perches edac_dbg(0, " Address range split per DCT: %s\n", 223372381bd5SBorislav Petkov (dct_high_range_enabled(pvt) ? "yes" : "no")); 223472381bd5SBorislav Petkov 2235956b9ba1SJoe Perches edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n", 223672381bd5SBorislav Petkov (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"), 223772381bd5SBorislav Petkov (dct_memory_cleared(pvt) ? "yes" : "no")); 223872381bd5SBorislav Petkov 2239956b9ba1SJoe Perches edac_dbg(0, " channel interleave: %s, " 224078da121eSBorislav Petkov "interleave bits selector: 0x%x\n", 224172381bd5SBorislav Petkov (dct_interleave_enabled(pvt) ? "enabled" : "disabled"), 22426163b5d4SDoug Thompson dct_sel_interleave_addr(pvt)); 22436163b5d4SDoug Thompson } 22446163b5d4SDoug Thompson 22457981a28fSAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F2, DCT_SEL_HI, &pvt->dct_sel_hi); 22466163b5d4SDoug Thompson } 22476163b5d4SDoug Thompson 2248f71d0a05SDoug Thompson /* 224918b94f66SAravind Gopalakrishnan * Determine channel (DCT) based on the interleaving mode (see F15h M30h BKDG, 225018b94f66SAravind Gopalakrishnan * 2.10.12 Memory Interleaving Modes). 225118b94f66SAravind Gopalakrishnan */ 225218b94f66SAravind Gopalakrishnan static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr, 225318b94f66SAravind Gopalakrishnan u8 intlv_en, int num_dcts_intlv, 225418b94f66SAravind Gopalakrishnan u32 dct_sel) 225518b94f66SAravind Gopalakrishnan { 225618b94f66SAravind Gopalakrishnan u8 channel = 0; 225718b94f66SAravind Gopalakrishnan u8 select; 225818b94f66SAravind Gopalakrishnan 225918b94f66SAravind Gopalakrishnan if (!(intlv_en)) 226018b94f66SAravind Gopalakrishnan return (u8)(dct_sel); 226118b94f66SAravind Gopalakrishnan 226218b94f66SAravind Gopalakrishnan if (num_dcts_intlv == 2) { 226318b94f66SAravind Gopalakrishnan select = (sys_addr >> 8) & 0x3; 226418b94f66SAravind Gopalakrishnan channel = select ? 0x3 : 0; 22659d0e8d83SAravind Gopalakrishnan } else if (num_dcts_intlv == 4) { 22669d0e8d83SAravind Gopalakrishnan u8 intlv_addr = dct_sel_interleave_addr(pvt); 22679d0e8d83SAravind Gopalakrishnan switch (intlv_addr) { 22689d0e8d83SAravind Gopalakrishnan case 0x4: 22699d0e8d83SAravind Gopalakrishnan channel = (sys_addr >> 8) & 0x3; 22709d0e8d83SAravind Gopalakrishnan break; 22719d0e8d83SAravind Gopalakrishnan case 0x5: 22729d0e8d83SAravind Gopalakrishnan channel = (sys_addr >> 9) & 0x3; 22739d0e8d83SAravind Gopalakrishnan break; 22749d0e8d83SAravind Gopalakrishnan } 22759d0e8d83SAravind Gopalakrishnan } 227618b94f66SAravind Gopalakrishnan return channel; 227718b94f66SAravind Gopalakrishnan } 227818b94f66SAravind Gopalakrishnan 227918b94f66SAravind Gopalakrishnan /* 2280229a7a11SBorislav Petkov * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory 2281f71d0a05SDoug Thompson * Interleaving Modes. 2282f71d0a05SDoug Thompson */ 2283b15f0fcaSBorislav Petkov static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr, 2284229a7a11SBorislav Petkov bool hi_range_sel, u8 intlv_en) 22856163b5d4SDoug Thompson { 2286151fa71cSBorislav Petkov u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1; 22876163b5d4SDoug Thompson 22886163b5d4SDoug Thompson if (dct_ganging_enabled(pvt)) 2289229a7a11SBorislav Petkov return 0; 2290229a7a11SBorislav Petkov 2291229a7a11SBorislav Petkov if (hi_range_sel) 2292229a7a11SBorislav Petkov return dct_sel_high; 2293229a7a11SBorislav Petkov 2294f71d0a05SDoug Thompson /* 2295f71d0a05SDoug Thompson * see F2x110[DctSelIntLvAddr] - channel interleave mode 2296f71d0a05SDoug Thompson */ 2297229a7a11SBorislav Petkov if (dct_interleave_enabled(pvt)) { 2298229a7a11SBorislav Petkov u8 intlv_addr = dct_sel_interleave_addr(pvt); 22996163b5d4SDoug Thompson 2300229a7a11SBorislav Petkov /* return DCT select function: 0=DCT0, 1=DCT1 */ 2301229a7a11SBorislav Petkov if (!intlv_addr) 2302229a7a11SBorislav Petkov return sys_addr >> 6 & 1; 23036163b5d4SDoug Thompson 2304229a7a11SBorislav Petkov if (intlv_addr & 0x2) { 2305229a7a11SBorislav Petkov u8 shift = intlv_addr & 0x1 ? 9 : 6; 2306dc0a50a8SYazen Ghannam u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) & 1; 2307229a7a11SBorislav Petkov 2308229a7a11SBorislav Petkov return ((sys_addr >> shift) & 1) ^ temp; 23096163b5d4SDoug Thompson } 23106163b5d4SDoug Thompson 2311dc0a50a8SYazen Ghannam if (intlv_addr & 0x4) { 2312dc0a50a8SYazen Ghannam u8 shift = intlv_addr & 0x1 ? 9 : 8; 2313dc0a50a8SYazen Ghannam 2314dc0a50a8SYazen Ghannam return (sys_addr >> shift) & 1; 2315dc0a50a8SYazen Ghannam } 2316dc0a50a8SYazen Ghannam 2317229a7a11SBorislav Petkov return (sys_addr >> (12 + hweight8(intlv_en))) & 1; 2318229a7a11SBorislav Petkov } 2319229a7a11SBorislav Petkov 2320229a7a11SBorislav Petkov if (dct_high_range_enabled(pvt)) 2321229a7a11SBorislav Petkov return ~dct_sel_high & 1; 23226163b5d4SDoug Thompson 23236163b5d4SDoug Thompson return 0; 23246163b5d4SDoug Thompson } 23256163b5d4SDoug Thompson 2326c8e518d5SBorislav Petkov /* Convert the sys_addr to the normalized DCT address */ 2327c7e5301aSDaniel J Blueman static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range, 2328c8e518d5SBorislav Petkov u64 sys_addr, bool hi_rng, 2329c8e518d5SBorislav Petkov u32 dct_sel_base_addr) 23306163b5d4SDoug Thompson { 23316163b5d4SDoug Thompson u64 chan_off; 2332c8e518d5SBorislav Petkov u64 dram_base = get_dram_base(pvt, range); 2333c8e518d5SBorislav Petkov u64 hole_off = f10_dhar_offset(pvt); 23346f3508f6SDan Carpenter u64 dct_sel_base_off = (u64)(pvt->dct_sel_hi & 0xFFFFFC00) << 16; 23356163b5d4SDoug Thompson 2336c8e518d5SBorislav Petkov if (hi_rng) { 2337c8e518d5SBorislav Petkov /* 2338c8e518d5SBorislav Petkov * if 2339c8e518d5SBorislav Petkov * base address of high range is below 4Gb 2340c8e518d5SBorislav Petkov * (bits [47:27] at [31:11]) 2341c8e518d5SBorislav Petkov * DRAM address space on this DCT is hoisted above 4Gb && 2342c8e518d5SBorislav Petkov * sys_addr > 4Gb 2343c8e518d5SBorislav Petkov * 2344c8e518d5SBorislav Petkov * remove hole offset from sys_addr 2345c8e518d5SBorislav Petkov * else 2346c8e518d5SBorislav Petkov * remove high range offset from sys_addr 2347c8e518d5SBorislav Petkov */ 2348c8e518d5SBorislav Petkov if ((!(dct_sel_base_addr >> 16) || 2349c8e518d5SBorislav Petkov dct_sel_base_addr < dhar_base(pvt)) && 2350972ea17aSBorislav Petkov dhar_valid(pvt) && 2351c8e518d5SBorislav Petkov (sys_addr >= BIT_64(32))) 2352bc21fa57SBorislav Petkov chan_off = hole_off; 23536163b5d4SDoug Thompson else 23546163b5d4SDoug Thompson chan_off = dct_sel_base_off; 23556163b5d4SDoug Thompson } else { 2356c8e518d5SBorislav Petkov /* 2357c8e518d5SBorislav Petkov * if 2358c8e518d5SBorislav Petkov * we have a valid hole && 2359c8e518d5SBorislav Petkov * sys_addr > 4Gb 2360c8e518d5SBorislav Petkov * 2361c8e518d5SBorislav Petkov * remove hole 2362c8e518d5SBorislav Petkov * else 2363c8e518d5SBorislav Petkov * remove dram base to normalize to DCT address 2364c8e518d5SBorislav Petkov */ 2365972ea17aSBorislav Petkov if (dhar_valid(pvt) && (sys_addr >= BIT_64(32))) 2366bc21fa57SBorislav Petkov chan_off = hole_off; 23676163b5d4SDoug Thompson else 2368c8e518d5SBorislav Petkov chan_off = dram_base; 23696163b5d4SDoug Thompson } 23706163b5d4SDoug Thompson 237110ef6b0dSChen, Gong return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23)); 23726163b5d4SDoug Thompson } 23736163b5d4SDoug Thompson 23746163b5d4SDoug Thompson /* 23756163b5d4SDoug Thompson * checks if the csrow passed in is marked as SPARED, if so returns the new 23766163b5d4SDoug Thompson * spare row 23776163b5d4SDoug Thompson */ 237811c75eadSBorislav Petkov static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow) 23796163b5d4SDoug Thompson { 2380614ec9d8SBorislav Petkov int tmp_cs; 23816163b5d4SDoug Thompson 2382614ec9d8SBorislav Petkov if (online_spare_swap_done(pvt, dct) && 2383614ec9d8SBorislav Petkov csrow == online_spare_bad_dramcs(pvt, dct)) { 2384614ec9d8SBorislav Petkov 2385614ec9d8SBorislav Petkov for_each_chip_select(tmp_cs, dct, pvt) { 2386614ec9d8SBorislav Petkov if (chip_select_base(tmp_cs, dct, pvt) & 0x2) { 2387614ec9d8SBorislav Petkov csrow = tmp_cs; 2388614ec9d8SBorislav Petkov break; 2389614ec9d8SBorislav Petkov } 2390614ec9d8SBorislav Petkov } 23916163b5d4SDoug Thompson } 23926163b5d4SDoug Thompson return csrow; 23936163b5d4SDoug Thompson } 23946163b5d4SDoug Thompson 23956163b5d4SDoug Thompson /* 23966163b5d4SDoug Thompson * Iterate over the DRAM DCT "base" and "mask" registers looking for a 23976163b5d4SDoug Thompson * SystemAddr match on the specified 'ChannelSelect' and 'NodeID' 23986163b5d4SDoug Thompson * 23996163b5d4SDoug Thompson * Return: 24006163b5d4SDoug Thompson * -EINVAL: NOT FOUND 24016163b5d4SDoug Thompson * 0..csrow = Chip-Select Row 24026163b5d4SDoug Thompson */ 2403c7e5301aSDaniel J Blueman static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct) 24046163b5d4SDoug Thompson { 24056163b5d4SDoug Thompson struct mem_ctl_info *mci; 24066163b5d4SDoug Thompson struct amd64_pvt *pvt; 240711c75eadSBorislav Petkov u64 cs_base, cs_mask; 24086163b5d4SDoug Thompson int cs_found = -EINVAL; 24096163b5d4SDoug Thompson int csrow; 24106163b5d4SDoug Thompson 24112ec591acSBorislav Petkov mci = edac_mc_find(nid); 24126163b5d4SDoug Thompson if (!mci) 24136163b5d4SDoug Thompson return cs_found; 24146163b5d4SDoug Thompson 24156163b5d4SDoug Thompson pvt = mci->pvt_info; 24166163b5d4SDoug Thompson 2417956b9ba1SJoe Perches edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct); 24186163b5d4SDoug Thompson 241911c75eadSBorislav Petkov for_each_chip_select(csrow, dct, pvt) { 242011c75eadSBorislav Petkov if (!csrow_enabled(csrow, dct, pvt)) 24216163b5d4SDoug Thompson continue; 24226163b5d4SDoug Thompson 242311c75eadSBorislav Petkov get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask); 24246163b5d4SDoug Thompson 2425956b9ba1SJoe Perches edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n", 24266163b5d4SDoug Thompson csrow, cs_base, cs_mask); 24276163b5d4SDoug Thompson 242811c75eadSBorislav Petkov cs_mask = ~cs_mask; 24296163b5d4SDoug Thompson 2430956b9ba1SJoe Perches edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n", 243111c75eadSBorislav Petkov (in_addr & cs_mask), (cs_base & cs_mask)); 24326163b5d4SDoug Thompson 243311c75eadSBorislav Petkov if ((in_addr & cs_mask) == (cs_base & cs_mask)) { 243418b94f66SAravind Gopalakrishnan if (pvt->fam == 0x15 && pvt->model >= 0x30) { 243518b94f66SAravind Gopalakrishnan cs_found = csrow; 243618b94f66SAravind Gopalakrishnan break; 243718b94f66SAravind Gopalakrishnan } 243811c75eadSBorislav Petkov cs_found = f10_process_possible_spare(pvt, dct, csrow); 24396163b5d4SDoug Thompson 2440956b9ba1SJoe Perches edac_dbg(1, " MATCH csrow=%d\n", cs_found); 24416163b5d4SDoug Thompson break; 24426163b5d4SDoug Thompson } 24436163b5d4SDoug Thompson } 24446163b5d4SDoug Thompson return cs_found; 24456163b5d4SDoug Thompson } 24466163b5d4SDoug Thompson 244795b0ef55SBorislav Petkov /* 244895b0ef55SBorislav Petkov * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is 244995b0ef55SBorislav Petkov * swapped with a region located at the bottom of memory so that the GPU can use 245095b0ef55SBorislav Petkov * the interleaved region and thus two channels. 245195b0ef55SBorislav Petkov */ 2452b15f0fcaSBorislav Petkov static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr) 245395b0ef55SBorislav Petkov { 245495b0ef55SBorislav Petkov u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr; 245595b0ef55SBorislav Petkov 2456a4b4bedcSBorislav Petkov if (pvt->fam == 0x10) { 245795b0ef55SBorislav Petkov /* only revC3 and revE have that feature */ 2458a4b4bedcSBorislav Petkov if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3)) 245995b0ef55SBorislav Petkov return sys_addr; 246095b0ef55SBorislav Petkov } 246195b0ef55SBorislav Petkov 24627981a28fSAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F2, SWAP_INTLV_REG, &swap_reg); 246395b0ef55SBorislav Petkov 246495b0ef55SBorislav Petkov if (!(swap_reg & 0x1)) 246595b0ef55SBorislav Petkov return sys_addr; 246695b0ef55SBorislav Petkov 246795b0ef55SBorislav Petkov swap_base = (swap_reg >> 3) & 0x7f; 246895b0ef55SBorislav Petkov swap_limit = (swap_reg >> 11) & 0x7f; 246995b0ef55SBorislav Petkov rgn_size = (swap_reg >> 20) & 0x7f; 247095b0ef55SBorislav Petkov tmp_addr = sys_addr >> 27; 247195b0ef55SBorislav Petkov 247295b0ef55SBorislav Petkov if (!(sys_addr >> 34) && 247395b0ef55SBorislav Petkov (((tmp_addr >= swap_base) && 247495b0ef55SBorislav Petkov (tmp_addr <= swap_limit)) || 247595b0ef55SBorislav Petkov (tmp_addr < rgn_size))) 247695b0ef55SBorislav Petkov return sys_addr ^ (u64)swap_base << 27; 247795b0ef55SBorislav Petkov 247895b0ef55SBorislav Petkov return sys_addr; 247995b0ef55SBorislav Petkov } 248095b0ef55SBorislav Petkov 2481f71d0a05SDoug Thompson /* For a given @dram_range, check if @sys_addr falls within it. */ 2482e761359aSBorislav Petkov static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range, 248333ca0643SBorislav Petkov u64 sys_addr, int *chan_sel) 2484f71d0a05SDoug Thompson { 2485229a7a11SBorislav Petkov int cs_found = -EINVAL; 2486c8e518d5SBorislav Petkov u64 chan_addr; 24875d4b58e8SBorislav Petkov u32 dct_sel_base; 248811c75eadSBorislav Petkov u8 channel; 2489229a7a11SBorislav Petkov bool high_range = false; 2490f71d0a05SDoug Thompson 24917f19bf75SBorislav Petkov u8 node_id = dram_dst_node(pvt, range); 2492229a7a11SBorislav Petkov u8 intlv_en = dram_intlv_en(pvt, range); 24937f19bf75SBorislav Petkov u32 intlv_sel = dram_intlv_sel(pvt, range); 2494f71d0a05SDoug Thompson 2495956b9ba1SJoe Perches edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n", 2496c8e518d5SBorislav Petkov range, sys_addr, get_dram_limit(pvt, range)); 2497f71d0a05SDoug Thompson 2498355fba60SBorislav Petkov if (dhar_valid(pvt) && 2499355fba60SBorislav Petkov dhar_base(pvt) <= sys_addr && 2500355fba60SBorislav Petkov sys_addr < BIT_64(32)) { 2501355fba60SBorislav Petkov amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n", 2502355fba60SBorislav Petkov sys_addr); 2503f71d0a05SDoug Thompson return -EINVAL; 2504355fba60SBorislav Petkov } 2505355fba60SBorislav Petkov 2506f030ddfbSBorislav Petkov if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en))) 2507355fba60SBorislav Petkov return -EINVAL; 2508f71d0a05SDoug Thompson 2509b15f0fcaSBorislav Petkov sys_addr = f1x_swap_interleaved_region(pvt, sys_addr); 251095b0ef55SBorislav Petkov 2511f71d0a05SDoug Thompson dct_sel_base = dct_sel_baseaddr(pvt); 2512f71d0a05SDoug Thompson 2513f71d0a05SDoug Thompson /* 2514f71d0a05SDoug Thompson * check whether addresses >= DctSelBaseAddr[47:27] are to be used to 2515f71d0a05SDoug Thompson * select between DCT0 and DCT1. 2516f71d0a05SDoug Thompson */ 2517f71d0a05SDoug Thompson if (dct_high_range_enabled(pvt) && 2518f71d0a05SDoug Thompson !dct_ganging_enabled(pvt) && 2519f71d0a05SDoug Thompson ((sys_addr >> 27) >= (dct_sel_base >> 11))) 2520229a7a11SBorislav Petkov high_range = true; 2521f71d0a05SDoug Thompson 2522b15f0fcaSBorislav Petkov channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en); 2523f71d0a05SDoug Thompson 2524b15f0fcaSBorislav Petkov chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr, 2525c8e518d5SBorislav Petkov high_range, dct_sel_base); 2526f71d0a05SDoug Thompson 2527e2f79dbdSBorislav Petkov /* Remove node interleaving, see F1x120 */ 2528e2f79dbdSBorislav Petkov if (intlv_en) 2529e2f79dbdSBorislav Petkov chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) | 2530e2f79dbdSBorislav Petkov (chan_addr & 0xfff); 2531f71d0a05SDoug Thompson 25325d4b58e8SBorislav Petkov /* remove channel interleave */ 2533f71d0a05SDoug Thompson if (dct_interleave_enabled(pvt) && 2534f71d0a05SDoug Thompson !dct_high_range_enabled(pvt) && 2535f71d0a05SDoug Thompson !dct_ganging_enabled(pvt)) { 25365d4b58e8SBorislav Petkov 25375d4b58e8SBorislav Petkov if (dct_sel_interleave_addr(pvt) != 1) { 25385d4b58e8SBorislav Petkov if (dct_sel_interleave_addr(pvt) == 0x3) 25395d4b58e8SBorislav Petkov /* hash 9 */ 25405d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 10) << 9) | 25415d4b58e8SBorislav Petkov (chan_addr & 0x1ff); 25425d4b58e8SBorislav Petkov else 25435d4b58e8SBorislav Petkov /* A[6] or hash 6 */ 25445d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 7) << 6) | 25455d4b58e8SBorislav Petkov (chan_addr & 0x3f); 25465d4b58e8SBorislav Petkov } else 25475d4b58e8SBorislav Petkov /* A[12] */ 25485d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 13) << 12) | 25495d4b58e8SBorislav Petkov (chan_addr & 0xfff); 2550f71d0a05SDoug Thompson } 2551f71d0a05SDoug Thompson 2552956b9ba1SJoe Perches edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr); 2553f71d0a05SDoug Thompson 2554b15f0fcaSBorislav Petkov cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel); 2555f71d0a05SDoug Thompson 255633ca0643SBorislav Petkov if (cs_found >= 0) 2557f71d0a05SDoug Thompson *chan_sel = channel; 255833ca0643SBorislav Petkov 2559f71d0a05SDoug Thompson return cs_found; 2560f71d0a05SDoug Thompson } 2561f71d0a05SDoug Thompson 256218b94f66SAravind Gopalakrishnan static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range, 256318b94f66SAravind Gopalakrishnan u64 sys_addr, int *chan_sel) 256418b94f66SAravind Gopalakrishnan { 256518b94f66SAravind Gopalakrishnan int cs_found = -EINVAL; 256618b94f66SAravind Gopalakrishnan int num_dcts_intlv = 0; 256718b94f66SAravind Gopalakrishnan u64 chan_addr, chan_offset; 256818b94f66SAravind Gopalakrishnan u64 dct_base, dct_limit; 256918b94f66SAravind Gopalakrishnan u32 dct_cont_base_reg, dct_cont_limit_reg, tmp; 257018b94f66SAravind Gopalakrishnan u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en; 257118b94f66SAravind Gopalakrishnan 257218b94f66SAravind Gopalakrishnan u64 dhar_offset = f10_dhar_offset(pvt); 257318b94f66SAravind Gopalakrishnan u8 intlv_addr = dct_sel_interleave_addr(pvt); 257418b94f66SAravind Gopalakrishnan u8 node_id = dram_dst_node(pvt, range); 257518b94f66SAravind Gopalakrishnan u8 intlv_en = dram_intlv_en(pvt, range); 257618b94f66SAravind Gopalakrishnan 257718b94f66SAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg); 257818b94f66SAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg); 257918b94f66SAravind Gopalakrishnan 258018b94f66SAravind Gopalakrishnan dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0)); 258118b94f66SAravind Gopalakrishnan dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7); 258218b94f66SAravind Gopalakrishnan 258318b94f66SAravind Gopalakrishnan edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n", 258418b94f66SAravind Gopalakrishnan range, sys_addr, get_dram_limit(pvt, range)); 258518b94f66SAravind Gopalakrishnan 258618b94f66SAravind Gopalakrishnan if (!(get_dram_base(pvt, range) <= sys_addr) && 258718b94f66SAravind Gopalakrishnan !(get_dram_limit(pvt, range) >= sys_addr)) 258818b94f66SAravind Gopalakrishnan return -EINVAL; 258918b94f66SAravind Gopalakrishnan 259018b94f66SAravind Gopalakrishnan if (dhar_valid(pvt) && 259118b94f66SAravind Gopalakrishnan dhar_base(pvt) <= sys_addr && 259218b94f66SAravind Gopalakrishnan sys_addr < BIT_64(32)) { 259318b94f66SAravind Gopalakrishnan amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n", 259418b94f66SAravind Gopalakrishnan sys_addr); 259518b94f66SAravind Gopalakrishnan return -EINVAL; 259618b94f66SAravind Gopalakrishnan } 259718b94f66SAravind Gopalakrishnan 259818b94f66SAravind Gopalakrishnan /* Verify sys_addr is within DCT Range. */ 25994fc06b31SAravind Gopalakrishnan dct_base = (u64) dct_sel_baseaddr(pvt); 26004fc06b31SAravind Gopalakrishnan dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF; 260118b94f66SAravind Gopalakrishnan 260218b94f66SAravind Gopalakrishnan if (!(dct_cont_base_reg & BIT(0)) && 26034fc06b31SAravind Gopalakrishnan !(dct_base <= (sys_addr >> 27) && 26044fc06b31SAravind Gopalakrishnan dct_limit >= (sys_addr >> 27))) 260518b94f66SAravind Gopalakrishnan return -EINVAL; 260618b94f66SAravind Gopalakrishnan 260718b94f66SAravind Gopalakrishnan /* Verify number of dct's that participate in channel interleaving. */ 260818b94f66SAravind Gopalakrishnan num_dcts_intlv = (int) hweight8(intlv_en); 260918b94f66SAravind Gopalakrishnan 261018b94f66SAravind Gopalakrishnan if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4)) 261118b94f66SAravind Gopalakrishnan return -EINVAL; 261218b94f66SAravind Gopalakrishnan 2613dc0a50a8SYazen Ghannam if (pvt->model >= 0x60) 2614dc0a50a8SYazen Ghannam channel = f1x_determine_channel(pvt, sys_addr, false, intlv_en); 2615dc0a50a8SYazen Ghannam else 261618b94f66SAravind Gopalakrishnan channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en, 261718b94f66SAravind Gopalakrishnan num_dcts_intlv, dct_sel); 261818b94f66SAravind Gopalakrishnan 261918b94f66SAravind Gopalakrishnan /* Verify we stay within the MAX number of channels allowed */ 26207f3f5240SAravind Gopalakrishnan if (channel > 3) 262118b94f66SAravind Gopalakrishnan return -EINVAL; 262218b94f66SAravind Gopalakrishnan 262318b94f66SAravind Gopalakrishnan leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0)); 262418b94f66SAravind Gopalakrishnan 262518b94f66SAravind Gopalakrishnan /* Get normalized DCT addr */ 262618b94f66SAravind Gopalakrishnan if (leg_mmio_hole && (sys_addr >= BIT_64(32))) 262718b94f66SAravind Gopalakrishnan chan_offset = dhar_offset; 262818b94f66SAravind Gopalakrishnan else 26294fc06b31SAravind Gopalakrishnan chan_offset = dct_base << 27; 263018b94f66SAravind Gopalakrishnan 263118b94f66SAravind Gopalakrishnan chan_addr = sys_addr - chan_offset; 263218b94f66SAravind Gopalakrishnan 263318b94f66SAravind Gopalakrishnan /* remove channel interleave */ 263418b94f66SAravind Gopalakrishnan if (num_dcts_intlv == 2) { 263518b94f66SAravind Gopalakrishnan if (intlv_addr == 0x4) 263618b94f66SAravind Gopalakrishnan chan_addr = ((chan_addr >> 9) << 8) | 263718b94f66SAravind Gopalakrishnan (chan_addr & 0xff); 263818b94f66SAravind Gopalakrishnan else if (intlv_addr == 0x5) 263918b94f66SAravind Gopalakrishnan chan_addr = ((chan_addr >> 10) << 9) | 264018b94f66SAravind Gopalakrishnan (chan_addr & 0x1ff); 264118b94f66SAravind Gopalakrishnan else 264218b94f66SAravind Gopalakrishnan return -EINVAL; 264318b94f66SAravind Gopalakrishnan 264418b94f66SAravind Gopalakrishnan } else if (num_dcts_intlv == 4) { 264518b94f66SAravind Gopalakrishnan if (intlv_addr == 0x4) 264618b94f66SAravind Gopalakrishnan chan_addr = ((chan_addr >> 10) << 8) | 264718b94f66SAravind Gopalakrishnan (chan_addr & 0xff); 264818b94f66SAravind Gopalakrishnan else if (intlv_addr == 0x5) 264918b94f66SAravind Gopalakrishnan chan_addr = ((chan_addr >> 11) << 9) | 265018b94f66SAravind Gopalakrishnan (chan_addr & 0x1ff); 265118b94f66SAravind Gopalakrishnan else 265218b94f66SAravind Gopalakrishnan return -EINVAL; 265318b94f66SAravind Gopalakrishnan } 265418b94f66SAravind Gopalakrishnan 265518b94f66SAravind Gopalakrishnan if (dct_offset_en) { 265618b94f66SAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F1, 265718b94f66SAravind Gopalakrishnan DRAM_CONT_HIGH_OFF + (int) channel * 4, 265818b94f66SAravind Gopalakrishnan &tmp); 26594fc06b31SAravind Gopalakrishnan chan_addr += (u64) ((tmp >> 11) & 0xfff) << 27; 266018b94f66SAravind Gopalakrishnan } 266118b94f66SAravind Gopalakrishnan 266218b94f66SAravind Gopalakrishnan f15h_select_dct(pvt, channel); 266318b94f66SAravind Gopalakrishnan 266418b94f66SAravind Gopalakrishnan edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr); 266518b94f66SAravind Gopalakrishnan 266618b94f66SAravind Gopalakrishnan /* 266718b94f66SAravind Gopalakrishnan * Find Chip select: 266818b94f66SAravind Gopalakrishnan * if channel = 3, then alias it to 1. This is because, in F15 M30h, 266918b94f66SAravind Gopalakrishnan * there is support for 4 DCT's, but only 2 are currently functional. 267018b94f66SAravind Gopalakrishnan * They are DCT0 and DCT3. But we have read all registers of DCT3 into 267118b94f66SAravind Gopalakrishnan * pvt->csels[1]. So we need to use '1' here to get correct info. 267218b94f66SAravind Gopalakrishnan * Refer F15 M30h BKDG Section 2.10 and 2.10.3 for clarifications. 267318b94f66SAravind Gopalakrishnan */ 267418b94f66SAravind Gopalakrishnan alias_channel = (channel == 3) ? 1 : channel; 267518b94f66SAravind Gopalakrishnan 267618b94f66SAravind Gopalakrishnan cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel); 267718b94f66SAravind Gopalakrishnan 267818b94f66SAravind Gopalakrishnan if (cs_found >= 0) 267918b94f66SAravind Gopalakrishnan *chan_sel = alias_channel; 268018b94f66SAravind Gopalakrishnan 268118b94f66SAravind Gopalakrishnan return cs_found; 268218b94f66SAravind Gopalakrishnan } 268318b94f66SAravind Gopalakrishnan 268418b94f66SAravind Gopalakrishnan static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt, 268518b94f66SAravind Gopalakrishnan u64 sys_addr, 268633ca0643SBorislav Petkov int *chan_sel) 2687f71d0a05SDoug Thompson { 2688e761359aSBorislav Petkov int cs_found = -EINVAL; 2689e761359aSBorislav Petkov unsigned range; 2690f71d0a05SDoug Thompson 26917f19bf75SBorislav Petkov for (range = 0; range < DRAM_RANGES; range++) { 26927f19bf75SBorislav Petkov if (!dram_rw(pvt, range)) 2693f71d0a05SDoug Thompson continue; 2694f71d0a05SDoug Thompson 269518b94f66SAravind Gopalakrishnan if (pvt->fam == 0x15 && pvt->model >= 0x30) 269618b94f66SAravind Gopalakrishnan cs_found = f15_m30h_match_to_this_node(pvt, range, 269718b94f66SAravind Gopalakrishnan sys_addr, 269818b94f66SAravind Gopalakrishnan chan_sel); 2699f71d0a05SDoug Thompson 270018b94f66SAravind Gopalakrishnan else if ((get_dram_base(pvt, range) <= sys_addr) && 270118b94f66SAravind Gopalakrishnan (get_dram_limit(pvt, range) >= sys_addr)) { 2702b15f0fcaSBorislav Petkov cs_found = f1x_match_to_this_node(pvt, range, 270333ca0643SBorislav Petkov sys_addr, chan_sel); 2704f71d0a05SDoug Thompson if (cs_found >= 0) 2705f71d0a05SDoug Thompson break; 2706f71d0a05SDoug Thompson } 2707f71d0a05SDoug Thompson } 2708f71d0a05SDoug Thompson return cs_found; 2709f71d0a05SDoug Thompson } 2710f71d0a05SDoug Thompson 2711f71d0a05SDoug Thompson /* 2712bdc30a0cSBorislav Petkov * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps 2713bdc30a0cSBorislav Petkov * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW). 2714f71d0a05SDoug Thompson * 2715bdc30a0cSBorislav Petkov * The @sys_addr is usually an error address received from the hardware 2716bdc30a0cSBorislav Petkov * (MCX_ADDR). 2717f71d0a05SDoug Thompson */ 2718b15f0fcaSBorislav Petkov static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr, 271933ca0643SBorislav Petkov struct err_info *err) 2720f71d0a05SDoug Thompson { 2721f71d0a05SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 2722f71d0a05SDoug Thompson 272333ca0643SBorislav Petkov error_address_to_page_and_offset(sys_addr, err); 2724ab5a503cSMauro Carvalho Chehab 272533ca0643SBorislav Petkov err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel); 272633ca0643SBorislav Petkov if (err->csrow < 0) { 272733ca0643SBorislav Petkov err->err_code = ERR_CSROW; 2728bdc30a0cSBorislav Petkov return; 2729bdc30a0cSBorislav Petkov } 2730bdc30a0cSBorislav Petkov 2731f71d0a05SDoug Thompson /* 2732bdc30a0cSBorislav Petkov * We need the syndromes for channel detection only when we're 2733bdc30a0cSBorislav Petkov * ganged. Otherwise @chan should already contain the channel at 2734bdc30a0cSBorislav Petkov * this point. 2735f71d0a05SDoug Thompson */ 2736a97fa68eSBorislav Petkov if (dct_ganging_enabled(pvt)) 273733ca0643SBorislav Petkov err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome); 2738f71d0a05SDoug Thompson } 2739f71d0a05SDoug Thompson 2740b1289d6fSDoug Thompson /* 2741bfc04aecSBorislav Petkov * These are tables of eigenvectors (one per line) which can be used for the 2742bfc04aecSBorislav Petkov * construction of the syndrome tables. The modified syndrome search algorithm 2743bfc04aecSBorislav Petkov * uses those to find the symbol in error and thus the DIMM. 2744b1289d6fSDoug Thompson * 2745bfc04aecSBorislav Petkov * Algorithm courtesy of Ross LaFetra from AMD. 2746b1289d6fSDoug Thompson */ 2747c7e5301aSDaniel J Blueman static const u16 x4_vectors[] = { 2748bfc04aecSBorislav Petkov 0x2f57, 0x1afe, 0x66cc, 0xdd88, 2749bfc04aecSBorislav Petkov 0x11eb, 0x3396, 0x7f4c, 0xeac8, 2750bfc04aecSBorislav Petkov 0x0001, 0x0002, 0x0004, 0x0008, 2751bfc04aecSBorislav Petkov 0x1013, 0x3032, 0x4044, 0x8088, 2752bfc04aecSBorislav Petkov 0x106b, 0x30d6, 0x70fc, 0xe0a8, 2753bfc04aecSBorislav Petkov 0x4857, 0xc4fe, 0x13cc, 0x3288, 2754bfc04aecSBorislav Petkov 0x1ac5, 0x2f4a, 0x5394, 0xa1e8, 2755bfc04aecSBorislav Petkov 0x1f39, 0x251e, 0xbd6c, 0x6bd8, 2756bfc04aecSBorislav Petkov 0x15c1, 0x2a42, 0x89ac, 0x4758, 2757bfc04aecSBorislav Petkov 0x2b03, 0x1602, 0x4f0c, 0xca08, 2758bfc04aecSBorislav Petkov 0x1f07, 0x3a0e, 0x6b04, 0xbd08, 2759bfc04aecSBorislav Petkov 0x8ba7, 0x465e, 0x244c, 0x1cc8, 2760bfc04aecSBorislav Petkov 0x2b87, 0x164e, 0x642c, 0xdc18, 2761bfc04aecSBorislav Petkov 0x40b9, 0x80de, 0x1094, 0x20e8, 2762bfc04aecSBorislav Petkov 0x27db, 0x1eb6, 0x9dac, 0x7b58, 2763bfc04aecSBorislav Petkov 0x11c1, 0x2242, 0x84ac, 0x4c58, 2764bfc04aecSBorislav Petkov 0x1be5, 0x2d7a, 0x5e34, 0xa718, 2765bfc04aecSBorislav Petkov 0x4b39, 0x8d1e, 0x14b4, 0x28d8, 2766bfc04aecSBorislav Petkov 0x4c97, 0xc87e, 0x11fc, 0x33a8, 2767bfc04aecSBorislav Petkov 0x8e97, 0x497e, 0x2ffc, 0x1aa8, 2768bfc04aecSBorislav Petkov 0x16b3, 0x3d62, 0x4f34, 0x8518, 2769bfc04aecSBorislav Petkov 0x1e2f, 0x391a, 0x5cac, 0xf858, 2770bfc04aecSBorislav Petkov 0x1d9f, 0x3b7a, 0x572c, 0xfe18, 2771bfc04aecSBorislav Petkov 0x15f5, 0x2a5a, 0x5264, 0xa3b8, 2772bfc04aecSBorislav Petkov 0x1dbb, 0x3b66, 0x715c, 0xe3f8, 2773bfc04aecSBorislav Petkov 0x4397, 0xc27e, 0x17fc, 0x3ea8, 2774bfc04aecSBorislav Petkov 0x1617, 0x3d3e, 0x6464, 0xb8b8, 2775bfc04aecSBorislav Petkov 0x23ff, 0x12aa, 0xab6c, 0x56d8, 2776bfc04aecSBorislav Petkov 0x2dfb, 0x1ba6, 0x913c, 0x7328, 2777bfc04aecSBorislav Petkov 0x185d, 0x2ca6, 0x7914, 0x9e28, 2778bfc04aecSBorislav Petkov 0x171b, 0x3e36, 0x7d7c, 0xebe8, 2779bfc04aecSBorislav Petkov 0x4199, 0x82ee, 0x19f4, 0x2e58, 2780bfc04aecSBorislav Petkov 0x4807, 0xc40e, 0x130c, 0x3208, 2781bfc04aecSBorislav Petkov 0x1905, 0x2e0a, 0x5804, 0xac08, 2782bfc04aecSBorislav Petkov 0x213f, 0x132a, 0xadfc, 0x5ba8, 2783bfc04aecSBorislav Petkov 0x19a9, 0x2efe, 0xb5cc, 0x6f88, 2784b1289d6fSDoug Thompson }; 2785b1289d6fSDoug Thompson 2786c7e5301aSDaniel J Blueman static const u16 x8_vectors[] = { 2787bfc04aecSBorislav Petkov 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480, 2788bfc04aecSBorislav Petkov 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80, 2789bfc04aecSBorislav Petkov 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80, 2790bfc04aecSBorislav Petkov 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80, 2791bfc04aecSBorislav Petkov 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780, 2792bfc04aecSBorislav Petkov 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080, 2793bfc04aecSBorislav Petkov 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080, 2794bfc04aecSBorislav Petkov 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080, 2795bfc04aecSBorislav Petkov 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80, 2796bfc04aecSBorislav Petkov 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580, 2797bfc04aecSBorislav Petkov 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880, 2798bfc04aecSBorislav Petkov 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280, 2799bfc04aecSBorislav Petkov 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180, 2800bfc04aecSBorislav Petkov 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580, 2801bfc04aecSBorislav Petkov 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280, 2802bfc04aecSBorislav Petkov 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180, 2803bfc04aecSBorislav Petkov 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080, 2804bfc04aecSBorislav Petkov 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 2805bfc04aecSBorislav Petkov 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000, 2806bfc04aecSBorislav Petkov }; 2807bfc04aecSBorislav Petkov 2808c7e5301aSDaniel J Blueman static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs, 2809d34a6ecdSBorislav Petkov unsigned v_dim) 2810b1289d6fSDoug Thompson { 2811bfc04aecSBorislav Petkov unsigned int i, err_sym; 2812b1289d6fSDoug Thompson 2813bfc04aecSBorislav Petkov for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) { 2814bfc04aecSBorislav Petkov u16 s = syndrome; 2815d34a6ecdSBorislav Petkov unsigned v_idx = err_sym * v_dim; 2816d34a6ecdSBorislav Petkov unsigned v_end = (err_sym + 1) * v_dim; 2817b1289d6fSDoug Thompson 2818bfc04aecSBorislav Petkov /* walk over all 16 bits of the syndrome */ 2819bfc04aecSBorislav Petkov for (i = 1; i < (1U << 16); i <<= 1) { 2820bfc04aecSBorislav Petkov 2821bfc04aecSBorislav Petkov /* if bit is set in that eigenvector... */ 2822bfc04aecSBorislav Petkov if (v_idx < v_end && vectors[v_idx] & i) { 2823bfc04aecSBorislav Petkov u16 ev_comp = vectors[v_idx++]; 2824bfc04aecSBorislav Petkov 2825bfc04aecSBorislav Petkov /* ... and bit set in the modified syndrome, */ 2826bfc04aecSBorislav Petkov if (s & i) { 2827bfc04aecSBorislav Petkov /* remove it. */ 2828bfc04aecSBorislav Petkov s ^= ev_comp; 2829bfc04aecSBorislav Petkov 2830bfc04aecSBorislav Petkov if (!s) 2831bfc04aecSBorislav Petkov return err_sym; 2832bfc04aecSBorislav Petkov } 2833bfc04aecSBorislav Petkov 2834bfc04aecSBorislav Petkov } else if (s & i) 2835bfc04aecSBorislav Petkov /* can't get to zero, move to next symbol */ 2836bfc04aecSBorislav Petkov break; 2837bfc04aecSBorislav Petkov } 2838b1289d6fSDoug Thompson } 2839b1289d6fSDoug Thompson 2840956b9ba1SJoe Perches edac_dbg(0, "syndrome(%x) not found\n", syndrome); 2841b1289d6fSDoug Thompson return -1; 2842b1289d6fSDoug Thompson } 2843d27bf6faSDoug Thompson 2844bfc04aecSBorislav Petkov static int map_err_sym_to_channel(int err_sym, int sym_size) 2845bfc04aecSBorislav Petkov { 2846bfc04aecSBorislav Petkov if (sym_size == 4) 2847bfc04aecSBorislav Petkov switch (err_sym) { 2848bfc04aecSBorislav Petkov case 0x20: 2849bfc04aecSBorislav Petkov case 0x21: 2850bfc04aecSBorislav Petkov return 0; 2851bfc04aecSBorislav Petkov case 0x22: 2852bfc04aecSBorislav Petkov case 0x23: 2853bfc04aecSBorislav Petkov return 1; 2854bfc04aecSBorislav Petkov default: 2855bfc04aecSBorislav Petkov return err_sym >> 4; 2856bfc04aecSBorislav Petkov } 2857bfc04aecSBorislav Petkov /* x8 symbols */ 2858bfc04aecSBorislav Petkov else 2859bfc04aecSBorislav Petkov switch (err_sym) { 2860bfc04aecSBorislav Petkov /* imaginary bits not in a DIMM */ 2861bfc04aecSBorislav Petkov case 0x10: 2862bfc04aecSBorislav Petkov WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n", 2863bfc04aecSBorislav Petkov err_sym); 2864bfc04aecSBorislav Petkov return -1; 2865bfc04aecSBorislav Petkov case 0x11: 2866bfc04aecSBorislav Petkov return 0; 2867bfc04aecSBorislav Petkov case 0x12: 2868bfc04aecSBorislav Petkov return 1; 2869bfc04aecSBorislav Petkov default: 2870bfc04aecSBorislav Petkov return err_sym >> 3; 2871bfc04aecSBorislav Petkov } 2872bfc04aecSBorislav Petkov return -1; 2873bfc04aecSBorislav Petkov } 2874bfc04aecSBorislav Petkov 2875bfc04aecSBorislav Petkov static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome) 2876bfc04aecSBorislav Petkov { 2877bfc04aecSBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 2878ad6a32e9SBorislav Petkov int err_sym = -1; 2879bfc04aecSBorislav Petkov 2880a3b7db09SBorislav Petkov if (pvt->ecc_sym_sz == 8) 2881bfc04aecSBorislav Petkov err_sym = decode_syndrome(syndrome, x8_vectors, 2882ad6a32e9SBorislav Petkov ARRAY_SIZE(x8_vectors), 2883a3b7db09SBorislav Petkov pvt->ecc_sym_sz); 2884a3b7db09SBorislav Petkov else if (pvt->ecc_sym_sz == 4) 2885ad6a32e9SBorislav Petkov err_sym = decode_syndrome(syndrome, x4_vectors, 2886ad6a32e9SBorislav Petkov ARRAY_SIZE(x4_vectors), 2887a3b7db09SBorislav Petkov pvt->ecc_sym_sz); 2888ad6a32e9SBorislav Petkov else { 2889a3b7db09SBorislav Petkov amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz); 2890ad6a32e9SBorislav Petkov return err_sym; 2891bfc04aecSBorislav Petkov } 2892ad6a32e9SBorislav Petkov 2893a3b7db09SBorislav Petkov return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz); 289441c31044SBorislav Petkov } 2895bfc04aecSBorislav Petkov 2896e70984d9SYazen Ghannam static void __log_ecc_error(struct mem_ctl_info *mci, struct err_info *err, 289733ca0643SBorislav Petkov u8 ecc_type) 2898d27bf6faSDoug Thompson { 289933ca0643SBorislav Petkov enum hw_event_mc_err_type err_type; 290033ca0643SBorislav Petkov const char *string; 2901d27bf6faSDoug Thompson 290233ca0643SBorislav Petkov if (ecc_type == 2) 290333ca0643SBorislav Petkov err_type = HW_EVENT_ERR_CORRECTED; 290433ca0643SBorislav Petkov else if (ecc_type == 1) 290533ca0643SBorislav Petkov err_type = HW_EVENT_ERR_UNCORRECTED; 2906d12a969eSYazen Ghannam else if (ecc_type == 3) 2907d12a969eSYazen Ghannam err_type = HW_EVENT_ERR_DEFERRED; 290833ca0643SBorislav Petkov else { 290933ca0643SBorislav Petkov WARN(1, "Something is rotten in the state of Denmark.\n"); 2910d27bf6faSDoug Thompson return; 2911d27bf6faSDoug Thompson } 2912d27bf6faSDoug Thompson 291333ca0643SBorislav Petkov switch (err->err_code) { 291433ca0643SBorislav Petkov case DECODE_OK: 291533ca0643SBorislav Petkov string = ""; 291633ca0643SBorislav Petkov break; 291733ca0643SBorislav Petkov case ERR_NODE: 291833ca0643SBorislav Petkov string = "Failed to map error addr to a node"; 291933ca0643SBorislav Petkov break; 292033ca0643SBorislav Petkov case ERR_CSROW: 292133ca0643SBorislav Petkov string = "Failed to map error addr to a csrow"; 292233ca0643SBorislav Petkov break; 292333ca0643SBorislav Petkov case ERR_CHANNEL: 2924713ad546SYazen Ghannam string = "Unknown syndrome - possible error reporting race"; 2925713ad546SYazen Ghannam break; 2926713ad546SYazen Ghannam case ERR_SYND: 2927713ad546SYazen Ghannam string = "MCA_SYND not valid - unknown syndrome and csrow"; 2928713ad546SYazen Ghannam break; 2929713ad546SYazen Ghannam case ERR_NORM_ADDR: 2930713ad546SYazen Ghannam string = "Cannot decode normalized address"; 293133ca0643SBorislav Petkov break; 293233ca0643SBorislav Petkov default: 293333ca0643SBorislav Petkov string = "WTF error"; 293433ca0643SBorislav Petkov break; 2935d27bf6faSDoug Thompson } 293633ca0643SBorislav Petkov 293733ca0643SBorislav Petkov edac_mc_handle_error(err_type, mci, 1, 293833ca0643SBorislav Petkov err->page, err->offset, err->syndrome, 293933ca0643SBorislav Petkov err->csrow, err->channel, -1, 294033ca0643SBorislav Petkov string, ""); 2941d27bf6faSDoug Thompson } 2942d27bf6faSDoug Thompson 2943df781d03SBorislav Petkov static inline void decode_bus_error(int node_id, struct mce *m) 2944d27bf6faSDoug Thompson { 29450c510cc8SDaniel J Blueman struct mem_ctl_info *mci; 29460c510cc8SDaniel J Blueman struct amd64_pvt *pvt; 2947f192c7b1SBorislav Petkov u8 ecc_type = (m->status >> 45) & 0x3; 294866fed2d4SBorislav Petkov u8 xec = XEC(m->status, 0x1f); 294966fed2d4SBorislav Petkov u16 ec = EC(m->status); 295033ca0643SBorislav Petkov u64 sys_addr; 295133ca0643SBorislav Petkov struct err_info err; 2952d27bf6faSDoug Thompson 29530c510cc8SDaniel J Blueman mci = edac_mc_find(node_id); 29540c510cc8SDaniel J Blueman if (!mci) 29550c510cc8SDaniel J Blueman return; 29560c510cc8SDaniel J Blueman 29570c510cc8SDaniel J Blueman pvt = mci->pvt_info; 29580c510cc8SDaniel J Blueman 295966fed2d4SBorislav Petkov /* Bail out early if this was an 'observed' error */ 29605980bb9cSBorislav Petkov if (PP(ec) == NBSL_PP_OBS) 2961b70ef010SBorislav Petkov return; 2962d27bf6faSDoug Thompson 2963ecaf5606SBorislav Petkov /* Do only ECC errors */ 2964ecaf5606SBorislav Petkov if (xec && xec != F10_NBSL_EXT_ERR_ECC) 2965d27bf6faSDoug Thompson return; 2966d27bf6faSDoug Thompson 296733ca0643SBorislav Petkov memset(&err, 0, sizeof(err)); 296833ca0643SBorislav Petkov 2969a4b4bedcSBorislav Petkov sys_addr = get_error_address(pvt, m); 297033ca0643SBorislav Petkov 2971ecaf5606SBorislav Petkov if (ecc_type == 2) 297233ca0643SBorislav Petkov err.syndrome = extract_syndrome(m->status); 297333ca0643SBorislav Petkov 297433ca0643SBorislav Petkov pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err); 297533ca0643SBorislav Petkov 2976e70984d9SYazen Ghannam __log_ecc_error(mci, &err, ecc_type); 2977d27bf6faSDoug Thompson } 2978d27bf6faSDoug Thompson 29790ec449eeSDoug Thompson /* 2980713ad546SYazen Ghannam * To find the UMC channel represented by this bank we need to match on its 2981713ad546SYazen Ghannam * instance_id. The instance_id of a bank is held in the lower 32 bits of its 2982713ad546SYazen Ghannam * IPID. 2983bdcee774SYazen Ghannam * 2984bdcee774SYazen Ghannam * Currently, we can derive the channel number by looking at the 6th nibble in 2985bdcee774SYazen Ghannam * the instance_id. For example, instance_id=0xYXXXXX where Y is the channel 2986bdcee774SYazen Ghannam * number. 2987713ad546SYazen Ghannam */ 2988bdcee774SYazen Ghannam static int find_umc_channel(struct mce *m) 2989713ad546SYazen Ghannam { 2990bdcee774SYazen Ghannam return (m->ipid & GENMASK(31, 0)) >> 20; 2991713ad546SYazen Ghannam } 2992713ad546SYazen Ghannam 2993713ad546SYazen Ghannam static void decode_umc_error(int node_id, struct mce *m) 2994713ad546SYazen Ghannam { 2995713ad546SYazen Ghannam u8 ecc_type = (m->status >> 45) & 0x3; 2996713ad546SYazen Ghannam struct mem_ctl_info *mci; 2997713ad546SYazen Ghannam struct amd64_pvt *pvt; 2998713ad546SYazen Ghannam struct err_info err; 2999713ad546SYazen Ghannam u64 sys_addr; 3000713ad546SYazen Ghannam 3001713ad546SYazen Ghannam mci = edac_mc_find(node_id); 3002713ad546SYazen Ghannam if (!mci) 3003713ad546SYazen Ghannam return; 3004713ad546SYazen Ghannam 3005713ad546SYazen Ghannam pvt = mci->pvt_info; 3006713ad546SYazen Ghannam 3007713ad546SYazen Ghannam memset(&err, 0, sizeof(err)); 3008713ad546SYazen Ghannam 3009713ad546SYazen Ghannam if (m->status & MCI_STATUS_DEFERRED) 3010713ad546SYazen Ghannam ecc_type = 3; 3011713ad546SYazen Ghannam 3012bdcee774SYazen Ghannam err.channel = find_umc_channel(m); 3013713ad546SYazen Ghannam 3014713ad546SYazen Ghannam if (!(m->status & MCI_STATUS_SYNDV)) { 3015713ad546SYazen Ghannam err.err_code = ERR_SYND; 3016713ad546SYazen Ghannam goto log_error; 3017713ad546SYazen Ghannam } 3018713ad546SYazen Ghannam 3019713ad546SYazen Ghannam if (ecc_type == 2) { 3020713ad546SYazen Ghannam u8 length = (m->synd >> 18) & 0x3f; 3021713ad546SYazen Ghannam 3022713ad546SYazen Ghannam if (length) 3023713ad546SYazen Ghannam err.syndrome = (m->synd >> 32) & GENMASK(length - 1, 0); 3024713ad546SYazen Ghannam else 3025713ad546SYazen Ghannam err.err_code = ERR_CHANNEL; 3026713ad546SYazen Ghannam } 3027713ad546SYazen Ghannam 3028713ad546SYazen Ghannam err.csrow = m->synd & 0x7; 3029713ad546SYazen Ghannam 30308a2eaab7SYazen Ghannam if (umc_normaddr_to_sysaddr(m->addr, pvt->mc_node_id, err.channel, &sys_addr)) { 30318a2eaab7SYazen Ghannam err.err_code = ERR_NORM_ADDR; 30328a2eaab7SYazen Ghannam goto log_error; 30338a2eaab7SYazen Ghannam } 30348a2eaab7SYazen Ghannam 30358a2eaab7SYazen Ghannam error_address_to_page_and_offset(sys_addr, &err); 30368a2eaab7SYazen Ghannam 3037713ad546SYazen Ghannam log_error: 3038713ad546SYazen Ghannam __log_ecc_error(mci, &err, ecc_type); 3039713ad546SYazen Ghannam } 3040713ad546SYazen Ghannam 3041713ad546SYazen Ghannam /* 30423f37a36bSBorislav Petkov * Use pvt->F3 which contains the F3 CPU PCI device to get the related 30433f37a36bSBorislav Petkov * F1 (AddrMap) and F2 (Dct) devices. Return negative value on error. 30440ec449eeSDoug Thompson */ 3045936fc3afSYazen Ghannam static int 3046936fc3afSYazen Ghannam reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2) 30470ec449eeSDoug Thompson { 30480ec449eeSDoug Thompson /* Reserve the ADDRESS MAP Device */ 3049936fc3afSYazen Ghannam pvt->F1 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3); 30508d5b5d9cSBorislav Petkov if (!pvt->F1) { 30516a4afe38SYazen Ghannam edac_dbg(1, "F1 not found: device 0x%x\n", pci_id1); 3052bbd0c1f6SBorislav Petkov return -ENODEV; 30530ec449eeSDoug Thompson } 30540ec449eeSDoug Thompson 30553f37a36bSBorislav Petkov /* Reserve the DCT Device */ 3056936fc3afSYazen Ghannam pvt->F2 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3); 30573f37a36bSBorislav Petkov if (!pvt->F2) { 30588d5b5d9cSBorislav Petkov pci_dev_put(pvt->F1); 30598d5b5d9cSBorislav Petkov pvt->F1 = NULL; 30600ec449eeSDoug Thompson 30616a4afe38SYazen Ghannam edac_dbg(1, "F2 not found: device 0x%x\n", pci_id2); 3062bbd0c1f6SBorislav Petkov return -ENODEV; 30630ec449eeSDoug Thompson } 3064936fc3afSYazen Ghannam 3065706657b1SBorislav Petkov if (!pci_ctl_dev) 3066706657b1SBorislav Petkov pci_ctl_dev = &pvt->F2->dev; 3067706657b1SBorislav Petkov 3068956b9ba1SJoe Perches edac_dbg(1, "F1: %s\n", pci_name(pvt->F1)); 3069956b9ba1SJoe Perches edac_dbg(1, "F2: %s\n", pci_name(pvt->F2)); 3070956b9ba1SJoe Perches edac_dbg(1, "F3: %s\n", pci_name(pvt->F3)); 30710ec449eeSDoug Thompson 30720ec449eeSDoug Thompson return 0; 30730ec449eeSDoug Thompson } 30740ec449eeSDoug Thompson 3075b64ce7cdSYazen Ghannam static void determine_ecc_sym_sz(struct amd64_pvt *pvt) 3076b64ce7cdSYazen Ghannam { 3077b64ce7cdSYazen Ghannam pvt->ecc_sym_sz = 4; 3078b64ce7cdSYazen Ghannam 30795a1adb37SYazen Ghannam if (pvt->fam >= 0x10) { 3080b64ce7cdSYazen Ghannam u32 tmp; 3081b64ce7cdSYazen Ghannam 3082b64ce7cdSYazen Ghannam amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp); 3083b64ce7cdSYazen Ghannam /* F16h has only DCT0, so no need to read dbam1. */ 3084b64ce7cdSYazen Ghannam if (pvt->fam != 0x16) 3085b64ce7cdSYazen Ghannam amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1); 3086b64ce7cdSYazen Ghannam 3087b64ce7cdSYazen Ghannam /* F10h, revD and later can do x8 ECC too. */ 3088b64ce7cdSYazen Ghannam if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25)) 3089b64ce7cdSYazen Ghannam pvt->ecc_sym_sz = 8; 3090b64ce7cdSYazen Ghannam } 3091b64ce7cdSYazen Ghannam } 3092b64ce7cdSYazen Ghannam 3093b64ce7cdSYazen Ghannam /* 3094b64ce7cdSYazen Ghannam * Retrieve the hardware registers of the memory controller. 3095b64ce7cdSYazen Ghannam */ 309632ecdf86SMuralidhara M K static void umc_read_mc_regs(struct amd64_pvt *pvt) 3097b64ce7cdSYazen Ghannam { 3098b64ce7cdSYazen Ghannam u8 nid = pvt->mc_node_id; 3099b64ce7cdSYazen Ghannam struct amd64_umc *umc; 3100b64ce7cdSYazen Ghannam u32 i, umc_base; 3101b64ce7cdSYazen Ghannam 3102b64ce7cdSYazen Ghannam /* Read registers from each UMC */ 31034d30d2bcSYazen Ghannam for_each_umc(i) { 3104b64ce7cdSYazen Ghannam 3105b64ce7cdSYazen Ghannam umc_base = get_umc_base(i); 3106b64ce7cdSYazen Ghannam umc = &pvt->umc[i]; 3107b64ce7cdSYazen Ghannam 3108ed623d55SMuralidhara M K amd_smn_read(nid, umc_base + get_umc_reg(pvt, UMCCH_DIMM_CFG), &umc->dimm_cfg); 310907ed82efSYazen Ghannam amd_smn_read(nid, umc_base + UMCCH_UMC_CFG, &umc->umc_cfg); 3110b64ce7cdSYazen Ghannam amd_smn_read(nid, umc_base + UMCCH_SDP_CTRL, &umc->sdp_ctrl); 3111b64ce7cdSYazen Ghannam amd_smn_read(nid, umc_base + UMCCH_ECC_CTRL, &umc->ecc_ctrl); 311207ed82efSYazen Ghannam amd_smn_read(nid, umc_base + UMCCH_UMC_CAP_HI, &umc->umc_cap_hi); 3113b64ce7cdSYazen Ghannam } 3114b64ce7cdSYazen Ghannam } 3115b64ce7cdSYazen Ghannam 31160ec449eeSDoug Thompson /* 31170ec449eeSDoug Thompson * Retrieve the hardware registers of the memory controller (this includes the 31180ec449eeSDoug Thompson * 'Address Map' and 'Misc' device regs) 31190ec449eeSDoug Thompson */ 312032ecdf86SMuralidhara M K static void dct_read_mc_regs(struct amd64_pvt *pvt) 31210ec449eeSDoug Thompson { 3122b64ce7cdSYazen Ghannam unsigned int range; 31230ec449eeSDoug Thompson u64 msr_val; 31240ec449eeSDoug Thompson 31250ec449eeSDoug Thompson /* 31260ec449eeSDoug Thompson * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since 3127b64ce7cdSYazen Ghannam * those are Read-As-Zero. 31280ec449eeSDoug Thompson */ 3129e97f8bb8SBorislav Petkov rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem); 3130956b9ba1SJoe Perches edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem); 31310ec449eeSDoug Thompson 3132b64ce7cdSYazen Ghannam /* Check first whether TOP_MEM2 is enabled: */ 3133059e5c32SBrijesh Singh rdmsrl(MSR_AMD64_SYSCFG, msr_val); 3134b64ce7cdSYazen Ghannam if (msr_val & BIT(21)) { 3135e97f8bb8SBorislav Petkov rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2); 3136956b9ba1SJoe Perches edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2); 3137b64ce7cdSYazen Ghannam } else { 3138956b9ba1SJoe Perches edac_dbg(0, " TOP_MEM2 disabled\n"); 3139b64ce7cdSYazen Ghannam } 3140b64ce7cdSYazen Ghannam 31415980bb9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap); 31420ec449eeSDoug Thompson 31435a5d2371SBorislav Petkov read_dram_ctl_register(pvt); 31440ec449eeSDoug Thompson 31457f19bf75SBorislav Petkov for (range = 0; range < DRAM_RANGES; range++) { 31467f19bf75SBorislav Petkov u8 rw; 31470ec449eeSDoug Thompson 31487f19bf75SBorislav Petkov /* read settings for this DRAM range */ 31497f19bf75SBorislav Petkov read_dram_base_limit_regs(pvt, range); 3150e97f8bb8SBorislav Petkov 31517f19bf75SBorislav Petkov rw = dram_rw(pvt, range); 31527f19bf75SBorislav Petkov if (!rw) 31537f19bf75SBorislav Petkov continue; 31547f19bf75SBorislav Petkov 3155956b9ba1SJoe Perches edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n", 31567f19bf75SBorislav Petkov range, 31577f19bf75SBorislav Petkov get_dram_base(pvt, range), 31587f19bf75SBorislav Petkov get_dram_limit(pvt, range)); 31597f19bf75SBorislav Petkov 3160956b9ba1SJoe Perches edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n", 31617f19bf75SBorislav Petkov dram_intlv_en(pvt, range) ? "Enabled" : "Disabled", 31627f19bf75SBorislav Petkov (rw & 0x1) ? "R" : "-", 31637f19bf75SBorislav Petkov (rw & 0x2) ? "W" : "-", 31647f19bf75SBorislav Petkov dram_intlv_sel(pvt, range), 31657f19bf75SBorislav Petkov dram_dst_node(pvt, range)); 31660ec449eeSDoug Thompson } 31670ec449eeSDoug Thompson 3168bc21fa57SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar); 31697981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 0, DBAM0, &pvt->dbam0); 31700ec449eeSDoug Thompson 31718d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare); 31720ec449eeSDoug Thompson 31737981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 0, DCLR0, &pvt->dclr0); 31747981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 0, DCHR0, &pvt->dchr0); 31750ec449eeSDoug Thompson 317678da121eSBorislav Petkov if (!dct_ganging_enabled(pvt)) { 31777981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 1, DCLR0, &pvt->dclr1); 31787981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1); 31790ec449eeSDoug Thompson } 3180b2b0c605SBorislav Petkov 3181b64ce7cdSYazen Ghannam determine_ecc_sym_sz(pvt); 31820ec449eeSDoug Thompson } 31830ec449eeSDoug Thompson 31840ec449eeSDoug Thompson /* 31850ec449eeSDoug Thompson * NOTE: CPU Revision Dependent code 31860ec449eeSDoug Thompson * 31870ec449eeSDoug Thompson * Input: 318811c75eadSBorislav Petkov * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1) 31890ec449eeSDoug Thompson * k8 private pointer to --> 31900ec449eeSDoug Thompson * DRAM Bank Address mapping register 31910ec449eeSDoug Thompson * node_id 31920ec449eeSDoug Thompson * DCL register where dual_channel_active is 31930ec449eeSDoug Thompson * 31940ec449eeSDoug Thompson * The DBAM register consists of 4 sets of 4 bits each definitions: 31950ec449eeSDoug Thompson * 31960ec449eeSDoug Thompson * Bits: CSROWs 31970ec449eeSDoug Thompson * 0-3 CSROWs 0 and 1 31980ec449eeSDoug Thompson * 4-7 CSROWs 2 and 3 31990ec449eeSDoug Thompson * 8-11 CSROWs 4 and 5 32000ec449eeSDoug Thompson * 12-15 CSROWs 6 and 7 32010ec449eeSDoug Thompson * 32020ec449eeSDoug Thompson * Values range from: 0 to 15 32030ec449eeSDoug Thompson * The meaning of the values depends on CPU revision and dual-channel state, 32040ec449eeSDoug Thompson * see relevant BKDG more info. 32050ec449eeSDoug Thompson * 32060ec449eeSDoug Thompson * The memory controller provides for total of only 8 CSROWs in its current 32070ec449eeSDoug Thompson * architecture. Each "pair" of CSROWs normally represents just one DIMM in 32080ec449eeSDoug Thompson * single channel or two (2) DIMMs in dual channel mode. 32090ec449eeSDoug Thompson * 32100ec449eeSDoug Thompson * The following code logic collapses the various tables for CSROW based on CPU 32110ec449eeSDoug Thompson * revision. 32120ec449eeSDoug Thompson * 32130ec449eeSDoug Thompson * Returns: 32140ec449eeSDoug Thompson * The number of PAGE_SIZE pages on the specified CSROW number it 32150ec449eeSDoug Thompson * encompasses 32160ec449eeSDoug Thompson * 32170ec449eeSDoug Thompson */ 3218c0984666SYazen Ghannam static u32 dct_get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr) 32190ec449eeSDoug Thompson { 3220f92cae45SAshish Shenoy u32 dbam = dct ? pvt->dbam1 : pvt->dbam0; 3221eb77e6b8SYazen Ghannam u32 cs_mode, nr_pages; 32220ec449eeSDoug Thompson 3223eb77e6b8SYazen Ghannam csrow_nr >>= 1; 3224eb77e6b8SYazen Ghannam cs_mode = DBAM_DIMM(csrow_nr, dbam); 32250ec449eeSDoug Thompson 3226eb77e6b8SYazen Ghannam nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, csrow_nr); 3227eb77e6b8SYazen Ghannam nr_pages <<= 20 - PAGE_SHIFT; 32280ec449eeSDoug Thompson 322910de6497SBorislav Petkov edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n", 3230c0984666SYazen Ghannam csrow_nr, dct, cs_mode); 3231c0984666SYazen Ghannam edac_dbg(0, "nr_pages/channel: %u\n", nr_pages); 3232c0984666SYazen Ghannam 3233c0984666SYazen Ghannam return nr_pages; 3234c0984666SYazen Ghannam } 3235c0984666SYazen Ghannam 3236c0984666SYazen Ghannam static u32 umc_get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr_orig) 3237c0984666SYazen Ghannam { 3238c0984666SYazen Ghannam int csrow_nr = csrow_nr_orig; 3239c0984666SYazen Ghannam u32 cs_mode, nr_pages; 3240c0984666SYazen Ghannam 3241c0984666SYazen Ghannam cs_mode = umc_get_cs_mode(csrow_nr >> 1, dct, pvt); 3242c0984666SYazen Ghannam 3243a2e59ab8SYazen Ghannam nr_pages = umc_addr_mask_to_cs_size(pvt, dct, cs_mode, csrow_nr); 3244c0984666SYazen Ghannam nr_pages <<= 20 - PAGE_SHIFT; 3245c0984666SYazen Ghannam 3246c0984666SYazen Ghannam edac_dbg(0, "csrow: %d, channel: %d, cs_mode %d\n", 3247eb77e6b8SYazen Ghannam csrow_nr_orig, dct, cs_mode); 324810de6497SBorislav Petkov edac_dbg(0, "nr_pages/channel: %u\n", nr_pages); 32490ec449eeSDoug Thompson 32500ec449eeSDoug Thompson return nr_pages; 32510ec449eeSDoug Thompson } 32520ec449eeSDoug Thompson 3253353a1fcbSYazen Ghannam static int init_csrows_df(struct mem_ctl_info *mci) 3254353a1fcbSYazen Ghannam { 3255353a1fcbSYazen Ghannam struct amd64_pvt *pvt = mci->pvt_info; 3256353a1fcbSYazen Ghannam enum edac_type edac_mode = EDAC_NONE; 3257353a1fcbSYazen Ghannam enum dev_type dev_type = DEV_UNKNOWN; 3258353a1fcbSYazen Ghannam struct dimm_info *dimm; 3259353a1fcbSYazen Ghannam int empty = 1; 3260353a1fcbSYazen Ghannam u8 umc, cs; 3261353a1fcbSYazen Ghannam 3262353a1fcbSYazen Ghannam if (mci->edac_ctl_cap & EDAC_FLAG_S16ECD16ED) { 3263353a1fcbSYazen Ghannam edac_mode = EDAC_S16ECD16ED; 3264353a1fcbSYazen Ghannam dev_type = DEV_X16; 3265353a1fcbSYazen Ghannam } else if (mci->edac_ctl_cap & EDAC_FLAG_S8ECD8ED) { 3266353a1fcbSYazen Ghannam edac_mode = EDAC_S8ECD8ED; 3267353a1fcbSYazen Ghannam dev_type = DEV_X8; 3268353a1fcbSYazen Ghannam } else if (mci->edac_ctl_cap & EDAC_FLAG_S4ECD4ED) { 3269353a1fcbSYazen Ghannam edac_mode = EDAC_S4ECD4ED; 3270353a1fcbSYazen Ghannam dev_type = DEV_X4; 3271353a1fcbSYazen Ghannam } else if (mci->edac_ctl_cap & EDAC_FLAG_SECDED) { 3272353a1fcbSYazen Ghannam edac_mode = EDAC_SECDED; 3273353a1fcbSYazen Ghannam } 3274353a1fcbSYazen Ghannam 3275353a1fcbSYazen Ghannam for_each_umc(umc) { 3276353a1fcbSYazen Ghannam for_each_chip_select(cs, umc, pvt) { 3277353a1fcbSYazen Ghannam if (!csrow_enabled(cs, umc, pvt)) 3278353a1fcbSYazen Ghannam continue; 3279353a1fcbSYazen Ghannam 3280353a1fcbSYazen Ghannam empty = 0; 3281353a1fcbSYazen Ghannam dimm = mci->csrows[cs]->channels[umc]->dimm; 3282353a1fcbSYazen Ghannam 3283353a1fcbSYazen Ghannam edac_dbg(1, "MC node: %d, csrow: %d\n", 3284353a1fcbSYazen Ghannam pvt->mc_node_id, cs); 3285353a1fcbSYazen Ghannam 3286c0984666SYazen Ghannam dimm->nr_pages = umc_get_csrow_nr_pages(pvt, umc, cs); 328775aeaaf2SYazen Ghannam dimm->mtype = pvt->umc[umc].dram_type; 3288353a1fcbSYazen Ghannam dimm->edac_mode = edac_mode; 3289353a1fcbSYazen Ghannam dimm->dtype = dev_type; 3290466503d6SYazen Ghannam dimm->grain = 64; 3291353a1fcbSYazen Ghannam } 3292353a1fcbSYazen Ghannam } 3293353a1fcbSYazen Ghannam 3294353a1fcbSYazen Ghannam return empty; 3295353a1fcbSYazen Ghannam } 3296353a1fcbSYazen Ghannam 32970ec449eeSDoug Thompson /* 32980ec449eeSDoug Thompson * Initialize the array of csrow attribute instances, based on the values 32990ec449eeSDoug Thompson * from pci config hardware registers. 33000ec449eeSDoug Thompson */ 3301360b7f3cSBorislav Petkov static int init_csrows(struct mem_ctl_info *mci) 33020ec449eeSDoug Thompson { 330310de6497SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 33042d09d8f3SYazen Ghannam enum edac_type edac_mode = EDAC_NONE; 33050ec449eeSDoug Thompson struct csrow_info *csrow; 3306de3910ebSMauro Carvalho Chehab struct dimm_info *dimm; 330710de6497SBorislav Petkov int i, j, empty = 1; 3308a895bf8bSMauro Carvalho Chehab int nr_pages = 0; 330910de6497SBorislav Petkov u32 val; 33100ec449eeSDoug Thompson 3311353a1fcbSYazen Ghannam if (pvt->umc) 3312353a1fcbSYazen Ghannam return init_csrows_df(mci); 3313353a1fcbSYazen Ghannam 3314a97fa68eSBorislav Petkov amd64_read_pci_cfg(pvt->F3, NBCFG, &val); 33150ec449eeSDoug Thompson 33162299ef71SBorislav Petkov pvt->nbcfg = val; 33170ec449eeSDoug Thompson 3318956b9ba1SJoe Perches edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n", 33192299ef71SBorislav Petkov pvt->mc_node_id, val, 3320a97fa68eSBorislav Petkov !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE)); 33210ec449eeSDoug Thompson 332210de6497SBorislav Petkov /* 332310de6497SBorislav Petkov * We iterate over DCT0 here but we look at DCT1 in parallel, if needed. 332410de6497SBorislav Petkov */ 332511c75eadSBorislav Petkov for_each_chip_select(i, 0, pvt) { 332610de6497SBorislav Petkov bool row_dct0 = !!csrow_enabled(i, 0, pvt); 332710de6497SBorislav Petkov bool row_dct1 = false; 33280ec449eeSDoug Thompson 3329a4b4bedcSBorislav Petkov if (pvt->fam != 0xf) 333010de6497SBorislav Petkov row_dct1 = !!csrow_enabled(i, 1, pvt); 333110de6497SBorislav Petkov 333210de6497SBorislav Petkov if (!row_dct0 && !row_dct1) 33330ec449eeSDoug Thompson continue; 33340ec449eeSDoug Thompson 333510de6497SBorislav Petkov csrow = mci->csrows[i]; 33360ec449eeSDoug Thompson empty = 0; 333711c75eadSBorislav Petkov 333810de6497SBorislav Petkov edac_dbg(1, "MC node: %d, csrow: %d\n", 333910de6497SBorislav Petkov pvt->mc_node_id, i); 334010de6497SBorislav Petkov 33411eef1282SMauro Carvalho Chehab if (row_dct0) { 3342c0984666SYazen Ghannam nr_pages = dct_get_csrow_nr_pages(pvt, 0, i); 33431eef1282SMauro Carvalho Chehab csrow->channels[0]->dimm->nr_pages = nr_pages; 33441eef1282SMauro Carvalho Chehab } 334510de6497SBorislav Petkov 334610de6497SBorislav Petkov /* K8 has only one DCT */ 3347a4b4bedcSBorislav Petkov if (pvt->fam != 0xf && row_dct1) { 3348c0984666SYazen Ghannam int row_dct1_pages = dct_get_csrow_nr_pages(pvt, 1, i); 33491eef1282SMauro Carvalho Chehab 33501eef1282SMauro Carvalho Chehab csrow->channels[1]->dimm->nr_pages = row_dct1_pages; 33511eef1282SMauro Carvalho Chehab nr_pages += row_dct1_pages; 33521eef1282SMauro Carvalho Chehab } 33530ec449eeSDoug Thompson 335410de6497SBorislav Petkov edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages); 33550ec449eeSDoug Thompson 33562d09d8f3SYazen Ghannam /* Determine DIMM ECC mode: */ 3357353a1fcbSYazen Ghannam if (pvt->nbcfg & NBCFG_ECC_ENABLE) { 33582d09d8f3SYazen Ghannam edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL) 33592d09d8f3SYazen Ghannam ? EDAC_S4ECD4ED 33602d09d8f3SYazen Ghannam : EDAC_SECDED; 33612d09d8f3SYazen Ghannam } 3362084a4fccSMauro Carvalho Chehab 3363ed623d55SMuralidhara M K for (j = 0; j < pvt->max_mcs; j++) { 3364de3910ebSMauro Carvalho Chehab dimm = csrow->channels[j]->dimm; 3365a597d2a5SAravind Gopalakrishnan dimm->mtype = pvt->dram_type; 3366de3910ebSMauro Carvalho Chehab dimm->edac_mode = edac_mode; 3367466503d6SYazen Ghannam dimm->grain = 64; 3368084a4fccSMauro Carvalho Chehab } 33690ec449eeSDoug Thompson } 33700ec449eeSDoug Thompson 33710ec449eeSDoug Thompson return empty; 33720ec449eeSDoug Thompson } 3373d27bf6faSDoug Thompson 337406724535SBorislav Petkov /* get all cores on this DCT */ 33758b84c8dfSDaniel J Blueman static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid) 3376f9431992SDoug Thompson { 337706724535SBorislav Petkov int cpu; 3378f9431992SDoug Thompson 337906724535SBorislav Petkov for_each_online_cpu(cpu) 3380db970bd2SYazen Ghannam if (topology_die_id(cpu) == nid) 338106724535SBorislav Petkov cpumask_set_cpu(cpu, mask); 3382f9431992SDoug Thompson } 3383f9431992SDoug Thompson 3384f9431992SDoug Thompson /* check MCG_CTL on all the cpus on this node */ 3385d1ea71cdSBorislav Petkov static bool nb_mce_bank_enabled_on_node(u16 nid) 3386f9431992SDoug Thompson { 3387ba578cb3SRusty Russell cpumask_var_t mask; 338850542251SBorislav Petkov int cpu, nbe; 338906724535SBorislav Petkov bool ret = false; 3390f9431992SDoug Thompson 3391ba578cb3SRusty Russell if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { 339224f9a7feSBorislav Petkov amd64_warn("%s: Error allocating mask\n", __func__); 339306724535SBorislav Petkov return false; 339406724535SBorislav Petkov } 339506724535SBorislav Petkov 3396ba578cb3SRusty Russell get_cpus_on_this_dct_cpumask(mask, nid); 339706724535SBorislav Petkov 3398ba578cb3SRusty Russell rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs); 3399ba578cb3SRusty Russell 3400ba578cb3SRusty Russell for_each_cpu(cpu, mask) { 340150542251SBorislav Petkov struct msr *reg = per_cpu_ptr(msrs, cpu); 34025980bb9cSBorislav Petkov nbe = reg->l & MSR_MCGCTL_NBE; 340306724535SBorislav Petkov 3404956b9ba1SJoe Perches edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n", 340550542251SBorislav Petkov cpu, reg->q, 340606724535SBorislav Petkov (nbe ? "enabled" : "disabled")); 340706724535SBorislav Petkov 340806724535SBorislav Petkov if (!nbe) 340906724535SBorislav Petkov goto out; 341006724535SBorislav Petkov } 341106724535SBorislav Petkov ret = true; 341206724535SBorislav Petkov 341306724535SBorislav Petkov out: 3414ba578cb3SRusty Russell free_cpumask_var(mask); 3415f9431992SDoug Thompson return ret; 3416f9431992SDoug Thompson } 3417f9431992SDoug Thompson 3418c7e5301aSDaniel J Blueman static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on) 3419f6d6ae96SBorislav Petkov { 3420f6d6ae96SBorislav Petkov cpumask_var_t cmask; 342150542251SBorislav Petkov int cpu; 3422f6d6ae96SBorislav Petkov 3423f6d6ae96SBorislav Petkov if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) { 342424f9a7feSBorislav Petkov amd64_warn("%s: error allocating mask\n", __func__); 34250de27884SPan Bian return -ENOMEM; 3426f6d6ae96SBorislav Petkov } 3427f6d6ae96SBorislav Petkov 3428ae7bb7c6SBorislav Petkov get_cpus_on_this_dct_cpumask(cmask, nid); 3429f6d6ae96SBorislav Petkov 3430f6d6ae96SBorislav Petkov rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); 3431f6d6ae96SBorislav Petkov 3432f6d6ae96SBorislav Petkov for_each_cpu(cpu, cmask) { 3433f6d6ae96SBorislav Petkov 343450542251SBorislav Petkov struct msr *reg = per_cpu_ptr(msrs, cpu); 343550542251SBorislav Petkov 3436f6d6ae96SBorislav Petkov if (on) { 34375980bb9cSBorislav Petkov if (reg->l & MSR_MCGCTL_NBE) 3438ae7bb7c6SBorislav Petkov s->flags.nb_mce_enable = 1; 3439f6d6ae96SBorislav Petkov 34405980bb9cSBorislav Petkov reg->l |= MSR_MCGCTL_NBE; 3441f6d6ae96SBorislav Petkov } else { 3442f6d6ae96SBorislav Petkov /* 3443d95cf4deSBorislav Petkov * Turn off NB MCE reporting only when it was off before 3444f6d6ae96SBorislav Petkov */ 3445ae7bb7c6SBorislav Petkov if (!s->flags.nb_mce_enable) 34465980bb9cSBorislav Petkov reg->l &= ~MSR_MCGCTL_NBE; 3447f6d6ae96SBorislav Petkov } 3448f6d6ae96SBorislav Petkov } 3449f6d6ae96SBorislav Petkov wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); 3450f6d6ae96SBorislav Petkov 3451f6d6ae96SBorislav Petkov free_cpumask_var(cmask); 3452f6d6ae96SBorislav Petkov 3453f6d6ae96SBorislav Petkov return 0; 3454f6d6ae96SBorislav Petkov } 3455f6d6ae96SBorislav Petkov 3456c7e5301aSDaniel J Blueman static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid, 34572299ef71SBorislav Petkov struct pci_dev *F3) 3458f6d6ae96SBorislav Petkov { 34592299ef71SBorislav Petkov bool ret = true; 3460c9f4f26eSBorislav Petkov u32 value, mask = 0x3; /* UECC/CECC enable */ 3461f6d6ae96SBorislav Petkov 34622299ef71SBorislav Petkov if (toggle_ecc_err_reporting(s, nid, ON)) { 34632299ef71SBorislav Petkov amd64_warn("Error enabling ECC reporting over MCGCTL!\n"); 34642299ef71SBorislav Petkov return false; 34652299ef71SBorislav Petkov } 34662299ef71SBorislav Petkov 3467c9f4f26eSBorislav Petkov amd64_read_pci_cfg(F3, NBCTL, &value); 3468f6d6ae96SBorislav Petkov 3469ae7bb7c6SBorislav Petkov s->old_nbctl = value & mask; 3470ae7bb7c6SBorislav Petkov s->nbctl_valid = true; 3471f6d6ae96SBorislav Petkov 3472f6d6ae96SBorislav Petkov value |= mask; 3473c9f4f26eSBorislav Petkov amd64_write_pci_cfg(F3, NBCTL, value); 3474f6d6ae96SBorislav Petkov 3475a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 3476f6d6ae96SBorislav Petkov 3477956b9ba1SJoe Perches edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n", 3478a97fa68eSBorislav Petkov nid, value, !!(value & NBCFG_ECC_ENABLE)); 3479f6d6ae96SBorislav Petkov 3480a97fa68eSBorislav Petkov if (!(value & NBCFG_ECC_ENABLE)) { 348124f9a7feSBorislav Petkov amd64_warn("DRAM ECC disabled on this node, enabling...\n"); 3482f6d6ae96SBorislav Petkov 3483ae7bb7c6SBorislav Petkov s->flags.nb_ecc_prev = 0; 3484d95cf4deSBorislav Petkov 3485f6d6ae96SBorislav Petkov /* Attempt to turn on DRAM ECC Enable */ 3486a97fa68eSBorislav Petkov value |= NBCFG_ECC_ENABLE; 3487a97fa68eSBorislav Petkov amd64_write_pci_cfg(F3, NBCFG, value); 3488f6d6ae96SBorislav Petkov 3489a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 3490f6d6ae96SBorislav Petkov 3491a97fa68eSBorislav Petkov if (!(value & NBCFG_ECC_ENABLE)) { 349224f9a7feSBorislav Petkov amd64_warn("Hardware rejected DRAM ECC enable," 349324f9a7feSBorislav Petkov "check memory DIMM configuration.\n"); 34942299ef71SBorislav Petkov ret = false; 3495f6d6ae96SBorislav Petkov } else { 349624f9a7feSBorislav Petkov amd64_info("Hardware accepted DRAM ECC Enable\n"); 3497f6d6ae96SBorislav Petkov } 3498d95cf4deSBorislav Petkov } else { 3499ae7bb7c6SBorislav Petkov s->flags.nb_ecc_prev = 1; 3500f6d6ae96SBorislav Petkov } 3501d95cf4deSBorislav Petkov 3502956b9ba1SJoe Perches edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n", 3503a97fa68eSBorislav Petkov nid, value, !!(value & NBCFG_ECC_ENABLE)); 3504f6d6ae96SBorislav Petkov 35052299ef71SBorislav Petkov return ret; 3506f6d6ae96SBorislav Petkov } 3507f6d6ae96SBorislav Petkov 3508c7e5301aSDaniel J Blueman static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid, 3509360b7f3cSBorislav Petkov struct pci_dev *F3) 3510f6d6ae96SBorislav Petkov { 3511c9f4f26eSBorislav Petkov u32 value, mask = 0x3; /* UECC/CECC enable */ 3512c9f4f26eSBorislav Petkov 3513ae7bb7c6SBorislav Petkov if (!s->nbctl_valid) 3514f6d6ae96SBorislav Petkov return; 3515f6d6ae96SBorislav Petkov 3516c9f4f26eSBorislav Petkov amd64_read_pci_cfg(F3, NBCTL, &value); 3517f6d6ae96SBorislav Petkov value &= ~mask; 3518ae7bb7c6SBorislav Petkov value |= s->old_nbctl; 3519f6d6ae96SBorislav Petkov 3520c9f4f26eSBorislav Petkov amd64_write_pci_cfg(F3, NBCTL, value); 3521f6d6ae96SBorislav Petkov 3522ae7bb7c6SBorislav Petkov /* restore previous BIOS DRAM ECC "off" setting we force-enabled */ 3523ae7bb7c6SBorislav Petkov if (!s->flags.nb_ecc_prev) { 3524a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 3525a97fa68eSBorislav Petkov value &= ~NBCFG_ECC_ENABLE; 3526a97fa68eSBorislav Petkov amd64_write_pci_cfg(F3, NBCFG, value); 3527d95cf4deSBorislav Petkov } 3528d95cf4deSBorislav Petkov 3529d95cf4deSBorislav Petkov /* restore the NB Enable MCGCTL bit */ 35302299ef71SBorislav Petkov if (toggle_ecc_err_reporting(s, nid, OFF)) 353124f9a7feSBorislav Petkov amd64_warn("Error restoring NB MCGCTL settings!\n"); 3532f6d6ae96SBorislav Petkov } 3533f6d6ae96SBorislav Petkov 3534eb2bcdfcSMuralidhara M K static bool dct_ecc_enabled(struct amd64_pvt *pvt) 3535f9431992SDoug Thompson { 35361c9b08baSYazen Ghannam u16 nid = pvt->mc_node_id; 353706724535SBorislav Petkov bool nb_mce_en = false; 3538eb2bcdfcSMuralidhara M K u8 ecc_en = 0; 3539196b79fcSYazen Ghannam u32 value; 3540f9431992SDoug Thompson 3541eb2bcdfcSMuralidhara M K amd64_read_pci_cfg(pvt->F3, NBCFG, &value); 3542eb2bcdfcSMuralidhara M K 3543eb2bcdfcSMuralidhara M K ecc_en = !!(value & NBCFG_ECC_ENABLE); 3544eb2bcdfcSMuralidhara M K 3545eb2bcdfcSMuralidhara M K nb_mce_en = nb_mce_bank_enabled_on_node(nid); 3546eb2bcdfcSMuralidhara M K if (!nb_mce_en) 3547eb2bcdfcSMuralidhara M K edac_dbg(0, "NB MCE bank disabled, set MSR 0x%08x[4] on node %d to enable.\n", 3548eb2bcdfcSMuralidhara M K MSR_IA32_MCG_CTL, nid); 3549eb2bcdfcSMuralidhara M K 3550eb2bcdfcSMuralidhara M K edac_dbg(3, "Node %d: DRAM ECC %s.\n", nid, (ecc_en ? "enabled" : "disabled")); 3551eb2bcdfcSMuralidhara M K 3552eb2bcdfcSMuralidhara M K if (!ecc_en || !nb_mce_en) 3553eb2bcdfcSMuralidhara M K return false; 3554eb2bcdfcSMuralidhara M K else 3555eb2bcdfcSMuralidhara M K return true; 3556eb2bcdfcSMuralidhara M K } 3557eb2bcdfcSMuralidhara M K 3558eb2bcdfcSMuralidhara M K static bool umc_ecc_enabled(struct amd64_pvt *pvt) 3559eb2bcdfcSMuralidhara M K { 3560196b79fcSYazen Ghannam u8 umc_en_mask = 0, ecc_en_mask = 0; 3561eb2bcdfcSMuralidhara M K u16 nid = pvt->mc_node_id; 35621c9b08baSYazen Ghannam struct amd64_umc *umc; 3563eb2bcdfcSMuralidhara M K u8 ecc_en = 0, i; 3564196b79fcSYazen Ghannam 35654d30d2bcSYazen Ghannam for_each_umc(i) { 35661c9b08baSYazen Ghannam umc = &pvt->umc[i]; 3567196b79fcSYazen Ghannam 3568196b79fcSYazen Ghannam /* Only check enabled UMCs. */ 35691c9b08baSYazen Ghannam if (!(umc->sdp_ctrl & UMC_SDP_INIT)) 3570196b79fcSYazen Ghannam continue; 3571196b79fcSYazen Ghannam 3572196b79fcSYazen Ghannam umc_en_mask |= BIT(i); 3573196b79fcSYazen Ghannam 35741c9b08baSYazen Ghannam if (umc->umc_cap_hi & UMC_ECC_ENABLED) 3575196b79fcSYazen Ghannam ecc_en_mask |= BIT(i); 3576196b79fcSYazen Ghannam } 3577196b79fcSYazen Ghannam 3578196b79fcSYazen Ghannam /* Check whether at least one UMC is enabled: */ 3579196b79fcSYazen Ghannam if (umc_en_mask) 3580196b79fcSYazen Ghannam ecc_en = umc_en_mask == ecc_en_mask; 358111ab1caeSYazen Ghannam else 358211ab1caeSYazen Ghannam edac_dbg(0, "Node %d: No enabled UMCs.\n", nid); 3583196b79fcSYazen Ghannam 35844cbcb73bSBorislav Petkov edac_dbg(3, "Node %d: DRAM ECC %s.\n", nid, (ecc_en ? "enabled" : "disabled")); 3585be3468e8SBorislav Petkov 3586eb2bcdfcSMuralidhara M K if (!ecc_en) 35872299ef71SBorislav Petkov return false; 35887fdfee92SBorislav Petkov else 35892299ef71SBorislav Petkov return true; 3590f9431992SDoug Thompson } 3591f9431992SDoug Thompson 35922d09d8f3SYazen Ghannam static inline void 35939369239eSYazen Ghannam umc_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt) 35942d09d8f3SYazen Ghannam { 3595f8be8e56SYazen Ghannam u8 i, ecc_en = 1, cpk_en = 1, dev_x4 = 1, dev_x16 = 1; 35962d09d8f3SYazen Ghannam 35974d30d2bcSYazen Ghannam for_each_umc(i) { 35982d09d8f3SYazen Ghannam if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) { 35992d09d8f3SYazen Ghannam ecc_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_ENABLED); 36002d09d8f3SYazen Ghannam cpk_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_CHIPKILL_CAP); 3601f8be8e56SYazen Ghannam 3602f8be8e56SYazen Ghannam dev_x4 &= !!(pvt->umc[i].dimm_cfg & BIT(6)); 3603f8be8e56SYazen Ghannam dev_x16 &= !!(pvt->umc[i].dimm_cfg & BIT(7)); 36042d09d8f3SYazen Ghannam } 36052d09d8f3SYazen Ghannam } 36062d09d8f3SYazen Ghannam 36072d09d8f3SYazen Ghannam /* Set chipkill only if ECC is enabled: */ 36082d09d8f3SYazen Ghannam if (ecc_en) { 36092d09d8f3SYazen Ghannam mci->edac_ctl_cap |= EDAC_FLAG_SECDED; 36102d09d8f3SYazen Ghannam 3611f8be8e56SYazen Ghannam if (!cpk_en) 3612f8be8e56SYazen Ghannam return; 3613f8be8e56SYazen Ghannam 3614f8be8e56SYazen Ghannam if (dev_x4) 36152d09d8f3SYazen Ghannam mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED; 3616f8be8e56SYazen Ghannam else if (dev_x16) 3617f8be8e56SYazen Ghannam mci->edac_ctl_cap |= EDAC_FLAG_S16ECD16ED; 3618f8be8e56SYazen Ghannam else 3619f8be8e56SYazen Ghannam mci->edac_ctl_cap |= EDAC_FLAG_S8ECD8ED; 36202d09d8f3SYazen Ghannam } 36212d09d8f3SYazen Ghannam } 36222d09d8f3SYazen Ghannam 36230a42a37fSMuralidhara M K static void dct_setup_mci_misc_attrs(struct mem_ctl_info *mci) 36247d6034d3SDoug Thompson { 36257d6034d3SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 36267d6034d3SDoug Thompson 36277d6034d3SDoug Thompson mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2; 36287d6034d3SDoug Thompson mci->edac_ctl_cap = EDAC_FLAG_NONE; 36297d6034d3SDoug Thompson 36305980bb9cSBorislav Petkov if (pvt->nbcap & NBCAP_SECDED) 36317d6034d3SDoug Thompson mci->edac_ctl_cap |= EDAC_FLAG_SECDED; 36327d6034d3SDoug Thompson 36335980bb9cSBorislav Petkov if (pvt->nbcap & NBCAP_CHIPKILL) 36347d6034d3SDoug Thompson mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED; 36357d6034d3SDoug Thompson 3636*f6a4b4a1SMuralidhara M K mci->edac_cap = dct_determine_edac_cap(pvt); 36377d6034d3SDoug Thompson mci->mod_name = EDAC_MOD_STR; 3638ed623d55SMuralidhara M K mci->ctl_name = pvt->ctl_name; 3639e7934b70SYazen Ghannam mci->dev_name = pci_name(pvt->F3); 36407d6034d3SDoug Thompson mci->ctl_page_to_phys = NULL; 36417d6034d3SDoug Thompson 36427d6034d3SDoug Thompson /* memory scrubber interface */ 3643d1ea71cdSBorislav Petkov mci->set_sdram_scrub_rate = set_scrub_rate; 3644d1ea71cdSBorislav Petkov mci->get_sdram_scrub_rate = get_scrub_rate; 36457d6034d3SDoug Thompson } 36467d6034d3SDoug Thompson 36470a42a37fSMuralidhara M K static void umc_setup_mci_misc_attrs(struct mem_ctl_info *mci) 36480a42a37fSMuralidhara M K { 36490a42a37fSMuralidhara M K struct amd64_pvt *pvt = mci->pvt_info; 36500a42a37fSMuralidhara M K 36510a42a37fSMuralidhara M K mci->mtype_cap = MEM_FLAG_DDR4 | MEM_FLAG_RDDR4; 36520a42a37fSMuralidhara M K mci->edac_ctl_cap = EDAC_FLAG_NONE; 36530a42a37fSMuralidhara M K 36549369239eSYazen Ghannam umc_determine_edac_ctl_cap(mci, pvt); 36550a42a37fSMuralidhara M K 3656*f6a4b4a1SMuralidhara M K mci->edac_cap = umc_determine_edac_cap(pvt); 36570a42a37fSMuralidhara M K mci->mod_name = EDAC_MOD_STR; 36580a42a37fSMuralidhara M K mci->ctl_name = pvt->ctl_name; 36590a42a37fSMuralidhara M K mci->dev_name = pci_name(pvt->F3); 36600a42a37fSMuralidhara M K mci->ctl_page_to_phys = NULL; 36610a42a37fSMuralidhara M K } 36620a42a37fSMuralidhara M K 36639a97a7f4SYazen Ghannam static int dct_hw_info_get(struct amd64_pvt *pvt) 36649a97a7f4SYazen Ghannam { 36659a97a7f4SYazen Ghannam int ret = reserve_mc_sibling_devs(pvt, pvt->f1_id, pvt->f2_id); 36669a97a7f4SYazen Ghannam 36679a97a7f4SYazen Ghannam if (ret) 36689a97a7f4SYazen Ghannam return ret; 36699a97a7f4SYazen Ghannam 3670637f60efSMuralidhara M K dct_prep_chip_selects(pvt); 3671b29dad9bSMuralidhara M K dct_read_base_mask(pvt); 367232ecdf86SMuralidhara M K dct_read_mc_regs(pvt); 367378ec161aSMuralidhara M K dct_determine_memory_type(pvt); 36749a97a7f4SYazen Ghannam 36759a97a7f4SYazen Ghannam return 0; 36769a97a7f4SYazen Ghannam } 36779a97a7f4SYazen Ghannam 36789a97a7f4SYazen Ghannam static int umc_hw_info_get(struct amd64_pvt *pvt) 36799a97a7f4SYazen Ghannam { 36809a97a7f4SYazen Ghannam pvt->umc = kcalloc(pvt->max_mcs, sizeof(struct amd64_umc), GFP_KERNEL); 36819a97a7f4SYazen Ghannam if (!pvt->umc) 36829a97a7f4SYazen Ghannam return -ENOMEM; 36839a97a7f4SYazen Ghannam 3684637f60efSMuralidhara M K umc_prep_chip_selects(pvt); 3685b29dad9bSMuralidhara M K umc_read_base_mask(pvt); 368632ecdf86SMuralidhara M K umc_read_mc_regs(pvt); 368778ec161aSMuralidhara M K umc_determine_memory_type(pvt); 36889a97a7f4SYazen Ghannam 36899a97a7f4SYazen Ghannam return 0; 36909a97a7f4SYazen Ghannam } 36919a97a7f4SYazen Ghannam 36929a97a7f4SYazen Ghannam static void hw_info_put(struct amd64_pvt *pvt) 36939a97a7f4SYazen Ghannam { 36949a97a7f4SYazen Ghannam pci_dev_put(pvt->F1); 36959a97a7f4SYazen Ghannam pci_dev_put(pvt->F2); 36969a97a7f4SYazen Ghannam kfree(pvt->umc); 36979a97a7f4SYazen Ghannam } 36989a97a7f4SYazen Ghannam 3699ed623d55SMuralidhara M K static struct low_ops umc_ops = { 37009a97a7f4SYazen Ghannam .hw_info_get = umc_hw_info_get, 3701eb2bcdfcSMuralidhara M K .ecc_enabled = umc_ecc_enabled, 37020a42a37fSMuralidhara M K .setup_mci_misc_attrs = umc_setup_mci_misc_attrs, 3703ed623d55SMuralidhara M K }; 3704ed623d55SMuralidhara M K 3705ed623d55SMuralidhara M K /* Use Family 16h versions for defaults and adjust as needed below. */ 3706ed623d55SMuralidhara M K static struct low_ops dct_ops = { 3707ed623d55SMuralidhara M K .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, 3708ed623d55SMuralidhara M K .dbam_to_cs = f16_dbam_to_chip_select, 37099a97a7f4SYazen Ghannam .hw_info_get = dct_hw_info_get, 3710eb2bcdfcSMuralidhara M K .ecc_enabled = dct_ecc_enabled, 37110a42a37fSMuralidhara M K .setup_mci_misc_attrs = dct_setup_mci_misc_attrs, 3712ed623d55SMuralidhara M K }; 3713ed623d55SMuralidhara M K 3714ed623d55SMuralidhara M K static int per_family_init(struct amd64_pvt *pvt) 3715395ae783SBorislav Petkov { 371618b94f66SAravind Gopalakrishnan pvt->ext_model = boot_cpu_data.x86_model >> 4; 3717b399151cSJia Zhang pvt->stepping = boot_cpu_data.x86_stepping; 371818b94f66SAravind Gopalakrishnan pvt->model = boot_cpu_data.x86_model; 371918b94f66SAravind Gopalakrishnan pvt->fam = boot_cpu_data.x86; 3720ed623d55SMuralidhara M K pvt->max_mcs = 2; 3721ed623d55SMuralidhara M K 3722ed623d55SMuralidhara M K /* 3723ed623d55SMuralidhara M K * Decide on which ops group to use here and do any family/model 3724ed623d55SMuralidhara M K * overrides below. 3725ed623d55SMuralidhara M K */ 3726ed623d55SMuralidhara M K if (pvt->fam >= 0x17) 3727ed623d55SMuralidhara M K pvt->ops = &umc_ops; 3728ed623d55SMuralidhara M K else 3729ed623d55SMuralidhara M K pvt->ops = &dct_ops; 373018b94f66SAravind Gopalakrishnan 373118b94f66SAravind Gopalakrishnan switch (pvt->fam) { 3732395ae783SBorislav Petkov case 0xf: 3733ed623d55SMuralidhara M K pvt->ctl_name = (pvt->ext_model >= K8_REV_F) ? 3734ed623d55SMuralidhara M K "K8 revF or later" : "K8 revE or earlier"; 3735ed623d55SMuralidhara M K pvt->f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP; 3736ed623d55SMuralidhara M K pvt->f2_id = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL; 3737ed623d55SMuralidhara M K pvt->ops->map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow; 3738ed623d55SMuralidhara M K pvt->ops->dbam_to_cs = k8_dbam_to_chip_select; 3739395ae783SBorislav Petkov break; 3740df71a053SBorislav Petkov 3741395ae783SBorislav Petkov case 0x10: 3742ed623d55SMuralidhara M K pvt->ctl_name = "F10h"; 3743ed623d55SMuralidhara M K pvt->f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP; 3744ed623d55SMuralidhara M K pvt->f2_id = PCI_DEVICE_ID_AMD_10H_NB_DRAM; 3745ed623d55SMuralidhara M K pvt->ops->dbam_to_cs = f10_dbam_to_chip_select; 3746df71a053SBorislav Petkov break; 3747df71a053SBorislav Petkov 3748df71a053SBorislav Petkov case 0x15: 3749ed623d55SMuralidhara M K switch (pvt->model) { 3750ed623d55SMuralidhara M K case 0x30: 3751ed623d55SMuralidhara M K pvt->ctl_name = "F15h_M30h"; 3752ed623d55SMuralidhara M K pvt->f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1; 3753ed623d55SMuralidhara M K pvt->f2_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2; 375418b94f66SAravind Gopalakrishnan break; 3755ed623d55SMuralidhara M K case 0x60: 3756ed623d55SMuralidhara M K pvt->ctl_name = "F15h_M60h"; 3757ed623d55SMuralidhara M K pvt->f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1; 3758ed623d55SMuralidhara M K pvt->f2_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F2; 3759ed623d55SMuralidhara M K pvt->ops->dbam_to_cs = f15_m60h_dbam_to_chip_select; 3760a597d2a5SAravind Gopalakrishnan break; 3761ed623d55SMuralidhara M K case 0x13: 37626c13d7ffSBorislav Petkov /* Richland is only client */ 3763ed623d55SMuralidhara M K return -ENODEV; 3764ed623d55SMuralidhara M K default: 3765ed623d55SMuralidhara M K pvt->ctl_name = "F15h"; 3766ed623d55SMuralidhara M K pvt->f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1; 3767ed623d55SMuralidhara M K pvt->f2_id = PCI_DEVICE_ID_AMD_15H_NB_F2; 3768ed623d55SMuralidhara M K pvt->ops->dbam_to_cs = f15_dbam_to_chip_select; 3769ed623d55SMuralidhara M K break; 37706c13d7ffSBorislav Petkov } 3771395ae783SBorislav Petkov break; 3772395ae783SBorislav Petkov 377394c1acf2SAravind Gopalakrishnan case 0x16: 3774ed623d55SMuralidhara M K switch (pvt->model) { 3775ed623d55SMuralidhara M K case 0x30: 3776ed623d55SMuralidhara M K pvt->ctl_name = "F16h_M30h"; 3777ed623d55SMuralidhara M K pvt->f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1; 3778ed623d55SMuralidhara M K pvt->f2_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2; 3779ed623d55SMuralidhara M K break; 3780ed623d55SMuralidhara M K default: 3781ed623d55SMuralidhara M K pvt->ctl_name = "F16h"; 3782ed623d55SMuralidhara M K pvt->f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1; 3783ed623d55SMuralidhara M K pvt->f2_id = PCI_DEVICE_ID_AMD_16H_NB_F2; 378485a8885bSAravind Gopalakrishnan break; 378585a8885bSAravind Gopalakrishnan } 378694c1acf2SAravind Gopalakrishnan break; 378794c1acf2SAravind Gopalakrishnan 3788f1cbbec9SYazen Ghannam case 0x17: 3789ed623d55SMuralidhara M K switch (pvt->model) { 3790ed623d55SMuralidhara M K case 0x10 ... 0x2f: 3791ed623d55SMuralidhara M K pvt->ctl_name = "F17h_M10h"; 37928960de4aSMichael Jin break; 3793ed623d55SMuralidhara M K case 0x30 ... 0x3f: 3794ed623d55SMuralidhara M K pvt->ctl_name = "F17h_M30h"; 3795ed623d55SMuralidhara M K pvt->max_mcs = 8; 37966e846239SYazen Ghannam break; 3797ed623d55SMuralidhara M K case 0x60 ... 0x6f: 3798ed623d55SMuralidhara M K pvt->ctl_name = "F17h_M60h"; 3799b6bea24dSAlexander Monakov break; 3800ed623d55SMuralidhara M K case 0x70 ... 0x7f: 3801ed623d55SMuralidhara M K pvt->ctl_name = "F17h_M70h"; 3802ed623d55SMuralidhara M K break; 3803ed623d55SMuralidhara M K default: 3804ed623d55SMuralidhara M K pvt->ctl_name = "F17h"; 38053e443eb3SIsaac Vaughn break; 38068960de4aSMichael Jin } 3807ed623d55SMuralidhara M K break; 3808c4a3e946SPu Wen 3809ed623d55SMuralidhara M K case 0x18: 3810ed623d55SMuralidhara M K pvt->ctl_name = "F18h"; 3811f1cbbec9SYazen Ghannam break; 3812f1cbbec9SYazen Ghannam 38132eb61c91SYazen Ghannam case 0x19: 3814ed623d55SMuralidhara M K switch (pvt->model) { 3815ed623d55SMuralidhara M K case 0x00 ... 0x0f: 3816ed623d55SMuralidhara M K pvt->ctl_name = "F19h"; 3817ed623d55SMuralidhara M K pvt->max_mcs = 8; 3818e2be5955SYazen Ghannam break; 3819ed623d55SMuralidhara M K case 0x10 ... 0x1f: 3820ed623d55SMuralidhara M K pvt->ctl_name = "F19h_M10h"; 3821ed623d55SMuralidhara M K pvt->max_mcs = 12; 3822ed623d55SMuralidhara M K pvt->flags.zn_regs_v2 = 1; 3823b4210eabSYazen Ghannam break; 3824ed623d55SMuralidhara M K case 0x20 ... 0x2f: 3825ed623d55SMuralidhara M K pvt->ctl_name = "F19h_M20h"; 38260b8bf9cbSMarc Bevand break; 3827ed623d55SMuralidhara M K case 0x50 ... 0x5f: 3828ed623d55SMuralidhara M K pvt->ctl_name = "F19h_M50h"; 3829ed623d55SMuralidhara M K break; 3830ed623d55SMuralidhara M K case 0xa0 ... 0xaf: 3831ed623d55SMuralidhara M K pvt->ctl_name = "F19h_MA0h"; 3832ed623d55SMuralidhara M K pvt->max_mcs = 12; 3833ed623d55SMuralidhara M K pvt->flags.zn_regs_v2 = 1; 3834e2be5955SYazen Ghannam break; 3835b4210eabSYazen Ghannam } 38362eb61c91SYazen Ghannam break; 38372eb61c91SYazen Ghannam 3838395ae783SBorislav Petkov default: 383924f9a7feSBorislav Petkov amd64_err("Unsupported family!\n"); 3840ed623d55SMuralidhara M K return -ENODEV; 3841395ae783SBorislav Petkov } 38420092b20dSBorislav Petkov 3843ed623d55SMuralidhara M K return 0; 3844395ae783SBorislav Petkov } 3845395ae783SBorislav Petkov 3846e339f1ecSTakashi Iwai static const struct attribute_group *amd64_edac_attr_groups[] = { 3847e339f1ecSTakashi Iwai #ifdef CONFIG_EDAC_DEBUG 38482a28ceefSBorislav Petkov &dbg_group, 384961810096SBorislav Petkov &inj_group, 3850e339f1ecSTakashi Iwai #endif 3851e339f1ecSTakashi Iwai NULL 3852e339f1ecSTakashi Iwai }; 3853e339f1ecSTakashi Iwai 385480355a3bSYazen Ghannam static int init_one_instance(struct amd64_pvt *pvt) 385580355a3bSYazen Ghannam { 385680355a3bSYazen Ghannam struct mem_ctl_info *mci = NULL; 385780355a3bSYazen Ghannam struct edac_mc_layer layers[2]; 3858c4605bdeSYazen Ghannam int ret = -ENOMEM; 385980355a3bSYazen Ghannam 3860ab5a503cSMauro Carvalho Chehab layers[0].type = EDAC_MC_LAYER_CHIP_SELECT; 3861ab5a503cSMauro Carvalho Chehab layers[0].size = pvt->csels[0].b_cnt; 3862ab5a503cSMauro Carvalho Chehab layers[0].is_virt_csrow = true; 3863ab5a503cSMauro Carvalho Chehab layers[1].type = EDAC_MC_LAYER_CHANNEL; 3864ed623d55SMuralidhara M K layers[1].size = pvt->max_mcs; 3865ab5a503cSMauro Carvalho Chehab layers[1].is_virt_csrow = false; 3866f0a56c48SBorislav Petkov 386780355a3bSYazen Ghannam mci = edac_mc_alloc(pvt->mc_node_id, ARRAY_SIZE(layers), layers, 0); 38687d6034d3SDoug Thompson if (!mci) 386980355a3bSYazen Ghannam return ret; 38707d6034d3SDoug Thompson 38717d6034d3SDoug Thompson mci->pvt_info = pvt; 38723f37a36bSBorislav Petkov mci->pdev = &pvt->F3->dev; 38737d6034d3SDoug Thompson 38740a42a37fSMuralidhara M K pvt->ops->setup_mci_misc_attrs(mci); 3875360b7f3cSBorislav Petkov 3876360b7f3cSBorislav Petkov if (init_csrows(mci)) 38777d6034d3SDoug Thompson mci->edac_cap = EDAC_FLAG_NONE; 38787d6034d3SDoug Thompson 38797d6034d3SDoug Thompson ret = -ENODEV; 3880e339f1ecSTakashi Iwai if (edac_mc_add_mc_with_groups(mci, amd64_edac_attr_groups)) { 3881956b9ba1SJoe Perches edac_dbg(1, "failed edac_mc_add_mc()\n"); 388280355a3bSYazen Ghannam edac_mc_free(mci); 388380355a3bSYazen Ghannam return ret; 38847d6034d3SDoug Thompson } 38857d6034d3SDoug Thompson 38867d6034d3SDoug Thompson return 0; 38877d6034d3SDoug Thompson } 38887d6034d3SDoug Thompson 3889582f94b5SYazen Ghannam static bool instance_has_memory(struct amd64_pvt *pvt) 3890582f94b5SYazen Ghannam { 3891582f94b5SYazen Ghannam bool cs_enabled = false; 3892582f94b5SYazen Ghannam int cs = 0, dct = 0; 3893582f94b5SYazen Ghannam 3894ed623d55SMuralidhara M K for (dct = 0; dct < pvt->max_mcs; dct++) { 3895582f94b5SYazen Ghannam for_each_chip_select(cs, dct, pvt) 3896582f94b5SYazen Ghannam cs_enabled |= csrow_enabled(cs, dct, pvt); 3897582f94b5SYazen Ghannam } 3898582f94b5SYazen Ghannam 3899582f94b5SYazen Ghannam return cs_enabled; 3900582f94b5SYazen Ghannam } 3901582f94b5SYazen Ghannam 39023f37a36bSBorislav Petkov static int probe_one_instance(unsigned int nid) 39037d6034d3SDoug Thompson { 39042299ef71SBorislav Petkov struct pci_dev *F3 = node_to_amd_nb(nid)->misc; 390580355a3bSYazen Ghannam struct amd64_pvt *pvt = NULL; 3906ae7bb7c6SBorislav Petkov struct ecc_settings *s; 39073f37a36bSBorislav Petkov int ret; 3908b8cfa02fSBorislav Petkov 3909ae7bb7c6SBorislav Petkov ret = -ENOMEM; 3910ae7bb7c6SBorislav Petkov s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL); 3911ae7bb7c6SBorislav Petkov if (!s) 39122299ef71SBorislav Petkov goto err_out; 3913ae7bb7c6SBorislav Petkov 3914ae7bb7c6SBorislav Petkov ecc_stngs[nid] = s; 3915ae7bb7c6SBorislav Petkov 391680355a3bSYazen Ghannam pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL); 391780355a3bSYazen Ghannam if (!pvt) 391880355a3bSYazen Ghannam goto err_settings; 391980355a3bSYazen Ghannam 392080355a3bSYazen Ghannam pvt->mc_node_id = nid; 392180355a3bSYazen Ghannam pvt->F3 = F3; 392280355a3bSYazen Ghannam 3923ed623d55SMuralidhara M K ret = per_family_init(pvt); 3924ed623d55SMuralidhara M K if (ret < 0) 392580355a3bSYazen Ghannam goto err_enable; 392680355a3bSYazen Ghannam 39279a97a7f4SYazen Ghannam ret = pvt->ops->hw_info_get(pvt); 392880355a3bSYazen Ghannam if (ret < 0) 392980355a3bSYazen Ghannam goto err_enable; 393080355a3bSYazen Ghannam 39314688c9b4SYazen Ghannam ret = 0; 3932582f94b5SYazen Ghannam if (!instance_has_memory(pvt)) { 3933582f94b5SYazen Ghannam amd64_info("Node %d: No DIMMs detected.\n", nid); 3934582f94b5SYazen Ghannam goto err_enable; 3935582f94b5SYazen Ghannam } 3936582f94b5SYazen Ghannam 3937eb2bcdfcSMuralidhara M K if (!pvt->ops->ecc_enabled(pvt)) { 3938582f94b5SYazen Ghannam ret = -ENODEV; 39392299ef71SBorislav Petkov 39402299ef71SBorislav Petkov if (!ecc_enable_override) 39412299ef71SBorislav Petkov goto err_enable; 39422299ef71SBorislav Petkov 3943044e7a41SYazen Ghannam if (boot_cpu_data.x86 >= 0x17) { 3944044e7a41SYazen Ghannam amd64_warn("Forcing ECC on is not recommended on newer systems. Please enable ECC in BIOS."); 3945044e7a41SYazen Ghannam goto err_enable; 3946044e7a41SYazen Ghannam } else 39472299ef71SBorislav Petkov amd64_warn("Forcing ECC on!\n"); 39482299ef71SBorislav Petkov 39492299ef71SBorislav Petkov if (!enable_ecc_error_reporting(s, nid, F3)) 39502299ef71SBorislav Petkov goto err_enable; 39512299ef71SBorislav Petkov } 39522299ef71SBorislav Petkov 395380355a3bSYazen Ghannam ret = init_one_instance(pvt); 3954360b7f3cSBorislav Petkov if (ret < 0) { 3955ae7bb7c6SBorislav Petkov amd64_err("Error probing instance: %d\n", nid); 3956044e7a41SYazen Ghannam 3957044e7a41SYazen Ghannam if (boot_cpu_data.x86 < 0x17) 3958360b7f3cSBorislav Petkov restore_ecc_error_reporting(s, nid, F3); 39592b9b2c46SYazen Ghannam 39602b9b2c46SYazen Ghannam goto err_enable; 3961360b7f3cSBorislav Petkov } 39627d6034d3SDoug Thompson 3963ed623d55SMuralidhara M K amd64_info("%s detected (node %d).\n", pvt->ctl_name, pvt->mc_node_id); 39644cbcb73bSBorislav Petkov 3965582f94b5SYazen Ghannam dump_misc_regs(pvt); 3966582f94b5SYazen Ghannam 39677d6034d3SDoug Thompson return ret; 39682299ef71SBorislav Petkov 39692299ef71SBorislav Petkov err_enable: 397080355a3bSYazen Ghannam hw_info_put(pvt); 397180355a3bSYazen Ghannam kfree(pvt); 397280355a3bSYazen Ghannam 397380355a3bSYazen Ghannam err_settings: 39742299ef71SBorislav Petkov kfree(s); 39752299ef71SBorislav Petkov ecc_stngs[nid] = NULL; 39762299ef71SBorislav Petkov 39772299ef71SBorislav Petkov err_out: 39782299ef71SBorislav Petkov return ret; 39797d6034d3SDoug Thompson } 39807d6034d3SDoug Thompson 39813f37a36bSBorislav Petkov static void remove_one_instance(unsigned int nid) 39827d6034d3SDoug Thompson { 3983360b7f3cSBorislav Petkov struct pci_dev *F3 = node_to_amd_nb(nid)->misc; 3984360b7f3cSBorislav Petkov struct ecc_settings *s = ecc_stngs[nid]; 39853f37a36bSBorislav Petkov struct mem_ctl_info *mci; 39863f37a36bSBorislav Petkov struct amd64_pvt *pvt; 39877d6034d3SDoug Thompson 39887d6034d3SDoug Thompson /* Remove from EDAC CORE tracking list */ 39893f37a36bSBorislav Petkov mci = edac_mc_del_mc(&F3->dev); 39907d6034d3SDoug Thompson if (!mci) 39917d6034d3SDoug Thompson return; 39927d6034d3SDoug Thompson 39937d6034d3SDoug Thompson pvt = mci->pvt_info; 39947d6034d3SDoug Thompson 3995360b7f3cSBorislav Petkov restore_ecc_error_reporting(s, nid, F3); 39967d6034d3SDoug Thompson 3997360b7f3cSBorislav Petkov kfree(ecc_stngs[nid]); 3998360b7f3cSBorislav Petkov ecc_stngs[nid] = NULL; 3999ae7bb7c6SBorislav Petkov 40007d6034d3SDoug Thompson /* Free the EDAC CORE resources */ 40018f68ed97SBorislav Petkov mci->pvt_info = NULL; 40028f68ed97SBorislav Petkov 400380355a3bSYazen Ghannam hw_info_put(pvt); 40048f68ed97SBorislav Petkov kfree(pvt); 40057d6034d3SDoug Thompson edac_mc_free(mci); 40067d6034d3SDoug Thompson } 40077d6034d3SDoug Thompson 4008360b7f3cSBorislav Petkov static void setup_pci_device(void) 40097d6034d3SDoug Thompson { 4010d1ea71cdSBorislav Petkov if (pci_ctl) 40117d6034d3SDoug Thompson return; 40127d6034d3SDoug Thompson 4013706657b1SBorislav Petkov pci_ctl = edac_pci_create_generic_ctl(pci_ctl_dev, EDAC_MOD_STR); 4014d1ea71cdSBorislav Petkov if (!pci_ctl) { 4015d1ea71cdSBorislav Petkov pr_warn("%s(): Unable to create PCI control\n", __func__); 4016d1ea71cdSBorislav Petkov pr_warn("%s(): PCI error report via EDAC not set\n", __func__); 40177d6034d3SDoug Thompson } 40187d6034d3SDoug Thompson } 40197d6034d3SDoug Thompson 4020d6efab74SYazen Ghannam static const struct x86_cpu_id amd64_cpuids[] = { 402129842621SThomas Gleixner X86_MATCH_VENDOR_FAM(AMD, 0x0F, NULL), 402229842621SThomas Gleixner X86_MATCH_VENDOR_FAM(AMD, 0x10, NULL), 402329842621SThomas Gleixner X86_MATCH_VENDOR_FAM(AMD, 0x15, NULL), 402429842621SThomas Gleixner X86_MATCH_VENDOR_FAM(AMD, 0x16, NULL), 402529842621SThomas Gleixner X86_MATCH_VENDOR_FAM(AMD, 0x17, NULL), 402629842621SThomas Gleixner X86_MATCH_VENDOR_FAM(HYGON, 0x18, NULL), 402729842621SThomas Gleixner X86_MATCH_VENDOR_FAM(AMD, 0x19, NULL), 4028d6efab74SYazen Ghannam { } 4029d6efab74SYazen Ghannam }; 4030d6efab74SYazen Ghannam MODULE_DEVICE_TABLE(x86cpu, amd64_cpuids); 4031d6efab74SYazen Ghannam 40327d6034d3SDoug Thompson static int __init amd64_edac_init(void) 40337d6034d3SDoug Thompson { 4034301375e7SToshi Kani const char *owner; 4035360b7f3cSBorislav Petkov int err = -ENODEV; 40363f37a36bSBorislav Petkov int i; 40377d6034d3SDoug Thompson 4038315bada6SJia He if (ghes_get_devices()) 4039315bada6SJia He return -EBUSY; 4040315bada6SJia He 4041301375e7SToshi Kani owner = edac_get_owner(); 4042301375e7SToshi Kani if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR))) 4043301375e7SToshi Kani return -EBUSY; 4044301375e7SToshi Kani 40451bd9900bSYazen Ghannam if (!x86_match_cpu(amd64_cpuids)) 40461bd9900bSYazen Ghannam return -ENODEV; 40471bd9900bSYazen Ghannam 4048e1907d37SMuralidhara M K if (!amd_nb_num()) 40491bd9900bSYazen Ghannam return -ENODEV; 40507d6034d3SDoug Thompson 40516ba92feaSBorislav Petkov opstate_init(); 40526ba92feaSBorislav Petkov 4053cc4d8860SBorislav Petkov err = -ENOMEM; 40546396bb22SKees Cook ecc_stngs = kcalloc(amd_nb_num(), sizeof(ecc_stngs[0]), GFP_KERNEL); 40552ec591acSBorislav Petkov if (!ecc_stngs) 4056a9f0fbe2SBorislav Petkov goto err_free; 4057cc4d8860SBorislav Petkov 405850542251SBorislav Petkov msrs = msrs_alloc(); 405956b34b91SBorislav Petkov if (!msrs) 4060360b7f3cSBorislav Petkov goto err_free; 406150542251SBorislav Petkov 40622287c636SYazen Ghannam for (i = 0; i < amd_nb_num(); i++) { 40632287c636SYazen Ghannam err = probe_one_instance(i); 40642287c636SYazen Ghannam if (err) { 40653f37a36bSBorislav Petkov /* unwind properly */ 40663f37a36bSBorislav Petkov while (--i >= 0) 40673f37a36bSBorislav Petkov remove_one_instance(i); 40687d6034d3SDoug Thompson 40693f37a36bSBorislav Petkov goto err_pci; 40703f37a36bSBorislav Petkov } 40712287c636SYazen Ghannam } 40727d6034d3SDoug Thompson 40734688c9b4SYazen Ghannam if (!edac_has_mcs()) { 40744688c9b4SYazen Ghannam err = -ENODEV; 40754688c9b4SYazen Ghannam goto err_pci; 40764688c9b4SYazen Ghannam } 40774688c9b4SYazen Ghannam 4078234365f5SYazen Ghannam /* register stuff with EDAC MCE */ 4079fdce765aSYazen Ghannam if (boot_cpu_data.x86 >= 0x17) { 4080234365f5SYazen Ghannam amd_register_ecc_decoder(decode_umc_error); 4081fdce765aSYazen Ghannam } else { 4082234365f5SYazen Ghannam amd_register_ecc_decoder(decode_bus_error); 4083360b7f3cSBorislav Petkov setup_pci_device(); 4084fdce765aSYazen Ghannam } 4085f5b10c45STomasz Pala 4086f5b10c45STomasz Pala #ifdef CONFIG_X86_32 4087f5b10c45STomasz Pala amd64_err("%s on 32-bit is unsupported. USE AT YOUR OWN RISK!\n", EDAC_MOD_STR); 4088f5b10c45STomasz Pala #endif 4089f5b10c45STomasz Pala 4090de0336b3SBorislav Petkov printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION); 4091de0336b3SBorislav Petkov 40927d6034d3SDoug Thompson return 0; 40937d6034d3SDoug Thompson 409456b34b91SBorislav Petkov err_pci: 4095706657b1SBorislav Petkov pci_ctl_dev = NULL; 4096706657b1SBorislav Petkov 409756b34b91SBorislav Petkov msrs_free(msrs); 409856b34b91SBorislav Petkov msrs = NULL; 4099cc4d8860SBorislav Petkov 4100360b7f3cSBorislav Petkov err_free: 4101360b7f3cSBorislav Petkov kfree(ecc_stngs); 4102360b7f3cSBorislav Petkov ecc_stngs = NULL; 4103360b7f3cSBorislav Petkov 41047d6034d3SDoug Thompson return err; 41057d6034d3SDoug Thompson } 41067d6034d3SDoug Thompson 41077d6034d3SDoug Thompson static void __exit amd64_edac_exit(void) 41087d6034d3SDoug Thompson { 41093f37a36bSBorislav Petkov int i; 41103f37a36bSBorislav Petkov 4111d1ea71cdSBorislav Petkov if (pci_ctl) 4112d1ea71cdSBorislav Petkov edac_pci_release_generic_ctl(pci_ctl); 41137d6034d3SDoug Thompson 4114234365f5SYazen Ghannam /* unregister from EDAC MCE */ 4115234365f5SYazen Ghannam if (boot_cpu_data.x86 >= 0x17) 4116234365f5SYazen Ghannam amd_unregister_ecc_decoder(decode_umc_error); 4117234365f5SYazen Ghannam else 4118234365f5SYazen Ghannam amd_unregister_ecc_decoder(decode_bus_error); 4119234365f5SYazen Ghannam 41203f37a36bSBorislav Petkov for (i = 0; i < amd_nb_num(); i++) 41213f37a36bSBorislav Petkov remove_one_instance(i); 412250542251SBorislav Petkov 4123ae7bb7c6SBorislav Petkov kfree(ecc_stngs); 4124ae7bb7c6SBorislav Petkov ecc_stngs = NULL; 4125ae7bb7c6SBorislav Petkov 4126706657b1SBorislav Petkov pci_ctl_dev = NULL; 4127706657b1SBorislav Petkov 412850542251SBorislav Petkov msrs_free(msrs); 412950542251SBorislav Petkov msrs = NULL; 41307d6034d3SDoug Thompson } 41317d6034d3SDoug Thompson 41327d6034d3SDoug Thompson module_init(amd64_edac_init); 41337d6034d3SDoug Thompson module_exit(amd64_edac_exit); 41347d6034d3SDoug Thompson 41357d6034d3SDoug Thompson MODULE_LICENSE("GPL"); 41367d6034d3SDoug Thompson MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, " 41377d6034d3SDoug Thompson "Dave Peterson, Thayne Harbaugh"); 41387d6034d3SDoug Thompson MODULE_DESCRIPTION("MC support for AMD64 memory controllers - " 41397d6034d3SDoug Thompson EDAC_AMD64_VERSION); 41407d6034d3SDoug Thompson 41417d6034d3SDoug Thompson module_param(edac_op_state, int, 0444); 41427d6034d3SDoug Thompson MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI"); 4143