109c434b8SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 22bc65418SDoug Thompson #include "amd64_edac.h" 323ac4ae8SAndreas Herrmann #include <asm/amd_nb.h> 42bc65418SDoug Thompson 5d1ea71cdSBorislav Petkov static struct edac_pci_ctl_info *pci_ctl; 62bc65418SDoug Thompson 72bc65418SDoug Thompson static int report_gart_errors; 82bc65418SDoug Thompson module_param(report_gart_errors, int, 0644); 92bc65418SDoug Thompson 102bc65418SDoug Thompson /* 112bc65418SDoug Thompson * Set by command line parameter. If BIOS has enabled the ECC, this override is 122bc65418SDoug Thompson * cleared to prevent re-enabling the hardware by this driver. 132bc65418SDoug Thompson */ 142bc65418SDoug Thompson static int ecc_enable_override; 152bc65418SDoug Thompson module_param(ecc_enable_override, int, 0644); 162bc65418SDoug Thompson 17a29d8b8eSTejun Heo static struct msr __percpu *msrs; 1850542251SBorislav Petkov 1938ddd4d1SYazen Ghannam static struct amd64_family_type *fam_type; 2038ddd4d1SYazen Ghannam 212ec591acSBorislav Petkov /* Per-node stuff */ 22ae7bb7c6SBorislav Petkov static struct ecc_settings **ecc_stngs; 232bc65418SDoug Thompson 242bc65418SDoug Thompson /* 25b70ef010SBorislav Petkov * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing 26b70ef010SBorislav Petkov * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching- 27b70ef010SBorislav Petkov * or higher value'. 28b70ef010SBorislav Petkov * 29b70ef010SBorislav Petkov *FIXME: Produce a better mapping/linearisation. 30b70ef010SBorislav Petkov */ 31c7e5301aSDaniel J Blueman static const struct scrubrate { 3239094443SBorislav Petkov u32 scrubval; /* bit pattern for scrub rate */ 3339094443SBorislav Petkov u32 bandwidth; /* bandwidth consumed (bytes/sec) */ 3439094443SBorislav Petkov } scrubrates[] = { 35b70ef010SBorislav Petkov { 0x01, 1600000000UL}, 36b70ef010SBorislav Petkov { 0x02, 800000000UL}, 37b70ef010SBorislav Petkov { 0x03, 400000000UL}, 38b70ef010SBorislav Petkov { 0x04, 200000000UL}, 39b70ef010SBorislav Petkov { 0x05, 100000000UL}, 40b70ef010SBorislav Petkov { 0x06, 50000000UL}, 41b70ef010SBorislav Petkov { 0x07, 25000000UL}, 42b70ef010SBorislav Petkov { 0x08, 12284069UL}, 43b70ef010SBorislav Petkov { 0x09, 6274509UL}, 44b70ef010SBorislav Petkov { 0x0A, 3121951UL}, 45b70ef010SBorislav Petkov { 0x0B, 1560975UL}, 46b70ef010SBorislav Petkov { 0x0C, 781440UL}, 47b70ef010SBorislav Petkov { 0x0D, 390720UL}, 48b70ef010SBorislav Petkov { 0x0E, 195300UL}, 49b70ef010SBorislav Petkov { 0x0F, 97650UL}, 50b70ef010SBorislav Petkov { 0x10, 48854UL}, 51b70ef010SBorislav Petkov { 0x11, 24427UL}, 52b70ef010SBorislav Petkov { 0x12, 12213UL}, 53b70ef010SBorislav Petkov { 0x13, 6101UL}, 54b70ef010SBorislav Petkov { 0x14, 3051UL}, 55b70ef010SBorislav Petkov { 0x15, 1523UL}, 56b70ef010SBorislav Petkov { 0x16, 761UL}, 57b70ef010SBorislav Petkov { 0x00, 0UL}, /* scrubbing off */ 58b70ef010SBorislav Petkov }; 59b70ef010SBorislav Petkov 6066fed2d4SBorislav Petkov int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset, 61b2b0c605SBorislav Petkov u32 *val, const char *func) 62b2b0c605SBorislav Petkov { 63b2b0c605SBorislav Petkov int err = 0; 64b2b0c605SBorislav Petkov 65b2b0c605SBorislav Petkov err = pci_read_config_dword(pdev, offset, val); 66b2b0c605SBorislav Petkov if (err) 67b2b0c605SBorislav Petkov amd64_warn("%s: error reading F%dx%03x.\n", 68b2b0c605SBorislav Petkov func, PCI_FUNC(pdev->devfn), offset); 69b2b0c605SBorislav Petkov 70b2b0c605SBorislav Petkov return err; 71b2b0c605SBorislav Petkov } 72b2b0c605SBorislav Petkov 73b2b0c605SBorislav Petkov int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset, 74b2b0c605SBorislav Petkov u32 val, const char *func) 75b2b0c605SBorislav Petkov { 76b2b0c605SBorislav Petkov int err = 0; 77b2b0c605SBorislav Petkov 78b2b0c605SBorislav Petkov err = pci_write_config_dword(pdev, offset, val); 79b2b0c605SBorislav Petkov if (err) 80b2b0c605SBorislav Petkov amd64_warn("%s: error writing to F%dx%03x.\n", 81b2b0c605SBorislav Petkov func, PCI_FUNC(pdev->devfn), offset); 82b2b0c605SBorislav Petkov 83b2b0c605SBorislav Petkov return err; 84b2b0c605SBorislav Petkov } 85b2b0c605SBorislav Petkov 86b2b0c605SBorislav Petkov /* 8773ba8593SBorislav Petkov * Select DCT to which PCI cfg accesses are routed 8873ba8593SBorislav Petkov */ 8973ba8593SBorislav Petkov static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct) 9073ba8593SBorislav Petkov { 9173ba8593SBorislav Petkov u32 reg = 0; 9273ba8593SBorislav Petkov 9373ba8593SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, ®); 947981a28fSAravind Gopalakrishnan reg &= (pvt->model == 0x30) ? ~3 : ~1; 9573ba8593SBorislav Petkov reg |= dct; 9673ba8593SBorislav Petkov amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg); 9773ba8593SBorislav Petkov } 9873ba8593SBorislav Petkov 997981a28fSAravind Gopalakrishnan /* 1007981a28fSAravind Gopalakrishnan * 1017981a28fSAravind Gopalakrishnan * Depending on the family, F2 DCT reads need special handling: 1027981a28fSAravind Gopalakrishnan * 1037981a28fSAravind Gopalakrishnan * K8: has a single DCT only and no address offsets >= 0x100 1047981a28fSAravind Gopalakrishnan * 1057981a28fSAravind Gopalakrishnan * F10h: each DCT has its own set of regs 1067981a28fSAravind Gopalakrishnan * DCT0 -> F2x040.. 1077981a28fSAravind Gopalakrishnan * DCT1 -> F2x140.. 1087981a28fSAravind Gopalakrishnan * 1097981a28fSAravind Gopalakrishnan * F16h: has only 1 DCT 1107981a28fSAravind Gopalakrishnan * 1117981a28fSAravind Gopalakrishnan * F15h: we select which DCT we access using F1x10C[DctCfgSel] 1127981a28fSAravind Gopalakrishnan */ 1137981a28fSAravind Gopalakrishnan static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct, 1147981a28fSAravind Gopalakrishnan int offset, u32 *val) 115b2b0c605SBorislav Petkov { 1167981a28fSAravind Gopalakrishnan switch (pvt->fam) { 1177981a28fSAravind Gopalakrishnan case 0xf: 1187981a28fSAravind Gopalakrishnan if (dct || offset >= 0x100) 1197981a28fSAravind Gopalakrishnan return -EINVAL; 1207981a28fSAravind Gopalakrishnan break; 121b2b0c605SBorislav Petkov 1227981a28fSAravind Gopalakrishnan case 0x10: 1237981a28fSAravind Gopalakrishnan if (dct) { 1247981a28fSAravind Gopalakrishnan /* 1257981a28fSAravind Gopalakrishnan * Note: If ganging is enabled, barring the regs 1267981a28fSAravind Gopalakrishnan * F2x[1,0]98 and F2x[1,0]9C; reads reads to F2x1xx 1277981a28fSAravind Gopalakrishnan * return 0. (cf. Section 2.8.1 F10h BKDG) 1287981a28fSAravind Gopalakrishnan */ 1297981a28fSAravind Gopalakrishnan if (dct_ganging_enabled(pvt)) 1307981a28fSAravind Gopalakrishnan return 0; 1317981a28fSAravind Gopalakrishnan 1327981a28fSAravind Gopalakrishnan offset += 0x100; 133b2b0c605SBorislav Petkov } 1347981a28fSAravind Gopalakrishnan break; 135b2b0c605SBorislav Petkov 1367981a28fSAravind Gopalakrishnan case 0x15: 1377981a28fSAravind Gopalakrishnan /* 1387981a28fSAravind Gopalakrishnan * F15h: F2x1xx addresses do not map explicitly to DCT1. 1397981a28fSAravind Gopalakrishnan * We should select which DCT we access using F1x10C[DctCfgSel] 1407981a28fSAravind Gopalakrishnan */ 1417981a28fSAravind Gopalakrishnan dct = (dct && pvt->model == 0x30) ? 3 : dct; 14273ba8593SBorislav Petkov f15h_select_dct(pvt, dct); 1437981a28fSAravind Gopalakrishnan break; 144b2b0c605SBorislav Petkov 1457981a28fSAravind Gopalakrishnan case 0x16: 1467981a28fSAravind Gopalakrishnan if (dct) 1477981a28fSAravind Gopalakrishnan return -EINVAL; 1487981a28fSAravind Gopalakrishnan break; 1497981a28fSAravind Gopalakrishnan 1507981a28fSAravind Gopalakrishnan default: 1517981a28fSAravind Gopalakrishnan break; 1527981a28fSAravind Gopalakrishnan } 1537981a28fSAravind Gopalakrishnan return amd64_read_pci_cfg(pvt->F2, offset, val); 154b2b0c605SBorislav Petkov } 155b2b0c605SBorislav Petkov 156b70ef010SBorislav Petkov /* 1572bc65418SDoug Thompson * Memory scrubber control interface. For K8, memory scrubbing is handled by 1582bc65418SDoug Thompson * hardware and can involve L2 cache, dcache as well as the main memory. With 1592bc65418SDoug Thompson * F10, this is extended to L3 cache scrubbing on CPU models sporting that 1602bc65418SDoug Thompson * functionality. 1612bc65418SDoug Thompson * 1622bc65418SDoug Thompson * This causes the "units" for the scrubbing speed to vary from 64 byte blocks 1632bc65418SDoug Thompson * (dram) over to cache lines. This is nasty, so we will use bandwidth in 1642bc65418SDoug Thompson * bytes/sec for the setting. 1652bc65418SDoug Thompson * 1662bc65418SDoug Thompson * Currently, we only do dram scrubbing. If the scrubbing is done in software on 1672bc65418SDoug Thompson * other archs, we might not have access to the caches directly. 1682bc65418SDoug Thompson */ 1692bc65418SDoug Thompson 1708051c0afSYazen Ghannam static inline void __f17h_set_scrubval(struct amd64_pvt *pvt, u32 scrubval) 1718051c0afSYazen Ghannam { 1722bc65418SDoug Thompson /* 1738051c0afSYazen Ghannam * Fam17h supports scrub values between 0x5 and 0x14. Also, the values 1748051c0afSYazen Ghannam * are shifted down by 0x5, so scrubval 0x5 is written to the register 1758051c0afSYazen Ghannam * as 0x0, scrubval 0x6 as 0x1, etc. 1768051c0afSYazen Ghannam */ 1778051c0afSYazen Ghannam if (scrubval >= 0x5 && scrubval <= 0x14) { 1788051c0afSYazen Ghannam scrubval -= 0x5; 1798051c0afSYazen Ghannam pci_write_bits32(pvt->F6, F17H_SCR_LIMIT_ADDR, scrubval, 0xF); 1808051c0afSYazen Ghannam pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 1, 0x1); 1818051c0afSYazen Ghannam } else { 1828051c0afSYazen Ghannam pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 0, 0x1); 1838051c0afSYazen Ghannam } 1848051c0afSYazen Ghannam } 1858051c0afSYazen Ghannam /* 1868051c0afSYazen Ghannam * Scan the scrub rate mapping table for a close or matching bandwidth value to 1872bc65418SDoug Thompson * issue. If requested is too big, then use last maximum value found. 1882bc65418SDoug Thompson */ 189da92110dSAravind Gopalakrishnan static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate) 1902bc65418SDoug Thompson { 1912bc65418SDoug Thompson u32 scrubval; 1922bc65418SDoug Thompson int i; 1932bc65418SDoug Thompson 1942bc65418SDoug Thompson /* 1952bc65418SDoug Thompson * map the configured rate (new_bw) to a value specific to the AMD64 1962bc65418SDoug Thompson * memory controller and apply to register. Search for the first 1972bc65418SDoug Thompson * bandwidth entry that is greater or equal than the setting requested 1982bc65418SDoug Thompson * and program that. If at last entry, turn off DRAM scrubbing. 199168bfeefSAndrew Morton * 200168bfeefSAndrew Morton * If no suitable bandwidth is found, turn off DRAM scrubbing entirely 201168bfeefSAndrew Morton * by falling back to the last element in scrubrates[]. 2022bc65418SDoug Thompson */ 203168bfeefSAndrew Morton for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) { 2042bc65418SDoug Thompson /* 2052bc65418SDoug Thompson * skip scrub rates which aren't recommended 2062bc65418SDoug Thompson * (see F10 BKDG, F3x58) 2072bc65418SDoug Thompson */ 208395ae783SBorislav Petkov if (scrubrates[i].scrubval < min_rate) 2092bc65418SDoug Thompson continue; 2102bc65418SDoug Thompson 2112bc65418SDoug Thompson if (scrubrates[i].bandwidth <= new_bw) 2122bc65418SDoug Thompson break; 2132bc65418SDoug Thompson } 2142bc65418SDoug Thompson 2152bc65418SDoug Thompson scrubval = scrubrates[i].scrubval; 2162bc65418SDoug Thompson 217c4a3e946SPu Wen if (pvt->fam == 0x17 || pvt->fam == 0x18) { 2188051c0afSYazen Ghannam __f17h_set_scrubval(pvt, scrubval); 2198051c0afSYazen Ghannam } else if (pvt->fam == 0x15 && pvt->model == 0x60) { 220da92110dSAravind Gopalakrishnan f15h_select_dct(pvt, 0); 221da92110dSAravind Gopalakrishnan pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F); 222da92110dSAravind Gopalakrishnan f15h_select_dct(pvt, 1); 223da92110dSAravind Gopalakrishnan pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F); 224da92110dSAravind Gopalakrishnan } else { 225da92110dSAravind Gopalakrishnan pci_write_bits32(pvt->F3, SCRCTRL, scrubval, 0x001F); 226da92110dSAravind Gopalakrishnan } 2272bc65418SDoug Thompson 22839094443SBorislav Petkov if (scrubval) 22939094443SBorislav Petkov return scrubrates[i].bandwidth; 23039094443SBorislav Petkov 2312bc65418SDoug Thompson return 0; 2322bc65418SDoug Thompson } 2332bc65418SDoug Thompson 234d1ea71cdSBorislav Petkov static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw) 2352bc65418SDoug Thompson { 2362bc65418SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 23787b3e0e6SBorislav Petkov u32 min_scrubrate = 0x5; 2382bc65418SDoug Thompson 239a4b4bedcSBorislav Petkov if (pvt->fam == 0xf) 24087b3e0e6SBorislav Petkov min_scrubrate = 0x0; 24187b3e0e6SBorislav Petkov 242da92110dSAravind Gopalakrishnan if (pvt->fam == 0x15) { 2433f0aba4fSBorislav Petkov /* Erratum #505 */ 244da92110dSAravind Gopalakrishnan if (pvt->model < 0x10) 24573ba8593SBorislav Petkov f15h_select_dct(pvt, 0); 24673ba8593SBorislav Petkov 247da92110dSAravind Gopalakrishnan if (pvt->model == 0x60) 248da92110dSAravind Gopalakrishnan min_scrubrate = 0x6; 249da92110dSAravind Gopalakrishnan } 250da92110dSAravind Gopalakrishnan return __set_scrub_rate(pvt, bw, min_scrubrate); 2512bc65418SDoug Thompson } 2522bc65418SDoug Thompson 253d1ea71cdSBorislav Petkov static int get_scrub_rate(struct mem_ctl_info *mci) 2542bc65418SDoug Thompson { 2552bc65418SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 25639094443SBorislav Petkov int i, retval = -EINVAL; 2578051c0afSYazen Ghannam u32 scrubval = 0; 2582bc65418SDoug Thompson 2598051c0afSYazen Ghannam switch (pvt->fam) { 2608051c0afSYazen Ghannam case 0x15: 2613f0aba4fSBorislav Petkov /* Erratum #505 */ 262da92110dSAravind Gopalakrishnan if (pvt->model < 0x10) 26373ba8593SBorislav Petkov f15h_select_dct(pvt, 0); 26473ba8593SBorislav Petkov 265da92110dSAravind Gopalakrishnan if (pvt->model == 0x60) 266da92110dSAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F2, F15H_M60H_SCRCTRL, &scrubval); 2678051c0afSYazen Ghannam break; 2688051c0afSYazen Ghannam 2698051c0afSYazen Ghannam case 0x17: 270c4a3e946SPu Wen case 0x18: 2718051c0afSYazen Ghannam amd64_read_pci_cfg(pvt->F6, F17H_SCR_BASE_ADDR, &scrubval); 2728051c0afSYazen Ghannam if (scrubval & BIT(0)) { 2738051c0afSYazen Ghannam amd64_read_pci_cfg(pvt->F6, F17H_SCR_LIMIT_ADDR, &scrubval); 2748051c0afSYazen Ghannam scrubval &= 0xF; 2758051c0afSYazen Ghannam scrubval += 0x5; 2768051c0afSYazen Ghannam } else { 2778051c0afSYazen Ghannam scrubval = 0; 2788051c0afSYazen Ghannam } 2798051c0afSYazen Ghannam break; 2808051c0afSYazen Ghannam 2818051c0afSYazen Ghannam default: 2825980bb9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval); 2838051c0afSYazen Ghannam break; 2848051c0afSYazen Ghannam } 2852bc65418SDoug Thompson 2862bc65418SDoug Thompson scrubval = scrubval & 0x001F; 2872bc65418SDoug Thompson 288926311fdSRoel Kluin for (i = 0; i < ARRAY_SIZE(scrubrates); i++) { 2892bc65418SDoug Thompson if (scrubrates[i].scrubval == scrubval) { 29039094443SBorislav Petkov retval = scrubrates[i].bandwidth; 2912bc65418SDoug Thompson break; 2922bc65418SDoug Thompson } 2932bc65418SDoug Thompson } 29439094443SBorislav Petkov return retval; 2952bc65418SDoug Thompson } 2962bc65418SDoug Thompson 2976775763aSDoug Thompson /* 2987f19bf75SBorislav Petkov * returns true if the SysAddr given by sys_addr matches the 2997f19bf75SBorislav Petkov * DRAM base/limit associated with node_id 3006775763aSDoug Thompson */ 301d1ea71cdSBorislav Petkov static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid) 3026775763aSDoug Thompson { 3037f19bf75SBorislav Petkov u64 addr; 3046775763aSDoug Thompson 3056775763aSDoug Thompson /* The K8 treats this as a 40-bit value. However, bits 63-40 will be 3066775763aSDoug Thompson * all ones if the most significant implemented address bit is 1. 3076775763aSDoug Thompson * Here we discard bits 63-40. See section 3.4.2 of AMD publication 3086775763aSDoug Thompson * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1 3096775763aSDoug Thompson * Application Programming. 3106775763aSDoug Thompson */ 3116775763aSDoug Thompson addr = sys_addr & 0x000000ffffffffffull; 3126775763aSDoug Thompson 3137f19bf75SBorislav Petkov return ((addr >= get_dram_base(pvt, nid)) && 3147f19bf75SBorislav Petkov (addr <= get_dram_limit(pvt, nid))); 3156775763aSDoug Thompson } 3166775763aSDoug Thompson 3176775763aSDoug Thompson /* 3186775763aSDoug Thompson * Attempt to map a SysAddr to a node. On success, return a pointer to the 3196775763aSDoug Thompson * mem_ctl_info structure for the node that the SysAddr maps to. 3206775763aSDoug Thompson * 3216775763aSDoug Thompson * On failure, return NULL. 3226775763aSDoug Thompson */ 3236775763aSDoug Thompson static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci, 3246775763aSDoug Thompson u64 sys_addr) 3256775763aSDoug Thompson { 3266775763aSDoug Thompson struct amd64_pvt *pvt; 327c7e5301aSDaniel J Blueman u8 node_id; 3286775763aSDoug Thompson u32 intlv_en, bits; 3296775763aSDoug Thompson 3306775763aSDoug Thompson /* 3316775763aSDoug Thompson * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section 3326775763aSDoug Thompson * 3.4.4.2) registers to map the SysAddr to a node ID. 3336775763aSDoug Thompson */ 3346775763aSDoug Thompson pvt = mci->pvt_info; 3356775763aSDoug Thompson 3366775763aSDoug Thompson /* 3376775763aSDoug Thompson * The value of this field should be the same for all DRAM Base 3386775763aSDoug Thompson * registers. Therefore we arbitrarily choose to read it from the 3396775763aSDoug Thompson * register for node 0. 3406775763aSDoug Thompson */ 3417f19bf75SBorislav Petkov intlv_en = dram_intlv_en(pvt, 0); 3426775763aSDoug Thompson 3436775763aSDoug Thompson if (intlv_en == 0) { 3447f19bf75SBorislav Petkov for (node_id = 0; node_id < DRAM_RANGES; node_id++) { 345d1ea71cdSBorislav Petkov if (base_limit_match(pvt, sys_addr, node_id)) 3466775763aSDoug Thompson goto found; 3476775763aSDoug Thompson } 3488edc5445SBorislav Petkov goto err_no_match; 3498edc5445SBorislav Petkov } 3506775763aSDoug Thompson 35172f158feSBorislav Petkov if (unlikely((intlv_en != 0x01) && 35272f158feSBorislav Petkov (intlv_en != 0x03) && 35372f158feSBorislav Petkov (intlv_en != 0x07))) { 35424f9a7feSBorislav Petkov amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en); 3556775763aSDoug Thompson return NULL; 3566775763aSDoug Thompson } 3576775763aSDoug Thompson 3586775763aSDoug Thompson bits = (((u32) sys_addr) >> 12) & intlv_en; 3596775763aSDoug Thompson 3606775763aSDoug Thompson for (node_id = 0; ; ) { 3617f19bf75SBorislav Petkov if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits) 3626775763aSDoug Thompson break; /* intlv_sel field matches */ 3636775763aSDoug Thompson 3647f19bf75SBorislav Petkov if (++node_id >= DRAM_RANGES) 3656775763aSDoug Thompson goto err_no_match; 3666775763aSDoug Thompson } 3676775763aSDoug Thompson 3686775763aSDoug Thompson /* sanity test for sys_addr */ 369d1ea71cdSBorislav Petkov if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) { 37024f9a7feSBorislav Petkov amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address" 37124f9a7feSBorislav Petkov "range for node %d with node interleaving enabled.\n", 3728edc5445SBorislav Petkov __func__, sys_addr, node_id); 3736775763aSDoug Thompson return NULL; 3746775763aSDoug Thompson } 3756775763aSDoug Thompson 3766775763aSDoug Thompson found: 377b487c33eSBorislav Petkov return edac_mc_find((int)node_id); 3786775763aSDoug Thompson 3796775763aSDoug Thompson err_no_match: 380956b9ba1SJoe Perches edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n", 3816775763aSDoug Thompson (unsigned long)sys_addr); 3826775763aSDoug Thompson 3836775763aSDoug Thompson return NULL; 3846775763aSDoug Thompson } 385e2ce7255SDoug Thompson 386e2ce7255SDoug Thompson /* 38711c75eadSBorislav Petkov * compute the CS base address of the @csrow on the DRAM controller @dct. 38811c75eadSBorislav Petkov * For details see F2x[5C:40] in the processor's BKDG 389e2ce7255SDoug Thompson */ 39011c75eadSBorislav Petkov static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct, 39111c75eadSBorislav Petkov u64 *base, u64 *mask) 392e2ce7255SDoug Thompson { 39311c75eadSBorislav Petkov u64 csbase, csmask, base_bits, mask_bits; 39411c75eadSBorislav Petkov u8 addr_shift; 39511c75eadSBorislav Petkov 39618b94f66SAravind Gopalakrishnan if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) { 39711c75eadSBorislav Petkov csbase = pvt->csels[dct].csbases[csrow]; 39811c75eadSBorislav Petkov csmask = pvt->csels[dct].csmasks[csrow]; 39910ef6b0dSChen, Gong base_bits = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9); 40010ef6b0dSChen, Gong mask_bits = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9); 40111c75eadSBorislav Petkov addr_shift = 4; 40294c1acf2SAravind Gopalakrishnan 40394c1acf2SAravind Gopalakrishnan /* 40418b94f66SAravind Gopalakrishnan * F16h and F15h, models 30h and later need two addr_shift values: 40518b94f66SAravind Gopalakrishnan * 8 for high and 6 for low (cf. F16h BKDG). 40694c1acf2SAravind Gopalakrishnan */ 40718b94f66SAravind Gopalakrishnan } else if (pvt->fam == 0x16 || 40818b94f66SAravind Gopalakrishnan (pvt->fam == 0x15 && pvt->model >= 0x30)) { 40994c1acf2SAravind Gopalakrishnan csbase = pvt->csels[dct].csbases[csrow]; 41094c1acf2SAravind Gopalakrishnan csmask = pvt->csels[dct].csmasks[csrow >> 1]; 41194c1acf2SAravind Gopalakrishnan 41210ef6b0dSChen, Gong *base = (csbase & GENMASK_ULL(15, 5)) << 6; 41310ef6b0dSChen, Gong *base |= (csbase & GENMASK_ULL(30, 19)) << 8; 41494c1acf2SAravind Gopalakrishnan 41594c1acf2SAravind Gopalakrishnan *mask = ~0ULL; 41694c1acf2SAravind Gopalakrishnan /* poke holes for the csmask */ 41710ef6b0dSChen, Gong *mask &= ~((GENMASK_ULL(15, 5) << 6) | 41810ef6b0dSChen, Gong (GENMASK_ULL(30, 19) << 8)); 41994c1acf2SAravind Gopalakrishnan 42010ef6b0dSChen, Gong *mask |= (csmask & GENMASK_ULL(15, 5)) << 6; 42110ef6b0dSChen, Gong *mask |= (csmask & GENMASK_ULL(30, 19)) << 8; 42294c1acf2SAravind Gopalakrishnan 42394c1acf2SAravind Gopalakrishnan return; 42411c75eadSBorislav Petkov } else { 42511c75eadSBorislav Petkov csbase = pvt->csels[dct].csbases[csrow]; 42611c75eadSBorislav Petkov csmask = pvt->csels[dct].csmasks[csrow >> 1]; 42711c75eadSBorislav Petkov addr_shift = 8; 42811c75eadSBorislav Petkov 429a4b4bedcSBorislav Petkov if (pvt->fam == 0x15) 43010ef6b0dSChen, Gong base_bits = mask_bits = 43110ef6b0dSChen, Gong GENMASK_ULL(30,19) | GENMASK_ULL(13,5); 43211c75eadSBorislav Petkov else 43310ef6b0dSChen, Gong base_bits = mask_bits = 43410ef6b0dSChen, Gong GENMASK_ULL(28,19) | GENMASK_ULL(13,5); 435e2ce7255SDoug Thompson } 436e2ce7255SDoug Thompson 43711c75eadSBorislav Petkov *base = (csbase & base_bits) << addr_shift; 438e2ce7255SDoug Thompson 43911c75eadSBorislav Petkov *mask = ~0ULL; 44011c75eadSBorislav Petkov /* poke holes for the csmask */ 44111c75eadSBorislav Petkov *mask &= ~(mask_bits << addr_shift); 44211c75eadSBorislav Petkov /* OR them in */ 44311c75eadSBorislav Petkov *mask |= (csmask & mask_bits) << addr_shift; 444e2ce7255SDoug Thompson } 445e2ce7255SDoug Thompson 44611c75eadSBorislav Petkov #define for_each_chip_select(i, dct, pvt) \ 44711c75eadSBorislav Petkov for (i = 0; i < pvt->csels[dct].b_cnt; i++) 44811c75eadSBorislav Petkov 449614ec9d8SBorislav Petkov #define chip_select_base(i, dct, pvt) \ 450614ec9d8SBorislav Petkov pvt->csels[dct].csbases[i] 451614ec9d8SBorislav Petkov 45211c75eadSBorislav Petkov #define for_each_chip_select_mask(i, dct, pvt) \ 45311c75eadSBorislav Petkov for (i = 0; i < pvt->csels[dct].m_cnt; i++) 45411c75eadSBorislav Petkov 4554d30d2bcSYazen Ghannam #define for_each_umc(i) \ 4565e4c5527SYazen Ghannam for (i = 0; i < fam_type->max_mcs; i++) 4574d30d2bcSYazen Ghannam 458e2ce7255SDoug Thompson /* 459e2ce7255SDoug Thompson * @input_addr is an InputAddr associated with the node given by mci. Return the 460e2ce7255SDoug Thompson * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr). 461e2ce7255SDoug Thompson */ 462e2ce7255SDoug Thompson static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr) 463e2ce7255SDoug Thompson { 464e2ce7255SDoug Thompson struct amd64_pvt *pvt; 465e2ce7255SDoug Thompson int csrow; 466e2ce7255SDoug Thompson u64 base, mask; 467e2ce7255SDoug Thompson 468e2ce7255SDoug Thompson pvt = mci->pvt_info; 469e2ce7255SDoug Thompson 47011c75eadSBorislav Petkov for_each_chip_select(csrow, 0, pvt) { 47111c75eadSBorislav Petkov if (!csrow_enabled(csrow, 0, pvt)) 472e2ce7255SDoug Thompson continue; 473e2ce7255SDoug Thompson 47411c75eadSBorislav Petkov get_cs_base_and_mask(pvt, csrow, 0, &base, &mask); 47511c75eadSBorislav Petkov 47611c75eadSBorislav Petkov mask = ~mask; 477e2ce7255SDoug Thompson 478e2ce7255SDoug Thompson if ((input_addr & mask) == (base & mask)) { 479956b9ba1SJoe Perches edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n", 480e2ce7255SDoug Thompson (unsigned long)input_addr, csrow, 481e2ce7255SDoug Thompson pvt->mc_node_id); 482e2ce7255SDoug Thompson 483e2ce7255SDoug Thompson return csrow; 484e2ce7255SDoug Thompson } 485e2ce7255SDoug Thompson } 486956b9ba1SJoe Perches edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n", 487e2ce7255SDoug Thompson (unsigned long)input_addr, pvt->mc_node_id); 488e2ce7255SDoug Thompson 489e2ce7255SDoug Thompson return -1; 490e2ce7255SDoug Thompson } 491e2ce7255SDoug Thompson 492e2ce7255SDoug Thompson /* 493e2ce7255SDoug Thompson * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094) 494e2ce7255SDoug Thompson * for the node represented by mci. Info is passed back in *hole_base, 495e2ce7255SDoug Thompson * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if 496e2ce7255SDoug Thompson * info is invalid. Info may be invalid for either of the following reasons: 497e2ce7255SDoug Thompson * 498e2ce7255SDoug Thompson * - The revision of the node is not E or greater. In this case, the DRAM Hole 499e2ce7255SDoug Thompson * Address Register does not exist. 500e2ce7255SDoug Thompson * 501e2ce7255SDoug Thompson * - The DramHoleValid bit is cleared in the DRAM Hole Address Register, 502e2ce7255SDoug Thompson * indicating that its contents are not valid. 503e2ce7255SDoug Thompson * 504e2ce7255SDoug Thompson * The values passed back in *hole_base, *hole_offset, and *hole_size are 505e2ce7255SDoug Thompson * complete 32-bit values despite the fact that the bitfields in the DHAR 506e2ce7255SDoug Thompson * only represent bits 31-24 of the base and offset values. 507e2ce7255SDoug Thompson */ 508e2ce7255SDoug Thompson int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base, 509e2ce7255SDoug Thompson u64 *hole_offset, u64 *hole_size) 510e2ce7255SDoug Thompson { 511e2ce7255SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 512e2ce7255SDoug Thompson 513e2ce7255SDoug Thompson /* only revE and later have the DRAM Hole Address Register */ 514a4b4bedcSBorislav Petkov if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) { 515956b9ba1SJoe Perches edac_dbg(1, " revision %d for node %d does not support DHAR\n", 516e2ce7255SDoug Thompson pvt->ext_model, pvt->mc_node_id); 517e2ce7255SDoug Thompson return 1; 518e2ce7255SDoug Thompson } 519e2ce7255SDoug Thompson 520bc21fa57SBorislav Petkov /* valid for Fam10h and above */ 521a4b4bedcSBorislav Petkov if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) { 522956b9ba1SJoe Perches edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n"); 523e2ce7255SDoug Thompson return 1; 524e2ce7255SDoug Thompson } 525e2ce7255SDoug Thompson 526c8e518d5SBorislav Petkov if (!dhar_valid(pvt)) { 527956b9ba1SJoe Perches edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n", 528e2ce7255SDoug Thompson pvt->mc_node_id); 529e2ce7255SDoug Thompson return 1; 530e2ce7255SDoug Thompson } 531e2ce7255SDoug Thompson 532e2ce7255SDoug Thompson /* This node has Memory Hoisting */ 533e2ce7255SDoug Thompson 534e2ce7255SDoug Thompson /* +------------------+--------------------+--------------------+----- 535e2ce7255SDoug Thompson * | memory | DRAM hole | relocated | 536e2ce7255SDoug Thompson * | [0, (x - 1)] | [x, 0xffffffff] | addresses from | 537e2ce7255SDoug Thompson * | | | DRAM hole | 538e2ce7255SDoug Thompson * | | | [0x100000000, | 539e2ce7255SDoug Thompson * | | | (0x100000000+ | 540e2ce7255SDoug Thompson * | | | (0xffffffff-x))] | 541e2ce7255SDoug Thompson * +------------------+--------------------+--------------------+----- 542e2ce7255SDoug Thompson * 543e2ce7255SDoug Thompson * Above is a diagram of physical memory showing the DRAM hole and the 544e2ce7255SDoug Thompson * relocated addresses from the DRAM hole. As shown, the DRAM hole 545e2ce7255SDoug Thompson * starts at address x (the base address) and extends through address 546e2ce7255SDoug Thompson * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the 547e2ce7255SDoug Thompson * addresses in the hole so that they start at 0x100000000. 548e2ce7255SDoug Thompson */ 549e2ce7255SDoug Thompson 5501f31677eSBorislav Petkov *hole_base = dhar_base(pvt); 5511f31677eSBorislav Petkov *hole_size = (1ULL << 32) - *hole_base; 552e2ce7255SDoug Thompson 553a4b4bedcSBorislav Petkov *hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt) 554a4b4bedcSBorislav Petkov : k8_dhar_offset(pvt); 555e2ce7255SDoug Thompson 556956b9ba1SJoe Perches edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n", 557e2ce7255SDoug Thompson pvt->mc_node_id, (unsigned long)*hole_base, 558e2ce7255SDoug Thompson (unsigned long)*hole_offset, (unsigned long)*hole_size); 559e2ce7255SDoug Thompson 560e2ce7255SDoug Thompson return 0; 561e2ce7255SDoug Thompson } 562e2ce7255SDoug Thompson EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info); 563e2ce7255SDoug Thompson 56493c2df58SDoug Thompson /* 56593c2df58SDoug Thompson * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is 56693c2df58SDoug Thompson * assumed that sys_addr maps to the node given by mci. 56793c2df58SDoug Thompson * 56893c2df58SDoug Thompson * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section 56993c2df58SDoug Thompson * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a 57093c2df58SDoug Thompson * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled, 57193c2df58SDoug Thompson * then it is also involved in translating a SysAddr to a DramAddr. Sections 57293c2df58SDoug Thompson * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting. 57393c2df58SDoug Thompson * These parts of the documentation are unclear. I interpret them as follows: 57493c2df58SDoug Thompson * 57593c2df58SDoug Thompson * When node n receives a SysAddr, it processes the SysAddr as follows: 57693c2df58SDoug Thompson * 57793c2df58SDoug Thompson * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM 57893c2df58SDoug Thompson * Limit registers for node n. If the SysAddr is not within the range 57993c2df58SDoug Thompson * specified by the base and limit values, then node n ignores the Sysaddr 58093c2df58SDoug Thompson * (since it does not map to node n). Otherwise continue to step 2 below. 58193c2df58SDoug Thompson * 58293c2df58SDoug Thompson * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is 58393c2df58SDoug Thompson * disabled so skip to step 3 below. Otherwise see if the SysAddr is within 58493c2df58SDoug Thompson * the range of relocated addresses (starting at 0x100000000) from the DRAM 58593c2df58SDoug Thompson * hole. If not, skip to step 3 below. Else get the value of the 58693c2df58SDoug Thompson * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the 58793c2df58SDoug Thompson * offset defined by this value from the SysAddr. 58893c2df58SDoug Thompson * 58993c2df58SDoug Thompson * 3. Obtain the base address for node n from the DRAMBase field of the DRAM 59093c2df58SDoug Thompson * Base register for node n. To obtain the DramAddr, subtract the base 59193c2df58SDoug Thompson * address from the SysAddr, as shown near the start of section 3.4.4 (p.70). 59293c2df58SDoug Thompson */ 59393c2df58SDoug Thompson static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr) 59493c2df58SDoug Thompson { 5957f19bf75SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 59693c2df58SDoug Thompson u64 dram_base, hole_base, hole_offset, hole_size, dram_addr; 5971f31677eSBorislav Petkov int ret; 59893c2df58SDoug Thompson 5997f19bf75SBorislav Petkov dram_base = get_dram_base(pvt, pvt->mc_node_id); 60093c2df58SDoug Thompson 60193c2df58SDoug Thompson ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset, 60293c2df58SDoug Thompson &hole_size); 60393c2df58SDoug Thompson if (!ret) { 6041f31677eSBorislav Petkov if ((sys_addr >= (1ULL << 32)) && 6051f31677eSBorislav Petkov (sys_addr < ((1ULL << 32) + hole_size))) { 60693c2df58SDoug Thompson /* use DHAR to translate SysAddr to DramAddr */ 60793c2df58SDoug Thompson dram_addr = sys_addr - hole_offset; 60893c2df58SDoug Thompson 609956b9ba1SJoe Perches edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n", 61093c2df58SDoug Thompson (unsigned long)sys_addr, 61193c2df58SDoug Thompson (unsigned long)dram_addr); 61293c2df58SDoug Thompson 61393c2df58SDoug Thompson return dram_addr; 61493c2df58SDoug Thompson } 61593c2df58SDoug Thompson } 61693c2df58SDoug Thompson 61793c2df58SDoug Thompson /* 61893c2df58SDoug Thompson * Translate the SysAddr to a DramAddr as shown near the start of 61993c2df58SDoug Thompson * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8 62093c2df58SDoug Thompson * only deals with 40-bit values. Therefore we discard bits 63-40 of 62193c2df58SDoug Thompson * sys_addr below. If bit 39 of sys_addr is 1 then the bits we 62293c2df58SDoug Thompson * discard are all 1s. Otherwise the bits we discard are all 0s. See 62393c2df58SDoug Thompson * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture 62493c2df58SDoug Thompson * Programmer's Manual Volume 1 Application Programming. 62593c2df58SDoug Thompson */ 62610ef6b0dSChen, Gong dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base; 62793c2df58SDoug Thompson 628956b9ba1SJoe Perches edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n", 629956b9ba1SJoe Perches (unsigned long)sys_addr, (unsigned long)dram_addr); 63093c2df58SDoug Thompson return dram_addr; 63193c2df58SDoug Thompson } 63293c2df58SDoug Thompson 63393c2df58SDoug Thompson /* 63493c2df58SDoug Thompson * @intlv_en is the value of the IntlvEn field from a DRAM Base register 63593c2df58SDoug Thompson * (section 3.4.4.1). Return the number of bits from a SysAddr that are used 63693c2df58SDoug Thompson * for node interleaving. 63793c2df58SDoug Thompson */ 63893c2df58SDoug Thompson static int num_node_interleave_bits(unsigned intlv_en) 63993c2df58SDoug Thompson { 64093c2df58SDoug Thompson static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 }; 64193c2df58SDoug Thompson int n; 64293c2df58SDoug Thompson 64393c2df58SDoug Thompson BUG_ON(intlv_en > 7); 64493c2df58SDoug Thompson n = intlv_shift_table[intlv_en]; 64593c2df58SDoug Thompson return n; 64693c2df58SDoug Thompson } 64793c2df58SDoug Thompson 64893c2df58SDoug Thompson /* Translate the DramAddr given by @dram_addr to an InputAddr. */ 64993c2df58SDoug Thompson static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr) 65093c2df58SDoug Thompson { 65193c2df58SDoug Thompson struct amd64_pvt *pvt; 65293c2df58SDoug Thompson int intlv_shift; 65393c2df58SDoug Thompson u64 input_addr; 65493c2df58SDoug Thompson 65593c2df58SDoug Thompson pvt = mci->pvt_info; 65693c2df58SDoug Thompson 65793c2df58SDoug Thompson /* 65893c2df58SDoug Thompson * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E) 65993c2df58SDoug Thompson * concerning translating a DramAddr to an InputAddr. 66093c2df58SDoug Thompson */ 6617f19bf75SBorislav Petkov intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0)); 66210ef6b0dSChen, Gong input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) + 66393c2df58SDoug Thompson (dram_addr & 0xfff); 66493c2df58SDoug Thompson 665956b9ba1SJoe Perches edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n", 66693c2df58SDoug Thompson intlv_shift, (unsigned long)dram_addr, 66793c2df58SDoug Thompson (unsigned long)input_addr); 66893c2df58SDoug Thompson 66993c2df58SDoug Thompson return input_addr; 67093c2df58SDoug Thompson } 67193c2df58SDoug Thompson 67293c2df58SDoug Thompson /* 67393c2df58SDoug Thompson * Translate the SysAddr represented by @sys_addr to an InputAddr. It is 67493c2df58SDoug Thompson * assumed that @sys_addr maps to the node given by mci. 67593c2df58SDoug Thompson */ 67693c2df58SDoug Thompson static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr) 67793c2df58SDoug Thompson { 67893c2df58SDoug Thompson u64 input_addr; 67993c2df58SDoug Thompson 68093c2df58SDoug Thompson input_addr = 68193c2df58SDoug Thompson dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr)); 68293c2df58SDoug Thompson 683c19ca6cbSMasanari Iida edac_dbg(2, "SysAddr 0x%lx translates to InputAddr 0x%lx\n", 68493c2df58SDoug Thompson (unsigned long)sys_addr, (unsigned long)input_addr); 68593c2df58SDoug Thompson 68693c2df58SDoug Thompson return input_addr; 68793c2df58SDoug Thompson } 68893c2df58SDoug Thompson 68993c2df58SDoug Thompson /* Map the Error address to a PAGE and PAGE OFFSET. */ 69093c2df58SDoug Thompson static inline void error_address_to_page_and_offset(u64 error_address, 69133ca0643SBorislav Petkov struct err_info *err) 69293c2df58SDoug Thompson { 69333ca0643SBorislav Petkov err->page = (u32) (error_address >> PAGE_SHIFT); 69433ca0643SBorislav Petkov err->offset = ((u32) error_address) & ~PAGE_MASK; 69593c2df58SDoug Thompson } 69693c2df58SDoug Thompson 69793c2df58SDoug Thompson /* 69893c2df58SDoug Thompson * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address 69993c2df58SDoug Thompson * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers 70093c2df58SDoug Thompson * of a node that detected an ECC memory error. mci represents the node that 70193c2df58SDoug Thompson * the error address maps to (possibly different from the node that detected 70293c2df58SDoug Thompson * the error). Return the number of the csrow that sys_addr maps to, or -1 on 70393c2df58SDoug Thompson * error. 70493c2df58SDoug Thompson */ 70593c2df58SDoug Thompson static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr) 70693c2df58SDoug Thompson { 70793c2df58SDoug Thompson int csrow; 70893c2df58SDoug Thompson 70993c2df58SDoug Thompson csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr)); 71093c2df58SDoug Thompson 71193c2df58SDoug Thompson if (csrow == -1) 71224f9a7feSBorislav Petkov amd64_mc_err(mci, "Failed to translate InputAddr to csrow for " 71393c2df58SDoug Thompson "address 0x%lx\n", (unsigned long)sys_addr); 71493c2df58SDoug Thompson return csrow; 71593c2df58SDoug Thompson } 716e2ce7255SDoug Thompson 717bfc04aecSBorislav Petkov static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16); 7182da11654SDoug Thompson 7192da11654SDoug Thompson /* 7202da11654SDoug Thompson * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs 7212da11654SDoug Thompson * are ECC capable. 7222da11654SDoug Thompson */ 723d1ea71cdSBorislav Petkov static unsigned long determine_edac_cap(struct amd64_pvt *pvt) 7242da11654SDoug Thompson { 7251f6189edSDan Carpenter unsigned long edac_cap = EDAC_FLAG_NONE; 726d27f3a34SYazen Ghannam u8 bit; 7272da11654SDoug Thompson 728d27f3a34SYazen Ghannam if (pvt->umc) { 729d27f3a34SYazen Ghannam u8 i, umc_en_mask = 0, dimm_ecc_en_mask = 0; 730d27f3a34SYazen Ghannam 7314d30d2bcSYazen Ghannam for_each_umc(i) { 732d27f3a34SYazen Ghannam if (!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT)) 733d27f3a34SYazen Ghannam continue; 734d27f3a34SYazen Ghannam 735d27f3a34SYazen Ghannam umc_en_mask |= BIT(i); 736d27f3a34SYazen Ghannam 737d27f3a34SYazen Ghannam /* UMC Configuration bit 12 (DimmEccEn) */ 738d27f3a34SYazen Ghannam if (pvt->umc[i].umc_cfg & BIT(12)) 739d27f3a34SYazen Ghannam dimm_ecc_en_mask |= BIT(i); 740d27f3a34SYazen Ghannam } 741d27f3a34SYazen Ghannam 742d27f3a34SYazen Ghannam if (umc_en_mask == dimm_ecc_en_mask) 743d27f3a34SYazen Ghannam edac_cap = EDAC_FLAG_SECDED; 744d27f3a34SYazen Ghannam } else { 745a4b4bedcSBorislav Petkov bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F) 7462da11654SDoug Thompson ? 19 7472da11654SDoug Thompson : 17; 7482da11654SDoug Thompson 749584fcff4SBorislav Petkov if (pvt->dclr0 & BIT(bit)) 7502da11654SDoug Thompson edac_cap = EDAC_FLAG_SECDED; 751d27f3a34SYazen Ghannam } 7522da11654SDoug Thompson 7532da11654SDoug Thompson return edac_cap; 7542da11654SDoug Thompson } 7552da11654SDoug Thompson 756d1ea71cdSBorislav Petkov static void debug_display_dimm_sizes(struct amd64_pvt *, u8); 7572da11654SDoug Thompson 758d1ea71cdSBorislav Petkov static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan) 75968798e17SBorislav Petkov { 760956b9ba1SJoe Perches edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr); 76168798e17SBorislav Petkov 762a597d2a5SAravind Gopalakrishnan if (pvt->dram_type == MEM_LRDDR3) { 763a597d2a5SAravind Gopalakrishnan u32 dcsm = pvt->csels[chan].csmasks[0]; 764a597d2a5SAravind Gopalakrishnan /* 765a597d2a5SAravind Gopalakrishnan * It's assumed all LRDIMMs in a DCT are going to be of 766a597d2a5SAravind Gopalakrishnan * same 'type' until proven otherwise. So, use a cs 767a597d2a5SAravind Gopalakrishnan * value of '0' here to get dcsm value. 768a597d2a5SAravind Gopalakrishnan */ 769a597d2a5SAravind Gopalakrishnan edac_dbg(1, " LRDIMM %dx rank multiply\n", (dcsm & 0x3)); 770a597d2a5SAravind Gopalakrishnan } 771a597d2a5SAravind Gopalakrishnan 772a597d2a5SAravind Gopalakrishnan edac_dbg(1, "All DIMMs support ECC:%s\n", 77368798e17SBorislav Petkov (dclr & BIT(19)) ? "yes" : "no"); 77468798e17SBorislav Petkov 775a597d2a5SAravind Gopalakrishnan 776956b9ba1SJoe Perches edac_dbg(1, " PAR/ERR parity: %s\n", 77768798e17SBorislav Petkov (dclr & BIT(8)) ? "enabled" : "disabled"); 77868798e17SBorislav Petkov 779a4b4bedcSBorislav Petkov if (pvt->fam == 0x10) 780956b9ba1SJoe Perches edac_dbg(1, " DCT 128bit mode width: %s\n", 78168798e17SBorislav Petkov (dclr & BIT(11)) ? "128b" : "64b"); 78268798e17SBorislav Petkov 783956b9ba1SJoe Perches edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n", 78468798e17SBorislav Petkov (dclr & BIT(12)) ? "yes" : "no", 78568798e17SBorislav Petkov (dclr & BIT(13)) ? "yes" : "no", 78668798e17SBorislav Petkov (dclr & BIT(14)) ? "yes" : "no", 78768798e17SBorislav Petkov (dclr & BIT(15)) ? "yes" : "no"); 78868798e17SBorislav Petkov } 78968798e17SBorislav Petkov 790e53a3b26SYazen Ghannam #define CS_EVEN_PRIMARY BIT(0) 791e53a3b26SYazen Ghannam #define CS_ODD_PRIMARY BIT(1) 79281f5090dSYazen Ghannam #define CS_EVEN_SECONDARY BIT(2) 79381f5090dSYazen Ghannam #define CS_ODD_SECONDARY BIT(3) 794e53a3b26SYazen Ghannam 79581f5090dSYazen Ghannam #define CS_EVEN (CS_EVEN_PRIMARY | CS_EVEN_SECONDARY) 79681f5090dSYazen Ghannam #define CS_ODD (CS_ODD_PRIMARY | CS_ODD_SECONDARY) 797e53a3b26SYazen Ghannam 798e53a3b26SYazen Ghannam static int f17_get_cs_mode(int dimm, u8 ctrl, struct amd64_pvt *pvt) 799fc00c6a4SYazen Ghannam { 800e53a3b26SYazen Ghannam int cs_mode = 0; 801fc00c6a4SYazen Ghannam 802e53a3b26SYazen Ghannam if (csrow_enabled(2 * dimm, ctrl, pvt)) 803e53a3b26SYazen Ghannam cs_mode |= CS_EVEN_PRIMARY; 804fc00c6a4SYazen Ghannam 805e53a3b26SYazen Ghannam if (csrow_enabled(2 * dimm + 1, ctrl, pvt)) 806e53a3b26SYazen Ghannam cs_mode |= CS_ODD_PRIMARY; 807e53a3b26SYazen Ghannam 80881f5090dSYazen Ghannam /* Asymmetric dual-rank DIMM support. */ 80981f5090dSYazen Ghannam if (csrow_sec_enabled(2 * dimm + 1, ctrl, pvt)) 81081f5090dSYazen Ghannam cs_mode |= CS_ODD_SECONDARY; 81181f5090dSYazen Ghannam 812e53a3b26SYazen Ghannam return cs_mode; 813fc00c6a4SYazen Ghannam } 814fc00c6a4SYazen Ghannam 81507ed82efSYazen Ghannam static void debug_display_dimm_sizes_df(struct amd64_pvt *pvt, u8 ctrl) 81607ed82efSYazen Ghannam { 817e53a3b26SYazen Ghannam int dimm, size0, size1, cs0, cs1, cs_mode; 81807ed82efSYazen Ghannam 81907ed82efSYazen Ghannam edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:\n", ctrl); 82007ed82efSYazen Ghannam 821d971e28eSYazen Ghannam for (dimm = 0; dimm < 2; dimm++) { 822eb77e6b8SYazen Ghannam cs0 = dimm * 2; 823eb77e6b8SYazen Ghannam cs1 = dimm * 2 + 1; 824eb77e6b8SYazen Ghannam 825e53a3b26SYazen Ghannam cs_mode = f17_get_cs_mode(dimm, ctrl, pvt); 826e53a3b26SYazen Ghannam 827e53a3b26SYazen Ghannam size0 = pvt->ops->dbam_to_cs(pvt, ctrl, cs_mode, cs0); 828e53a3b26SYazen Ghannam size1 = pvt->ops->dbam_to_cs(pvt, ctrl, cs_mode, cs1); 82907ed82efSYazen Ghannam 83007ed82efSYazen Ghannam amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n", 831eb77e6b8SYazen Ghannam cs0, size0, 832eb77e6b8SYazen Ghannam cs1, size1); 83307ed82efSYazen Ghannam } 83407ed82efSYazen Ghannam } 83507ed82efSYazen Ghannam 83607ed82efSYazen Ghannam static void __dump_misc_regs_df(struct amd64_pvt *pvt) 83707ed82efSYazen Ghannam { 83807ed82efSYazen Ghannam struct amd64_umc *umc; 83907ed82efSYazen Ghannam u32 i, tmp, umc_base; 84007ed82efSYazen Ghannam 8414d30d2bcSYazen Ghannam for_each_umc(i) { 84207ed82efSYazen Ghannam umc_base = get_umc_base(i); 84307ed82efSYazen Ghannam umc = &pvt->umc[i]; 84407ed82efSYazen Ghannam 84507ed82efSYazen Ghannam edac_dbg(1, "UMC%d DIMM cfg: 0x%x\n", i, umc->dimm_cfg); 84607ed82efSYazen Ghannam edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i, umc->umc_cfg); 84707ed82efSYazen Ghannam edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i, umc->sdp_ctrl); 84807ed82efSYazen Ghannam edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i, umc->ecc_ctrl); 84907ed82efSYazen Ghannam 85007ed82efSYazen Ghannam amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ECC_BAD_SYMBOL, &tmp); 85107ed82efSYazen Ghannam edac_dbg(1, "UMC%d ECC bad symbol: 0x%x\n", i, tmp); 85207ed82efSYazen Ghannam 85307ed82efSYazen Ghannam amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_UMC_CAP, &tmp); 85407ed82efSYazen Ghannam edac_dbg(1, "UMC%d UMC cap: 0x%x\n", i, tmp); 85507ed82efSYazen Ghannam edac_dbg(1, "UMC%d UMC cap high: 0x%x\n", i, umc->umc_cap_hi); 85607ed82efSYazen Ghannam 85707ed82efSYazen Ghannam edac_dbg(1, "UMC%d ECC capable: %s, ChipKill ECC capable: %s\n", 85807ed82efSYazen Ghannam i, (umc->umc_cap_hi & BIT(30)) ? "yes" : "no", 85907ed82efSYazen Ghannam (umc->umc_cap_hi & BIT(31)) ? "yes" : "no"); 86007ed82efSYazen Ghannam edac_dbg(1, "UMC%d All DIMMs support ECC: %s\n", 86107ed82efSYazen Ghannam i, (umc->umc_cfg & BIT(12)) ? "yes" : "no"); 86207ed82efSYazen Ghannam edac_dbg(1, "UMC%d x4 DIMMs present: %s\n", 86307ed82efSYazen Ghannam i, (umc->dimm_cfg & BIT(6)) ? "yes" : "no"); 86407ed82efSYazen Ghannam edac_dbg(1, "UMC%d x16 DIMMs present: %s\n", 86507ed82efSYazen Ghannam i, (umc->dimm_cfg & BIT(7)) ? "yes" : "no"); 86607ed82efSYazen Ghannam 86707ed82efSYazen Ghannam if (pvt->dram_type == MEM_LRDDR4) { 86807ed82efSYazen Ghannam amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ADDR_CFG, &tmp); 86907ed82efSYazen Ghannam edac_dbg(1, "UMC%d LRDIMM %dx rank multiply\n", 87007ed82efSYazen Ghannam i, 1 << ((tmp >> 4) & 0x3)); 87107ed82efSYazen Ghannam } 87207ed82efSYazen Ghannam 87307ed82efSYazen Ghannam debug_display_dimm_sizes_df(pvt, i); 87407ed82efSYazen Ghannam } 87507ed82efSYazen Ghannam 87607ed82efSYazen Ghannam edac_dbg(1, "F0x104 (DRAM Hole Address): 0x%08x, base: 0x%08x\n", 87707ed82efSYazen Ghannam pvt->dhar, dhar_base(pvt)); 87807ed82efSYazen Ghannam } 87907ed82efSYazen Ghannam 8802da11654SDoug Thompson /* Display and decode various NB registers for debug purposes. */ 88107ed82efSYazen Ghannam static void __dump_misc_regs(struct amd64_pvt *pvt) 8822da11654SDoug Thompson { 883956b9ba1SJoe Perches edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap); 8842da11654SDoug Thompson 885956b9ba1SJoe Perches edac_dbg(1, " NB two channel DRAM capable: %s\n", 8865980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no"); 88768798e17SBorislav Petkov 888956b9ba1SJoe Perches edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n", 8895980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no", 8905980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no"); 89168798e17SBorislav Petkov 892d1ea71cdSBorislav Petkov debug_dump_dramcfg_low(pvt, pvt->dclr0, 0); 8932da11654SDoug Thompson 894956b9ba1SJoe Perches edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare); 8952da11654SDoug Thompson 896956b9ba1SJoe Perches edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n", 897bc21fa57SBorislav Petkov pvt->dhar, dhar_base(pvt), 898a4b4bedcSBorislav Petkov (pvt->fam == 0xf) ? k8_dhar_offset(pvt) 899bc21fa57SBorislav Petkov : f10_dhar_offset(pvt)); 9002da11654SDoug Thompson 901d1ea71cdSBorislav Petkov debug_display_dimm_sizes(pvt, 0); 9024d796364SBorislav Petkov 9034d796364SBorislav Petkov /* everything below this point is Fam10h and above */ 904a4b4bedcSBorislav Petkov if (pvt->fam == 0xf) 9052da11654SDoug Thompson return; 9064d796364SBorislav Petkov 907d1ea71cdSBorislav Petkov debug_display_dimm_sizes(pvt, 1); 9082da11654SDoug Thompson 9098de1d91eSBorislav Petkov /* Only if NOT ganged does dclr1 have valid info */ 91068798e17SBorislav Petkov if (!dct_ganging_enabled(pvt)) 911d1ea71cdSBorislav Petkov debug_dump_dramcfg_low(pvt, pvt->dclr1, 1); 9122da11654SDoug Thompson } 9132da11654SDoug Thompson 91407ed82efSYazen Ghannam /* Display and decode various NB registers for debug purposes. */ 91507ed82efSYazen Ghannam static void dump_misc_regs(struct amd64_pvt *pvt) 91607ed82efSYazen Ghannam { 91707ed82efSYazen Ghannam if (pvt->umc) 91807ed82efSYazen Ghannam __dump_misc_regs_df(pvt); 91907ed82efSYazen Ghannam else 92007ed82efSYazen Ghannam __dump_misc_regs(pvt); 92107ed82efSYazen Ghannam 92207ed82efSYazen Ghannam edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no"); 92307ed82efSYazen Ghannam 9247835961dSYazen Ghannam amd64_info("using x%u syndromes.\n", pvt->ecc_sym_sz); 92507ed82efSYazen Ghannam } 92607ed82efSYazen Ghannam 92794be4bffSDoug Thompson /* 92818b94f66SAravind Gopalakrishnan * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60] 92994be4bffSDoug Thompson */ 93011c75eadSBorislav Petkov static void prep_chip_selects(struct amd64_pvt *pvt) 93194be4bffSDoug Thompson { 93218b94f66SAravind Gopalakrishnan if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) { 93311c75eadSBorislav Petkov pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8; 93411c75eadSBorislav Petkov pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8; 935a597d2a5SAravind Gopalakrishnan } else if (pvt->fam == 0x15 && pvt->model == 0x30) { 93618b94f66SAravind Gopalakrishnan pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4; 93718b94f66SAravind Gopalakrishnan pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2; 938d971e28eSYazen Ghannam } else if (pvt->fam >= 0x17) { 939d971e28eSYazen Ghannam int umc; 940d971e28eSYazen Ghannam 941d971e28eSYazen Ghannam for_each_umc(umc) { 942d971e28eSYazen Ghannam pvt->csels[umc].b_cnt = 4; 943d971e28eSYazen Ghannam pvt->csels[umc].m_cnt = 2; 944d971e28eSYazen Ghannam } 945d971e28eSYazen Ghannam 9469d858bb1SBorislav Petkov } else { 94711c75eadSBorislav Petkov pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8; 94811c75eadSBorislav Petkov pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4; 9499d858bb1SBorislav Petkov } 95094be4bffSDoug Thompson } 95194be4bffSDoug Thompson 952d971e28eSYazen Ghannam static void read_umc_base_mask(struct amd64_pvt *pvt) 953d971e28eSYazen Ghannam { 9547574729eSYazen Ghannam u32 umc_base_reg, umc_base_reg_sec; 9557574729eSYazen Ghannam u32 umc_mask_reg, umc_mask_reg_sec; 9567574729eSYazen Ghannam u32 base_reg, base_reg_sec; 9577574729eSYazen Ghannam u32 mask_reg, mask_reg_sec; 9587574729eSYazen Ghannam u32 *base, *base_sec; 9597574729eSYazen Ghannam u32 *mask, *mask_sec; 960d971e28eSYazen Ghannam int cs, umc; 961d971e28eSYazen Ghannam 962d971e28eSYazen Ghannam for_each_umc(umc) { 963d971e28eSYazen Ghannam umc_base_reg = get_umc_base(umc) + UMCCH_BASE_ADDR; 9647574729eSYazen Ghannam umc_base_reg_sec = get_umc_base(umc) + UMCCH_BASE_ADDR_SEC; 965d971e28eSYazen Ghannam 966d971e28eSYazen Ghannam for_each_chip_select(cs, umc, pvt) { 967d971e28eSYazen Ghannam base = &pvt->csels[umc].csbases[cs]; 9687574729eSYazen Ghannam base_sec = &pvt->csels[umc].csbases_sec[cs]; 969d971e28eSYazen Ghannam 970d971e28eSYazen Ghannam base_reg = umc_base_reg + (cs * 4); 9717574729eSYazen Ghannam base_reg_sec = umc_base_reg_sec + (cs * 4); 972d971e28eSYazen Ghannam 973d971e28eSYazen Ghannam if (!amd_smn_read(pvt->mc_node_id, base_reg, base)) 974d971e28eSYazen Ghannam edac_dbg(0, " DCSB%d[%d]=0x%08x reg: 0x%x\n", 975d971e28eSYazen Ghannam umc, cs, *base, base_reg); 9767574729eSYazen Ghannam 9777574729eSYazen Ghannam if (!amd_smn_read(pvt->mc_node_id, base_reg_sec, base_sec)) 9787574729eSYazen Ghannam edac_dbg(0, " DCSB_SEC%d[%d]=0x%08x reg: 0x%x\n", 9797574729eSYazen Ghannam umc, cs, *base_sec, base_reg_sec); 980d971e28eSYazen Ghannam } 981d971e28eSYazen Ghannam 982d971e28eSYazen Ghannam umc_mask_reg = get_umc_base(umc) + UMCCH_ADDR_MASK; 9837574729eSYazen Ghannam umc_mask_reg_sec = get_umc_base(umc) + UMCCH_ADDR_MASK_SEC; 984d971e28eSYazen Ghannam 985d971e28eSYazen Ghannam for_each_chip_select_mask(cs, umc, pvt) { 986d971e28eSYazen Ghannam mask = &pvt->csels[umc].csmasks[cs]; 9877574729eSYazen Ghannam mask_sec = &pvt->csels[umc].csmasks_sec[cs]; 988d971e28eSYazen Ghannam 989d971e28eSYazen Ghannam mask_reg = umc_mask_reg + (cs * 4); 9907574729eSYazen Ghannam mask_reg_sec = umc_mask_reg_sec + (cs * 4); 991d971e28eSYazen Ghannam 992d971e28eSYazen Ghannam if (!amd_smn_read(pvt->mc_node_id, mask_reg, mask)) 993d971e28eSYazen Ghannam edac_dbg(0, " DCSM%d[%d]=0x%08x reg: 0x%x\n", 994d971e28eSYazen Ghannam umc, cs, *mask, mask_reg); 9957574729eSYazen Ghannam 9967574729eSYazen Ghannam if (!amd_smn_read(pvt->mc_node_id, mask_reg_sec, mask_sec)) 9977574729eSYazen Ghannam edac_dbg(0, " DCSM_SEC%d[%d]=0x%08x reg: 0x%x\n", 9987574729eSYazen Ghannam umc, cs, *mask_sec, mask_reg_sec); 999d971e28eSYazen Ghannam } 1000d971e28eSYazen Ghannam } 1001d971e28eSYazen Ghannam } 1002d971e28eSYazen Ghannam 100394be4bffSDoug Thompson /* 100411c75eadSBorislav Petkov * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers 100594be4bffSDoug Thompson */ 1006b2b0c605SBorislav Petkov static void read_dct_base_mask(struct amd64_pvt *pvt) 100794be4bffSDoug Thompson { 1008d971e28eSYazen Ghannam int cs; 100994be4bffSDoug Thompson 101011c75eadSBorislav Petkov prep_chip_selects(pvt); 101194be4bffSDoug Thompson 1012d971e28eSYazen Ghannam if (pvt->umc) 1013d971e28eSYazen Ghannam return read_umc_base_mask(pvt); 1014b64ce7cdSYazen Ghannam 101511c75eadSBorislav Petkov for_each_chip_select(cs, 0, pvt) { 1016d971e28eSYazen Ghannam int reg0 = DCSB0 + (cs * 4); 1017d971e28eSYazen Ghannam int reg1 = DCSB1 + (cs * 4); 101811c75eadSBorislav Petkov u32 *base0 = &pvt->csels[0].csbases[cs]; 101911c75eadSBorislav Petkov u32 *base1 = &pvt->csels[1].csbases[cs]; 1020b2b0c605SBorislav Petkov 10217981a28fSAravind Gopalakrishnan if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0)) 1022956b9ba1SJoe Perches edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n", 102311c75eadSBorislav Petkov cs, *base0, reg0); 102494be4bffSDoug Thompson 10257981a28fSAravind Gopalakrishnan if (pvt->fam == 0xf) 102611c75eadSBorislav Petkov continue; 1027b2b0c605SBorislav Petkov 10287981a28fSAravind Gopalakrishnan if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1)) 1029956b9ba1SJoe Perches edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n", 10307981a28fSAravind Gopalakrishnan cs, *base1, (pvt->fam == 0x10) ? reg1 10317981a28fSAravind Gopalakrishnan : reg0); 103294be4bffSDoug Thompson } 103394be4bffSDoug Thompson 103411c75eadSBorislav Petkov for_each_chip_select_mask(cs, 0, pvt) { 1035d971e28eSYazen Ghannam int reg0 = DCSM0 + (cs * 4); 1036d971e28eSYazen Ghannam int reg1 = DCSM1 + (cs * 4); 103711c75eadSBorislav Petkov u32 *mask0 = &pvt->csels[0].csmasks[cs]; 103811c75eadSBorislav Petkov u32 *mask1 = &pvt->csels[1].csmasks[cs]; 1039b2b0c605SBorislav Petkov 10407981a28fSAravind Gopalakrishnan if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0)) 1041956b9ba1SJoe Perches edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n", 104211c75eadSBorislav Petkov cs, *mask0, reg0); 104394be4bffSDoug Thompson 10447981a28fSAravind Gopalakrishnan if (pvt->fam == 0xf) 104511c75eadSBorislav Petkov continue; 1046b2b0c605SBorislav Petkov 10477981a28fSAravind Gopalakrishnan if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1)) 1048956b9ba1SJoe Perches edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n", 10497981a28fSAravind Gopalakrishnan cs, *mask1, (pvt->fam == 0x10) ? reg1 10507981a28fSAravind Gopalakrishnan : reg0); 105194be4bffSDoug Thompson } 10526ba5dcdcSBorislav Petkov } 105394be4bffSDoug Thompson 1054a597d2a5SAravind Gopalakrishnan static void determine_memory_type(struct amd64_pvt *pvt) 105594be4bffSDoug Thompson { 1056a597d2a5SAravind Gopalakrishnan u32 dram_ctrl, dcsm; 105794be4bffSDoug Thompson 1058a597d2a5SAravind Gopalakrishnan switch (pvt->fam) { 1059a597d2a5SAravind Gopalakrishnan case 0xf: 1060a597d2a5SAravind Gopalakrishnan if (pvt->ext_model >= K8_REV_F) 1061a597d2a5SAravind Gopalakrishnan goto ddr3; 1062a597d2a5SAravind Gopalakrishnan 1063a597d2a5SAravind Gopalakrishnan pvt->dram_type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR; 1064a597d2a5SAravind Gopalakrishnan return; 1065a597d2a5SAravind Gopalakrishnan 1066a597d2a5SAravind Gopalakrishnan case 0x10: 10676b4c0bdeSBorislav Petkov if (pvt->dchr0 & DDR3_MODE) 1068a597d2a5SAravind Gopalakrishnan goto ddr3; 1069a597d2a5SAravind Gopalakrishnan 1070a597d2a5SAravind Gopalakrishnan pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2; 1071a597d2a5SAravind Gopalakrishnan return; 1072a597d2a5SAravind Gopalakrishnan 1073a597d2a5SAravind Gopalakrishnan case 0x15: 1074a597d2a5SAravind Gopalakrishnan if (pvt->model < 0x60) 1075a597d2a5SAravind Gopalakrishnan goto ddr3; 1076a597d2a5SAravind Gopalakrishnan 1077a597d2a5SAravind Gopalakrishnan /* 1078a597d2a5SAravind Gopalakrishnan * Model 0x60h needs special handling: 1079a597d2a5SAravind Gopalakrishnan * 1080a597d2a5SAravind Gopalakrishnan * We use a Chip Select value of '0' to obtain dcsm. 1081a597d2a5SAravind Gopalakrishnan * Theoretically, it is possible to populate LRDIMMs of different 1082a597d2a5SAravind Gopalakrishnan * 'Rank' value on a DCT. But this is not the common case. So, 1083a597d2a5SAravind Gopalakrishnan * it's reasonable to assume all DIMMs are going to be of same 1084a597d2a5SAravind Gopalakrishnan * 'type' until proven otherwise. 1085a597d2a5SAravind Gopalakrishnan */ 1086a597d2a5SAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 0, DRAM_CONTROL, &dram_ctrl); 1087a597d2a5SAravind Gopalakrishnan dcsm = pvt->csels[0].csmasks[0]; 1088a597d2a5SAravind Gopalakrishnan 1089a597d2a5SAravind Gopalakrishnan if (((dram_ctrl >> 8) & 0x7) == 0x2) 1090a597d2a5SAravind Gopalakrishnan pvt->dram_type = MEM_DDR4; 1091a597d2a5SAravind Gopalakrishnan else if (pvt->dclr0 & BIT(16)) 1092a597d2a5SAravind Gopalakrishnan pvt->dram_type = MEM_DDR3; 1093a597d2a5SAravind Gopalakrishnan else if (dcsm & 0x3) 1094a597d2a5SAravind Gopalakrishnan pvt->dram_type = MEM_LRDDR3; 10956b4c0bdeSBorislav Petkov else 1096a597d2a5SAravind Gopalakrishnan pvt->dram_type = MEM_RDDR3; 1097a597d2a5SAravind Gopalakrishnan 1098a597d2a5SAravind Gopalakrishnan return; 1099a597d2a5SAravind Gopalakrishnan 1100a597d2a5SAravind Gopalakrishnan case 0x16: 1101a597d2a5SAravind Gopalakrishnan goto ddr3; 1102a597d2a5SAravind Gopalakrishnan 1103b64ce7cdSYazen Ghannam case 0x17: 1104c4a3e946SPu Wen case 0x18: 1105b64ce7cdSYazen Ghannam if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(5)) 1106b64ce7cdSYazen Ghannam pvt->dram_type = MEM_LRDDR4; 1107b64ce7cdSYazen Ghannam else if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(4)) 1108b64ce7cdSYazen Ghannam pvt->dram_type = MEM_RDDR4; 1109b64ce7cdSYazen Ghannam else 1110b64ce7cdSYazen Ghannam pvt->dram_type = MEM_DDR4; 1111b64ce7cdSYazen Ghannam return; 1112b64ce7cdSYazen Ghannam 1113a597d2a5SAravind Gopalakrishnan default: 1114a597d2a5SAravind Gopalakrishnan WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam); 1115a597d2a5SAravind Gopalakrishnan pvt->dram_type = MEM_EMPTY; 111694be4bffSDoug Thompson } 1117a597d2a5SAravind Gopalakrishnan return; 111894be4bffSDoug Thompson 1119a597d2a5SAravind Gopalakrishnan ddr3: 1120a597d2a5SAravind Gopalakrishnan pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3; 112194be4bffSDoug Thompson } 112294be4bffSDoug Thompson 1123cb328507SBorislav Petkov /* Get the number of DCT channels the memory controller is using. */ 1124ddff876dSDoug Thompson static int k8_early_channel_count(struct amd64_pvt *pvt) 1125ddff876dSDoug Thompson { 1126cb328507SBorislav Petkov int flag; 1127ddff876dSDoug Thompson 11289f56da0eSBorislav Petkov if (pvt->ext_model >= K8_REV_F) 1129ddff876dSDoug Thompson /* RevF (NPT) and later */ 113041d8bfabSBorislav Petkov flag = pvt->dclr0 & WIDTH_128; 11319f56da0eSBorislav Petkov else 1132ddff876dSDoug Thompson /* RevE and earlier */ 1133ddff876dSDoug Thompson flag = pvt->dclr0 & REVE_WIDTH_128; 1134ddff876dSDoug Thompson 1135ddff876dSDoug Thompson /* not used */ 1136ddff876dSDoug Thompson pvt->dclr1 = 0; 1137ddff876dSDoug Thompson 1138ddff876dSDoug Thompson return (flag) ? 2 : 1; 1139ddff876dSDoug Thompson } 1140ddff876dSDoug Thompson 114170046624SBorislav Petkov /* On F10h and later ErrAddr is MC4_ADDR[47:1] */ 1142a4b4bedcSBorislav Petkov static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m) 1143ddff876dSDoug Thompson { 11442ec591acSBorislav Petkov u16 mce_nid = amd_get_nb_id(m->extcpu); 11452ec591acSBorislav Petkov struct mem_ctl_info *mci; 114670046624SBorislav Petkov u8 start_bit = 1; 114770046624SBorislav Petkov u8 end_bit = 47; 11482ec591acSBorislav Petkov u64 addr; 11492ec591acSBorislav Petkov 11502ec591acSBorislav Petkov mci = edac_mc_find(mce_nid); 11512ec591acSBorislav Petkov if (!mci) 11522ec591acSBorislav Petkov return 0; 11532ec591acSBorislav Petkov 11542ec591acSBorislav Petkov pvt = mci->pvt_info; 115570046624SBorislav Petkov 1156a4b4bedcSBorislav Petkov if (pvt->fam == 0xf) { 115770046624SBorislav Petkov start_bit = 3; 115870046624SBorislav Petkov end_bit = 39; 115970046624SBorislav Petkov } 116070046624SBorislav Petkov 116110ef6b0dSChen, Gong addr = m->addr & GENMASK_ULL(end_bit, start_bit); 1162c1ae6830SBorislav Petkov 1163c1ae6830SBorislav Petkov /* 1164c1ae6830SBorislav Petkov * Erratum 637 workaround 1165c1ae6830SBorislav Petkov */ 1166a4b4bedcSBorislav Petkov if (pvt->fam == 0x15) { 1167c1ae6830SBorislav Petkov u64 cc6_base, tmp_addr; 1168c1ae6830SBorislav Petkov u32 tmp; 11698b84c8dfSDaniel J Blueman u8 intlv_en; 1170c1ae6830SBorislav Petkov 117110ef6b0dSChen, Gong if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7) 1172c1ae6830SBorislav Petkov return addr; 1173c1ae6830SBorislav Petkov 1174c1ae6830SBorislav Petkov 1175c1ae6830SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp); 1176c1ae6830SBorislav Petkov intlv_en = tmp >> 21 & 0x7; 1177c1ae6830SBorislav Petkov 1178c1ae6830SBorislav Petkov /* add [47:27] + 3 trailing bits */ 117910ef6b0dSChen, Gong cc6_base = (tmp & GENMASK_ULL(20, 0)) << 3; 1180c1ae6830SBorislav Petkov 1181c1ae6830SBorislav Petkov /* reverse and add DramIntlvEn */ 1182c1ae6830SBorislav Petkov cc6_base |= intlv_en ^ 0x7; 1183c1ae6830SBorislav Petkov 1184c1ae6830SBorislav Petkov /* pin at [47:24] */ 1185c1ae6830SBorislav Petkov cc6_base <<= 24; 1186c1ae6830SBorislav Petkov 1187c1ae6830SBorislav Petkov if (!intlv_en) 118810ef6b0dSChen, Gong return cc6_base | (addr & GENMASK_ULL(23, 0)); 1189c1ae6830SBorislav Petkov 1190c1ae6830SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp); 1191c1ae6830SBorislav Petkov 1192c1ae6830SBorislav Petkov /* faster log2 */ 119310ef6b0dSChen, Gong tmp_addr = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1); 1194c1ae6830SBorislav Petkov 1195c1ae6830SBorislav Petkov /* OR DramIntlvSel into bits [14:12] */ 119610ef6b0dSChen, Gong tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9; 1197c1ae6830SBorislav Petkov 1198c1ae6830SBorislav Petkov /* add remaining [11:0] bits from original MC4_ADDR */ 119910ef6b0dSChen, Gong tmp_addr |= addr & GENMASK_ULL(11, 0); 1200c1ae6830SBorislav Petkov 1201c1ae6830SBorislav Petkov return cc6_base | tmp_addr; 1202c1ae6830SBorislav Petkov } 1203c1ae6830SBorislav Petkov 1204c1ae6830SBorislav Petkov return addr; 1205ddff876dSDoug Thompson } 1206ddff876dSDoug Thompson 1207e2c0bffeSDaniel J Blueman static struct pci_dev *pci_get_related_function(unsigned int vendor, 1208e2c0bffeSDaniel J Blueman unsigned int device, 1209e2c0bffeSDaniel J Blueman struct pci_dev *related) 1210e2c0bffeSDaniel J Blueman { 1211e2c0bffeSDaniel J Blueman struct pci_dev *dev = NULL; 1212e2c0bffeSDaniel J Blueman 1213e2c0bffeSDaniel J Blueman while ((dev = pci_get_device(vendor, device, dev))) { 1214e2c0bffeSDaniel J Blueman if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) && 1215e2c0bffeSDaniel J Blueman (dev->bus->number == related->bus->number) && 1216e2c0bffeSDaniel J Blueman (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn))) 1217e2c0bffeSDaniel J Blueman break; 1218e2c0bffeSDaniel J Blueman } 1219e2c0bffeSDaniel J Blueman 1220e2c0bffeSDaniel J Blueman return dev; 1221e2c0bffeSDaniel J Blueman } 1222e2c0bffeSDaniel J Blueman 12237f19bf75SBorislav Petkov static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range) 1224ddff876dSDoug Thompson { 1225e2c0bffeSDaniel J Blueman struct amd_northbridge *nb; 122618b94f66SAravind Gopalakrishnan struct pci_dev *f1 = NULL; 122718b94f66SAravind Gopalakrishnan unsigned int pci_func; 122871d2a32eSBorislav Petkov int off = range << 3; 1229e2c0bffeSDaniel J Blueman u32 llim; 1230ddff876dSDoug Thompson 12317f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo); 12327f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo); 1233ddff876dSDoug Thompson 123418b94f66SAravind Gopalakrishnan if (pvt->fam == 0xf) 12357f19bf75SBorislav Petkov return; 1236ddff876dSDoug Thompson 12377f19bf75SBorislav Petkov if (!dram_rw(pvt, range)) 12387f19bf75SBorislav Petkov return; 1239ddff876dSDoug Thompson 12407f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi); 12417f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi); 1242f08e457cSBorislav Petkov 1243e2c0bffeSDaniel J Blueman /* F15h: factor in CC6 save area by reading dst node's limit reg */ 124418b94f66SAravind Gopalakrishnan if (pvt->fam != 0x15) 1245e2c0bffeSDaniel J Blueman return; 1246f08e457cSBorislav Petkov 1247e2c0bffeSDaniel J Blueman nb = node_to_amd_nb(dram_dst_node(pvt, range)); 1248e2c0bffeSDaniel J Blueman if (WARN_ON(!nb)) 1249e2c0bffeSDaniel J Blueman return; 1250e2c0bffeSDaniel J Blueman 1251a597d2a5SAravind Gopalakrishnan if (pvt->model == 0x60) 1252a597d2a5SAravind Gopalakrishnan pci_func = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1; 1253a597d2a5SAravind Gopalakrishnan else if (pvt->model == 0x30) 1254a597d2a5SAravind Gopalakrishnan pci_func = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1; 1255a597d2a5SAravind Gopalakrishnan else 1256a597d2a5SAravind Gopalakrishnan pci_func = PCI_DEVICE_ID_AMD_15H_NB_F1; 125718b94f66SAravind Gopalakrishnan 125818b94f66SAravind Gopalakrishnan f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc); 1259f08e457cSBorislav Petkov if (WARN_ON(!f1)) 1260f08e457cSBorislav Petkov return; 1261f08e457cSBorislav Petkov 1262f08e457cSBorislav Petkov amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim); 1263f08e457cSBorislav Petkov 126410ef6b0dSChen, Gong pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0); 1265f08e457cSBorislav Petkov 1266f08e457cSBorislav Petkov /* {[39:27],111b} */ 1267f08e457cSBorislav Petkov pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16; 1268f08e457cSBorislav Petkov 126910ef6b0dSChen, Gong pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0); 1270f08e457cSBorislav Petkov 1271f08e457cSBorislav Petkov /* [47:40] */ 1272f08e457cSBorislav Petkov pvt->ranges[range].lim.hi |= llim >> 13; 1273f08e457cSBorislav Petkov 1274f08e457cSBorislav Petkov pci_dev_put(f1); 1275f08e457cSBorislav Petkov } 1276ddff876dSDoug Thompson 1277f192c7b1SBorislav Petkov static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr, 127833ca0643SBorislav Petkov struct err_info *err) 1279ddff876dSDoug Thompson { 1280f192c7b1SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 1281ddff876dSDoug Thompson 128233ca0643SBorislav Petkov error_address_to_page_and_offset(sys_addr, err); 1283ab5a503cSMauro Carvalho Chehab 1284ab5a503cSMauro Carvalho Chehab /* 1285ab5a503cSMauro Carvalho Chehab * Find out which node the error address belongs to. This may be 1286ab5a503cSMauro Carvalho Chehab * different from the node that detected the error. 1287ab5a503cSMauro Carvalho Chehab */ 128833ca0643SBorislav Petkov err->src_mci = find_mc_by_sys_addr(mci, sys_addr); 128933ca0643SBorislav Petkov if (!err->src_mci) { 1290ab5a503cSMauro Carvalho Chehab amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n", 1291ab5a503cSMauro Carvalho Chehab (unsigned long)sys_addr); 129233ca0643SBorislav Petkov err->err_code = ERR_NODE; 1293ab5a503cSMauro Carvalho Chehab return; 1294ab5a503cSMauro Carvalho Chehab } 1295ab5a503cSMauro Carvalho Chehab 1296ab5a503cSMauro Carvalho Chehab /* Now map the sys_addr to a CSROW */ 129733ca0643SBorislav Petkov err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr); 129833ca0643SBorislav Petkov if (err->csrow < 0) { 129933ca0643SBorislav Petkov err->err_code = ERR_CSROW; 1300ab5a503cSMauro Carvalho Chehab return; 1301ab5a503cSMauro Carvalho Chehab } 1302ab5a503cSMauro Carvalho Chehab 1303ddff876dSDoug Thompson /* CHIPKILL enabled */ 1304f192c7b1SBorislav Petkov if (pvt->nbcfg & NBCFG_CHIPKILL) { 130533ca0643SBorislav Petkov err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome); 130633ca0643SBorislav Petkov if (err->channel < 0) { 1307ddff876dSDoug Thompson /* 1308ddff876dSDoug Thompson * Syndrome didn't map, so we don't know which of the 1309ddff876dSDoug Thompson * 2 DIMMs is in error. So we need to ID 'both' of them 1310ddff876dSDoug Thompson * as suspect. 1311ddff876dSDoug Thompson */ 131233ca0643SBorislav Petkov amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - " 1313ab5a503cSMauro Carvalho Chehab "possible error reporting race\n", 131433ca0643SBorislav Petkov err->syndrome); 131533ca0643SBorislav Petkov err->err_code = ERR_CHANNEL; 1316ddff876dSDoug Thompson return; 1317ddff876dSDoug Thompson } 1318ddff876dSDoug Thompson } else { 1319ddff876dSDoug Thompson /* 1320ddff876dSDoug Thompson * non-chipkill ecc mode 1321ddff876dSDoug Thompson * 1322ddff876dSDoug Thompson * The k8 documentation is unclear about how to determine the 1323ddff876dSDoug Thompson * channel number when using non-chipkill memory. This method 1324ddff876dSDoug Thompson * was obtained from email communication with someone at AMD. 1325ddff876dSDoug Thompson * (Wish the email was placed in this comment - norsk) 1326ddff876dSDoug Thompson */ 132733ca0643SBorislav Petkov err->channel = ((sys_addr & BIT(3)) != 0); 1328ddff876dSDoug Thompson } 1329ddff876dSDoug Thompson } 1330ddff876dSDoug Thompson 133141d8bfabSBorislav Petkov static int ddr2_cs_size(unsigned i, bool dct_width) 1332ddff876dSDoug Thompson { 133341d8bfabSBorislav Petkov unsigned shift = 0; 1334ddff876dSDoug Thompson 133541d8bfabSBorislav Petkov if (i <= 2) 133641d8bfabSBorislav Petkov shift = i; 133741d8bfabSBorislav Petkov else if (!(i & 0x1)) 133841d8bfabSBorislav Petkov shift = i >> 1; 13391433eb99SBorislav Petkov else 134041d8bfabSBorislav Petkov shift = (i + 1) >> 1; 1341ddff876dSDoug Thompson 134241d8bfabSBorislav Petkov return 128 << (shift + !!dct_width); 134341d8bfabSBorislav Petkov } 134441d8bfabSBorislav Petkov 134541d8bfabSBorislav Petkov static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 1346a597d2a5SAravind Gopalakrishnan unsigned cs_mode, int cs_mask_nr) 134741d8bfabSBorislav Petkov { 134841d8bfabSBorislav Petkov u32 dclr = dct ? pvt->dclr1 : pvt->dclr0; 134941d8bfabSBorislav Petkov 135041d8bfabSBorislav Petkov if (pvt->ext_model >= K8_REV_F) { 135141d8bfabSBorislav Petkov WARN_ON(cs_mode > 11); 135241d8bfabSBorislav Petkov return ddr2_cs_size(cs_mode, dclr & WIDTH_128); 135341d8bfabSBorislav Petkov } 135441d8bfabSBorislav Petkov else if (pvt->ext_model >= K8_REV_D) { 135511b0a314SBorislav Petkov unsigned diff; 135641d8bfabSBorislav Petkov WARN_ON(cs_mode > 10); 135741d8bfabSBorislav Petkov 135811b0a314SBorislav Petkov /* 135911b0a314SBorislav Petkov * the below calculation, besides trying to win an obfuscated C 136011b0a314SBorislav Petkov * contest, maps cs_mode values to DIMM chip select sizes. The 136111b0a314SBorislav Petkov * mappings are: 136211b0a314SBorislav Petkov * 136311b0a314SBorislav Petkov * cs_mode CS size (mb) 136411b0a314SBorislav Petkov * ======= ============ 136511b0a314SBorislav Petkov * 0 32 136611b0a314SBorislav Petkov * 1 64 136711b0a314SBorislav Petkov * 2 128 136811b0a314SBorislav Petkov * 3 128 136911b0a314SBorislav Petkov * 4 256 137011b0a314SBorislav Petkov * 5 512 137111b0a314SBorislav Petkov * 6 256 137211b0a314SBorislav Petkov * 7 512 137311b0a314SBorislav Petkov * 8 1024 137411b0a314SBorislav Petkov * 9 1024 137511b0a314SBorislav Petkov * 10 2048 137611b0a314SBorislav Petkov * 137711b0a314SBorislav Petkov * Basically, it calculates a value with which to shift the 137811b0a314SBorislav Petkov * smallest CS size of 32MB. 137911b0a314SBorislav Petkov * 138011b0a314SBorislav Petkov * ddr[23]_cs_size have a similar purpose. 138111b0a314SBorislav Petkov */ 138211b0a314SBorislav Petkov diff = cs_mode/3 + (unsigned)(cs_mode > 5); 138311b0a314SBorislav Petkov 138411b0a314SBorislav Petkov return 32 << (cs_mode - diff); 138541d8bfabSBorislav Petkov } 138641d8bfabSBorislav Petkov else { 138741d8bfabSBorislav Petkov WARN_ON(cs_mode > 6); 138841d8bfabSBorislav Petkov return 32 << cs_mode; 138941d8bfabSBorislav Petkov } 1390ddff876dSDoug Thompson } 1391ddff876dSDoug Thompson 13921afd3c98SDoug Thompson /* 13931afd3c98SDoug Thompson * Get the number of DCT channels in use. 13941afd3c98SDoug Thompson * 13951afd3c98SDoug Thompson * Return: 13961afd3c98SDoug Thompson * number of Memory Channels in operation 13971afd3c98SDoug Thompson * Pass back: 13981afd3c98SDoug Thompson * contents of the DCL0_LOW register 13991afd3c98SDoug Thompson */ 14007d20d14dSBorislav Petkov static int f1x_early_channel_count(struct amd64_pvt *pvt) 14011afd3c98SDoug Thompson { 14026ba5dcdcSBorislav Petkov int i, j, channels = 0; 1403ddff876dSDoug Thompson 14047d20d14dSBorislav Petkov /* On F10h, if we are in 128 bit mode, then we are using 2 channels */ 1405a4b4bedcSBorislav Petkov if (pvt->fam == 0x10 && (pvt->dclr0 & WIDTH_128)) 14067d20d14dSBorislav Petkov return 2; 14071afd3c98SDoug Thompson 14081afd3c98SDoug Thompson /* 1409d16149e8SBorislav Petkov * Need to check if in unganged mode: In such, there are 2 channels, 1410d16149e8SBorislav Petkov * but they are not in 128 bit mode and thus the above 'dclr0' status 1411d16149e8SBorislav Petkov * bit will be OFF. 14121afd3c98SDoug Thompson * 14131afd3c98SDoug Thompson * Need to check DCT0[0] and DCT1[0] to see if only one of them has 14141afd3c98SDoug Thompson * their CSEnable bit on. If so, then SINGLE DIMM case. 14151afd3c98SDoug Thompson */ 1416956b9ba1SJoe Perches edac_dbg(0, "Data width is not 128 bits - need more decoding\n"); 14171afd3c98SDoug Thompson 14181afd3c98SDoug Thompson /* 14191afd3c98SDoug Thompson * Check DRAM Bank Address Mapping values for each DIMM to see if there 14201afd3c98SDoug Thompson * is more than just one DIMM present in unganged mode. Need to check 14211afd3c98SDoug Thompson * both controllers since DIMMs can be placed in either one. 14221afd3c98SDoug Thompson */ 1423525a1b20SBorislav Petkov for (i = 0; i < 2; i++) { 1424525a1b20SBorislav Petkov u32 dbam = (i ? pvt->dbam1 : pvt->dbam0); 14251afd3c98SDoug Thompson 142657a30854SWan Wei for (j = 0; j < 4; j++) { 142757a30854SWan Wei if (DBAM_DIMM(j, dbam) > 0) { 14281afd3c98SDoug Thompson channels++; 142957a30854SWan Wei break; 14301afd3c98SDoug Thompson } 143157a30854SWan Wei } 143257a30854SWan Wei } 14331afd3c98SDoug Thompson 1434d16149e8SBorislav Petkov if (channels > 2) 1435d16149e8SBorislav Petkov channels = 2; 1436d16149e8SBorislav Petkov 143724f9a7feSBorislav Petkov amd64_info("MCT channel count: %d\n", channels); 14381afd3c98SDoug Thompson 14391afd3c98SDoug Thompson return channels; 14401afd3c98SDoug Thompson } 14411afd3c98SDoug Thompson 1442f1cbbec9SYazen Ghannam static int f17_early_channel_count(struct amd64_pvt *pvt) 1443f1cbbec9SYazen Ghannam { 1444f1cbbec9SYazen Ghannam int i, channels = 0; 1445f1cbbec9SYazen Ghannam 1446f1cbbec9SYazen Ghannam /* SDP Control bit 31 (SdpInit) is clear for unused UMC channels */ 14474d30d2bcSYazen Ghannam for_each_umc(i) 1448f1cbbec9SYazen Ghannam channels += !!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT); 1449f1cbbec9SYazen Ghannam 1450f1cbbec9SYazen Ghannam amd64_info("MCT channel count: %d\n", channels); 1451f1cbbec9SYazen Ghannam 1452f1cbbec9SYazen Ghannam return channels; 1453f1cbbec9SYazen Ghannam } 1454f1cbbec9SYazen Ghannam 145541d8bfabSBorislav Petkov static int ddr3_cs_size(unsigned i, bool dct_width) 14561afd3c98SDoug Thompson { 145741d8bfabSBorislav Petkov unsigned shift = 0; 145841d8bfabSBorislav Petkov int cs_size = 0; 145941d8bfabSBorislav Petkov 146041d8bfabSBorislav Petkov if (i == 0 || i == 3 || i == 4) 146141d8bfabSBorislav Petkov cs_size = -1; 146241d8bfabSBorislav Petkov else if (i <= 2) 146341d8bfabSBorislav Petkov shift = i; 146441d8bfabSBorislav Petkov else if (i == 12) 146541d8bfabSBorislav Petkov shift = 7; 146641d8bfabSBorislav Petkov else if (!(i & 0x1)) 146741d8bfabSBorislav Petkov shift = i >> 1; 146841d8bfabSBorislav Petkov else 146941d8bfabSBorislav Petkov shift = (i + 1) >> 1; 147041d8bfabSBorislav Petkov 147141d8bfabSBorislav Petkov if (cs_size != -1) 147241d8bfabSBorislav Petkov cs_size = (128 * (1 << !!dct_width)) << shift; 147341d8bfabSBorislav Petkov 147441d8bfabSBorislav Petkov return cs_size; 147541d8bfabSBorislav Petkov } 147641d8bfabSBorislav Petkov 1477a597d2a5SAravind Gopalakrishnan static int ddr3_lrdimm_cs_size(unsigned i, unsigned rank_multiply) 1478a597d2a5SAravind Gopalakrishnan { 1479a597d2a5SAravind Gopalakrishnan unsigned shift = 0; 1480a597d2a5SAravind Gopalakrishnan int cs_size = 0; 1481a597d2a5SAravind Gopalakrishnan 1482a597d2a5SAravind Gopalakrishnan if (i < 4 || i == 6) 1483a597d2a5SAravind Gopalakrishnan cs_size = -1; 1484a597d2a5SAravind Gopalakrishnan else if (i == 12) 1485a597d2a5SAravind Gopalakrishnan shift = 7; 1486a597d2a5SAravind Gopalakrishnan else if (!(i & 0x1)) 1487a597d2a5SAravind Gopalakrishnan shift = i >> 1; 1488a597d2a5SAravind Gopalakrishnan else 1489a597d2a5SAravind Gopalakrishnan shift = (i + 1) >> 1; 1490a597d2a5SAravind Gopalakrishnan 1491a597d2a5SAravind Gopalakrishnan if (cs_size != -1) 1492a597d2a5SAravind Gopalakrishnan cs_size = rank_multiply * (128 << shift); 1493a597d2a5SAravind Gopalakrishnan 1494a597d2a5SAravind Gopalakrishnan return cs_size; 1495a597d2a5SAravind Gopalakrishnan } 1496a597d2a5SAravind Gopalakrishnan 1497a597d2a5SAravind Gopalakrishnan static int ddr4_cs_size(unsigned i) 1498a597d2a5SAravind Gopalakrishnan { 1499a597d2a5SAravind Gopalakrishnan int cs_size = 0; 1500a597d2a5SAravind Gopalakrishnan 1501a597d2a5SAravind Gopalakrishnan if (i == 0) 1502a597d2a5SAravind Gopalakrishnan cs_size = -1; 1503a597d2a5SAravind Gopalakrishnan else if (i == 1) 1504a597d2a5SAravind Gopalakrishnan cs_size = 1024; 1505a597d2a5SAravind Gopalakrishnan else 1506a597d2a5SAravind Gopalakrishnan /* Min cs_size = 1G */ 1507a597d2a5SAravind Gopalakrishnan cs_size = 1024 * (1 << (i >> 1)); 1508a597d2a5SAravind Gopalakrishnan 1509a597d2a5SAravind Gopalakrishnan return cs_size; 1510a597d2a5SAravind Gopalakrishnan } 1511a597d2a5SAravind Gopalakrishnan 151241d8bfabSBorislav Petkov static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 1513a597d2a5SAravind Gopalakrishnan unsigned cs_mode, int cs_mask_nr) 151441d8bfabSBorislav Petkov { 151541d8bfabSBorislav Petkov u32 dclr = dct ? pvt->dclr1 : pvt->dclr0; 151641d8bfabSBorislav Petkov 151741d8bfabSBorislav Petkov WARN_ON(cs_mode > 11); 15181433eb99SBorislav Petkov 15191433eb99SBorislav Petkov if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE) 152041d8bfabSBorislav Petkov return ddr3_cs_size(cs_mode, dclr & WIDTH_128); 15211433eb99SBorislav Petkov else 152241d8bfabSBorislav Petkov return ddr2_cs_size(cs_mode, dclr & WIDTH_128); 152341d8bfabSBorislav Petkov } 15241433eb99SBorislav Petkov 152541d8bfabSBorislav Petkov /* 152641d8bfabSBorislav Petkov * F15h supports only 64bit DCT interfaces 152741d8bfabSBorislav Petkov */ 152841d8bfabSBorislav Petkov static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 1529a597d2a5SAravind Gopalakrishnan unsigned cs_mode, int cs_mask_nr) 153041d8bfabSBorislav Petkov { 153141d8bfabSBorislav Petkov WARN_ON(cs_mode > 12); 153241d8bfabSBorislav Petkov 153341d8bfabSBorislav Petkov return ddr3_cs_size(cs_mode, false); 15341afd3c98SDoug Thompson } 15351afd3c98SDoug Thompson 1536a597d2a5SAravind Gopalakrishnan /* F15h M60h supports DDR4 mapping as well.. */ 1537a597d2a5SAravind Gopalakrishnan static int f15_m60h_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 1538a597d2a5SAravind Gopalakrishnan unsigned cs_mode, int cs_mask_nr) 1539a597d2a5SAravind Gopalakrishnan { 1540a597d2a5SAravind Gopalakrishnan int cs_size; 1541a597d2a5SAravind Gopalakrishnan u32 dcsm = pvt->csels[dct].csmasks[cs_mask_nr]; 1542a597d2a5SAravind Gopalakrishnan 1543a597d2a5SAravind Gopalakrishnan WARN_ON(cs_mode > 12); 1544a597d2a5SAravind Gopalakrishnan 1545a597d2a5SAravind Gopalakrishnan if (pvt->dram_type == MEM_DDR4) { 1546a597d2a5SAravind Gopalakrishnan if (cs_mode > 9) 1547a597d2a5SAravind Gopalakrishnan return -1; 1548a597d2a5SAravind Gopalakrishnan 1549a597d2a5SAravind Gopalakrishnan cs_size = ddr4_cs_size(cs_mode); 1550a597d2a5SAravind Gopalakrishnan } else if (pvt->dram_type == MEM_LRDDR3) { 1551a597d2a5SAravind Gopalakrishnan unsigned rank_multiply = dcsm & 0xf; 1552a597d2a5SAravind Gopalakrishnan 1553a597d2a5SAravind Gopalakrishnan if (rank_multiply == 3) 1554a597d2a5SAravind Gopalakrishnan rank_multiply = 4; 1555a597d2a5SAravind Gopalakrishnan cs_size = ddr3_lrdimm_cs_size(cs_mode, rank_multiply); 1556a597d2a5SAravind Gopalakrishnan } else { 1557a597d2a5SAravind Gopalakrishnan /* Minimum cs size is 512mb for F15hM60h*/ 1558a597d2a5SAravind Gopalakrishnan if (cs_mode == 0x1) 1559a597d2a5SAravind Gopalakrishnan return -1; 1560a597d2a5SAravind Gopalakrishnan 1561a597d2a5SAravind Gopalakrishnan cs_size = ddr3_cs_size(cs_mode, false); 1562a597d2a5SAravind Gopalakrishnan } 1563a597d2a5SAravind Gopalakrishnan 1564a597d2a5SAravind Gopalakrishnan return cs_size; 1565a597d2a5SAravind Gopalakrishnan } 1566a597d2a5SAravind Gopalakrishnan 156794c1acf2SAravind Gopalakrishnan /* 156818b94f66SAravind Gopalakrishnan * F16h and F15h model 30h have only limited cs_modes. 156994c1acf2SAravind Gopalakrishnan */ 157094c1acf2SAravind Gopalakrishnan static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 1571a597d2a5SAravind Gopalakrishnan unsigned cs_mode, int cs_mask_nr) 157294c1acf2SAravind Gopalakrishnan { 157394c1acf2SAravind Gopalakrishnan WARN_ON(cs_mode > 12); 157494c1acf2SAravind Gopalakrishnan 157594c1acf2SAravind Gopalakrishnan if (cs_mode == 6 || cs_mode == 8 || 157694c1acf2SAravind Gopalakrishnan cs_mode == 9 || cs_mode == 12) 157794c1acf2SAravind Gopalakrishnan return -1; 157894c1acf2SAravind Gopalakrishnan else 157994c1acf2SAravind Gopalakrishnan return ddr3_cs_size(cs_mode, false); 158094c1acf2SAravind Gopalakrishnan } 158194c1acf2SAravind Gopalakrishnan 1582e53a3b26SYazen Ghannam static int f17_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc, 1583f1cbbec9SYazen Ghannam unsigned int cs_mode, int csrow_nr) 1584f1cbbec9SYazen Ghannam { 1585e53a3b26SYazen Ghannam u32 addr_mask_orig, addr_mask_deinterleaved; 1586e53a3b26SYazen Ghannam u32 msb, weight, num_zero_bits; 1587e53a3b26SYazen Ghannam int dimm, size = 0; 1588f1cbbec9SYazen Ghannam 1589e53a3b26SYazen Ghannam /* No Chip Selects are enabled. */ 1590e53a3b26SYazen Ghannam if (!cs_mode) 1591e53a3b26SYazen Ghannam return size; 1592e53a3b26SYazen Ghannam 1593e53a3b26SYazen Ghannam /* Requested size of an even CS but none are enabled. */ 1594e53a3b26SYazen Ghannam if (!(cs_mode & CS_EVEN) && !(csrow_nr & 1)) 1595e53a3b26SYazen Ghannam return size; 1596e53a3b26SYazen Ghannam 1597e53a3b26SYazen Ghannam /* Requested size of an odd CS but none are enabled. */ 1598e53a3b26SYazen Ghannam if (!(cs_mode & CS_ODD) && (csrow_nr & 1)) 1599e53a3b26SYazen Ghannam return size; 1600e53a3b26SYazen Ghannam 1601e53a3b26SYazen Ghannam /* 1602e53a3b26SYazen Ghannam * There is one mask per DIMM, and two Chip Selects per DIMM. 1603e53a3b26SYazen Ghannam * CS0 and CS1 -> DIMM0 1604e53a3b26SYazen Ghannam * CS2 and CS3 -> DIMM1 1605e53a3b26SYazen Ghannam */ 1606e53a3b26SYazen Ghannam dimm = csrow_nr >> 1; 1607e53a3b26SYazen Ghannam 160881f5090dSYazen Ghannam /* Asymmetric dual-rank DIMM support. */ 160981f5090dSYazen Ghannam if ((csrow_nr & 1) && (cs_mode & CS_ODD_SECONDARY)) 161081f5090dSYazen Ghannam addr_mask_orig = pvt->csels[umc].csmasks_sec[dimm]; 161181f5090dSYazen Ghannam else 1612e53a3b26SYazen Ghannam addr_mask_orig = pvt->csels[umc].csmasks[dimm]; 1613e53a3b26SYazen Ghannam 1614e53a3b26SYazen Ghannam /* 1615e53a3b26SYazen Ghannam * The number of zero bits in the mask is equal to the number of bits 1616e53a3b26SYazen Ghannam * in a full mask minus the number of bits in the current mask. 1617e53a3b26SYazen Ghannam * 1618e53a3b26SYazen Ghannam * The MSB is the number of bits in the full mask because BIT[0] is 1619e53a3b26SYazen Ghannam * always 0. 1620e53a3b26SYazen Ghannam */ 1621e53a3b26SYazen Ghannam msb = fls(addr_mask_orig) - 1; 1622e53a3b26SYazen Ghannam weight = hweight_long(addr_mask_orig); 1623e53a3b26SYazen Ghannam num_zero_bits = msb - weight; 1624e53a3b26SYazen Ghannam 1625e53a3b26SYazen Ghannam /* Take the number of zero bits off from the top of the mask. */ 1626e53a3b26SYazen Ghannam addr_mask_deinterleaved = GENMASK_ULL(msb - num_zero_bits, 1); 1627e53a3b26SYazen Ghannam 1628e53a3b26SYazen Ghannam edac_dbg(1, "CS%d DIMM%d AddrMasks:\n", csrow_nr, dimm); 1629e53a3b26SYazen Ghannam edac_dbg(1, " Original AddrMask: 0x%x\n", addr_mask_orig); 1630e53a3b26SYazen Ghannam edac_dbg(1, " Deinterleaved AddrMask: 0x%x\n", addr_mask_deinterleaved); 1631f1cbbec9SYazen Ghannam 1632f1cbbec9SYazen Ghannam /* Register [31:1] = Address [39:9]. Size is in kBs here. */ 1633e53a3b26SYazen Ghannam size = (addr_mask_deinterleaved >> 2) + 1; 1634f1cbbec9SYazen Ghannam 1635f1cbbec9SYazen Ghannam /* Return size in MBs. */ 1636f1cbbec9SYazen Ghannam return size >> 10; 1637f1cbbec9SYazen Ghannam } 1638f1cbbec9SYazen Ghannam 16395a5d2371SBorislav Petkov static void read_dram_ctl_register(struct amd64_pvt *pvt) 16406163b5d4SDoug Thompson { 16416163b5d4SDoug Thompson 1642a4b4bedcSBorislav Petkov if (pvt->fam == 0xf) 16435a5d2371SBorislav Petkov return; 16445a5d2371SBorislav Petkov 16457981a28fSAravind Gopalakrishnan if (!amd64_read_pci_cfg(pvt->F2, DCT_SEL_LO, &pvt->dct_sel_lo)) { 1646956b9ba1SJoe Perches edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n", 164778da121eSBorislav Petkov pvt->dct_sel_lo, dct_sel_baseaddr(pvt)); 16486163b5d4SDoug Thompson 1649956b9ba1SJoe Perches edac_dbg(0, " DCTs operate in %s mode\n", 16505a5d2371SBorislav Petkov (dct_ganging_enabled(pvt) ? "ganged" : "unganged")); 16516163b5d4SDoug Thompson 165272381bd5SBorislav Petkov if (!dct_ganging_enabled(pvt)) 1653956b9ba1SJoe Perches edac_dbg(0, " Address range split per DCT: %s\n", 165472381bd5SBorislav Petkov (dct_high_range_enabled(pvt) ? "yes" : "no")); 165572381bd5SBorislav Petkov 1656956b9ba1SJoe Perches edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n", 165772381bd5SBorislav Petkov (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"), 165872381bd5SBorislav Petkov (dct_memory_cleared(pvt) ? "yes" : "no")); 165972381bd5SBorislav Petkov 1660956b9ba1SJoe Perches edac_dbg(0, " channel interleave: %s, " 166178da121eSBorislav Petkov "interleave bits selector: 0x%x\n", 166272381bd5SBorislav Petkov (dct_interleave_enabled(pvt) ? "enabled" : "disabled"), 16636163b5d4SDoug Thompson dct_sel_interleave_addr(pvt)); 16646163b5d4SDoug Thompson } 16656163b5d4SDoug Thompson 16667981a28fSAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F2, DCT_SEL_HI, &pvt->dct_sel_hi); 16676163b5d4SDoug Thompson } 16686163b5d4SDoug Thompson 1669f71d0a05SDoug Thompson /* 167018b94f66SAravind Gopalakrishnan * Determine channel (DCT) based on the interleaving mode (see F15h M30h BKDG, 167118b94f66SAravind Gopalakrishnan * 2.10.12 Memory Interleaving Modes). 167218b94f66SAravind Gopalakrishnan */ 167318b94f66SAravind Gopalakrishnan static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr, 167418b94f66SAravind Gopalakrishnan u8 intlv_en, int num_dcts_intlv, 167518b94f66SAravind Gopalakrishnan u32 dct_sel) 167618b94f66SAravind Gopalakrishnan { 167718b94f66SAravind Gopalakrishnan u8 channel = 0; 167818b94f66SAravind Gopalakrishnan u8 select; 167918b94f66SAravind Gopalakrishnan 168018b94f66SAravind Gopalakrishnan if (!(intlv_en)) 168118b94f66SAravind Gopalakrishnan return (u8)(dct_sel); 168218b94f66SAravind Gopalakrishnan 168318b94f66SAravind Gopalakrishnan if (num_dcts_intlv == 2) { 168418b94f66SAravind Gopalakrishnan select = (sys_addr >> 8) & 0x3; 168518b94f66SAravind Gopalakrishnan channel = select ? 0x3 : 0; 16869d0e8d83SAravind Gopalakrishnan } else if (num_dcts_intlv == 4) { 16879d0e8d83SAravind Gopalakrishnan u8 intlv_addr = dct_sel_interleave_addr(pvt); 16889d0e8d83SAravind Gopalakrishnan switch (intlv_addr) { 16899d0e8d83SAravind Gopalakrishnan case 0x4: 16909d0e8d83SAravind Gopalakrishnan channel = (sys_addr >> 8) & 0x3; 16919d0e8d83SAravind Gopalakrishnan break; 16929d0e8d83SAravind Gopalakrishnan case 0x5: 16939d0e8d83SAravind Gopalakrishnan channel = (sys_addr >> 9) & 0x3; 16949d0e8d83SAravind Gopalakrishnan break; 16959d0e8d83SAravind Gopalakrishnan } 16969d0e8d83SAravind Gopalakrishnan } 169718b94f66SAravind Gopalakrishnan return channel; 169818b94f66SAravind Gopalakrishnan } 169918b94f66SAravind Gopalakrishnan 170018b94f66SAravind Gopalakrishnan /* 1701229a7a11SBorislav Petkov * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory 1702f71d0a05SDoug Thompson * Interleaving Modes. 1703f71d0a05SDoug Thompson */ 1704b15f0fcaSBorislav Petkov static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr, 1705229a7a11SBorislav Petkov bool hi_range_sel, u8 intlv_en) 17066163b5d4SDoug Thompson { 1707151fa71cSBorislav Petkov u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1; 17086163b5d4SDoug Thompson 17096163b5d4SDoug Thompson if (dct_ganging_enabled(pvt)) 1710229a7a11SBorislav Petkov return 0; 1711229a7a11SBorislav Petkov 1712229a7a11SBorislav Petkov if (hi_range_sel) 1713229a7a11SBorislav Petkov return dct_sel_high; 1714229a7a11SBorislav Petkov 1715f71d0a05SDoug Thompson /* 1716f71d0a05SDoug Thompson * see F2x110[DctSelIntLvAddr] - channel interleave mode 1717f71d0a05SDoug Thompson */ 1718229a7a11SBorislav Petkov if (dct_interleave_enabled(pvt)) { 1719229a7a11SBorislav Petkov u8 intlv_addr = dct_sel_interleave_addr(pvt); 17206163b5d4SDoug Thompson 1721229a7a11SBorislav Petkov /* return DCT select function: 0=DCT0, 1=DCT1 */ 1722229a7a11SBorislav Petkov if (!intlv_addr) 1723229a7a11SBorislav Petkov return sys_addr >> 6 & 1; 17246163b5d4SDoug Thompson 1725229a7a11SBorislav Petkov if (intlv_addr & 0x2) { 1726229a7a11SBorislav Petkov u8 shift = intlv_addr & 0x1 ? 9 : 6; 1727dc0a50a8SYazen Ghannam u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) & 1; 1728229a7a11SBorislav Petkov 1729229a7a11SBorislav Petkov return ((sys_addr >> shift) & 1) ^ temp; 17306163b5d4SDoug Thompson } 17316163b5d4SDoug Thompson 1732dc0a50a8SYazen Ghannam if (intlv_addr & 0x4) { 1733dc0a50a8SYazen Ghannam u8 shift = intlv_addr & 0x1 ? 9 : 8; 1734dc0a50a8SYazen Ghannam 1735dc0a50a8SYazen Ghannam return (sys_addr >> shift) & 1; 1736dc0a50a8SYazen Ghannam } 1737dc0a50a8SYazen Ghannam 1738229a7a11SBorislav Petkov return (sys_addr >> (12 + hweight8(intlv_en))) & 1; 1739229a7a11SBorislav Petkov } 1740229a7a11SBorislav Petkov 1741229a7a11SBorislav Petkov if (dct_high_range_enabled(pvt)) 1742229a7a11SBorislav Petkov return ~dct_sel_high & 1; 17436163b5d4SDoug Thompson 17446163b5d4SDoug Thompson return 0; 17456163b5d4SDoug Thompson } 17466163b5d4SDoug Thompson 1747c8e518d5SBorislav Petkov /* Convert the sys_addr to the normalized DCT address */ 1748c7e5301aSDaniel J Blueman static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range, 1749c8e518d5SBorislav Petkov u64 sys_addr, bool hi_rng, 1750c8e518d5SBorislav Petkov u32 dct_sel_base_addr) 17516163b5d4SDoug Thompson { 17526163b5d4SDoug Thompson u64 chan_off; 1753c8e518d5SBorislav Petkov u64 dram_base = get_dram_base(pvt, range); 1754c8e518d5SBorislav Petkov u64 hole_off = f10_dhar_offset(pvt); 17556f3508f6SDan Carpenter u64 dct_sel_base_off = (u64)(pvt->dct_sel_hi & 0xFFFFFC00) << 16; 17566163b5d4SDoug Thompson 1757c8e518d5SBorislav Petkov if (hi_rng) { 1758c8e518d5SBorislav Petkov /* 1759c8e518d5SBorislav Petkov * if 1760c8e518d5SBorislav Petkov * base address of high range is below 4Gb 1761c8e518d5SBorislav Petkov * (bits [47:27] at [31:11]) 1762c8e518d5SBorislav Petkov * DRAM address space on this DCT is hoisted above 4Gb && 1763c8e518d5SBorislav Petkov * sys_addr > 4Gb 1764c8e518d5SBorislav Petkov * 1765c8e518d5SBorislav Petkov * remove hole offset from sys_addr 1766c8e518d5SBorislav Petkov * else 1767c8e518d5SBorislav Petkov * remove high range offset from sys_addr 1768c8e518d5SBorislav Petkov */ 1769c8e518d5SBorislav Petkov if ((!(dct_sel_base_addr >> 16) || 1770c8e518d5SBorislav Petkov dct_sel_base_addr < dhar_base(pvt)) && 1771972ea17aSBorislav Petkov dhar_valid(pvt) && 1772c8e518d5SBorislav Petkov (sys_addr >= BIT_64(32))) 1773bc21fa57SBorislav Petkov chan_off = hole_off; 17746163b5d4SDoug Thompson else 17756163b5d4SDoug Thompson chan_off = dct_sel_base_off; 17766163b5d4SDoug Thompson } else { 1777c8e518d5SBorislav Petkov /* 1778c8e518d5SBorislav Petkov * if 1779c8e518d5SBorislav Petkov * we have a valid hole && 1780c8e518d5SBorislav Petkov * sys_addr > 4Gb 1781c8e518d5SBorislav Petkov * 1782c8e518d5SBorislav Petkov * remove hole 1783c8e518d5SBorislav Petkov * else 1784c8e518d5SBorislav Petkov * remove dram base to normalize to DCT address 1785c8e518d5SBorislav Petkov */ 1786972ea17aSBorislav Petkov if (dhar_valid(pvt) && (sys_addr >= BIT_64(32))) 1787bc21fa57SBorislav Petkov chan_off = hole_off; 17886163b5d4SDoug Thompson else 1789c8e518d5SBorislav Petkov chan_off = dram_base; 17906163b5d4SDoug Thompson } 17916163b5d4SDoug Thompson 179210ef6b0dSChen, Gong return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23)); 17936163b5d4SDoug Thompson } 17946163b5d4SDoug Thompson 17956163b5d4SDoug Thompson /* 17966163b5d4SDoug Thompson * checks if the csrow passed in is marked as SPARED, if so returns the new 17976163b5d4SDoug Thompson * spare row 17986163b5d4SDoug Thompson */ 179911c75eadSBorislav Petkov static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow) 18006163b5d4SDoug Thompson { 1801614ec9d8SBorislav Petkov int tmp_cs; 18026163b5d4SDoug Thompson 1803614ec9d8SBorislav Petkov if (online_spare_swap_done(pvt, dct) && 1804614ec9d8SBorislav Petkov csrow == online_spare_bad_dramcs(pvt, dct)) { 1805614ec9d8SBorislav Petkov 1806614ec9d8SBorislav Petkov for_each_chip_select(tmp_cs, dct, pvt) { 1807614ec9d8SBorislav Petkov if (chip_select_base(tmp_cs, dct, pvt) & 0x2) { 1808614ec9d8SBorislav Petkov csrow = tmp_cs; 1809614ec9d8SBorislav Petkov break; 1810614ec9d8SBorislav Petkov } 1811614ec9d8SBorislav Petkov } 18126163b5d4SDoug Thompson } 18136163b5d4SDoug Thompson return csrow; 18146163b5d4SDoug Thompson } 18156163b5d4SDoug Thompson 18166163b5d4SDoug Thompson /* 18176163b5d4SDoug Thompson * Iterate over the DRAM DCT "base" and "mask" registers looking for a 18186163b5d4SDoug Thompson * SystemAddr match on the specified 'ChannelSelect' and 'NodeID' 18196163b5d4SDoug Thompson * 18206163b5d4SDoug Thompson * Return: 18216163b5d4SDoug Thompson * -EINVAL: NOT FOUND 18226163b5d4SDoug Thompson * 0..csrow = Chip-Select Row 18236163b5d4SDoug Thompson */ 1824c7e5301aSDaniel J Blueman static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct) 18256163b5d4SDoug Thompson { 18266163b5d4SDoug Thompson struct mem_ctl_info *mci; 18276163b5d4SDoug Thompson struct amd64_pvt *pvt; 182811c75eadSBorislav Petkov u64 cs_base, cs_mask; 18296163b5d4SDoug Thompson int cs_found = -EINVAL; 18306163b5d4SDoug Thompson int csrow; 18316163b5d4SDoug Thompson 18322ec591acSBorislav Petkov mci = edac_mc_find(nid); 18336163b5d4SDoug Thompson if (!mci) 18346163b5d4SDoug Thompson return cs_found; 18356163b5d4SDoug Thompson 18366163b5d4SDoug Thompson pvt = mci->pvt_info; 18376163b5d4SDoug Thompson 1838956b9ba1SJoe Perches edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct); 18396163b5d4SDoug Thompson 184011c75eadSBorislav Petkov for_each_chip_select(csrow, dct, pvt) { 184111c75eadSBorislav Petkov if (!csrow_enabled(csrow, dct, pvt)) 18426163b5d4SDoug Thompson continue; 18436163b5d4SDoug Thompson 184411c75eadSBorislav Petkov get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask); 18456163b5d4SDoug Thompson 1846956b9ba1SJoe Perches edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n", 18476163b5d4SDoug Thompson csrow, cs_base, cs_mask); 18486163b5d4SDoug Thompson 184911c75eadSBorislav Petkov cs_mask = ~cs_mask; 18506163b5d4SDoug Thompson 1851956b9ba1SJoe Perches edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n", 185211c75eadSBorislav Petkov (in_addr & cs_mask), (cs_base & cs_mask)); 18536163b5d4SDoug Thompson 185411c75eadSBorislav Petkov if ((in_addr & cs_mask) == (cs_base & cs_mask)) { 185518b94f66SAravind Gopalakrishnan if (pvt->fam == 0x15 && pvt->model >= 0x30) { 185618b94f66SAravind Gopalakrishnan cs_found = csrow; 185718b94f66SAravind Gopalakrishnan break; 185818b94f66SAravind Gopalakrishnan } 185911c75eadSBorislav Petkov cs_found = f10_process_possible_spare(pvt, dct, csrow); 18606163b5d4SDoug Thompson 1861956b9ba1SJoe Perches edac_dbg(1, " MATCH csrow=%d\n", cs_found); 18626163b5d4SDoug Thompson break; 18636163b5d4SDoug Thompson } 18646163b5d4SDoug Thompson } 18656163b5d4SDoug Thompson return cs_found; 18666163b5d4SDoug Thompson } 18676163b5d4SDoug Thompson 186895b0ef55SBorislav Petkov /* 186995b0ef55SBorislav Petkov * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is 187095b0ef55SBorislav Petkov * swapped with a region located at the bottom of memory so that the GPU can use 187195b0ef55SBorislav Petkov * the interleaved region and thus two channels. 187295b0ef55SBorislav Petkov */ 1873b15f0fcaSBorislav Petkov static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr) 187495b0ef55SBorislav Petkov { 187595b0ef55SBorislav Petkov u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr; 187695b0ef55SBorislav Petkov 1877a4b4bedcSBorislav Petkov if (pvt->fam == 0x10) { 187895b0ef55SBorislav Petkov /* only revC3 and revE have that feature */ 1879a4b4bedcSBorislav Petkov if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3)) 188095b0ef55SBorislav Petkov return sys_addr; 188195b0ef55SBorislav Petkov } 188295b0ef55SBorislav Petkov 18837981a28fSAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F2, SWAP_INTLV_REG, &swap_reg); 188495b0ef55SBorislav Petkov 188595b0ef55SBorislav Petkov if (!(swap_reg & 0x1)) 188695b0ef55SBorislav Petkov return sys_addr; 188795b0ef55SBorislav Petkov 188895b0ef55SBorislav Petkov swap_base = (swap_reg >> 3) & 0x7f; 188995b0ef55SBorislav Petkov swap_limit = (swap_reg >> 11) & 0x7f; 189095b0ef55SBorislav Petkov rgn_size = (swap_reg >> 20) & 0x7f; 189195b0ef55SBorislav Petkov tmp_addr = sys_addr >> 27; 189295b0ef55SBorislav Petkov 189395b0ef55SBorislav Petkov if (!(sys_addr >> 34) && 189495b0ef55SBorislav Petkov (((tmp_addr >= swap_base) && 189595b0ef55SBorislav Petkov (tmp_addr <= swap_limit)) || 189695b0ef55SBorislav Petkov (tmp_addr < rgn_size))) 189795b0ef55SBorislav Petkov return sys_addr ^ (u64)swap_base << 27; 189895b0ef55SBorislav Petkov 189995b0ef55SBorislav Petkov return sys_addr; 190095b0ef55SBorislav Petkov } 190195b0ef55SBorislav Petkov 1902f71d0a05SDoug Thompson /* For a given @dram_range, check if @sys_addr falls within it. */ 1903e761359aSBorislav Petkov static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range, 190433ca0643SBorislav Petkov u64 sys_addr, int *chan_sel) 1905f71d0a05SDoug Thompson { 1906229a7a11SBorislav Petkov int cs_found = -EINVAL; 1907c8e518d5SBorislav Petkov u64 chan_addr; 19085d4b58e8SBorislav Petkov u32 dct_sel_base; 190911c75eadSBorislav Petkov u8 channel; 1910229a7a11SBorislav Petkov bool high_range = false; 1911f71d0a05SDoug Thompson 19127f19bf75SBorislav Petkov u8 node_id = dram_dst_node(pvt, range); 1913229a7a11SBorislav Petkov u8 intlv_en = dram_intlv_en(pvt, range); 19147f19bf75SBorislav Petkov u32 intlv_sel = dram_intlv_sel(pvt, range); 1915f71d0a05SDoug Thompson 1916956b9ba1SJoe Perches edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n", 1917c8e518d5SBorislav Petkov range, sys_addr, get_dram_limit(pvt, range)); 1918f71d0a05SDoug Thompson 1919355fba60SBorislav Petkov if (dhar_valid(pvt) && 1920355fba60SBorislav Petkov dhar_base(pvt) <= sys_addr && 1921355fba60SBorislav Petkov sys_addr < BIT_64(32)) { 1922355fba60SBorislav Petkov amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n", 1923355fba60SBorislav Petkov sys_addr); 1924f71d0a05SDoug Thompson return -EINVAL; 1925355fba60SBorislav Petkov } 1926355fba60SBorislav Petkov 1927f030ddfbSBorislav Petkov if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en))) 1928355fba60SBorislav Petkov return -EINVAL; 1929f71d0a05SDoug Thompson 1930b15f0fcaSBorislav Petkov sys_addr = f1x_swap_interleaved_region(pvt, sys_addr); 193195b0ef55SBorislav Petkov 1932f71d0a05SDoug Thompson dct_sel_base = dct_sel_baseaddr(pvt); 1933f71d0a05SDoug Thompson 1934f71d0a05SDoug Thompson /* 1935f71d0a05SDoug Thompson * check whether addresses >= DctSelBaseAddr[47:27] are to be used to 1936f71d0a05SDoug Thompson * select between DCT0 and DCT1. 1937f71d0a05SDoug Thompson */ 1938f71d0a05SDoug Thompson if (dct_high_range_enabled(pvt) && 1939f71d0a05SDoug Thompson !dct_ganging_enabled(pvt) && 1940f71d0a05SDoug Thompson ((sys_addr >> 27) >= (dct_sel_base >> 11))) 1941229a7a11SBorislav Petkov high_range = true; 1942f71d0a05SDoug Thompson 1943b15f0fcaSBorislav Petkov channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en); 1944f71d0a05SDoug Thompson 1945b15f0fcaSBorislav Petkov chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr, 1946c8e518d5SBorislav Petkov high_range, dct_sel_base); 1947f71d0a05SDoug Thompson 1948e2f79dbdSBorislav Petkov /* Remove node interleaving, see F1x120 */ 1949e2f79dbdSBorislav Petkov if (intlv_en) 1950e2f79dbdSBorislav Petkov chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) | 1951e2f79dbdSBorislav Petkov (chan_addr & 0xfff); 1952f71d0a05SDoug Thompson 19535d4b58e8SBorislav Petkov /* remove channel interleave */ 1954f71d0a05SDoug Thompson if (dct_interleave_enabled(pvt) && 1955f71d0a05SDoug Thompson !dct_high_range_enabled(pvt) && 1956f71d0a05SDoug Thompson !dct_ganging_enabled(pvt)) { 19575d4b58e8SBorislav Petkov 19585d4b58e8SBorislav Petkov if (dct_sel_interleave_addr(pvt) != 1) { 19595d4b58e8SBorislav Petkov if (dct_sel_interleave_addr(pvt) == 0x3) 19605d4b58e8SBorislav Petkov /* hash 9 */ 19615d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 10) << 9) | 19625d4b58e8SBorislav Petkov (chan_addr & 0x1ff); 19635d4b58e8SBorislav Petkov else 19645d4b58e8SBorislav Petkov /* A[6] or hash 6 */ 19655d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 7) << 6) | 19665d4b58e8SBorislav Petkov (chan_addr & 0x3f); 19675d4b58e8SBorislav Petkov } else 19685d4b58e8SBorislav Petkov /* A[12] */ 19695d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 13) << 12) | 19705d4b58e8SBorislav Petkov (chan_addr & 0xfff); 1971f71d0a05SDoug Thompson } 1972f71d0a05SDoug Thompson 1973956b9ba1SJoe Perches edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr); 1974f71d0a05SDoug Thompson 1975b15f0fcaSBorislav Petkov cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel); 1976f71d0a05SDoug Thompson 197733ca0643SBorislav Petkov if (cs_found >= 0) 1978f71d0a05SDoug Thompson *chan_sel = channel; 197933ca0643SBorislav Petkov 1980f71d0a05SDoug Thompson return cs_found; 1981f71d0a05SDoug Thompson } 1982f71d0a05SDoug Thompson 198318b94f66SAravind Gopalakrishnan static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range, 198418b94f66SAravind Gopalakrishnan u64 sys_addr, int *chan_sel) 198518b94f66SAravind Gopalakrishnan { 198618b94f66SAravind Gopalakrishnan int cs_found = -EINVAL; 198718b94f66SAravind Gopalakrishnan int num_dcts_intlv = 0; 198818b94f66SAravind Gopalakrishnan u64 chan_addr, chan_offset; 198918b94f66SAravind Gopalakrishnan u64 dct_base, dct_limit; 199018b94f66SAravind Gopalakrishnan u32 dct_cont_base_reg, dct_cont_limit_reg, tmp; 199118b94f66SAravind Gopalakrishnan u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en; 199218b94f66SAravind Gopalakrishnan 199318b94f66SAravind Gopalakrishnan u64 dhar_offset = f10_dhar_offset(pvt); 199418b94f66SAravind Gopalakrishnan u8 intlv_addr = dct_sel_interleave_addr(pvt); 199518b94f66SAravind Gopalakrishnan u8 node_id = dram_dst_node(pvt, range); 199618b94f66SAravind Gopalakrishnan u8 intlv_en = dram_intlv_en(pvt, range); 199718b94f66SAravind Gopalakrishnan 199818b94f66SAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg); 199918b94f66SAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg); 200018b94f66SAravind Gopalakrishnan 200118b94f66SAravind Gopalakrishnan dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0)); 200218b94f66SAravind Gopalakrishnan dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7); 200318b94f66SAravind Gopalakrishnan 200418b94f66SAravind Gopalakrishnan edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n", 200518b94f66SAravind Gopalakrishnan range, sys_addr, get_dram_limit(pvt, range)); 200618b94f66SAravind Gopalakrishnan 200718b94f66SAravind Gopalakrishnan if (!(get_dram_base(pvt, range) <= sys_addr) && 200818b94f66SAravind Gopalakrishnan !(get_dram_limit(pvt, range) >= sys_addr)) 200918b94f66SAravind Gopalakrishnan return -EINVAL; 201018b94f66SAravind Gopalakrishnan 201118b94f66SAravind Gopalakrishnan if (dhar_valid(pvt) && 201218b94f66SAravind Gopalakrishnan dhar_base(pvt) <= sys_addr && 201318b94f66SAravind Gopalakrishnan sys_addr < BIT_64(32)) { 201418b94f66SAravind Gopalakrishnan amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n", 201518b94f66SAravind Gopalakrishnan sys_addr); 201618b94f66SAravind Gopalakrishnan return -EINVAL; 201718b94f66SAravind Gopalakrishnan } 201818b94f66SAravind Gopalakrishnan 201918b94f66SAravind Gopalakrishnan /* Verify sys_addr is within DCT Range. */ 20204fc06b31SAravind Gopalakrishnan dct_base = (u64) dct_sel_baseaddr(pvt); 20214fc06b31SAravind Gopalakrishnan dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF; 202218b94f66SAravind Gopalakrishnan 202318b94f66SAravind Gopalakrishnan if (!(dct_cont_base_reg & BIT(0)) && 20244fc06b31SAravind Gopalakrishnan !(dct_base <= (sys_addr >> 27) && 20254fc06b31SAravind Gopalakrishnan dct_limit >= (sys_addr >> 27))) 202618b94f66SAravind Gopalakrishnan return -EINVAL; 202718b94f66SAravind Gopalakrishnan 202818b94f66SAravind Gopalakrishnan /* Verify number of dct's that participate in channel interleaving. */ 202918b94f66SAravind Gopalakrishnan num_dcts_intlv = (int) hweight8(intlv_en); 203018b94f66SAravind Gopalakrishnan 203118b94f66SAravind Gopalakrishnan if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4)) 203218b94f66SAravind Gopalakrishnan return -EINVAL; 203318b94f66SAravind Gopalakrishnan 2034dc0a50a8SYazen Ghannam if (pvt->model >= 0x60) 2035dc0a50a8SYazen Ghannam channel = f1x_determine_channel(pvt, sys_addr, false, intlv_en); 2036dc0a50a8SYazen Ghannam else 203718b94f66SAravind Gopalakrishnan channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en, 203818b94f66SAravind Gopalakrishnan num_dcts_intlv, dct_sel); 203918b94f66SAravind Gopalakrishnan 204018b94f66SAravind Gopalakrishnan /* Verify we stay within the MAX number of channels allowed */ 20417f3f5240SAravind Gopalakrishnan if (channel > 3) 204218b94f66SAravind Gopalakrishnan return -EINVAL; 204318b94f66SAravind Gopalakrishnan 204418b94f66SAravind Gopalakrishnan leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0)); 204518b94f66SAravind Gopalakrishnan 204618b94f66SAravind Gopalakrishnan /* Get normalized DCT addr */ 204718b94f66SAravind Gopalakrishnan if (leg_mmio_hole && (sys_addr >= BIT_64(32))) 204818b94f66SAravind Gopalakrishnan chan_offset = dhar_offset; 204918b94f66SAravind Gopalakrishnan else 20504fc06b31SAravind Gopalakrishnan chan_offset = dct_base << 27; 205118b94f66SAravind Gopalakrishnan 205218b94f66SAravind Gopalakrishnan chan_addr = sys_addr - chan_offset; 205318b94f66SAravind Gopalakrishnan 205418b94f66SAravind Gopalakrishnan /* remove channel interleave */ 205518b94f66SAravind Gopalakrishnan if (num_dcts_intlv == 2) { 205618b94f66SAravind Gopalakrishnan if (intlv_addr == 0x4) 205718b94f66SAravind Gopalakrishnan chan_addr = ((chan_addr >> 9) << 8) | 205818b94f66SAravind Gopalakrishnan (chan_addr & 0xff); 205918b94f66SAravind Gopalakrishnan else if (intlv_addr == 0x5) 206018b94f66SAravind Gopalakrishnan chan_addr = ((chan_addr >> 10) << 9) | 206118b94f66SAravind Gopalakrishnan (chan_addr & 0x1ff); 206218b94f66SAravind Gopalakrishnan else 206318b94f66SAravind Gopalakrishnan return -EINVAL; 206418b94f66SAravind Gopalakrishnan 206518b94f66SAravind Gopalakrishnan } else if (num_dcts_intlv == 4) { 206618b94f66SAravind Gopalakrishnan if (intlv_addr == 0x4) 206718b94f66SAravind Gopalakrishnan chan_addr = ((chan_addr >> 10) << 8) | 206818b94f66SAravind Gopalakrishnan (chan_addr & 0xff); 206918b94f66SAravind Gopalakrishnan else if (intlv_addr == 0x5) 207018b94f66SAravind Gopalakrishnan chan_addr = ((chan_addr >> 11) << 9) | 207118b94f66SAravind Gopalakrishnan (chan_addr & 0x1ff); 207218b94f66SAravind Gopalakrishnan else 207318b94f66SAravind Gopalakrishnan return -EINVAL; 207418b94f66SAravind Gopalakrishnan } 207518b94f66SAravind Gopalakrishnan 207618b94f66SAravind Gopalakrishnan if (dct_offset_en) { 207718b94f66SAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F1, 207818b94f66SAravind Gopalakrishnan DRAM_CONT_HIGH_OFF + (int) channel * 4, 207918b94f66SAravind Gopalakrishnan &tmp); 20804fc06b31SAravind Gopalakrishnan chan_addr += (u64) ((tmp >> 11) & 0xfff) << 27; 208118b94f66SAravind Gopalakrishnan } 208218b94f66SAravind Gopalakrishnan 208318b94f66SAravind Gopalakrishnan f15h_select_dct(pvt, channel); 208418b94f66SAravind Gopalakrishnan 208518b94f66SAravind Gopalakrishnan edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr); 208618b94f66SAravind Gopalakrishnan 208718b94f66SAravind Gopalakrishnan /* 208818b94f66SAravind Gopalakrishnan * Find Chip select: 208918b94f66SAravind Gopalakrishnan * if channel = 3, then alias it to 1. This is because, in F15 M30h, 209018b94f66SAravind Gopalakrishnan * there is support for 4 DCT's, but only 2 are currently functional. 209118b94f66SAravind Gopalakrishnan * They are DCT0 and DCT3. But we have read all registers of DCT3 into 209218b94f66SAravind Gopalakrishnan * pvt->csels[1]. So we need to use '1' here to get correct info. 209318b94f66SAravind Gopalakrishnan * Refer F15 M30h BKDG Section 2.10 and 2.10.3 for clarifications. 209418b94f66SAravind Gopalakrishnan */ 209518b94f66SAravind Gopalakrishnan alias_channel = (channel == 3) ? 1 : channel; 209618b94f66SAravind Gopalakrishnan 209718b94f66SAravind Gopalakrishnan cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel); 209818b94f66SAravind Gopalakrishnan 209918b94f66SAravind Gopalakrishnan if (cs_found >= 0) 210018b94f66SAravind Gopalakrishnan *chan_sel = alias_channel; 210118b94f66SAravind Gopalakrishnan 210218b94f66SAravind Gopalakrishnan return cs_found; 210318b94f66SAravind Gopalakrishnan } 210418b94f66SAravind Gopalakrishnan 210518b94f66SAravind Gopalakrishnan static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt, 210618b94f66SAravind Gopalakrishnan u64 sys_addr, 210733ca0643SBorislav Petkov int *chan_sel) 2108f71d0a05SDoug Thompson { 2109e761359aSBorislav Petkov int cs_found = -EINVAL; 2110e761359aSBorislav Petkov unsigned range; 2111f71d0a05SDoug Thompson 21127f19bf75SBorislav Petkov for (range = 0; range < DRAM_RANGES; range++) { 21137f19bf75SBorislav Petkov if (!dram_rw(pvt, range)) 2114f71d0a05SDoug Thompson continue; 2115f71d0a05SDoug Thompson 211618b94f66SAravind Gopalakrishnan if (pvt->fam == 0x15 && pvt->model >= 0x30) 211718b94f66SAravind Gopalakrishnan cs_found = f15_m30h_match_to_this_node(pvt, range, 211818b94f66SAravind Gopalakrishnan sys_addr, 211918b94f66SAravind Gopalakrishnan chan_sel); 2120f71d0a05SDoug Thompson 212118b94f66SAravind Gopalakrishnan else if ((get_dram_base(pvt, range) <= sys_addr) && 212218b94f66SAravind Gopalakrishnan (get_dram_limit(pvt, range) >= sys_addr)) { 2123b15f0fcaSBorislav Petkov cs_found = f1x_match_to_this_node(pvt, range, 212433ca0643SBorislav Petkov sys_addr, chan_sel); 2125f71d0a05SDoug Thompson if (cs_found >= 0) 2126f71d0a05SDoug Thompson break; 2127f71d0a05SDoug Thompson } 2128f71d0a05SDoug Thompson } 2129f71d0a05SDoug Thompson return cs_found; 2130f71d0a05SDoug Thompson } 2131f71d0a05SDoug Thompson 2132f71d0a05SDoug Thompson /* 2133bdc30a0cSBorislav Petkov * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps 2134bdc30a0cSBorislav Petkov * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW). 2135f71d0a05SDoug Thompson * 2136bdc30a0cSBorislav Petkov * The @sys_addr is usually an error address received from the hardware 2137bdc30a0cSBorislav Petkov * (MCX_ADDR). 2138f71d0a05SDoug Thompson */ 2139b15f0fcaSBorislav Petkov static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr, 214033ca0643SBorislav Petkov struct err_info *err) 2141f71d0a05SDoug Thompson { 2142f71d0a05SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 2143f71d0a05SDoug Thompson 214433ca0643SBorislav Petkov error_address_to_page_and_offset(sys_addr, err); 2145ab5a503cSMauro Carvalho Chehab 214633ca0643SBorislav Petkov err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel); 214733ca0643SBorislav Petkov if (err->csrow < 0) { 214833ca0643SBorislav Petkov err->err_code = ERR_CSROW; 2149bdc30a0cSBorislav Petkov return; 2150bdc30a0cSBorislav Petkov } 2151bdc30a0cSBorislav Petkov 2152f71d0a05SDoug Thompson /* 2153bdc30a0cSBorislav Petkov * We need the syndromes for channel detection only when we're 2154bdc30a0cSBorislav Petkov * ganged. Otherwise @chan should already contain the channel at 2155bdc30a0cSBorislav Petkov * this point. 2156f71d0a05SDoug Thompson */ 2157a97fa68eSBorislav Petkov if (dct_ganging_enabled(pvt)) 215833ca0643SBorislav Petkov err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome); 2159f71d0a05SDoug Thompson } 2160f71d0a05SDoug Thompson 2161f71d0a05SDoug Thompson /* 21628566c4dfSBorislav Petkov * debug routine to display the memory sizes of all logical DIMMs and its 2163cb328507SBorislav Petkov * CSROWs 2164f71d0a05SDoug Thompson */ 2165d1ea71cdSBorislav Petkov static void debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl) 2166f71d0a05SDoug Thompson { 2167bb89f5a0SBorislav Petkov int dimm, size0, size1; 2168525a1b20SBorislav Petkov u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases; 2169525a1b20SBorislav Petkov u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0; 2170f71d0a05SDoug Thompson 2171a4b4bedcSBorislav Petkov if (pvt->fam == 0xf) { 21728566c4dfSBorislav Petkov /* K8 families < revF not supported yet */ 21731433eb99SBorislav Petkov if (pvt->ext_model < K8_REV_F) 21748566c4dfSBorislav Petkov return; 21758566c4dfSBorislav Petkov else 21768566c4dfSBorislav Petkov WARN_ON(ctrl != 0); 21778566c4dfSBorislav Petkov } 21788566c4dfSBorislav Petkov 21797981a28fSAravind Gopalakrishnan if (pvt->fam == 0x10) { 21807981a28fSAravind Gopalakrishnan dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 21817981a28fSAravind Gopalakrishnan : pvt->dbam0; 21827981a28fSAravind Gopalakrishnan dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? 21837981a28fSAravind Gopalakrishnan pvt->csels[1].csbases : 21847981a28fSAravind Gopalakrishnan pvt->csels[0].csbases; 21857981a28fSAravind Gopalakrishnan } else if (ctrl) { 21867981a28fSAravind Gopalakrishnan dbam = pvt->dbam0; 21877981a28fSAravind Gopalakrishnan dcsb = pvt->csels[1].csbases; 21887981a28fSAravind Gopalakrishnan } 2189956b9ba1SJoe Perches edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", 2190956b9ba1SJoe Perches ctrl, dbam); 2191f71d0a05SDoug Thompson 21928566c4dfSBorislav Petkov edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl); 21938566c4dfSBorislav Petkov 2194f71d0a05SDoug Thompson /* Dump memory sizes for DIMM and its CSROWs */ 2195f71d0a05SDoug Thompson for (dimm = 0; dimm < 4; dimm++) { 2196f71d0a05SDoug Thompson 2197f71d0a05SDoug Thompson size0 = 0; 219811c75eadSBorislav Petkov if (dcsb[dimm*2] & DCSB_CS_ENABLE) 219907ed82efSYazen Ghannam /* 220007ed82efSYazen Ghannam * For F15m60h, we need multiplier for LRDIMM cs_size 220107ed82efSYazen Ghannam * calculation. We pass dimm value to the dbam_to_cs 2202a597d2a5SAravind Gopalakrishnan * mapper so we can find the multiplier from the 2203a597d2a5SAravind Gopalakrishnan * corresponding DCSM. 2204a597d2a5SAravind Gopalakrishnan */ 220541d8bfabSBorislav Petkov size0 = pvt->ops->dbam_to_cs(pvt, ctrl, 2206a597d2a5SAravind Gopalakrishnan DBAM_DIMM(dimm, dbam), 2207a597d2a5SAravind Gopalakrishnan dimm); 2208f71d0a05SDoug Thompson 2209f71d0a05SDoug Thompson size1 = 0; 221011c75eadSBorislav Petkov if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE) 221141d8bfabSBorislav Petkov size1 = pvt->ops->dbam_to_cs(pvt, ctrl, 2212a597d2a5SAravind Gopalakrishnan DBAM_DIMM(dimm, dbam), 2213a597d2a5SAravind Gopalakrishnan dimm); 2214f71d0a05SDoug Thompson 221524f9a7feSBorislav Petkov amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n", 2216bb89f5a0SBorislav Petkov dimm * 2, size0, 2217bb89f5a0SBorislav Petkov dimm * 2 + 1, size1); 2218f71d0a05SDoug Thompson } 2219f71d0a05SDoug Thompson } 2220f71d0a05SDoug Thompson 2221d1ea71cdSBorislav Petkov static struct amd64_family_type family_types[] = { 22224d37607aSDoug Thompson [K8_CPUS] = { 22230092b20dSBorislav Petkov .ctl_name = "K8", 22248d5b5d9cSBorislav Petkov .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP, 22253f37a36bSBorislav Petkov .f2_id = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL, 22265e4c5527SYazen Ghannam .max_mcs = 2, 22274d37607aSDoug Thompson .ops = { 22284d37607aSDoug Thompson .early_channel_count = k8_early_channel_count, 22294d37607aSDoug Thompson .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow, 22301433eb99SBorislav Petkov .dbam_to_cs = k8_dbam_to_chip_select, 22314d37607aSDoug Thompson } 22324d37607aSDoug Thompson }, 22334d37607aSDoug Thompson [F10_CPUS] = { 22340092b20dSBorislav Petkov .ctl_name = "F10h", 22358d5b5d9cSBorislav Petkov .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP, 22363f37a36bSBorislav Petkov .f2_id = PCI_DEVICE_ID_AMD_10H_NB_DRAM, 22375e4c5527SYazen Ghannam .max_mcs = 2, 22384d37607aSDoug Thompson .ops = { 22397d20d14dSBorislav Petkov .early_channel_count = f1x_early_channel_count, 2240b15f0fcaSBorislav Petkov .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, 22411433eb99SBorislav Petkov .dbam_to_cs = f10_dbam_to_chip_select, 2242b2b0c605SBorislav Petkov } 2243b2b0c605SBorislav Petkov }, 2244b2b0c605SBorislav Petkov [F15_CPUS] = { 2245b2b0c605SBorislav Petkov .ctl_name = "F15h", 2246df71a053SBorislav Petkov .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1, 22473f37a36bSBorislav Petkov .f2_id = PCI_DEVICE_ID_AMD_15H_NB_F2, 22485e4c5527SYazen Ghannam .max_mcs = 2, 2249b2b0c605SBorislav Petkov .ops = { 22507d20d14dSBorislav Petkov .early_channel_count = f1x_early_channel_count, 2251b15f0fcaSBorislav Petkov .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, 225241d8bfabSBorislav Petkov .dbam_to_cs = f15_dbam_to_chip_select, 22534d37607aSDoug Thompson } 22544d37607aSDoug Thompson }, 225518b94f66SAravind Gopalakrishnan [F15_M30H_CPUS] = { 225618b94f66SAravind Gopalakrishnan .ctl_name = "F15h_M30h", 225718b94f66SAravind Gopalakrishnan .f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1, 22583f37a36bSBorislav Petkov .f2_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2, 22595e4c5527SYazen Ghannam .max_mcs = 2, 226018b94f66SAravind Gopalakrishnan .ops = { 226118b94f66SAravind Gopalakrishnan .early_channel_count = f1x_early_channel_count, 226218b94f66SAravind Gopalakrishnan .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, 226318b94f66SAravind Gopalakrishnan .dbam_to_cs = f16_dbam_to_chip_select, 226418b94f66SAravind Gopalakrishnan } 226518b94f66SAravind Gopalakrishnan }, 2266a597d2a5SAravind Gopalakrishnan [F15_M60H_CPUS] = { 2267a597d2a5SAravind Gopalakrishnan .ctl_name = "F15h_M60h", 2268a597d2a5SAravind Gopalakrishnan .f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1, 22693f37a36bSBorislav Petkov .f2_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F2, 22705e4c5527SYazen Ghannam .max_mcs = 2, 2271a597d2a5SAravind Gopalakrishnan .ops = { 2272a597d2a5SAravind Gopalakrishnan .early_channel_count = f1x_early_channel_count, 2273a597d2a5SAravind Gopalakrishnan .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, 2274a597d2a5SAravind Gopalakrishnan .dbam_to_cs = f15_m60h_dbam_to_chip_select, 2275a597d2a5SAravind Gopalakrishnan } 2276a597d2a5SAravind Gopalakrishnan }, 227794c1acf2SAravind Gopalakrishnan [F16_CPUS] = { 227894c1acf2SAravind Gopalakrishnan .ctl_name = "F16h", 227994c1acf2SAravind Gopalakrishnan .f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1, 22803f37a36bSBorislav Petkov .f2_id = PCI_DEVICE_ID_AMD_16H_NB_F2, 22815e4c5527SYazen Ghannam .max_mcs = 2, 228294c1acf2SAravind Gopalakrishnan .ops = { 228394c1acf2SAravind Gopalakrishnan .early_channel_count = f1x_early_channel_count, 228494c1acf2SAravind Gopalakrishnan .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, 228594c1acf2SAravind Gopalakrishnan .dbam_to_cs = f16_dbam_to_chip_select, 228694c1acf2SAravind Gopalakrishnan } 228794c1acf2SAravind Gopalakrishnan }, 228885a8885bSAravind Gopalakrishnan [F16_M30H_CPUS] = { 228985a8885bSAravind Gopalakrishnan .ctl_name = "F16h_M30h", 229085a8885bSAravind Gopalakrishnan .f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1, 22913f37a36bSBorislav Petkov .f2_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2, 22925e4c5527SYazen Ghannam .max_mcs = 2, 229385a8885bSAravind Gopalakrishnan .ops = { 229485a8885bSAravind Gopalakrishnan .early_channel_count = f1x_early_channel_count, 229585a8885bSAravind Gopalakrishnan .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, 229685a8885bSAravind Gopalakrishnan .dbam_to_cs = f16_dbam_to_chip_select, 229785a8885bSAravind Gopalakrishnan } 229885a8885bSAravind Gopalakrishnan }, 2299f1cbbec9SYazen Ghannam [F17_CPUS] = { 2300f1cbbec9SYazen Ghannam .ctl_name = "F17h", 2301f1cbbec9SYazen Ghannam .f0_id = PCI_DEVICE_ID_AMD_17H_DF_F0, 2302f1cbbec9SYazen Ghannam .f6_id = PCI_DEVICE_ID_AMD_17H_DF_F6, 23035e4c5527SYazen Ghannam .max_mcs = 2, 2304f1cbbec9SYazen Ghannam .ops = { 2305f1cbbec9SYazen Ghannam .early_channel_count = f17_early_channel_count, 2306e53a3b26SYazen Ghannam .dbam_to_cs = f17_addr_mask_to_cs_size, 2307f1cbbec9SYazen Ghannam } 2308f1cbbec9SYazen Ghannam }, 23098960de4aSMichael Jin [F17_M10H_CPUS] = { 23108960de4aSMichael Jin .ctl_name = "F17h_M10h", 23118960de4aSMichael Jin .f0_id = PCI_DEVICE_ID_AMD_17H_M10H_DF_F0, 23128960de4aSMichael Jin .f6_id = PCI_DEVICE_ID_AMD_17H_M10H_DF_F6, 23135e4c5527SYazen Ghannam .max_mcs = 2, 23148960de4aSMichael Jin .ops = { 23158960de4aSMichael Jin .early_channel_count = f17_early_channel_count, 2316e53a3b26SYazen Ghannam .dbam_to_cs = f17_addr_mask_to_cs_size, 23178960de4aSMichael Jin } 23188960de4aSMichael Jin }, 23196e846239SYazen Ghannam [F17_M30H_CPUS] = { 23206e846239SYazen Ghannam .ctl_name = "F17h_M30h", 23216e846239SYazen Ghannam .f0_id = PCI_DEVICE_ID_AMD_17H_M30H_DF_F0, 23226e846239SYazen Ghannam .f6_id = PCI_DEVICE_ID_AMD_17H_M30H_DF_F6, 23235e4c5527SYazen Ghannam .max_mcs = 8, 23246e846239SYazen Ghannam .ops = { 23256e846239SYazen Ghannam .early_channel_count = f17_early_channel_count, 2326e53a3b26SYazen Ghannam .dbam_to_cs = f17_addr_mask_to_cs_size, 23276e846239SYazen Ghannam } 23286e846239SYazen Ghannam }, 23293e443eb3SIsaac Vaughn [F17_M70H_CPUS] = { 23303e443eb3SIsaac Vaughn .ctl_name = "F17h_M70h", 23313e443eb3SIsaac Vaughn .f0_id = PCI_DEVICE_ID_AMD_17H_M70H_DF_F0, 23323e443eb3SIsaac Vaughn .f6_id = PCI_DEVICE_ID_AMD_17H_M70H_DF_F6, 23335e4c5527SYazen Ghannam .max_mcs = 2, 23343e443eb3SIsaac Vaughn .ops = { 23353e443eb3SIsaac Vaughn .early_channel_count = f17_early_channel_count, 23363e443eb3SIsaac Vaughn .dbam_to_cs = f17_addr_mask_to_cs_size, 23373e443eb3SIsaac Vaughn } 23383e443eb3SIsaac Vaughn }, 23394d37607aSDoug Thompson }; 23404d37607aSDoug Thompson 2341b1289d6fSDoug Thompson /* 2342bfc04aecSBorislav Petkov * These are tables of eigenvectors (one per line) which can be used for the 2343bfc04aecSBorislav Petkov * construction of the syndrome tables. The modified syndrome search algorithm 2344bfc04aecSBorislav Petkov * uses those to find the symbol in error and thus the DIMM. 2345b1289d6fSDoug Thompson * 2346bfc04aecSBorislav Petkov * Algorithm courtesy of Ross LaFetra from AMD. 2347b1289d6fSDoug Thompson */ 2348c7e5301aSDaniel J Blueman static const u16 x4_vectors[] = { 2349bfc04aecSBorislav Petkov 0x2f57, 0x1afe, 0x66cc, 0xdd88, 2350bfc04aecSBorislav Petkov 0x11eb, 0x3396, 0x7f4c, 0xeac8, 2351bfc04aecSBorislav Petkov 0x0001, 0x0002, 0x0004, 0x0008, 2352bfc04aecSBorislav Petkov 0x1013, 0x3032, 0x4044, 0x8088, 2353bfc04aecSBorislav Petkov 0x106b, 0x30d6, 0x70fc, 0xe0a8, 2354bfc04aecSBorislav Petkov 0x4857, 0xc4fe, 0x13cc, 0x3288, 2355bfc04aecSBorislav Petkov 0x1ac5, 0x2f4a, 0x5394, 0xa1e8, 2356bfc04aecSBorislav Petkov 0x1f39, 0x251e, 0xbd6c, 0x6bd8, 2357bfc04aecSBorislav Petkov 0x15c1, 0x2a42, 0x89ac, 0x4758, 2358bfc04aecSBorislav Petkov 0x2b03, 0x1602, 0x4f0c, 0xca08, 2359bfc04aecSBorislav Petkov 0x1f07, 0x3a0e, 0x6b04, 0xbd08, 2360bfc04aecSBorislav Petkov 0x8ba7, 0x465e, 0x244c, 0x1cc8, 2361bfc04aecSBorislav Petkov 0x2b87, 0x164e, 0x642c, 0xdc18, 2362bfc04aecSBorislav Petkov 0x40b9, 0x80de, 0x1094, 0x20e8, 2363bfc04aecSBorislav Petkov 0x27db, 0x1eb6, 0x9dac, 0x7b58, 2364bfc04aecSBorislav Petkov 0x11c1, 0x2242, 0x84ac, 0x4c58, 2365bfc04aecSBorislav Petkov 0x1be5, 0x2d7a, 0x5e34, 0xa718, 2366bfc04aecSBorislav Petkov 0x4b39, 0x8d1e, 0x14b4, 0x28d8, 2367bfc04aecSBorislav Petkov 0x4c97, 0xc87e, 0x11fc, 0x33a8, 2368bfc04aecSBorislav Petkov 0x8e97, 0x497e, 0x2ffc, 0x1aa8, 2369bfc04aecSBorislav Petkov 0x16b3, 0x3d62, 0x4f34, 0x8518, 2370bfc04aecSBorislav Petkov 0x1e2f, 0x391a, 0x5cac, 0xf858, 2371bfc04aecSBorislav Petkov 0x1d9f, 0x3b7a, 0x572c, 0xfe18, 2372bfc04aecSBorislav Petkov 0x15f5, 0x2a5a, 0x5264, 0xa3b8, 2373bfc04aecSBorislav Petkov 0x1dbb, 0x3b66, 0x715c, 0xe3f8, 2374bfc04aecSBorislav Petkov 0x4397, 0xc27e, 0x17fc, 0x3ea8, 2375bfc04aecSBorislav Petkov 0x1617, 0x3d3e, 0x6464, 0xb8b8, 2376bfc04aecSBorislav Petkov 0x23ff, 0x12aa, 0xab6c, 0x56d8, 2377bfc04aecSBorislav Petkov 0x2dfb, 0x1ba6, 0x913c, 0x7328, 2378bfc04aecSBorislav Petkov 0x185d, 0x2ca6, 0x7914, 0x9e28, 2379bfc04aecSBorislav Petkov 0x171b, 0x3e36, 0x7d7c, 0xebe8, 2380bfc04aecSBorislav Petkov 0x4199, 0x82ee, 0x19f4, 0x2e58, 2381bfc04aecSBorislav Petkov 0x4807, 0xc40e, 0x130c, 0x3208, 2382bfc04aecSBorislav Petkov 0x1905, 0x2e0a, 0x5804, 0xac08, 2383bfc04aecSBorislav Petkov 0x213f, 0x132a, 0xadfc, 0x5ba8, 2384bfc04aecSBorislav Petkov 0x19a9, 0x2efe, 0xb5cc, 0x6f88, 2385b1289d6fSDoug Thompson }; 2386b1289d6fSDoug Thompson 2387c7e5301aSDaniel J Blueman static const u16 x8_vectors[] = { 2388bfc04aecSBorislav Petkov 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480, 2389bfc04aecSBorislav Petkov 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80, 2390bfc04aecSBorislav Petkov 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80, 2391bfc04aecSBorislav Petkov 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80, 2392bfc04aecSBorislav Petkov 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780, 2393bfc04aecSBorislav Petkov 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080, 2394bfc04aecSBorislav Petkov 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080, 2395bfc04aecSBorislav Petkov 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080, 2396bfc04aecSBorislav Petkov 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80, 2397bfc04aecSBorislav Petkov 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580, 2398bfc04aecSBorislav Petkov 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880, 2399bfc04aecSBorislav Petkov 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280, 2400bfc04aecSBorislav Petkov 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180, 2401bfc04aecSBorislav Petkov 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580, 2402bfc04aecSBorislav Petkov 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280, 2403bfc04aecSBorislav Petkov 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180, 2404bfc04aecSBorislav Petkov 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080, 2405bfc04aecSBorislav Petkov 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 2406bfc04aecSBorislav Petkov 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000, 2407bfc04aecSBorislav Petkov }; 2408bfc04aecSBorislav Petkov 2409c7e5301aSDaniel J Blueman static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs, 2410d34a6ecdSBorislav Petkov unsigned v_dim) 2411b1289d6fSDoug Thompson { 2412bfc04aecSBorislav Petkov unsigned int i, err_sym; 2413b1289d6fSDoug Thompson 2414bfc04aecSBorislav Petkov for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) { 2415bfc04aecSBorislav Petkov u16 s = syndrome; 2416d34a6ecdSBorislav Petkov unsigned v_idx = err_sym * v_dim; 2417d34a6ecdSBorislav Petkov unsigned v_end = (err_sym + 1) * v_dim; 2418b1289d6fSDoug Thompson 2419bfc04aecSBorislav Petkov /* walk over all 16 bits of the syndrome */ 2420bfc04aecSBorislav Petkov for (i = 1; i < (1U << 16); i <<= 1) { 2421bfc04aecSBorislav Petkov 2422bfc04aecSBorislav Petkov /* if bit is set in that eigenvector... */ 2423bfc04aecSBorislav Petkov if (v_idx < v_end && vectors[v_idx] & i) { 2424bfc04aecSBorislav Petkov u16 ev_comp = vectors[v_idx++]; 2425bfc04aecSBorislav Petkov 2426bfc04aecSBorislav Petkov /* ... and bit set in the modified syndrome, */ 2427bfc04aecSBorislav Petkov if (s & i) { 2428bfc04aecSBorislav Petkov /* remove it. */ 2429bfc04aecSBorislav Petkov s ^= ev_comp; 2430bfc04aecSBorislav Petkov 2431bfc04aecSBorislav Petkov if (!s) 2432bfc04aecSBorislav Petkov return err_sym; 2433bfc04aecSBorislav Petkov } 2434bfc04aecSBorislav Petkov 2435bfc04aecSBorislav Petkov } else if (s & i) 2436bfc04aecSBorislav Petkov /* can't get to zero, move to next symbol */ 2437bfc04aecSBorislav Petkov break; 2438bfc04aecSBorislav Petkov } 2439b1289d6fSDoug Thompson } 2440b1289d6fSDoug Thompson 2441956b9ba1SJoe Perches edac_dbg(0, "syndrome(%x) not found\n", syndrome); 2442b1289d6fSDoug Thompson return -1; 2443b1289d6fSDoug Thompson } 2444d27bf6faSDoug Thompson 2445bfc04aecSBorislav Petkov static int map_err_sym_to_channel(int err_sym, int sym_size) 2446bfc04aecSBorislav Petkov { 2447bfc04aecSBorislav Petkov if (sym_size == 4) 2448bfc04aecSBorislav Petkov switch (err_sym) { 2449bfc04aecSBorislav Petkov case 0x20: 2450bfc04aecSBorislav Petkov case 0x21: 2451bfc04aecSBorislav Petkov return 0; 2452bfc04aecSBorislav Petkov break; 2453bfc04aecSBorislav Petkov case 0x22: 2454bfc04aecSBorislav Petkov case 0x23: 2455bfc04aecSBorislav Petkov return 1; 2456bfc04aecSBorislav Petkov break; 2457bfc04aecSBorislav Petkov default: 2458bfc04aecSBorislav Petkov return err_sym >> 4; 2459bfc04aecSBorislav Petkov break; 2460bfc04aecSBorislav Petkov } 2461bfc04aecSBorislav Petkov /* x8 symbols */ 2462bfc04aecSBorislav Petkov else 2463bfc04aecSBorislav Petkov switch (err_sym) { 2464bfc04aecSBorislav Petkov /* imaginary bits not in a DIMM */ 2465bfc04aecSBorislav Petkov case 0x10: 2466bfc04aecSBorislav Petkov WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n", 2467bfc04aecSBorislav Petkov err_sym); 2468bfc04aecSBorislav Petkov return -1; 2469bfc04aecSBorislav Petkov break; 2470bfc04aecSBorislav Petkov 2471bfc04aecSBorislav Petkov case 0x11: 2472bfc04aecSBorislav Petkov return 0; 2473bfc04aecSBorislav Petkov break; 2474bfc04aecSBorislav Petkov case 0x12: 2475bfc04aecSBorislav Petkov return 1; 2476bfc04aecSBorislav Petkov break; 2477bfc04aecSBorislav Petkov default: 2478bfc04aecSBorislav Petkov return err_sym >> 3; 2479bfc04aecSBorislav Petkov break; 2480bfc04aecSBorislav Petkov } 2481bfc04aecSBorislav Petkov return -1; 2482bfc04aecSBorislav Petkov } 2483bfc04aecSBorislav Petkov 2484bfc04aecSBorislav Petkov static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome) 2485bfc04aecSBorislav Petkov { 2486bfc04aecSBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 2487ad6a32e9SBorislav Petkov int err_sym = -1; 2488bfc04aecSBorislav Petkov 2489a3b7db09SBorislav Petkov if (pvt->ecc_sym_sz == 8) 2490bfc04aecSBorislav Petkov err_sym = decode_syndrome(syndrome, x8_vectors, 2491ad6a32e9SBorislav Petkov ARRAY_SIZE(x8_vectors), 2492a3b7db09SBorislav Petkov pvt->ecc_sym_sz); 2493a3b7db09SBorislav Petkov else if (pvt->ecc_sym_sz == 4) 2494ad6a32e9SBorislav Petkov err_sym = decode_syndrome(syndrome, x4_vectors, 2495ad6a32e9SBorislav Petkov ARRAY_SIZE(x4_vectors), 2496a3b7db09SBorislav Petkov pvt->ecc_sym_sz); 2497ad6a32e9SBorislav Petkov else { 2498a3b7db09SBorislav Petkov amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz); 2499ad6a32e9SBorislav Petkov return err_sym; 2500bfc04aecSBorislav Petkov } 2501ad6a32e9SBorislav Petkov 2502a3b7db09SBorislav Petkov return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz); 250341c31044SBorislav Petkov } 2504bfc04aecSBorislav Petkov 2505e70984d9SYazen Ghannam static void __log_ecc_error(struct mem_ctl_info *mci, struct err_info *err, 250633ca0643SBorislav Petkov u8 ecc_type) 2507d27bf6faSDoug Thompson { 250833ca0643SBorislav Petkov enum hw_event_mc_err_type err_type; 250933ca0643SBorislav Petkov const char *string; 2510d27bf6faSDoug Thompson 251133ca0643SBorislav Petkov if (ecc_type == 2) 251233ca0643SBorislav Petkov err_type = HW_EVENT_ERR_CORRECTED; 251333ca0643SBorislav Petkov else if (ecc_type == 1) 251433ca0643SBorislav Petkov err_type = HW_EVENT_ERR_UNCORRECTED; 2515d12a969eSYazen Ghannam else if (ecc_type == 3) 2516d12a969eSYazen Ghannam err_type = HW_EVENT_ERR_DEFERRED; 251733ca0643SBorislav Petkov else { 251833ca0643SBorislav Petkov WARN(1, "Something is rotten in the state of Denmark.\n"); 2519d27bf6faSDoug Thompson return; 2520d27bf6faSDoug Thompson } 2521d27bf6faSDoug Thompson 252233ca0643SBorislav Petkov switch (err->err_code) { 252333ca0643SBorislav Petkov case DECODE_OK: 252433ca0643SBorislav Petkov string = ""; 252533ca0643SBorislav Petkov break; 252633ca0643SBorislav Petkov case ERR_NODE: 252733ca0643SBorislav Petkov string = "Failed to map error addr to a node"; 252833ca0643SBorislav Petkov break; 252933ca0643SBorislav Petkov case ERR_CSROW: 253033ca0643SBorislav Petkov string = "Failed to map error addr to a csrow"; 253133ca0643SBorislav Petkov break; 253233ca0643SBorislav Petkov case ERR_CHANNEL: 2533713ad546SYazen Ghannam string = "Unknown syndrome - possible error reporting race"; 2534713ad546SYazen Ghannam break; 2535713ad546SYazen Ghannam case ERR_SYND: 2536713ad546SYazen Ghannam string = "MCA_SYND not valid - unknown syndrome and csrow"; 2537713ad546SYazen Ghannam break; 2538713ad546SYazen Ghannam case ERR_NORM_ADDR: 2539713ad546SYazen Ghannam string = "Cannot decode normalized address"; 254033ca0643SBorislav Petkov break; 254133ca0643SBorislav Petkov default: 254233ca0643SBorislav Petkov string = "WTF error"; 254333ca0643SBorislav Petkov break; 2544d27bf6faSDoug Thompson } 254533ca0643SBorislav Petkov 254633ca0643SBorislav Petkov edac_mc_handle_error(err_type, mci, 1, 254733ca0643SBorislav Petkov err->page, err->offset, err->syndrome, 254833ca0643SBorislav Petkov err->csrow, err->channel, -1, 254933ca0643SBorislav Petkov string, ""); 2550d27bf6faSDoug Thompson } 2551d27bf6faSDoug Thompson 2552df781d03SBorislav Petkov static inline void decode_bus_error(int node_id, struct mce *m) 2553d27bf6faSDoug Thompson { 25540c510cc8SDaniel J Blueman struct mem_ctl_info *mci; 25550c510cc8SDaniel J Blueman struct amd64_pvt *pvt; 2556f192c7b1SBorislav Petkov u8 ecc_type = (m->status >> 45) & 0x3; 255766fed2d4SBorislav Petkov u8 xec = XEC(m->status, 0x1f); 255866fed2d4SBorislav Petkov u16 ec = EC(m->status); 255933ca0643SBorislav Petkov u64 sys_addr; 256033ca0643SBorislav Petkov struct err_info err; 2561d27bf6faSDoug Thompson 25620c510cc8SDaniel J Blueman mci = edac_mc_find(node_id); 25630c510cc8SDaniel J Blueman if (!mci) 25640c510cc8SDaniel J Blueman return; 25650c510cc8SDaniel J Blueman 25660c510cc8SDaniel J Blueman pvt = mci->pvt_info; 25670c510cc8SDaniel J Blueman 256866fed2d4SBorislav Petkov /* Bail out early if this was an 'observed' error */ 25695980bb9cSBorislav Petkov if (PP(ec) == NBSL_PP_OBS) 2570b70ef010SBorislav Petkov return; 2571d27bf6faSDoug Thompson 2572ecaf5606SBorislav Petkov /* Do only ECC errors */ 2573ecaf5606SBorislav Petkov if (xec && xec != F10_NBSL_EXT_ERR_ECC) 2574d27bf6faSDoug Thompson return; 2575d27bf6faSDoug Thompson 257633ca0643SBorislav Petkov memset(&err, 0, sizeof(err)); 257733ca0643SBorislav Petkov 2578a4b4bedcSBorislav Petkov sys_addr = get_error_address(pvt, m); 257933ca0643SBorislav Petkov 2580ecaf5606SBorislav Petkov if (ecc_type == 2) 258133ca0643SBorislav Petkov err.syndrome = extract_syndrome(m->status); 258233ca0643SBorislav Petkov 258333ca0643SBorislav Petkov pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err); 258433ca0643SBorislav Petkov 2585e70984d9SYazen Ghannam __log_ecc_error(mci, &err, ecc_type); 2586d27bf6faSDoug Thompson } 2587d27bf6faSDoug Thompson 25880ec449eeSDoug Thompson /* 2589713ad546SYazen Ghannam * To find the UMC channel represented by this bank we need to match on its 2590713ad546SYazen Ghannam * instance_id. The instance_id of a bank is held in the lower 32 bits of its 2591713ad546SYazen Ghannam * IPID. 2592bdcee774SYazen Ghannam * 2593bdcee774SYazen Ghannam * Currently, we can derive the channel number by looking at the 6th nibble in 2594bdcee774SYazen Ghannam * the instance_id. For example, instance_id=0xYXXXXX where Y is the channel 2595bdcee774SYazen Ghannam * number. 2596713ad546SYazen Ghannam */ 2597bdcee774SYazen Ghannam static int find_umc_channel(struct mce *m) 2598713ad546SYazen Ghannam { 2599bdcee774SYazen Ghannam return (m->ipid & GENMASK(31, 0)) >> 20; 2600713ad546SYazen Ghannam } 2601713ad546SYazen Ghannam 2602713ad546SYazen Ghannam static void decode_umc_error(int node_id, struct mce *m) 2603713ad546SYazen Ghannam { 2604713ad546SYazen Ghannam u8 ecc_type = (m->status >> 45) & 0x3; 2605713ad546SYazen Ghannam struct mem_ctl_info *mci; 2606713ad546SYazen Ghannam struct amd64_pvt *pvt; 2607713ad546SYazen Ghannam struct err_info err; 2608713ad546SYazen Ghannam u64 sys_addr; 2609713ad546SYazen Ghannam 2610713ad546SYazen Ghannam mci = edac_mc_find(node_id); 2611713ad546SYazen Ghannam if (!mci) 2612713ad546SYazen Ghannam return; 2613713ad546SYazen Ghannam 2614713ad546SYazen Ghannam pvt = mci->pvt_info; 2615713ad546SYazen Ghannam 2616713ad546SYazen Ghannam memset(&err, 0, sizeof(err)); 2617713ad546SYazen Ghannam 2618713ad546SYazen Ghannam if (m->status & MCI_STATUS_DEFERRED) 2619713ad546SYazen Ghannam ecc_type = 3; 2620713ad546SYazen Ghannam 2621bdcee774SYazen Ghannam err.channel = find_umc_channel(m); 2622713ad546SYazen Ghannam 2623713ad546SYazen Ghannam if (!(m->status & MCI_STATUS_SYNDV)) { 2624713ad546SYazen Ghannam err.err_code = ERR_SYND; 2625713ad546SYazen Ghannam goto log_error; 2626713ad546SYazen Ghannam } 2627713ad546SYazen Ghannam 2628713ad546SYazen Ghannam if (ecc_type == 2) { 2629713ad546SYazen Ghannam u8 length = (m->synd >> 18) & 0x3f; 2630713ad546SYazen Ghannam 2631713ad546SYazen Ghannam if (length) 2632713ad546SYazen Ghannam err.syndrome = (m->synd >> 32) & GENMASK(length - 1, 0); 2633713ad546SYazen Ghannam else 2634713ad546SYazen Ghannam err.err_code = ERR_CHANNEL; 2635713ad546SYazen Ghannam } 2636713ad546SYazen Ghannam 2637713ad546SYazen Ghannam err.csrow = m->synd & 0x7; 2638713ad546SYazen Ghannam 26398a2eaab7SYazen Ghannam if (umc_normaddr_to_sysaddr(m->addr, pvt->mc_node_id, err.channel, &sys_addr)) { 26408a2eaab7SYazen Ghannam err.err_code = ERR_NORM_ADDR; 26418a2eaab7SYazen Ghannam goto log_error; 26428a2eaab7SYazen Ghannam } 26438a2eaab7SYazen Ghannam 26448a2eaab7SYazen Ghannam error_address_to_page_and_offset(sys_addr, &err); 26458a2eaab7SYazen Ghannam 2646713ad546SYazen Ghannam log_error: 2647713ad546SYazen Ghannam __log_ecc_error(mci, &err, ecc_type); 2648713ad546SYazen Ghannam } 2649713ad546SYazen Ghannam 2650713ad546SYazen Ghannam /* 26513f37a36bSBorislav Petkov * Use pvt->F3 which contains the F3 CPU PCI device to get the related 26523f37a36bSBorislav Petkov * F1 (AddrMap) and F2 (Dct) devices. Return negative value on error. 2653936fc3afSYazen Ghannam * Reserve F0 and F6 on systems with a UMC. 26540ec449eeSDoug Thompson */ 2655936fc3afSYazen Ghannam static int 2656936fc3afSYazen Ghannam reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2) 26570ec449eeSDoug Thompson { 2658936fc3afSYazen Ghannam if (pvt->umc) { 2659936fc3afSYazen Ghannam pvt->F0 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3); 2660936fc3afSYazen Ghannam if (!pvt->F0) { 26615246c540SBorislav Petkov amd64_err("F0 not found, device 0x%x (broken BIOS?)\n", pci_id1); 2662936fc3afSYazen Ghannam return -ENODEV; 2663936fc3afSYazen Ghannam } 2664936fc3afSYazen Ghannam 2665936fc3afSYazen Ghannam pvt->F6 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3); 2666936fc3afSYazen Ghannam if (!pvt->F6) { 2667936fc3afSYazen Ghannam pci_dev_put(pvt->F0); 2668936fc3afSYazen Ghannam pvt->F0 = NULL; 2669936fc3afSYazen Ghannam 26705246c540SBorislav Petkov amd64_err("F6 not found: device 0x%x (broken BIOS?)\n", pci_id2); 2671936fc3afSYazen Ghannam return -ENODEV; 2672936fc3afSYazen Ghannam } 26735246c540SBorislav Petkov 2674936fc3afSYazen Ghannam edac_dbg(1, "F0: %s\n", pci_name(pvt->F0)); 2675936fc3afSYazen Ghannam edac_dbg(1, "F3: %s\n", pci_name(pvt->F3)); 2676936fc3afSYazen Ghannam edac_dbg(1, "F6: %s\n", pci_name(pvt->F6)); 2677936fc3afSYazen Ghannam 2678936fc3afSYazen Ghannam return 0; 2679936fc3afSYazen Ghannam } 2680936fc3afSYazen Ghannam 26810ec449eeSDoug Thompson /* Reserve the ADDRESS MAP Device */ 2682936fc3afSYazen Ghannam pvt->F1 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3); 26838d5b5d9cSBorislav Petkov if (!pvt->F1) { 26845246c540SBorislav Petkov amd64_err("F1 not found: device 0x%x (broken BIOS?)\n", pci_id1); 2685bbd0c1f6SBorislav Petkov return -ENODEV; 26860ec449eeSDoug Thompson } 26870ec449eeSDoug Thompson 26883f37a36bSBorislav Petkov /* Reserve the DCT Device */ 2689936fc3afSYazen Ghannam pvt->F2 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3); 26903f37a36bSBorislav Petkov if (!pvt->F2) { 26918d5b5d9cSBorislav Petkov pci_dev_put(pvt->F1); 26928d5b5d9cSBorislav Petkov pvt->F1 = NULL; 26930ec449eeSDoug Thompson 26945246c540SBorislav Petkov amd64_err("F2 not found: device 0x%x (broken BIOS?)\n", pci_id2); 2695bbd0c1f6SBorislav Petkov return -ENODEV; 26960ec449eeSDoug Thompson } 2697936fc3afSYazen Ghannam 2698956b9ba1SJoe Perches edac_dbg(1, "F1: %s\n", pci_name(pvt->F1)); 2699956b9ba1SJoe Perches edac_dbg(1, "F2: %s\n", pci_name(pvt->F2)); 2700956b9ba1SJoe Perches edac_dbg(1, "F3: %s\n", pci_name(pvt->F3)); 27010ec449eeSDoug Thompson 27020ec449eeSDoug Thompson return 0; 27030ec449eeSDoug Thompson } 27040ec449eeSDoug Thompson 2705360b7f3cSBorislav Petkov static void free_mc_sibling_devs(struct amd64_pvt *pvt) 27060ec449eeSDoug Thompson { 2707936fc3afSYazen Ghannam if (pvt->umc) { 2708936fc3afSYazen Ghannam pci_dev_put(pvt->F0); 2709936fc3afSYazen Ghannam pci_dev_put(pvt->F6); 2710936fc3afSYazen Ghannam } else { 27118d5b5d9cSBorislav Petkov pci_dev_put(pvt->F1); 27123f37a36bSBorislav Petkov pci_dev_put(pvt->F2); 27130ec449eeSDoug Thompson } 2714936fc3afSYazen Ghannam } 27150ec449eeSDoug Thompson 2716b64ce7cdSYazen Ghannam static void determine_ecc_sym_sz(struct amd64_pvt *pvt) 2717b64ce7cdSYazen Ghannam { 2718b64ce7cdSYazen Ghannam pvt->ecc_sym_sz = 4; 2719b64ce7cdSYazen Ghannam 2720b64ce7cdSYazen Ghannam if (pvt->umc) { 2721b64ce7cdSYazen Ghannam u8 i; 2722b64ce7cdSYazen Ghannam 27234d30d2bcSYazen Ghannam for_each_umc(i) { 2724b64ce7cdSYazen Ghannam /* Check enabled channels only: */ 27257835961dSYazen Ghannam if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) { 27267835961dSYazen Ghannam if (pvt->umc[i].ecc_ctrl & BIT(9)) { 27277835961dSYazen Ghannam pvt->ecc_sym_sz = 16; 27287835961dSYazen Ghannam return; 27297835961dSYazen Ghannam } else if (pvt->umc[i].ecc_ctrl & BIT(7)) { 2730b64ce7cdSYazen Ghannam pvt->ecc_sym_sz = 8; 2731b64ce7cdSYazen Ghannam return; 2732b64ce7cdSYazen Ghannam } 27337835961dSYazen Ghannam } 27347835961dSYazen Ghannam } 27357835961dSYazen Ghannam } else if (pvt->fam >= 0x10) { 2736b64ce7cdSYazen Ghannam u32 tmp; 2737b64ce7cdSYazen Ghannam 2738b64ce7cdSYazen Ghannam amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp); 2739b64ce7cdSYazen Ghannam /* F16h has only DCT0, so no need to read dbam1. */ 2740b64ce7cdSYazen Ghannam if (pvt->fam != 0x16) 2741b64ce7cdSYazen Ghannam amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1); 2742b64ce7cdSYazen Ghannam 2743b64ce7cdSYazen Ghannam /* F10h, revD and later can do x8 ECC too. */ 2744b64ce7cdSYazen Ghannam if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25)) 2745b64ce7cdSYazen Ghannam pvt->ecc_sym_sz = 8; 2746b64ce7cdSYazen Ghannam } 2747b64ce7cdSYazen Ghannam } 2748b64ce7cdSYazen Ghannam 2749b64ce7cdSYazen Ghannam /* 2750b64ce7cdSYazen Ghannam * Retrieve the hardware registers of the memory controller. 2751b64ce7cdSYazen Ghannam */ 2752b64ce7cdSYazen Ghannam static void __read_mc_regs_df(struct amd64_pvt *pvt) 2753b64ce7cdSYazen Ghannam { 2754b64ce7cdSYazen Ghannam u8 nid = pvt->mc_node_id; 2755b64ce7cdSYazen Ghannam struct amd64_umc *umc; 2756b64ce7cdSYazen Ghannam u32 i, umc_base; 2757b64ce7cdSYazen Ghannam 2758b64ce7cdSYazen Ghannam /* Read registers from each UMC */ 27594d30d2bcSYazen Ghannam for_each_umc(i) { 2760b64ce7cdSYazen Ghannam 2761b64ce7cdSYazen Ghannam umc_base = get_umc_base(i); 2762b64ce7cdSYazen Ghannam umc = &pvt->umc[i]; 2763b64ce7cdSYazen Ghannam 276407ed82efSYazen Ghannam amd_smn_read(nid, umc_base + UMCCH_DIMM_CFG, &umc->dimm_cfg); 276507ed82efSYazen Ghannam amd_smn_read(nid, umc_base + UMCCH_UMC_CFG, &umc->umc_cfg); 2766b64ce7cdSYazen Ghannam amd_smn_read(nid, umc_base + UMCCH_SDP_CTRL, &umc->sdp_ctrl); 2767b64ce7cdSYazen Ghannam amd_smn_read(nid, umc_base + UMCCH_ECC_CTRL, &umc->ecc_ctrl); 276807ed82efSYazen Ghannam amd_smn_read(nid, umc_base + UMCCH_UMC_CAP_HI, &umc->umc_cap_hi); 2769b64ce7cdSYazen Ghannam } 2770b64ce7cdSYazen Ghannam } 2771b64ce7cdSYazen Ghannam 27720ec449eeSDoug Thompson /* 27730ec449eeSDoug Thompson * Retrieve the hardware registers of the memory controller (this includes the 27740ec449eeSDoug Thompson * 'Address Map' and 'Misc' device regs) 27750ec449eeSDoug Thompson */ 2776360b7f3cSBorislav Petkov static void read_mc_regs(struct amd64_pvt *pvt) 27770ec449eeSDoug Thompson { 2778b64ce7cdSYazen Ghannam unsigned int range; 27790ec449eeSDoug Thompson u64 msr_val; 27800ec449eeSDoug Thompson 27810ec449eeSDoug Thompson /* 27820ec449eeSDoug Thompson * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since 2783b64ce7cdSYazen Ghannam * those are Read-As-Zero. 27840ec449eeSDoug Thompson */ 2785e97f8bb8SBorislav Petkov rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem); 2786956b9ba1SJoe Perches edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem); 27870ec449eeSDoug Thompson 2788b64ce7cdSYazen Ghannam /* Check first whether TOP_MEM2 is enabled: */ 27890ec449eeSDoug Thompson rdmsrl(MSR_K8_SYSCFG, msr_val); 2790b64ce7cdSYazen Ghannam if (msr_val & BIT(21)) { 2791e97f8bb8SBorislav Petkov rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2); 2792956b9ba1SJoe Perches edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2); 2793b64ce7cdSYazen Ghannam } else { 2794956b9ba1SJoe Perches edac_dbg(0, " TOP_MEM2 disabled\n"); 2795b64ce7cdSYazen Ghannam } 2796b64ce7cdSYazen Ghannam 2797b64ce7cdSYazen Ghannam if (pvt->umc) { 2798b64ce7cdSYazen Ghannam __read_mc_regs_df(pvt); 2799b64ce7cdSYazen Ghannam amd64_read_pci_cfg(pvt->F0, DF_DHAR, &pvt->dhar); 2800b64ce7cdSYazen Ghannam 2801b64ce7cdSYazen Ghannam goto skip; 2802b64ce7cdSYazen Ghannam } 28030ec449eeSDoug Thompson 28045980bb9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap); 28050ec449eeSDoug Thompson 28065a5d2371SBorislav Petkov read_dram_ctl_register(pvt); 28070ec449eeSDoug Thompson 28087f19bf75SBorislav Petkov for (range = 0; range < DRAM_RANGES; range++) { 28097f19bf75SBorislav Petkov u8 rw; 28100ec449eeSDoug Thompson 28117f19bf75SBorislav Petkov /* read settings for this DRAM range */ 28127f19bf75SBorislav Petkov read_dram_base_limit_regs(pvt, range); 2813e97f8bb8SBorislav Petkov 28147f19bf75SBorislav Petkov rw = dram_rw(pvt, range); 28157f19bf75SBorislav Petkov if (!rw) 28167f19bf75SBorislav Petkov continue; 28177f19bf75SBorislav Petkov 2818956b9ba1SJoe Perches edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n", 28197f19bf75SBorislav Petkov range, 28207f19bf75SBorislav Petkov get_dram_base(pvt, range), 28217f19bf75SBorislav Petkov get_dram_limit(pvt, range)); 28227f19bf75SBorislav Petkov 2823956b9ba1SJoe Perches edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n", 28247f19bf75SBorislav Petkov dram_intlv_en(pvt, range) ? "Enabled" : "Disabled", 28257f19bf75SBorislav Petkov (rw & 0x1) ? "R" : "-", 28267f19bf75SBorislav Petkov (rw & 0x2) ? "W" : "-", 28277f19bf75SBorislav Petkov dram_intlv_sel(pvt, range), 28287f19bf75SBorislav Petkov dram_dst_node(pvt, range)); 28290ec449eeSDoug Thompson } 28300ec449eeSDoug Thompson 2831bc21fa57SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar); 28327981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 0, DBAM0, &pvt->dbam0); 28330ec449eeSDoug Thompson 28348d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare); 28350ec449eeSDoug Thompson 28367981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 0, DCLR0, &pvt->dclr0); 28377981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 0, DCHR0, &pvt->dchr0); 28380ec449eeSDoug Thompson 283978da121eSBorislav Petkov if (!dct_ganging_enabled(pvt)) { 28407981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 1, DCLR0, &pvt->dclr1); 28417981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1); 28420ec449eeSDoug Thompson } 2843b2b0c605SBorislav Petkov 2844b64ce7cdSYazen Ghannam skip: 2845b64ce7cdSYazen Ghannam read_dct_base_mask(pvt); 2846b64ce7cdSYazen Ghannam 2847a597d2a5SAravind Gopalakrishnan determine_memory_type(pvt); 2848a597d2a5SAravind Gopalakrishnan edac_dbg(1, " DIMM type: %s\n", edac_mem_types[pvt->dram_type]); 2849a3b7db09SBorislav Petkov 2850b64ce7cdSYazen Ghannam determine_ecc_sym_sz(pvt); 28510ec449eeSDoug Thompson } 28520ec449eeSDoug Thompson 28530ec449eeSDoug Thompson /* 28540ec449eeSDoug Thompson * NOTE: CPU Revision Dependent code 28550ec449eeSDoug Thompson * 28560ec449eeSDoug Thompson * Input: 285711c75eadSBorislav Petkov * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1) 28580ec449eeSDoug Thompson * k8 private pointer to --> 28590ec449eeSDoug Thompson * DRAM Bank Address mapping register 28600ec449eeSDoug Thompson * node_id 28610ec449eeSDoug Thompson * DCL register where dual_channel_active is 28620ec449eeSDoug Thompson * 28630ec449eeSDoug Thompson * The DBAM register consists of 4 sets of 4 bits each definitions: 28640ec449eeSDoug Thompson * 28650ec449eeSDoug Thompson * Bits: CSROWs 28660ec449eeSDoug Thompson * 0-3 CSROWs 0 and 1 28670ec449eeSDoug Thompson * 4-7 CSROWs 2 and 3 28680ec449eeSDoug Thompson * 8-11 CSROWs 4 and 5 28690ec449eeSDoug Thompson * 12-15 CSROWs 6 and 7 28700ec449eeSDoug Thompson * 28710ec449eeSDoug Thompson * Values range from: 0 to 15 28720ec449eeSDoug Thompson * The meaning of the values depends on CPU revision and dual-channel state, 28730ec449eeSDoug Thompson * see relevant BKDG more info. 28740ec449eeSDoug Thompson * 28750ec449eeSDoug Thompson * The memory controller provides for total of only 8 CSROWs in its current 28760ec449eeSDoug Thompson * architecture. Each "pair" of CSROWs normally represents just one DIMM in 28770ec449eeSDoug Thompson * single channel or two (2) DIMMs in dual channel mode. 28780ec449eeSDoug Thompson * 28790ec449eeSDoug Thompson * The following code logic collapses the various tables for CSROW based on CPU 28800ec449eeSDoug Thompson * revision. 28810ec449eeSDoug Thompson * 28820ec449eeSDoug Thompson * Returns: 28830ec449eeSDoug Thompson * The number of PAGE_SIZE pages on the specified CSROW number it 28840ec449eeSDoug Thompson * encompasses 28850ec449eeSDoug Thompson * 28860ec449eeSDoug Thompson */ 2887eb77e6b8SYazen Ghannam static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr_orig) 28880ec449eeSDoug Thompson { 2889f92cae45SAshish Shenoy u32 dbam = dct ? pvt->dbam1 : pvt->dbam0; 2890eb77e6b8SYazen Ghannam int csrow_nr = csrow_nr_orig; 2891eb77e6b8SYazen Ghannam u32 cs_mode, nr_pages; 28920ec449eeSDoug Thompson 2893e53a3b26SYazen Ghannam if (!pvt->umc) { 2894eb77e6b8SYazen Ghannam csrow_nr >>= 1; 2895eb77e6b8SYazen Ghannam cs_mode = DBAM_DIMM(csrow_nr, dbam); 2896e53a3b26SYazen Ghannam } else { 2897e53a3b26SYazen Ghannam cs_mode = f17_get_cs_mode(csrow_nr >> 1, dct, pvt); 2898e53a3b26SYazen Ghannam } 28990ec449eeSDoug Thompson 2900eb77e6b8SYazen Ghannam nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, csrow_nr); 2901eb77e6b8SYazen Ghannam nr_pages <<= 20 - PAGE_SHIFT; 29020ec449eeSDoug Thompson 290310de6497SBorislav Petkov edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n", 2904eb77e6b8SYazen Ghannam csrow_nr_orig, dct, cs_mode); 290510de6497SBorislav Petkov edac_dbg(0, "nr_pages/channel: %u\n", nr_pages); 29060ec449eeSDoug Thompson 29070ec449eeSDoug Thompson return nr_pages; 29080ec449eeSDoug Thompson } 29090ec449eeSDoug Thompson 2910353a1fcbSYazen Ghannam static int init_csrows_df(struct mem_ctl_info *mci) 2911353a1fcbSYazen Ghannam { 2912353a1fcbSYazen Ghannam struct amd64_pvt *pvt = mci->pvt_info; 2913353a1fcbSYazen Ghannam enum edac_type edac_mode = EDAC_NONE; 2914353a1fcbSYazen Ghannam enum dev_type dev_type = DEV_UNKNOWN; 2915353a1fcbSYazen Ghannam struct dimm_info *dimm; 2916353a1fcbSYazen Ghannam int empty = 1; 2917353a1fcbSYazen Ghannam u8 umc, cs; 2918353a1fcbSYazen Ghannam 2919353a1fcbSYazen Ghannam if (mci->edac_ctl_cap & EDAC_FLAG_S16ECD16ED) { 2920353a1fcbSYazen Ghannam edac_mode = EDAC_S16ECD16ED; 2921353a1fcbSYazen Ghannam dev_type = DEV_X16; 2922353a1fcbSYazen Ghannam } else if (mci->edac_ctl_cap & EDAC_FLAG_S8ECD8ED) { 2923353a1fcbSYazen Ghannam edac_mode = EDAC_S8ECD8ED; 2924353a1fcbSYazen Ghannam dev_type = DEV_X8; 2925353a1fcbSYazen Ghannam } else if (mci->edac_ctl_cap & EDAC_FLAG_S4ECD4ED) { 2926353a1fcbSYazen Ghannam edac_mode = EDAC_S4ECD4ED; 2927353a1fcbSYazen Ghannam dev_type = DEV_X4; 2928353a1fcbSYazen Ghannam } else if (mci->edac_ctl_cap & EDAC_FLAG_SECDED) { 2929353a1fcbSYazen Ghannam edac_mode = EDAC_SECDED; 2930353a1fcbSYazen Ghannam } 2931353a1fcbSYazen Ghannam 2932353a1fcbSYazen Ghannam for_each_umc(umc) { 2933353a1fcbSYazen Ghannam for_each_chip_select(cs, umc, pvt) { 2934353a1fcbSYazen Ghannam if (!csrow_enabled(cs, umc, pvt)) 2935353a1fcbSYazen Ghannam continue; 2936353a1fcbSYazen Ghannam 2937353a1fcbSYazen Ghannam empty = 0; 2938353a1fcbSYazen Ghannam dimm = mci->csrows[cs]->channels[umc]->dimm; 2939353a1fcbSYazen Ghannam 2940353a1fcbSYazen Ghannam edac_dbg(1, "MC node: %d, csrow: %d\n", 2941353a1fcbSYazen Ghannam pvt->mc_node_id, cs); 2942353a1fcbSYazen Ghannam 2943353a1fcbSYazen Ghannam dimm->nr_pages = get_csrow_nr_pages(pvt, umc, cs); 2944353a1fcbSYazen Ghannam dimm->mtype = pvt->dram_type; 2945353a1fcbSYazen Ghannam dimm->edac_mode = edac_mode; 2946353a1fcbSYazen Ghannam dimm->dtype = dev_type; 2947466503d6SYazen Ghannam dimm->grain = 64; 2948353a1fcbSYazen Ghannam } 2949353a1fcbSYazen Ghannam } 2950353a1fcbSYazen Ghannam 2951353a1fcbSYazen Ghannam return empty; 2952353a1fcbSYazen Ghannam } 2953353a1fcbSYazen Ghannam 29540ec449eeSDoug Thompson /* 29550ec449eeSDoug Thompson * Initialize the array of csrow attribute instances, based on the values 29560ec449eeSDoug Thompson * from pci config hardware registers. 29570ec449eeSDoug Thompson */ 2958360b7f3cSBorislav Petkov static int init_csrows(struct mem_ctl_info *mci) 29590ec449eeSDoug Thompson { 296010de6497SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 29612d09d8f3SYazen Ghannam enum edac_type edac_mode = EDAC_NONE; 29620ec449eeSDoug Thompson struct csrow_info *csrow; 2963de3910ebSMauro Carvalho Chehab struct dimm_info *dimm; 296410de6497SBorislav Petkov int i, j, empty = 1; 2965a895bf8bSMauro Carvalho Chehab int nr_pages = 0; 296610de6497SBorislav Petkov u32 val; 29670ec449eeSDoug Thompson 2968353a1fcbSYazen Ghannam if (pvt->umc) 2969353a1fcbSYazen Ghannam return init_csrows_df(mci); 2970353a1fcbSYazen Ghannam 2971a97fa68eSBorislav Petkov amd64_read_pci_cfg(pvt->F3, NBCFG, &val); 29720ec449eeSDoug Thompson 29732299ef71SBorislav Petkov pvt->nbcfg = val; 29740ec449eeSDoug Thompson 2975956b9ba1SJoe Perches edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n", 29762299ef71SBorislav Petkov pvt->mc_node_id, val, 2977a97fa68eSBorislav Petkov !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE)); 29780ec449eeSDoug Thompson 297910de6497SBorislav Petkov /* 298010de6497SBorislav Petkov * We iterate over DCT0 here but we look at DCT1 in parallel, if needed. 298110de6497SBorislav Petkov */ 298211c75eadSBorislav Petkov for_each_chip_select(i, 0, pvt) { 298310de6497SBorislav Petkov bool row_dct0 = !!csrow_enabled(i, 0, pvt); 298410de6497SBorislav Petkov bool row_dct1 = false; 29850ec449eeSDoug Thompson 2986a4b4bedcSBorislav Petkov if (pvt->fam != 0xf) 298710de6497SBorislav Petkov row_dct1 = !!csrow_enabled(i, 1, pvt); 298810de6497SBorislav Petkov 298910de6497SBorislav Petkov if (!row_dct0 && !row_dct1) 29900ec449eeSDoug Thompson continue; 29910ec449eeSDoug Thompson 299210de6497SBorislav Petkov csrow = mci->csrows[i]; 29930ec449eeSDoug Thompson empty = 0; 299411c75eadSBorislav Petkov 299510de6497SBorislav Petkov edac_dbg(1, "MC node: %d, csrow: %d\n", 299610de6497SBorislav Petkov pvt->mc_node_id, i); 299710de6497SBorislav Petkov 29981eef1282SMauro Carvalho Chehab if (row_dct0) { 2999d1ea71cdSBorislav Petkov nr_pages = get_csrow_nr_pages(pvt, 0, i); 30001eef1282SMauro Carvalho Chehab csrow->channels[0]->dimm->nr_pages = nr_pages; 30011eef1282SMauro Carvalho Chehab } 300210de6497SBorislav Petkov 300310de6497SBorislav Petkov /* K8 has only one DCT */ 3004a4b4bedcSBorislav Petkov if (pvt->fam != 0xf && row_dct1) { 3005d1ea71cdSBorislav Petkov int row_dct1_pages = get_csrow_nr_pages(pvt, 1, i); 30061eef1282SMauro Carvalho Chehab 30071eef1282SMauro Carvalho Chehab csrow->channels[1]->dimm->nr_pages = row_dct1_pages; 30081eef1282SMauro Carvalho Chehab nr_pages += row_dct1_pages; 30091eef1282SMauro Carvalho Chehab } 30100ec449eeSDoug Thompson 301110de6497SBorislav Petkov edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages); 30120ec449eeSDoug Thompson 30132d09d8f3SYazen Ghannam /* Determine DIMM ECC mode: */ 3014353a1fcbSYazen Ghannam if (pvt->nbcfg & NBCFG_ECC_ENABLE) { 30152d09d8f3SYazen Ghannam edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL) 30162d09d8f3SYazen Ghannam ? EDAC_S4ECD4ED 30172d09d8f3SYazen Ghannam : EDAC_SECDED; 30182d09d8f3SYazen Ghannam } 3019084a4fccSMauro Carvalho Chehab 3020084a4fccSMauro Carvalho Chehab for (j = 0; j < pvt->channel_count; j++) { 3021de3910ebSMauro Carvalho Chehab dimm = csrow->channels[j]->dimm; 3022a597d2a5SAravind Gopalakrishnan dimm->mtype = pvt->dram_type; 3023de3910ebSMauro Carvalho Chehab dimm->edac_mode = edac_mode; 3024466503d6SYazen Ghannam dimm->grain = 64; 3025084a4fccSMauro Carvalho Chehab } 30260ec449eeSDoug Thompson } 30270ec449eeSDoug Thompson 30280ec449eeSDoug Thompson return empty; 30290ec449eeSDoug Thompson } 3030d27bf6faSDoug Thompson 303106724535SBorislav Petkov /* get all cores on this DCT */ 30328b84c8dfSDaniel J Blueman static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid) 3033f9431992SDoug Thompson { 303406724535SBorislav Petkov int cpu; 3035f9431992SDoug Thompson 303606724535SBorislav Petkov for_each_online_cpu(cpu) 303706724535SBorislav Petkov if (amd_get_nb_id(cpu) == nid) 303806724535SBorislav Petkov cpumask_set_cpu(cpu, mask); 3039f9431992SDoug Thompson } 3040f9431992SDoug Thompson 3041f9431992SDoug Thompson /* check MCG_CTL on all the cpus on this node */ 3042d1ea71cdSBorislav Petkov static bool nb_mce_bank_enabled_on_node(u16 nid) 3043f9431992SDoug Thompson { 3044ba578cb3SRusty Russell cpumask_var_t mask; 304550542251SBorislav Petkov int cpu, nbe; 304606724535SBorislav Petkov bool ret = false; 3047f9431992SDoug Thompson 3048ba578cb3SRusty Russell if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { 304924f9a7feSBorislav Petkov amd64_warn("%s: Error allocating mask\n", __func__); 305006724535SBorislav Petkov return false; 305106724535SBorislav Petkov } 305206724535SBorislav Petkov 3053ba578cb3SRusty Russell get_cpus_on_this_dct_cpumask(mask, nid); 305406724535SBorislav Petkov 3055ba578cb3SRusty Russell rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs); 3056ba578cb3SRusty Russell 3057ba578cb3SRusty Russell for_each_cpu(cpu, mask) { 305850542251SBorislav Petkov struct msr *reg = per_cpu_ptr(msrs, cpu); 30595980bb9cSBorislav Petkov nbe = reg->l & MSR_MCGCTL_NBE; 306006724535SBorislav Petkov 3061956b9ba1SJoe Perches edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n", 306250542251SBorislav Petkov cpu, reg->q, 306306724535SBorislav Petkov (nbe ? "enabled" : "disabled")); 306406724535SBorislav Petkov 306506724535SBorislav Petkov if (!nbe) 306606724535SBorislav Petkov goto out; 306706724535SBorislav Petkov } 306806724535SBorislav Petkov ret = true; 306906724535SBorislav Petkov 307006724535SBorislav Petkov out: 3071ba578cb3SRusty Russell free_cpumask_var(mask); 3072f9431992SDoug Thompson return ret; 3073f9431992SDoug Thompson } 3074f9431992SDoug Thompson 3075c7e5301aSDaniel J Blueman static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on) 3076f6d6ae96SBorislav Petkov { 3077f6d6ae96SBorislav Petkov cpumask_var_t cmask; 307850542251SBorislav Petkov int cpu; 3079f6d6ae96SBorislav Petkov 3080f6d6ae96SBorislav Petkov if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) { 308124f9a7feSBorislav Petkov amd64_warn("%s: error allocating mask\n", __func__); 30820de27884SPan Bian return -ENOMEM; 3083f6d6ae96SBorislav Petkov } 3084f6d6ae96SBorislav Petkov 3085ae7bb7c6SBorislav Petkov get_cpus_on_this_dct_cpumask(cmask, nid); 3086f6d6ae96SBorislav Petkov 3087f6d6ae96SBorislav Petkov rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); 3088f6d6ae96SBorislav Petkov 3089f6d6ae96SBorislav Petkov for_each_cpu(cpu, cmask) { 3090f6d6ae96SBorislav Petkov 309150542251SBorislav Petkov struct msr *reg = per_cpu_ptr(msrs, cpu); 309250542251SBorislav Petkov 3093f6d6ae96SBorislav Petkov if (on) { 30945980bb9cSBorislav Petkov if (reg->l & MSR_MCGCTL_NBE) 3095ae7bb7c6SBorislav Petkov s->flags.nb_mce_enable = 1; 3096f6d6ae96SBorislav Petkov 30975980bb9cSBorislav Petkov reg->l |= MSR_MCGCTL_NBE; 3098f6d6ae96SBorislav Petkov } else { 3099f6d6ae96SBorislav Petkov /* 3100d95cf4deSBorislav Petkov * Turn off NB MCE reporting only when it was off before 3101f6d6ae96SBorislav Petkov */ 3102ae7bb7c6SBorislav Petkov if (!s->flags.nb_mce_enable) 31035980bb9cSBorislav Petkov reg->l &= ~MSR_MCGCTL_NBE; 3104f6d6ae96SBorislav Petkov } 3105f6d6ae96SBorislav Petkov } 3106f6d6ae96SBorislav Petkov wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); 3107f6d6ae96SBorislav Petkov 3108f6d6ae96SBorislav Petkov free_cpumask_var(cmask); 3109f6d6ae96SBorislav Petkov 3110f6d6ae96SBorislav Petkov return 0; 3111f6d6ae96SBorislav Petkov } 3112f6d6ae96SBorislav Petkov 3113c7e5301aSDaniel J Blueman static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid, 31142299ef71SBorislav Petkov struct pci_dev *F3) 3115f6d6ae96SBorislav Petkov { 31162299ef71SBorislav Petkov bool ret = true; 3117c9f4f26eSBorislav Petkov u32 value, mask = 0x3; /* UECC/CECC enable */ 3118f6d6ae96SBorislav Petkov 31192299ef71SBorislav Petkov if (toggle_ecc_err_reporting(s, nid, ON)) { 31202299ef71SBorislav Petkov amd64_warn("Error enabling ECC reporting over MCGCTL!\n"); 31212299ef71SBorislav Petkov return false; 31222299ef71SBorislav Petkov } 31232299ef71SBorislav Petkov 3124c9f4f26eSBorislav Petkov amd64_read_pci_cfg(F3, NBCTL, &value); 3125f6d6ae96SBorislav Petkov 3126ae7bb7c6SBorislav Petkov s->old_nbctl = value & mask; 3127ae7bb7c6SBorislav Petkov s->nbctl_valid = true; 3128f6d6ae96SBorislav Petkov 3129f6d6ae96SBorislav Petkov value |= mask; 3130c9f4f26eSBorislav Petkov amd64_write_pci_cfg(F3, NBCTL, value); 3131f6d6ae96SBorislav Petkov 3132a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 3133f6d6ae96SBorislav Petkov 3134956b9ba1SJoe Perches edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n", 3135a97fa68eSBorislav Petkov nid, value, !!(value & NBCFG_ECC_ENABLE)); 3136f6d6ae96SBorislav Petkov 3137a97fa68eSBorislav Petkov if (!(value & NBCFG_ECC_ENABLE)) { 313824f9a7feSBorislav Petkov amd64_warn("DRAM ECC disabled on this node, enabling...\n"); 3139f6d6ae96SBorislav Petkov 3140ae7bb7c6SBorislav Petkov s->flags.nb_ecc_prev = 0; 3141d95cf4deSBorislav Petkov 3142f6d6ae96SBorislav Petkov /* Attempt to turn on DRAM ECC Enable */ 3143a97fa68eSBorislav Petkov value |= NBCFG_ECC_ENABLE; 3144a97fa68eSBorislav Petkov amd64_write_pci_cfg(F3, NBCFG, value); 3145f6d6ae96SBorislav Petkov 3146a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 3147f6d6ae96SBorislav Petkov 3148a97fa68eSBorislav Petkov if (!(value & NBCFG_ECC_ENABLE)) { 314924f9a7feSBorislav Petkov amd64_warn("Hardware rejected DRAM ECC enable," 315024f9a7feSBorislav Petkov "check memory DIMM configuration.\n"); 31512299ef71SBorislav Petkov ret = false; 3152f6d6ae96SBorislav Petkov } else { 315324f9a7feSBorislav Petkov amd64_info("Hardware accepted DRAM ECC Enable\n"); 3154f6d6ae96SBorislav Petkov } 3155d95cf4deSBorislav Petkov } else { 3156ae7bb7c6SBorislav Petkov s->flags.nb_ecc_prev = 1; 3157f6d6ae96SBorislav Petkov } 3158d95cf4deSBorislav Petkov 3159956b9ba1SJoe Perches edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n", 3160a97fa68eSBorislav Petkov nid, value, !!(value & NBCFG_ECC_ENABLE)); 3161f6d6ae96SBorislav Petkov 31622299ef71SBorislav Petkov return ret; 3163f6d6ae96SBorislav Petkov } 3164f6d6ae96SBorislav Petkov 3165c7e5301aSDaniel J Blueman static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid, 3166360b7f3cSBorislav Petkov struct pci_dev *F3) 3167f6d6ae96SBorislav Petkov { 3168c9f4f26eSBorislav Petkov u32 value, mask = 0x3; /* UECC/CECC enable */ 3169c9f4f26eSBorislav Petkov 3170ae7bb7c6SBorislav Petkov if (!s->nbctl_valid) 3171f6d6ae96SBorislav Petkov return; 3172f6d6ae96SBorislav Petkov 3173c9f4f26eSBorislav Petkov amd64_read_pci_cfg(F3, NBCTL, &value); 3174f6d6ae96SBorislav Petkov value &= ~mask; 3175ae7bb7c6SBorislav Petkov value |= s->old_nbctl; 3176f6d6ae96SBorislav Petkov 3177c9f4f26eSBorislav Petkov amd64_write_pci_cfg(F3, NBCTL, value); 3178f6d6ae96SBorislav Petkov 3179ae7bb7c6SBorislav Petkov /* restore previous BIOS DRAM ECC "off" setting we force-enabled */ 3180ae7bb7c6SBorislav Petkov if (!s->flags.nb_ecc_prev) { 3181a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 3182a97fa68eSBorislav Petkov value &= ~NBCFG_ECC_ENABLE; 3183a97fa68eSBorislav Petkov amd64_write_pci_cfg(F3, NBCFG, value); 3184d95cf4deSBorislav Petkov } 3185d95cf4deSBorislav Petkov 3186d95cf4deSBorislav Petkov /* restore the NB Enable MCGCTL bit */ 31872299ef71SBorislav Petkov if (toggle_ecc_err_reporting(s, nid, OFF)) 318824f9a7feSBorislav Petkov amd64_warn("Error restoring NB MCGCTL settings!\n"); 3189f6d6ae96SBorislav Petkov } 3190f6d6ae96SBorislav Petkov 31911c9b08baSYazen Ghannam static bool ecc_enabled(struct amd64_pvt *pvt) 3192f9431992SDoug Thompson { 31931c9b08baSYazen Ghannam u16 nid = pvt->mc_node_id; 319406724535SBorislav Petkov bool nb_mce_en = false; 3195196b79fcSYazen Ghannam u8 ecc_en = 0, i; 3196196b79fcSYazen Ghannam u32 value; 3197f9431992SDoug Thompson 3198196b79fcSYazen Ghannam if (boot_cpu_data.x86 >= 0x17) { 3199196b79fcSYazen Ghannam u8 umc_en_mask = 0, ecc_en_mask = 0; 32001c9b08baSYazen Ghannam struct amd64_umc *umc; 3201196b79fcSYazen Ghannam 32024d30d2bcSYazen Ghannam for_each_umc(i) { 32031c9b08baSYazen Ghannam umc = &pvt->umc[i]; 3204196b79fcSYazen Ghannam 3205196b79fcSYazen Ghannam /* Only check enabled UMCs. */ 32061c9b08baSYazen Ghannam if (!(umc->sdp_ctrl & UMC_SDP_INIT)) 3207196b79fcSYazen Ghannam continue; 3208196b79fcSYazen Ghannam 3209196b79fcSYazen Ghannam umc_en_mask |= BIT(i); 3210196b79fcSYazen Ghannam 32111c9b08baSYazen Ghannam if (umc->umc_cap_hi & UMC_ECC_ENABLED) 3212196b79fcSYazen Ghannam ecc_en_mask |= BIT(i); 3213196b79fcSYazen Ghannam } 3214196b79fcSYazen Ghannam 3215196b79fcSYazen Ghannam /* Check whether at least one UMC is enabled: */ 3216196b79fcSYazen Ghannam if (umc_en_mask) 3217196b79fcSYazen Ghannam ecc_en = umc_en_mask == ecc_en_mask; 321811ab1caeSYazen Ghannam else 321911ab1caeSYazen Ghannam edac_dbg(0, "Node %d: No enabled UMCs.\n", nid); 3220196b79fcSYazen Ghannam 3221196b79fcSYazen Ghannam /* Assume UMC MCA banks are enabled. */ 3222196b79fcSYazen Ghannam nb_mce_en = true; 3223196b79fcSYazen Ghannam } else { 32241c9b08baSYazen Ghannam amd64_read_pci_cfg(pvt->F3, NBCFG, &value); 3225f9431992SDoug Thompson 3226a97fa68eSBorislav Petkov ecc_en = !!(value & NBCFG_ECC_ENABLE); 3227be3468e8SBorislav Petkov 3228d1ea71cdSBorislav Petkov nb_mce_en = nb_mce_bank_enabled_on_node(nid); 322906724535SBorislav Petkov if (!nb_mce_en) 323011ab1caeSYazen Ghannam edac_dbg(0, "NB MCE bank disabled, set MSR 0x%08x[4] on node %d to enable.\n", 32312299ef71SBorislav Petkov MSR_IA32_MCG_CTL, nid); 3232196b79fcSYazen Ghannam } 3233196b79fcSYazen Ghannam 323411ab1caeSYazen Ghannam amd64_info("Node %d: DRAM ECC %s.\n", 323511ab1caeSYazen Ghannam nid, (ecc_en ? "enabled" : "disabled")); 3236be3468e8SBorislav Petkov 32377fdfee92SBorislav Petkov if (!ecc_en || !nb_mce_en) 32382299ef71SBorislav Petkov return false; 32397fdfee92SBorislav Petkov else 32402299ef71SBorislav Petkov return true; 3241f9431992SDoug Thompson } 3242f9431992SDoug Thompson 32432d09d8f3SYazen Ghannam static inline void 32442d09d8f3SYazen Ghannam f17h_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt) 32452d09d8f3SYazen Ghannam { 3246f8be8e56SYazen Ghannam u8 i, ecc_en = 1, cpk_en = 1, dev_x4 = 1, dev_x16 = 1; 32472d09d8f3SYazen Ghannam 32484d30d2bcSYazen Ghannam for_each_umc(i) { 32492d09d8f3SYazen Ghannam if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) { 32502d09d8f3SYazen Ghannam ecc_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_ENABLED); 32512d09d8f3SYazen Ghannam cpk_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_CHIPKILL_CAP); 3252f8be8e56SYazen Ghannam 3253f8be8e56SYazen Ghannam dev_x4 &= !!(pvt->umc[i].dimm_cfg & BIT(6)); 3254f8be8e56SYazen Ghannam dev_x16 &= !!(pvt->umc[i].dimm_cfg & BIT(7)); 32552d09d8f3SYazen Ghannam } 32562d09d8f3SYazen Ghannam } 32572d09d8f3SYazen Ghannam 32582d09d8f3SYazen Ghannam /* Set chipkill only if ECC is enabled: */ 32592d09d8f3SYazen Ghannam if (ecc_en) { 32602d09d8f3SYazen Ghannam mci->edac_ctl_cap |= EDAC_FLAG_SECDED; 32612d09d8f3SYazen Ghannam 3262f8be8e56SYazen Ghannam if (!cpk_en) 3263f8be8e56SYazen Ghannam return; 3264f8be8e56SYazen Ghannam 3265f8be8e56SYazen Ghannam if (dev_x4) 32662d09d8f3SYazen Ghannam mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED; 3267f8be8e56SYazen Ghannam else if (dev_x16) 3268f8be8e56SYazen Ghannam mci->edac_ctl_cap |= EDAC_FLAG_S16ECD16ED; 3269f8be8e56SYazen Ghannam else 3270f8be8e56SYazen Ghannam mci->edac_ctl_cap |= EDAC_FLAG_S8ECD8ED; 32712d09d8f3SYazen Ghannam } 32722d09d8f3SYazen Ghannam } 32732d09d8f3SYazen Ghannam 327438ddd4d1SYazen Ghannam static void setup_mci_misc_attrs(struct mem_ctl_info *mci) 32757d6034d3SDoug Thompson { 32767d6034d3SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 32777d6034d3SDoug Thompson 32787d6034d3SDoug Thompson mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2; 32797d6034d3SDoug Thompson mci->edac_ctl_cap = EDAC_FLAG_NONE; 32807d6034d3SDoug Thompson 32812d09d8f3SYazen Ghannam if (pvt->umc) { 32822d09d8f3SYazen Ghannam f17h_determine_edac_ctl_cap(mci, pvt); 32832d09d8f3SYazen Ghannam } else { 32845980bb9cSBorislav Petkov if (pvt->nbcap & NBCAP_SECDED) 32857d6034d3SDoug Thompson mci->edac_ctl_cap |= EDAC_FLAG_SECDED; 32867d6034d3SDoug Thompson 32875980bb9cSBorislav Petkov if (pvt->nbcap & NBCAP_CHIPKILL) 32887d6034d3SDoug Thompson mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED; 32892d09d8f3SYazen Ghannam } 32907d6034d3SDoug Thompson 3291d1ea71cdSBorislav Petkov mci->edac_cap = determine_edac_cap(pvt); 32927d6034d3SDoug Thompson mci->mod_name = EDAC_MOD_STR; 329338ddd4d1SYazen Ghannam mci->ctl_name = fam_type->ctl_name; 3294e7934b70SYazen Ghannam mci->dev_name = pci_name(pvt->F3); 32957d6034d3SDoug Thompson mci->ctl_page_to_phys = NULL; 32967d6034d3SDoug Thompson 32977d6034d3SDoug Thompson /* memory scrubber interface */ 3298d1ea71cdSBorislav Petkov mci->set_sdram_scrub_rate = set_scrub_rate; 3299d1ea71cdSBorislav Petkov mci->get_sdram_scrub_rate = get_scrub_rate; 33007d6034d3SDoug Thompson } 33017d6034d3SDoug Thompson 33020092b20dSBorislav Petkov /* 33030092b20dSBorislav Petkov * returns a pointer to the family descriptor on success, NULL otherwise. 33040092b20dSBorislav Petkov */ 3305d1ea71cdSBorislav Petkov static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt) 3306395ae783SBorislav Petkov { 330718b94f66SAravind Gopalakrishnan pvt->ext_model = boot_cpu_data.x86_model >> 4; 3308b399151cSJia Zhang pvt->stepping = boot_cpu_data.x86_stepping; 330918b94f66SAravind Gopalakrishnan pvt->model = boot_cpu_data.x86_model; 331018b94f66SAravind Gopalakrishnan pvt->fam = boot_cpu_data.x86; 331118b94f66SAravind Gopalakrishnan 331218b94f66SAravind Gopalakrishnan switch (pvt->fam) { 3313395ae783SBorislav Petkov case 0xf: 3314d1ea71cdSBorislav Petkov fam_type = &family_types[K8_CPUS]; 3315d1ea71cdSBorislav Petkov pvt->ops = &family_types[K8_CPUS].ops; 3316395ae783SBorislav Petkov break; 3317df71a053SBorislav Petkov 3318395ae783SBorislav Petkov case 0x10: 3319d1ea71cdSBorislav Petkov fam_type = &family_types[F10_CPUS]; 3320d1ea71cdSBorislav Petkov pvt->ops = &family_types[F10_CPUS].ops; 3321df71a053SBorislav Petkov break; 3322df71a053SBorislav Petkov 3323df71a053SBorislav Petkov case 0x15: 332418b94f66SAravind Gopalakrishnan if (pvt->model == 0x30) { 3325d1ea71cdSBorislav Petkov fam_type = &family_types[F15_M30H_CPUS]; 3326d1ea71cdSBorislav Petkov pvt->ops = &family_types[F15_M30H_CPUS].ops; 332718b94f66SAravind Gopalakrishnan break; 3328a597d2a5SAravind Gopalakrishnan } else if (pvt->model == 0x60) { 3329a597d2a5SAravind Gopalakrishnan fam_type = &family_types[F15_M60H_CPUS]; 3330a597d2a5SAravind Gopalakrishnan pvt->ops = &family_types[F15_M60H_CPUS].ops; 3331a597d2a5SAravind Gopalakrishnan break; 333218b94f66SAravind Gopalakrishnan } 333318b94f66SAravind Gopalakrishnan 3334d1ea71cdSBorislav Petkov fam_type = &family_types[F15_CPUS]; 3335d1ea71cdSBorislav Petkov pvt->ops = &family_types[F15_CPUS].ops; 3336395ae783SBorislav Petkov break; 3337395ae783SBorislav Petkov 333894c1acf2SAravind Gopalakrishnan case 0x16: 333985a8885bSAravind Gopalakrishnan if (pvt->model == 0x30) { 334085a8885bSAravind Gopalakrishnan fam_type = &family_types[F16_M30H_CPUS]; 334185a8885bSAravind Gopalakrishnan pvt->ops = &family_types[F16_M30H_CPUS].ops; 334285a8885bSAravind Gopalakrishnan break; 334385a8885bSAravind Gopalakrishnan } 3344d1ea71cdSBorislav Petkov fam_type = &family_types[F16_CPUS]; 3345d1ea71cdSBorislav Petkov pvt->ops = &family_types[F16_CPUS].ops; 334694c1acf2SAravind Gopalakrishnan break; 334794c1acf2SAravind Gopalakrishnan 3348f1cbbec9SYazen Ghannam case 0x17: 33498960de4aSMichael Jin if (pvt->model >= 0x10 && pvt->model <= 0x2f) { 33508960de4aSMichael Jin fam_type = &family_types[F17_M10H_CPUS]; 33518960de4aSMichael Jin pvt->ops = &family_types[F17_M10H_CPUS].ops; 33528960de4aSMichael Jin break; 33536e846239SYazen Ghannam } else if (pvt->model >= 0x30 && pvt->model <= 0x3f) { 33546e846239SYazen Ghannam fam_type = &family_types[F17_M30H_CPUS]; 33556e846239SYazen Ghannam pvt->ops = &family_types[F17_M30H_CPUS].ops; 33566e846239SYazen Ghannam break; 33573e443eb3SIsaac Vaughn } else if (pvt->model >= 0x70 && pvt->model <= 0x7f) { 33583e443eb3SIsaac Vaughn fam_type = &family_types[F17_M70H_CPUS]; 33593e443eb3SIsaac Vaughn pvt->ops = &family_types[F17_M70H_CPUS].ops; 33603e443eb3SIsaac Vaughn break; 33618960de4aSMichael Jin } 3362c4a3e946SPu Wen /* fall through */ 3363c4a3e946SPu Wen case 0x18: 3364f1cbbec9SYazen Ghannam fam_type = &family_types[F17_CPUS]; 3365f1cbbec9SYazen Ghannam pvt->ops = &family_types[F17_CPUS].ops; 3366c4a3e946SPu Wen 3367c4a3e946SPu Wen if (pvt->fam == 0x18) 3368c4a3e946SPu Wen family_types[F17_CPUS].ctl_name = "F18h"; 3369f1cbbec9SYazen Ghannam break; 3370f1cbbec9SYazen Ghannam 3371395ae783SBorislav Petkov default: 337224f9a7feSBorislav Petkov amd64_err("Unsupported family!\n"); 33730092b20dSBorislav Petkov return NULL; 3374395ae783SBorislav Petkov } 33750092b20dSBorislav Petkov 3376df71a053SBorislav Petkov amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name, 337718b94f66SAravind Gopalakrishnan (pvt->fam == 0xf ? 33780092b20dSBorislav Petkov (pvt->ext_model >= K8_REV_F ? "revF or later " 33790092b20dSBorislav Petkov : "revE or earlier ") 338024f9a7feSBorislav Petkov : ""), pvt->mc_node_id); 33810092b20dSBorislav Petkov return fam_type; 3382395ae783SBorislav Petkov } 3383395ae783SBorislav Petkov 3384e339f1ecSTakashi Iwai static const struct attribute_group *amd64_edac_attr_groups[] = { 3385e339f1ecSTakashi Iwai #ifdef CONFIG_EDAC_DEBUG 3386e339f1ecSTakashi Iwai &amd64_edac_dbg_group, 3387e339f1ecSTakashi Iwai #endif 3388e339f1ecSTakashi Iwai #ifdef CONFIG_EDAC_AMD64_ERROR_INJECTION 3389e339f1ecSTakashi Iwai &amd64_edac_inj_group, 3390e339f1ecSTakashi Iwai #endif 3391e339f1ecSTakashi Iwai NULL 3392e339f1ecSTakashi Iwai }; 3393e339f1ecSTakashi Iwai 339480355a3bSYazen Ghannam static int hw_info_get(struct amd64_pvt *pvt) 33957d6034d3SDoug Thompson { 3396936fc3afSYazen Ghannam u16 pci_id1, pci_id2; 339780355a3bSYazen Ghannam int ret = -EINVAL; 3398395ae783SBorislav Petkov 3399936fc3afSYazen Ghannam if (pvt->fam >= 0x17) { 34005e4c5527SYazen Ghannam pvt->umc = kcalloc(fam_type->max_mcs, sizeof(struct amd64_umc), GFP_KERNEL); 340180355a3bSYazen Ghannam if (!pvt->umc) 340280355a3bSYazen Ghannam return -ENOMEM; 3403936fc3afSYazen Ghannam 3404936fc3afSYazen Ghannam pci_id1 = fam_type->f0_id; 3405936fc3afSYazen Ghannam pci_id2 = fam_type->f6_id; 3406936fc3afSYazen Ghannam } else { 3407936fc3afSYazen Ghannam pci_id1 = fam_type->f1_id; 3408936fc3afSYazen Ghannam pci_id2 = fam_type->f2_id; 3409936fc3afSYazen Ghannam } 3410936fc3afSYazen Ghannam 341180355a3bSYazen Ghannam ret = reserve_mc_sibling_devs(pvt, pci_id1, pci_id2); 341280355a3bSYazen Ghannam if (ret) 341380355a3bSYazen Ghannam return ret; 34147d6034d3SDoug Thompson 3415360b7f3cSBorislav Petkov read_mc_regs(pvt); 34167d6034d3SDoug Thompson 341780355a3bSYazen Ghannam return 0; 341880355a3bSYazen Ghannam } 341980355a3bSYazen Ghannam 342080355a3bSYazen Ghannam static void hw_info_put(struct amd64_pvt *pvt) 342180355a3bSYazen Ghannam { 342280355a3bSYazen Ghannam if (pvt->F0 || pvt->F1) 342380355a3bSYazen Ghannam free_mc_sibling_devs(pvt); 342480355a3bSYazen Ghannam 342580355a3bSYazen Ghannam kfree(pvt->umc); 342680355a3bSYazen Ghannam } 342780355a3bSYazen Ghannam 342880355a3bSYazen Ghannam static int init_one_instance(struct amd64_pvt *pvt) 342980355a3bSYazen Ghannam { 343080355a3bSYazen Ghannam struct mem_ctl_info *mci = NULL; 343180355a3bSYazen Ghannam struct edac_mc_layer layers[2]; 343280355a3bSYazen Ghannam int ret = -EINVAL; 343380355a3bSYazen Ghannam 34347d6034d3SDoug Thompson /* 34357d6034d3SDoug Thompson * We need to determine how many memory channels there are. Then use 34367d6034d3SDoug Thompson * that information for calculating the size of the dynamic instance 3437360b7f3cSBorislav Petkov * tables in the 'mci' structure. 34387d6034d3SDoug Thompson */ 34397d6034d3SDoug Thompson pvt->channel_count = pvt->ops->early_channel_count(pvt); 34407d6034d3SDoug Thompson if (pvt->channel_count < 0) 344180355a3bSYazen Ghannam return ret; 34427d6034d3SDoug Thompson 34437d6034d3SDoug Thompson ret = -ENOMEM; 3444ab5a503cSMauro Carvalho Chehab layers[0].type = EDAC_MC_LAYER_CHIP_SELECT; 3445ab5a503cSMauro Carvalho Chehab layers[0].size = pvt->csels[0].b_cnt; 3446ab5a503cSMauro Carvalho Chehab layers[0].is_virt_csrow = true; 3447ab5a503cSMauro Carvalho Chehab layers[1].type = EDAC_MC_LAYER_CHANNEL; 3448f0a56c48SBorislav Petkov 3449f0a56c48SBorislav Petkov /* 3450f0a56c48SBorislav Petkov * Always allocate two channels since we can have setups with DIMMs on 3451f0a56c48SBorislav Petkov * only one channel. Also, this simplifies handling later for the price 3452f0a56c48SBorislav Petkov * of a couple of KBs tops. 3453f0a56c48SBorislav Petkov */ 34545e4c5527SYazen Ghannam layers[1].size = fam_type->max_mcs; 3455ab5a503cSMauro Carvalho Chehab layers[1].is_virt_csrow = false; 3456f0a56c48SBorislav Petkov 345780355a3bSYazen Ghannam mci = edac_mc_alloc(pvt->mc_node_id, ARRAY_SIZE(layers), layers, 0); 34587d6034d3SDoug Thompson if (!mci) 345980355a3bSYazen Ghannam return ret; 34607d6034d3SDoug Thompson 34617d6034d3SDoug Thompson mci->pvt_info = pvt; 34623f37a36bSBorislav Petkov mci->pdev = &pvt->F3->dev; 34637d6034d3SDoug Thompson 346438ddd4d1SYazen Ghannam setup_mci_misc_attrs(mci); 3465360b7f3cSBorislav Petkov 3466360b7f3cSBorislav Petkov if (init_csrows(mci)) 34677d6034d3SDoug Thompson mci->edac_cap = EDAC_FLAG_NONE; 34687d6034d3SDoug Thompson 34697d6034d3SDoug Thompson ret = -ENODEV; 3470e339f1ecSTakashi Iwai if (edac_mc_add_mc_with_groups(mci, amd64_edac_attr_groups)) { 3471956b9ba1SJoe Perches edac_dbg(1, "failed edac_mc_add_mc()\n"); 347280355a3bSYazen Ghannam edac_mc_free(mci); 347380355a3bSYazen Ghannam return ret; 34747d6034d3SDoug Thompson } 34757d6034d3SDoug Thompson 34767d6034d3SDoug Thompson return 0; 34777d6034d3SDoug Thompson } 34787d6034d3SDoug Thompson 3479582f94b5SYazen Ghannam static bool instance_has_memory(struct amd64_pvt *pvt) 3480582f94b5SYazen Ghannam { 3481582f94b5SYazen Ghannam bool cs_enabled = false; 3482582f94b5SYazen Ghannam int cs = 0, dct = 0; 3483582f94b5SYazen Ghannam 3484582f94b5SYazen Ghannam for (dct = 0; dct < fam_type->max_mcs; dct++) { 3485582f94b5SYazen Ghannam for_each_chip_select(cs, dct, pvt) 3486582f94b5SYazen Ghannam cs_enabled |= csrow_enabled(cs, dct, pvt); 3487582f94b5SYazen Ghannam } 3488582f94b5SYazen Ghannam 3489582f94b5SYazen Ghannam return cs_enabled; 3490582f94b5SYazen Ghannam } 3491582f94b5SYazen Ghannam 34923f37a36bSBorislav Petkov static int probe_one_instance(unsigned int nid) 34937d6034d3SDoug Thompson { 34942299ef71SBorislav Petkov struct pci_dev *F3 = node_to_amd_nb(nid)->misc; 349580355a3bSYazen Ghannam struct amd64_pvt *pvt = NULL; 3496ae7bb7c6SBorislav Petkov struct ecc_settings *s; 34973f37a36bSBorislav Petkov int ret; 3498b8cfa02fSBorislav Petkov 3499ae7bb7c6SBorislav Petkov ret = -ENOMEM; 3500ae7bb7c6SBorislav Petkov s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL); 3501ae7bb7c6SBorislav Petkov if (!s) 35022299ef71SBorislav Petkov goto err_out; 3503ae7bb7c6SBorislav Petkov 3504ae7bb7c6SBorislav Petkov ecc_stngs[nid] = s; 3505ae7bb7c6SBorislav Petkov 350680355a3bSYazen Ghannam pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL); 350780355a3bSYazen Ghannam if (!pvt) 350880355a3bSYazen Ghannam goto err_settings; 350980355a3bSYazen Ghannam 351080355a3bSYazen Ghannam pvt->mc_node_id = nid; 351180355a3bSYazen Ghannam pvt->F3 = F3; 351280355a3bSYazen Ghannam 351380355a3bSYazen Ghannam fam_type = per_family_init(pvt); 351480355a3bSYazen Ghannam if (!fam_type) 351580355a3bSYazen Ghannam goto err_enable; 351680355a3bSYazen Ghannam 351780355a3bSYazen Ghannam ret = hw_info_get(pvt); 351880355a3bSYazen Ghannam if (ret < 0) 351980355a3bSYazen Ghannam goto err_enable; 352080355a3bSYazen Ghannam 35214688c9b4SYazen Ghannam ret = 0; 3522582f94b5SYazen Ghannam if (!instance_has_memory(pvt)) { 3523582f94b5SYazen Ghannam amd64_info("Node %d: No DIMMs detected.\n", nid); 3524582f94b5SYazen Ghannam goto err_enable; 3525582f94b5SYazen Ghannam } 3526582f94b5SYazen Ghannam 3527582f94b5SYazen Ghannam if (!ecc_enabled(pvt)) { 3528582f94b5SYazen Ghannam ret = -ENODEV; 35292299ef71SBorislav Petkov 35302299ef71SBorislav Petkov if (!ecc_enable_override) 35312299ef71SBorislav Petkov goto err_enable; 35322299ef71SBorislav Petkov 3533044e7a41SYazen Ghannam if (boot_cpu_data.x86 >= 0x17) { 3534044e7a41SYazen Ghannam amd64_warn("Forcing ECC on is not recommended on newer systems. Please enable ECC in BIOS."); 3535044e7a41SYazen Ghannam goto err_enable; 3536044e7a41SYazen Ghannam } else 35372299ef71SBorislav Petkov amd64_warn("Forcing ECC on!\n"); 35382299ef71SBorislav Petkov 35392299ef71SBorislav Petkov if (!enable_ecc_error_reporting(s, nid, F3)) 35402299ef71SBorislav Petkov goto err_enable; 35412299ef71SBorislav Petkov } 35422299ef71SBorislav Petkov 354380355a3bSYazen Ghannam ret = init_one_instance(pvt); 3544360b7f3cSBorislav Petkov if (ret < 0) { 3545ae7bb7c6SBorislav Petkov amd64_err("Error probing instance: %d\n", nid); 3546044e7a41SYazen Ghannam 3547044e7a41SYazen Ghannam if (boot_cpu_data.x86 < 0x17) 3548360b7f3cSBorislav Petkov restore_ecc_error_reporting(s, nid, F3); 35492b9b2c46SYazen Ghannam 35502b9b2c46SYazen Ghannam goto err_enable; 3551360b7f3cSBorislav Petkov } 35527d6034d3SDoug Thompson 3553582f94b5SYazen Ghannam dump_misc_regs(pvt); 3554582f94b5SYazen Ghannam 35557d6034d3SDoug Thompson return ret; 35562299ef71SBorislav Petkov 35572299ef71SBorislav Petkov err_enable: 355880355a3bSYazen Ghannam hw_info_put(pvt); 355980355a3bSYazen Ghannam kfree(pvt); 356080355a3bSYazen Ghannam 356180355a3bSYazen Ghannam err_settings: 35622299ef71SBorislav Petkov kfree(s); 35632299ef71SBorislav Petkov ecc_stngs[nid] = NULL; 35642299ef71SBorislav Petkov 35652299ef71SBorislav Petkov err_out: 35662299ef71SBorislav Petkov return ret; 35677d6034d3SDoug Thompson } 35687d6034d3SDoug Thompson 35693f37a36bSBorislav Petkov static void remove_one_instance(unsigned int nid) 35707d6034d3SDoug Thompson { 3571360b7f3cSBorislav Petkov struct pci_dev *F3 = node_to_amd_nb(nid)->misc; 3572360b7f3cSBorislav Petkov struct ecc_settings *s = ecc_stngs[nid]; 35733f37a36bSBorislav Petkov struct mem_ctl_info *mci; 35743f37a36bSBorislav Petkov struct amd64_pvt *pvt; 35757d6034d3SDoug Thompson 35763f37a36bSBorislav Petkov mci = find_mci_by_dev(&F3->dev); 3577a4b4bedcSBorislav Petkov WARN_ON(!mci); 3578a4b4bedcSBorislav Petkov 35797d6034d3SDoug Thompson /* Remove from EDAC CORE tracking list */ 35803f37a36bSBorislav Petkov mci = edac_mc_del_mc(&F3->dev); 35817d6034d3SDoug Thompson if (!mci) 35827d6034d3SDoug Thompson return; 35837d6034d3SDoug Thompson 35847d6034d3SDoug Thompson pvt = mci->pvt_info; 35857d6034d3SDoug Thompson 3586360b7f3cSBorislav Petkov restore_ecc_error_reporting(s, nid, F3); 35877d6034d3SDoug Thompson 3588360b7f3cSBorislav Petkov kfree(ecc_stngs[nid]); 3589360b7f3cSBorislav Petkov ecc_stngs[nid] = NULL; 3590ae7bb7c6SBorislav Petkov 35917d6034d3SDoug Thompson /* Free the EDAC CORE resources */ 35928f68ed97SBorislav Petkov mci->pvt_info = NULL; 35938f68ed97SBorislav Petkov 359480355a3bSYazen Ghannam hw_info_put(pvt); 35958f68ed97SBorislav Petkov kfree(pvt); 35967d6034d3SDoug Thompson edac_mc_free(mci); 35977d6034d3SDoug Thompson } 35987d6034d3SDoug Thompson 3599360b7f3cSBorislav Petkov static void setup_pci_device(void) 36007d6034d3SDoug Thompson { 36017d6034d3SDoug Thompson struct mem_ctl_info *mci; 36027d6034d3SDoug Thompson struct amd64_pvt *pvt; 36037d6034d3SDoug Thompson 3604d1ea71cdSBorislav Petkov if (pci_ctl) 36057d6034d3SDoug Thompson return; 36067d6034d3SDoug Thompson 36072ec591acSBorislav Petkov mci = edac_mc_find(0); 3608d1ea71cdSBorislav Petkov if (!mci) 3609d1ea71cdSBorislav Petkov return; 36107d6034d3SDoug Thompson 36117d6034d3SDoug Thompson pvt = mci->pvt_info; 3612936fc3afSYazen Ghannam if (pvt->umc) 3613936fc3afSYazen Ghannam pci_ctl = edac_pci_create_generic_ctl(&pvt->F0->dev, EDAC_MOD_STR); 3614936fc3afSYazen Ghannam else 3615d1ea71cdSBorislav Petkov pci_ctl = edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR); 3616d1ea71cdSBorislav Petkov if (!pci_ctl) { 3617d1ea71cdSBorislav Petkov pr_warn("%s(): Unable to create PCI control\n", __func__); 3618d1ea71cdSBorislav Petkov pr_warn("%s(): PCI error report via EDAC not set\n", __func__); 36197d6034d3SDoug Thompson } 36207d6034d3SDoug Thompson } 36217d6034d3SDoug Thompson 3622d6efab74SYazen Ghannam static const struct x86_cpu_id amd64_cpuids[] = { 3623d6efab74SYazen Ghannam { X86_VENDOR_AMD, 0xF, X86_MODEL_ANY, X86_FEATURE_ANY, 0 }, 3624d6efab74SYazen Ghannam { X86_VENDOR_AMD, 0x10, X86_MODEL_ANY, X86_FEATURE_ANY, 0 }, 3625d6efab74SYazen Ghannam { X86_VENDOR_AMD, 0x15, X86_MODEL_ANY, X86_FEATURE_ANY, 0 }, 3626d6efab74SYazen Ghannam { X86_VENDOR_AMD, 0x16, X86_MODEL_ANY, X86_FEATURE_ANY, 0 }, 362795d3af6bSYazen Ghannam { X86_VENDOR_AMD, 0x17, X86_MODEL_ANY, X86_FEATURE_ANY, 0 }, 3628c4a3e946SPu Wen { X86_VENDOR_HYGON, 0x18, X86_MODEL_ANY, X86_FEATURE_ANY, 0 }, 3629d6efab74SYazen Ghannam { } 3630d6efab74SYazen Ghannam }; 3631d6efab74SYazen Ghannam MODULE_DEVICE_TABLE(x86cpu, amd64_cpuids); 3632d6efab74SYazen Ghannam 36337d6034d3SDoug Thompson static int __init amd64_edac_init(void) 36347d6034d3SDoug Thompson { 3635301375e7SToshi Kani const char *owner; 3636360b7f3cSBorislav Petkov int err = -ENODEV; 36373f37a36bSBorislav Petkov int i; 36387d6034d3SDoug Thompson 3639301375e7SToshi Kani owner = edac_get_owner(); 3640301375e7SToshi Kani if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR))) 3641301375e7SToshi Kani return -EBUSY; 3642301375e7SToshi Kani 36431bd9900bSYazen Ghannam if (!x86_match_cpu(amd64_cpuids)) 36441bd9900bSYazen Ghannam return -ENODEV; 36451bd9900bSYazen Ghannam 36469653a5c7SHans Rosenfeld if (amd_cache_northbridges() < 0) 36471bd9900bSYazen Ghannam return -ENODEV; 36487d6034d3SDoug Thompson 36496ba92feaSBorislav Petkov opstate_init(); 36506ba92feaSBorislav Petkov 3651cc4d8860SBorislav Petkov err = -ENOMEM; 36526396bb22SKees Cook ecc_stngs = kcalloc(amd_nb_num(), sizeof(ecc_stngs[0]), GFP_KERNEL); 36532ec591acSBorislav Petkov if (!ecc_stngs) 3654a9f0fbe2SBorislav Petkov goto err_free; 3655cc4d8860SBorislav Petkov 365650542251SBorislav Petkov msrs = msrs_alloc(); 365756b34b91SBorislav Petkov if (!msrs) 3658360b7f3cSBorislav Petkov goto err_free; 365950542251SBorislav Petkov 36602287c636SYazen Ghannam for (i = 0; i < amd_nb_num(); i++) { 36612287c636SYazen Ghannam err = probe_one_instance(i); 36622287c636SYazen Ghannam if (err) { 36633f37a36bSBorislav Petkov /* unwind properly */ 36643f37a36bSBorislav Petkov while (--i >= 0) 36653f37a36bSBorislav Petkov remove_one_instance(i); 36667d6034d3SDoug Thompson 36673f37a36bSBorislav Petkov goto err_pci; 36683f37a36bSBorislav Petkov } 36692287c636SYazen Ghannam } 36707d6034d3SDoug Thompson 36714688c9b4SYazen Ghannam if (!edac_has_mcs()) { 36724688c9b4SYazen Ghannam err = -ENODEV; 36734688c9b4SYazen Ghannam goto err_pci; 36744688c9b4SYazen Ghannam } 36754688c9b4SYazen Ghannam 3676234365f5SYazen Ghannam /* register stuff with EDAC MCE */ 3677234365f5SYazen Ghannam if (report_gart_errors) 3678234365f5SYazen Ghannam amd_report_gart_errors(true); 3679234365f5SYazen Ghannam 3680234365f5SYazen Ghannam if (boot_cpu_data.x86 >= 0x17) 3681234365f5SYazen Ghannam amd_register_ecc_decoder(decode_umc_error); 3682234365f5SYazen Ghannam else 3683234365f5SYazen Ghannam amd_register_ecc_decoder(decode_bus_error); 3684234365f5SYazen Ghannam 3685360b7f3cSBorislav Petkov setup_pci_device(); 3686f5b10c45STomasz Pala 3687f5b10c45STomasz Pala #ifdef CONFIG_X86_32 3688f5b10c45STomasz Pala amd64_err("%s on 32-bit is unsupported. USE AT YOUR OWN RISK!\n", EDAC_MOD_STR); 3689f5b10c45STomasz Pala #endif 3690f5b10c45STomasz Pala 3691de0336b3SBorislav Petkov printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION); 3692de0336b3SBorislav Petkov 36937d6034d3SDoug Thompson return 0; 36947d6034d3SDoug Thompson 369556b34b91SBorislav Petkov err_pci: 369656b34b91SBorislav Petkov msrs_free(msrs); 369756b34b91SBorislav Petkov msrs = NULL; 3698cc4d8860SBorislav Petkov 3699360b7f3cSBorislav Petkov err_free: 3700360b7f3cSBorislav Petkov kfree(ecc_stngs); 3701360b7f3cSBorislav Petkov ecc_stngs = NULL; 3702360b7f3cSBorislav Petkov 37037d6034d3SDoug Thompson return err; 37047d6034d3SDoug Thompson } 37057d6034d3SDoug Thompson 37067d6034d3SDoug Thompson static void __exit amd64_edac_exit(void) 37077d6034d3SDoug Thompson { 37083f37a36bSBorislav Petkov int i; 37093f37a36bSBorislav Petkov 3710d1ea71cdSBorislav Petkov if (pci_ctl) 3711d1ea71cdSBorislav Petkov edac_pci_release_generic_ctl(pci_ctl); 37127d6034d3SDoug Thompson 3713234365f5SYazen Ghannam /* unregister from EDAC MCE */ 3714234365f5SYazen Ghannam amd_report_gart_errors(false); 3715234365f5SYazen Ghannam 3716234365f5SYazen Ghannam if (boot_cpu_data.x86 >= 0x17) 3717234365f5SYazen Ghannam amd_unregister_ecc_decoder(decode_umc_error); 3718234365f5SYazen Ghannam else 3719234365f5SYazen Ghannam amd_unregister_ecc_decoder(decode_bus_error); 3720234365f5SYazen Ghannam 37213f37a36bSBorislav Petkov for (i = 0; i < amd_nb_num(); i++) 37223f37a36bSBorislav Petkov remove_one_instance(i); 372350542251SBorislav Petkov 3724ae7bb7c6SBorislav Petkov kfree(ecc_stngs); 3725ae7bb7c6SBorislav Petkov ecc_stngs = NULL; 3726ae7bb7c6SBorislav Petkov 372750542251SBorislav Petkov msrs_free(msrs); 372850542251SBorislav Petkov msrs = NULL; 37297d6034d3SDoug Thompson } 37307d6034d3SDoug Thompson 37317d6034d3SDoug Thompson module_init(amd64_edac_init); 37327d6034d3SDoug Thompson module_exit(amd64_edac_exit); 37337d6034d3SDoug Thompson 37347d6034d3SDoug Thompson MODULE_LICENSE("GPL"); 37357d6034d3SDoug Thompson MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, " 37367d6034d3SDoug Thompson "Dave Peterson, Thayne Harbaugh"); 37377d6034d3SDoug Thompson MODULE_DESCRIPTION("MC support for AMD64 memory controllers - " 37387d6034d3SDoug Thompson EDAC_AMD64_VERSION); 37397d6034d3SDoug Thompson 37407d6034d3SDoug Thompson module_param(edac_op_state, int, 0444); 37417d6034d3SDoug Thompson MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI"); 3742