109c434b8SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
22bc65418SDoug Thompson #include "amd64_edac.h"
323ac4ae8SAndreas Herrmann #include <asm/amd_nb.h>
42bc65418SDoug Thompson
5d1ea71cdSBorislav Petkov static struct edac_pci_ctl_info *pci_ctl;
62bc65418SDoug Thompson
72bc65418SDoug Thompson /*
82bc65418SDoug Thompson * Set by command line parameter. If BIOS has enabled the ECC, this override is
92bc65418SDoug Thompson * cleared to prevent re-enabling the hardware by this driver.
102bc65418SDoug Thompson */
112bc65418SDoug Thompson static int ecc_enable_override;
122bc65418SDoug Thompson module_param(ecc_enable_override, int, 0644);
132bc65418SDoug Thompson
14a29d8b8eSTejun Heo static struct msr __percpu *msrs;
1550542251SBorislav Petkov
get_umc_reg(struct amd64_pvt * pvt,u32 reg)16ed623d55SMuralidhara M K static inline u32 get_umc_reg(struct amd64_pvt *pvt, u32 reg)
172151c84eSYazen Ghannam {
18ed623d55SMuralidhara M K if (!pvt->flags.zn_regs_v2)
192151c84eSYazen Ghannam return reg;
202151c84eSYazen Ghannam
212151c84eSYazen Ghannam switch (reg) {
222151c84eSYazen Ghannam case UMCCH_ADDR_CFG: return UMCCH_ADDR_CFG_DDR5;
232151c84eSYazen Ghannam case UMCCH_ADDR_MASK_SEC: return UMCCH_ADDR_MASK_SEC_DDR5;
242151c84eSYazen Ghannam case UMCCH_DIMM_CFG: return UMCCH_DIMM_CFG_DDR5;
252151c84eSYazen Ghannam }
262151c84eSYazen Ghannam
272151c84eSYazen Ghannam WARN_ONCE(1, "%s: unknown register 0x%x", __func__, reg);
282151c84eSYazen Ghannam return 0;
292151c84eSYazen Ghannam }
302151c84eSYazen Ghannam
312ec591acSBorislav Petkov /* Per-node stuff */
32ae7bb7c6SBorislav Petkov static struct ecc_settings **ecc_stngs;
332bc65418SDoug Thompson
34706657b1SBorislav Petkov /* Device for the PCI component */
35706657b1SBorislav Petkov static struct device *pci_ctl_dev;
36706657b1SBorislav Petkov
372bc65418SDoug Thompson /*
38b70ef010SBorislav Petkov * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
39b70ef010SBorislav Petkov * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
40b70ef010SBorislav Petkov * or higher value'.
41b70ef010SBorislav Petkov *
42b70ef010SBorislav Petkov *FIXME: Produce a better mapping/linearisation.
43b70ef010SBorislav Petkov */
44c7e5301aSDaniel J Blueman static const struct scrubrate {
4539094443SBorislav Petkov u32 scrubval; /* bit pattern for scrub rate */
4639094443SBorislav Petkov u32 bandwidth; /* bandwidth consumed (bytes/sec) */
4739094443SBorislav Petkov } scrubrates[] = {
48b70ef010SBorislav Petkov { 0x01, 1600000000UL},
49b70ef010SBorislav Petkov { 0x02, 800000000UL},
50b70ef010SBorislav Petkov { 0x03, 400000000UL},
51b70ef010SBorislav Petkov { 0x04, 200000000UL},
52b70ef010SBorislav Petkov { 0x05, 100000000UL},
53b70ef010SBorislav Petkov { 0x06, 50000000UL},
54b70ef010SBorislav Petkov { 0x07, 25000000UL},
55b70ef010SBorislav Petkov { 0x08, 12284069UL},
56b70ef010SBorislav Petkov { 0x09, 6274509UL},
57b70ef010SBorislav Petkov { 0x0A, 3121951UL},
58b70ef010SBorislav Petkov { 0x0B, 1560975UL},
59b70ef010SBorislav Petkov { 0x0C, 781440UL},
60b70ef010SBorislav Petkov { 0x0D, 390720UL},
61b70ef010SBorislav Petkov { 0x0E, 195300UL},
62b70ef010SBorislav Petkov { 0x0F, 97650UL},
63b70ef010SBorislav Petkov { 0x10, 48854UL},
64b70ef010SBorislav Petkov { 0x11, 24427UL},
65b70ef010SBorislav Petkov { 0x12, 12213UL},
66b70ef010SBorislav Petkov { 0x13, 6101UL},
67b70ef010SBorislav Petkov { 0x14, 3051UL},
68b70ef010SBorislav Petkov { 0x15, 1523UL},
69b70ef010SBorislav Petkov { 0x16, 761UL},
70b70ef010SBorislav Petkov { 0x00, 0UL}, /* scrubbing off */
71b70ef010SBorislav Petkov };
72b70ef010SBorislav Petkov
__amd64_read_pci_cfg_dword(struct pci_dev * pdev,int offset,u32 * val,const char * func)7366fed2d4SBorislav Petkov int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
74b2b0c605SBorislav Petkov u32 *val, const char *func)
75b2b0c605SBorislav Petkov {
76b2b0c605SBorislav Petkov int err = 0;
77b2b0c605SBorislav Petkov
78b2b0c605SBorislav Petkov err = pci_read_config_dword(pdev, offset, val);
79b2b0c605SBorislav Petkov if (err)
80b2b0c605SBorislav Petkov amd64_warn("%s: error reading F%dx%03x.\n",
81b2b0c605SBorislav Petkov func, PCI_FUNC(pdev->devfn), offset);
82b2b0c605SBorislav Petkov
838f84ae50SIlpo Järvinen return pcibios_err_to_errno(err);
84b2b0c605SBorislav Petkov }
85b2b0c605SBorislav Petkov
__amd64_write_pci_cfg_dword(struct pci_dev * pdev,int offset,u32 val,const char * func)86b2b0c605SBorislav Petkov int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
87b2b0c605SBorislav Petkov u32 val, const char *func)
88b2b0c605SBorislav Petkov {
89b2b0c605SBorislav Petkov int err = 0;
90b2b0c605SBorislav Petkov
91b2b0c605SBorislav Petkov err = pci_write_config_dword(pdev, offset, val);
92b2b0c605SBorislav Petkov if (err)
93b2b0c605SBorislav Petkov amd64_warn("%s: error writing to F%dx%03x.\n",
94b2b0c605SBorislav Petkov func, PCI_FUNC(pdev->devfn), offset);
95b2b0c605SBorislav Petkov
968f84ae50SIlpo Järvinen return pcibios_err_to_errno(err);
97b2b0c605SBorislav Petkov }
98b2b0c605SBorislav Petkov
99b2b0c605SBorislav Petkov /*
10073ba8593SBorislav Petkov * Select DCT to which PCI cfg accesses are routed
10173ba8593SBorislav Petkov */
f15h_select_dct(struct amd64_pvt * pvt,u8 dct)10273ba8593SBorislav Petkov static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
10373ba8593SBorislav Petkov {
10473ba8593SBorislav Petkov u32 reg = 0;
10573ba8593SBorislav Petkov
10673ba8593SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, ®);
1077981a28fSAravind Gopalakrishnan reg &= (pvt->model == 0x30) ? ~3 : ~1;
10873ba8593SBorislav Petkov reg |= dct;
10973ba8593SBorislav Petkov amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
11073ba8593SBorislav Petkov }
11173ba8593SBorislav Petkov
1127981a28fSAravind Gopalakrishnan /*
1137981a28fSAravind Gopalakrishnan *
1147981a28fSAravind Gopalakrishnan * Depending on the family, F2 DCT reads need special handling:
1157981a28fSAravind Gopalakrishnan *
1167981a28fSAravind Gopalakrishnan * K8: has a single DCT only and no address offsets >= 0x100
1177981a28fSAravind Gopalakrishnan *
1187981a28fSAravind Gopalakrishnan * F10h: each DCT has its own set of regs
1197981a28fSAravind Gopalakrishnan * DCT0 -> F2x040..
1207981a28fSAravind Gopalakrishnan * DCT1 -> F2x140..
1217981a28fSAravind Gopalakrishnan *
1227981a28fSAravind Gopalakrishnan * F16h: has only 1 DCT
1237981a28fSAravind Gopalakrishnan *
1247981a28fSAravind Gopalakrishnan * F15h: we select which DCT we access using F1x10C[DctCfgSel]
1257981a28fSAravind Gopalakrishnan */
amd64_read_dct_pci_cfg(struct amd64_pvt * pvt,u8 dct,int offset,u32 * val)1267981a28fSAravind Gopalakrishnan static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct,
1277981a28fSAravind Gopalakrishnan int offset, u32 *val)
128b2b0c605SBorislav Petkov {
1297981a28fSAravind Gopalakrishnan switch (pvt->fam) {
1307981a28fSAravind Gopalakrishnan case 0xf:
1317981a28fSAravind Gopalakrishnan if (dct || offset >= 0x100)
1327981a28fSAravind Gopalakrishnan return -EINVAL;
1337981a28fSAravind Gopalakrishnan break;
134b2b0c605SBorislav Petkov
1357981a28fSAravind Gopalakrishnan case 0x10:
1367981a28fSAravind Gopalakrishnan if (dct) {
1377981a28fSAravind Gopalakrishnan /*
1387981a28fSAravind Gopalakrishnan * Note: If ganging is enabled, barring the regs
1397981a28fSAravind Gopalakrishnan * F2x[1,0]98 and F2x[1,0]9C; reads reads to F2x1xx
1407981a28fSAravind Gopalakrishnan * return 0. (cf. Section 2.8.1 F10h BKDG)
1417981a28fSAravind Gopalakrishnan */
1427981a28fSAravind Gopalakrishnan if (dct_ganging_enabled(pvt))
1437981a28fSAravind Gopalakrishnan return 0;
1447981a28fSAravind Gopalakrishnan
1457981a28fSAravind Gopalakrishnan offset += 0x100;
146b2b0c605SBorislav Petkov }
1477981a28fSAravind Gopalakrishnan break;
148b2b0c605SBorislav Petkov
1497981a28fSAravind Gopalakrishnan case 0x15:
1507981a28fSAravind Gopalakrishnan /*
1517981a28fSAravind Gopalakrishnan * F15h: F2x1xx addresses do not map explicitly to DCT1.
1527981a28fSAravind Gopalakrishnan * We should select which DCT we access using F1x10C[DctCfgSel]
1537981a28fSAravind Gopalakrishnan */
1547981a28fSAravind Gopalakrishnan dct = (dct && pvt->model == 0x30) ? 3 : dct;
15573ba8593SBorislav Petkov f15h_select_dct(pvt, dct);
1567981a28fSAravind Gopalakrishnan break;
157b2b0c605SBorislav Petkov
1587981a28fSAravind Gopalakrishnan case 0x16:
1597981a28fSAravind Gopalakrishnan if (dct)
1607981a28fSAravind Gopalakrishnan return -EINVAL;
1617981a28fSAravind Gopalakrishnan break;
1627981a28fSAravind Gopalakrishnan
1637981a28fSAravind Gopalakrishnan default:
1647981a28fSAravind Gopalakrishnan break;
1657981a28fSAravind Gopalakrishnan }
1667981a28fSAravind Gopalakrishnan return amd64_read_pci_cfg(pvt->F2, offset, val);
167b2b0c605SBorislav Petkov }
168b2b0c605SBorislav Petkov
169b70ef010SBorislav Petkov /*
1702bc65418SDoug Thompson * Memory scrubber control interface. For K8, memory scrubbing is handled by
1712bc65418SDoug Thompson * hardware and can involve L2 cache, dcache as well as the main memory. With
1722bc65418SDoug Thompson * F10, this is extended to L3 cache scrubbing on CPU models sporting that
1732bc65418SDoug Thompson * functionality.
1742bc65418SDoug Thompson *
1752bc65418SDoug Thompson * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
1762bc65418SDoug Thompson * (dram) over to cache lines. This is nasty, so we will use bandwidth in
1772bc65418SDoug Thompson * bytes/sec for the setting.
1782bc65418SDoug Thompson *
1792bc65418SDoug Thompson * Currently, we only do dram scrubbing. If the scrubbing is done in software on
1802bc65418SDoug Thompson * other archs, we might not have access to the caches directly.
1812bc65418SDoug Thompson */
1822bc65418SDoug Thompson
1838051c0afSYazen Ghannam /*
1848051c0afSYazen Ghannam * Scan the scrub rate mapping table for a close or matching bandwidth value to
1852bc65418SDoug Thompson * issue. If requested is too big, then use last maximum value found.
1862bc65418SDoug Thompson */
__set_scrub_rate(struct amd64_pvt * pvt,u32 new_bw,u32 min_rate)187da92110dSAravind Gopalakrishnan static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate)
1882bc65418SDoug Thompson {
1892bc65418SDoug Thompson u32 scrubval;
1902bc65418SDoug Thompson int i;
1912bc65418SDoug Thompson
1922bc65418SDoug Thompson /*
1932bc65418SDoug Thompson * map the configured rate (new_bw) to a value specific to the AMD64
1942bc65418SDoug Thompson * memory controller and apply to register. Search for the first
1952bc65418SDoug Thompson * bandwidth entry that is greater or equal than the setting requested
1962bc65418SDoug Thompson * and program that. If at last entry, turn off DRAM scrubbing.
197168bfeefSAndrew Morton *
198168bfeefSAndrew Morton * If no suitable bandwidth is found, turn off DRAM scrubbing entirely
199168bfeefSAndrew Morton * by falling back to the last element in scrubrates[].
2002bc65418SDoug Thompson */
201168bfeefSAndrew Morton for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {
2022bc65418SDoug Thompson /*
2032bc65418SDoug Thompson * skip scrub rates which aren't recommended
2042bc65418SDoug Thompson * (see F10 BKDG, F3x58)
2052bc65418SDoug Thompson */
206395ae783SBorislav Petkov if (scrubrates[i].scrubval < min_rate)
2072bc65418SDoug Thompson continue;
2082bc65418SDoug Thompson
2092bc65418SDoug Thompson if (scrubrates[i].bandwidth <= new_bw)
2102bc65418SDoug Thompson break;
2112bc65418SDoug Thompson }
2122bc65418SDoug Thompson
2132bc65418SDoug Thompson scrubval = scrubrates[i].scrubval;
2142bc65418SDoug Thompson
2156e241bc9SYazen Ghannam if (pvt->fam == 0x15 && pvt->model == 0x60) {
216da92110dSAravind Gopalakrishnan f15h_select_dct(pvt, 0);
217da92110dSAravind Gopalakrishnan pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
218da92110dSAravind Gopalakrishnan f15h_select_dct(pvt, 1);
219da92110dSAravind Gopalakrishnan pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
220da92110dSAravind Gopalakrishnan } else {
221da92110dSAravind Gopalakrishnan pci_write_bits32(pvt->F3, SCRCTRL, scrubval, 0x001F);
222da92110dSAravind Gopalakrishnan }
2232bc65418SDoug Thompson
22439094443SBorislav Petkov if (scrubval)
22539094443SBorislav Petkov return scrubrates[i].bandwidth;
22639094443SBorislav Petkov
2272bc65418SDoug Thompson return 0;
2282bc65418SDoug Thompson }
2292bc65418SDoug Thompson
set_scrub_rate(struct mem_ctl_info * mci,u32 bw)230d1ea71cdSBorislav Petkov static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
2312bc65418SDoug Thompson {
2322bc65418SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info;
23387b3e0e6SBorislav Petkov u32 min_scrubrate = 0x5;
2342bc65418SDoug Thompson
235a4b4bedcSBorislav Petkov if (pvt->fam == 0xf)
23687b3e0e6SBorislav Petkov min_scrubrate = 0x0;
23787b3e0e6SBorislav Petkov
238da92110dSAravind Gopalakrishnan if (pvt->fam == 0x15) {
2393f0aba4fSBorislav Petkov /* Erratum #505 */
240da92110dSAravind Gopalakrishnan if (pvt->model < 0x10)
24173ba8593SBorislav Petkov f15h_select_dct(pvt, 0);
24273ba8593SBorislav Petkov
243da92110dSAravind Gopalakrishnan if (pvt->model == 0x60)
244da92110dSAravind Gopalakrishnan min_scrubrate = 0x6;
245da92110dSAravind Gopalakrishnan }
246da92110dSAravind Gopalakrishnan return __set_scrub_rate(pvt, bw, min_scrubrate);
2472bc65418SDoug Thompson }
2482bc65418SDoug Thompson
get_scrub_rate(struct mem_ctl_info * mci)249d1ea71cdSBorislav Petkov static int get_scrub_rate(struct mem_ctl_info *mci)
2502bc65418SDoug Thompson {
2512bc65418SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info;
25239094443SBorislav Petkov int i, retval = -EINVAL;
2538051c0afSYazen Ghannam u32 scrubval = 0;
2542bc65418SDoug Thompson
2556e241bc9SYazen Ghannam if (pvt->fam == 0x15) {
256dcd01394SYazen Ghannam /* Erratum #505 */
257dcd01394SYazen Ghannam if (pvt->model < 0x10)
258dcd01394SYazen Ghannam f15h_select_dct(pvt, 0);
2598051c0afSYazen Ghannam
260dcd01394SYazen Ghannam if (pvt->model == 0x60)
261dcd01394SYazen Ghannam amd64_read_pci_cfg(pvt->F2, F15H_M60H_SCRCTRL, &scrubval);
262ee470bb2SBorislav Petkov else
263ee470bb2SBorislav Petkov amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
264dcd01394SYazen Ghannam } else {
2655980bb9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
2668051c0afSYazen Ghannam }
2672bc65418SDoug Thompson
2682bc65418SDoug Thompson scrubval = scrubval & 0x001F;
2692bc65418SDoug Thompson
270926311fdSRoel Kluin for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
2712bc65418SDoug Thompson if (scrubrates[i].scrubval == scrubval) {
27239094443SBorislav Petkov retval = scrubrates[i].bandwidth;
2732bc65418SDoug Thompson break;
2742bc65418SDoug Thompson }
2752bc65418SDoug Thompson }
27639094443SBorislav Petkov return retval;
2772bc65418SDoug Thompson }
2782bc65418SDoug Thompson
2796775763aSDoug Thompson /*
2807f19bf75SBorislav Petkov * returns true if the SysAddr given by sys_addr matches the
2817f19bf75SBorislav Petkov * DRAM base/limit associated with node_id
2826775763aSDoug Thompson */
base_limit_match(struct amd64_pvt * pvt,u64 sys_addr,u8 nid)283d1ea71cdSBorislav Petkov static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid)
2846775763aSDoug Thompson {
2857f19bf75SBorislav Petkov u64 addr;
2866775763aSDoug Thompson
2876775763aSDoug Thompson /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
2886775763aSDoug Thompson * all ones if the most significant implemented address bit is 1.
2896775763aSDoug Thompson * Here we discard bits 63-40. See section 3.4.2 of AMD publication
2906775763aSDoug Thompson * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
2916775763aSDoug Thompson * Application Programming.
2926775763aSDoug Thompson */
2936775763aSDoug Thompson addr = sys_addr & 0x000000ffffffffffull;
2946775763aSDoug Thompson
2957f19bf75SBorislav Petkov return ((addr >= get_dram_base(pvt, nid)) &&
2967f19bf75SBorislav Petkov (addr <= get_dram_limit(pvt, nid)));
2976775763aSDoug Thompson }
2986775763aSDoug Thompson
2996775763aSDoug Thompson /*
3006775763aSDoug Thompson * Attempt to map a SysAddr to a node. On success, return a pointer to the
3016775763aSDoug Thompson * mem_ctl_info structure for the node that the SysAddr maps to.
3026775763aSDoug Thompson *
3036775763aSDoug Thompson * On failure, return NULL.
3046775763aSDoug Thompson */
find_mc_by_sys_addr(struct mem_ctl_info * mci,u64 sys_addr)3056775763aSDoug Thompson static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
3066775763aSDoug Thompson u64 sys_addr)
3076775763aSDoug Thompson {
3086775763aSDoug Thompson struct amd64_pvt *pvt;
309c7e5301aSDaniel J Blueman u8 node_id;
3106775763aSDoug Thompson u32 intlv_en, bits;
3116775763aSDoug Thompson
3126775763aSDoug Thompson /*
3136775763aSDoug Thompson * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
3146775763aSDoug Thompson * 3.4.4.2) registers to map the SysAddr to a node ID.
3156775763aSDoug Thompson */
3166775763aSDoug Thompson pvt = mci->pvt_info;
3176775763aSDoug Thompson
3186775763aSDoug Thompson /*
3196775763aSDoug Thompson * The value of this field should be the same for all DRAM Base
3206775763aSDoug Thompson * registers. Therefore we arbitrarily choose to read it from the
3216775763aSDoug Thompson * register for node 0.
3226775763aSDoug Thompson */
3237f19bf75SBorislav Petkov intlv_en = dram_intlv_en(pvt, 0);
3246775763aSDoug Thompson
3256775763aSDoug Thompson if (intlv_en == 0) {
3267f19bf75SBorislav Petkov for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
327d1ea71cdSBorislav Petkov if (base_limit_match(pvt, sys_addr, node_id))
3286775763aSDoug Thompson goto found;
3296775763aSDoug Thompson }
3308edc5445SBorislav Petkov goto err_no_match;
3318edc5445SBorislav Petkov }
3326775763aSDoug Thompson
33372f158feSBorislav Petkov if (unlikely((intlv_en != 0x01) &&
33472f158feSBorislav Petkov (intlv_en != 0x03) &&
33572f158feSBorislav Petkov (intlv_en != 0x07))) {
33624f9a7feSBorislav Petkov amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
3376775763aSDoug Thompson return NULL;
3386775763aSDoug Thompson }
3396775763aSDoug Thompson
3406775763aSDoug Thompson bits = (((u32) sys_addr) >> 12) & intlv_en;
3416775763aSDoug Thompson
3426775763aSDoug Thompson for (node_id = 0; ; ) {
3437f19bf75SBorislav Petkov if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
3446775763aSDoug Thompson break; /* intlv_sel field matches */
3456775763aSDoug Thompson
3467f19bf75SBorislav Petkov if (++node_id >= DRAM_RANGES)
3476775763aSDoug Thompson goto err_no_match;
3486775763aSDoug Thompson }
3496775763aSDoug Thompson
3506775763aSDoug Thompson /* sanity test for sys_addr */
351d1ea71cdSBorislav Petkov if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) {
35224f9a7feSBorislav Petkov amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
35324f9a7feSBorislav Petkov "range for node %d with node interleaving enabled.\n",
3548edc5445SBorislav Petkov __func__, sys_addr, node_id);
3556775763aSDoug Thompson return NULL;
3566775763aSDoug Thompson }
3576775763aSDoug Thompson
3586775763aSDoug Thompson found:
359b487c33eSBorislav Petkov return edac_mc_find((int)node_id);
3606775763aSDoug Thompson
3616775763aSDoug Thompson err_no_match:
362956b9ba1SJoe Perches edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n",
3636775763aSDoug Thompson (unsigned long)sys_addr);
3646775763aSDoug Thompson
3656775763aSDoug Thompson return NULL;
3666775763aSDoug Thompson }
367e2ce7255SDoug Thompson
368e2ce7255SDoug Thompson /*
36911c75eadSBorislav Petkov * compute the CS base address of the @csrow on the DRAM controller @dct.
37011c75eadSBorislav Petkov * For details see F2x[5C:40] in the processor's BKDG
371e2ce7255SDoug Thompson */
get_cs_base_and_mask(struct amd64_pvt * pvt,int csrow,u8 dct,u64 * base,u64 * mask)37211c75eadSBorislav Petkov static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
37311c75eadSBorislav Petkov u64 *base, u64 *mask)
374e2ce7255SDoug Thompson {
37511c75eadSBorislav Petkov u64 csbase, csmask, base_bits, mask_bits;
37611c75eadSBorislav Petkov u8 addr_shift;
37711c75eadSBorislav Petkov
37818b94f66SAravind Gopalakrishnan if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
37911c75eadSBorislav Petkov csbase = pvt->csels[dct].csbases[csrow];
38011c75eadSBorislav Petkov csmask = pvt->csels[dct].csmasks[csrow];
38110ef6b0dSChen, Gong base_bits = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9);
38210ef6b0dSChen, Gong mask_bits = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9);
38311c75eadSBorislav Petkov addr_shift = 4;
38494c1acf2SAravind Gopalakrishnan
38594c1acf2SAravind Gopalakrishnan /*
38618b94f66SAravind Gopalakrishnan * F16h and F15h, models 30h and later need two addr_shift values:
38718b94f66SAravind Gopalakrishnan * 8 for high and 6 for low (cf. F16h BKDG).
38894c1acf2SAravind Gopalakrishnan */
38918b94f66SAravind Gopalakrishnan } else if (pvt->fam == 0x16 ||
39018b94f66SAravind Gopalakrishnan (pvt->fam == 0x15 && pvt->model >= 0x30)) {
39194c1acf2SAravind Gopalakrishnan csbase = pvt->csels[dct].csbases[csrow];
39294c1acf2SAravind Gopalakrishnan csmask = pvt->csels[dct].csmasks[csrow >> 1];
39394c1acf2SAravind Gopalakrishnan
39410ef6b0dSChen, Gong *base = (csbase & GENMASK_ULL(15, 5)) << 6;
39510ef6b0dSChen, Gong *base |= (csbase & GENMASK_ULL(30, 19)) << 8;
39694c1acf2SAravind Gopalakrishnan
39794c1acf2SAravind Gopalakrishnan *mask = ~0ULL;
39894c1acf2SAravind Gopalakrishnan /* poke holes for the csmask */
39910ef6b0dSChen, Gong *mask &= ~((GENMASK_ULL(15, 5) << 6) |
40010ef6b0dSChen, Gong (GENMASK_ULL(30, 19) << 8));
40194c1acf2SAravind Gopalakrishnan
40210ef6b0dSChen, Gong *mask |= (csmask & GENMASK_ULL(15, 5)) << 6;
40310ef6b0dSChen, Gong *mask |= (csmask & GENMASK_ULL(30, 19)) << 8;
40494c1acf2SAravind Gopalakrishnan
40594c1acf2SAravind Gopalakrishnan return;
40611c75eadSBorislav Petkov } else {
40711c75eadSBorislav Petkov csbase = pvt->csels[dct].csbases[csrow];
40811c75eadSBorislav Petkov csmask = pvt->csels[dct].csmasks[csrow >> 1];
40911c75eadSBorislav Petkov addr_shift = 8;
41011c75eadSBorislav Petkov
411a4b4bedcSBorislav Petkov if (pvt->fam == 0x15)
41210ef6b0dSChen, Gong base_bits = mask_bits =
41310ef6b0dSChen, Gong GENMASK_ULL(30,19) | GENMASK_ULL(13,5);
41411c75eadSBorislav Petkov else
41510ef6b0dSChen, Gong base_bits = mask_bits =
41610ef6b0dSChen, Gong GENMASK_ULL(28,19) | GENMASK_ULL(13,5);
417e2ce7255SDoug Thompson }
418e2ce7255SDoug Thompson
41911c75eadSBorislav Petkov *base = (csbase & base_bits) << addr_shift;
420e2ce7255SDoug Thompson
42111c75eadSBorislav Petkov *mask = ~0ULL;
42211c75eadSBorislav Petkov /* poke holes for the csmask */
42311c75eadSBorislav Petkov *mask &= ~(mask_bits << addr_shift);
42411c75eadSBorislav Petkov /* OR them in */
42511c75eadSBorislav Petkov *mask |= (csmask & mask_bits) << addr_shift;
426e2ce7255SDoug Thompson }
427e2ce7255SDoug Thompson
42811c75eadSBorislav Petkov #define for_each_chip_select(i, dct, pvt) \
42911c75eadSBorislav Petkov for (i = 0; i < pvt->csels[dct].b_cnt; i++)
43011c75eadSBorislav Petkov
431614ec9d8SBorislav Petkov #define chip_select_base(i, dct, pvt) \
432614ec9d8SBorislav Petkov pvt->csels[dct].csbases[i]
433614ec9d8SBorislav Petkov
43411c75eadSBorislav Petkov #define for_each_chip_select_mask(i, dct, pvt) \
43511c75eadSBorislav Petkov for (i = 0; i < pvt->csels[dct].m_cnt; i++)
43611c75eadSBorislav Petkov
4374d30d2bcSYazen Ghannam #define for_each_umc(i) \
438ed623d55SMuralidhara M K for (i = 0; i < pvt->max_mcs; i++)
4394d30d2bcSYazen Ghannam
440e2ce7255SDoug Thompson /*
441e2ce7255SDoug Thompson * @input_addr is an InputAddr associated with the node given by mci. Return the
442e2ce7255SDoug Thompson * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
443e2ce7255SDoug Thompson */
input_addr_to_csrow(struct mem_ctl_info * mci,u64 input_addr)444e2ce7255SDoug Thompson static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
445e2ce7255SDoug Thompson {
446e2ce7255SDoug Thompson struct amd64_pvt *pvt;
447e2ce7255SDoug Thompson int csrow;
448e2ce7255SDoug Thompson u64 base, mask;
449e2ce7255SDoug Thompson
450e2ce7255SDoug Thompson pvt = mci->pvt_info;
451e2ce7255SDoug Thompson
45211c75eadSBorislav Petkov for_each_chip_select(csrow, 0, pvt) {
45311c75eadSBorislav Petkov if (!csrow_enabled(csrow, 0, pvt))
454e2ce7255SDoug Thompson continue;
455e2ce7255SDoug Thompson
45611c75eadSBorislav Petkov get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
45711c75eadSBorislav Petkov
45811c75eadSBorislav Petkov mask = ~mask;
459e2ce7255SDoug Thompson
460e2ce7255SDoug Thompson if ((input_addr & mask) == (base & mask)) {
461956b9ba1SJoe Perches edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n",
462e2ce7255SDoug Thompson (unsigned long)input_addr, csrow,
463e2ce7255SDoug Thompson pvt->mc_node_id);
464e2ce7255SDoug Thompson
465e2ce7255SDoug Thompson return csrow;
466e2ce7255SDoug Thompson }
467e2ce7255SDoug Thompson }
468956b9ba1SJoe Perches edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n",
469e2ce7255SDoug Thompson (unsigned long)input_addr, pvt->mc_node_id);
470e2ce7255SDoug Thompson
471e2ce7255SDoug Thompson return -1;
472e2ce7255SDoug Thompson }
473e2ce7255SDoug Thompson
474e2ce7255SDoug Thompson /*
475e2ce7255SDoug Thompson * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
476e2ce7255SDoug Thompson * for the node represented by mci. Info is passed back in *hole_base,
477e2ce7255SDoug Thompson * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
478e2ce7255SDoug Thompson * info is invalid. Info may be invalid for either of the following reasons:
479e2ce7255SDoug Thompson *
480e2ce7255SDoug Thompson * - The revision of the node is not E or greater. In this case, the DRAM Hole
481e2ce7255SDoug Thompson * Address Register does not exist.
482e2ce7255SDoug Thompson *
483e2ce7255SDoug Thompson * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
484e2ce7255SDoug Thompson * indicating that its contents are not valid.
485e2ce7255SDoug Thompson *
486e2ce7255SDoug Thompson * The values passed back in *hole_base, *hole_offset, and *hole_size are
487e2ce7255SDoug Thompson * complete 32-bit values despite the fact that the bitfields in the DHAR
488e2ce7255SDoug Thompson * only represent bits 31-24 of the base and offset values.
489e2ce7255SDoug Thompson */
get_dram_hole_info(struct mem_ctl_info * mci,u64 * hole_base,u64 * hole_offset,u64 * hole_size)4902a28ceefSBorislav Petkov static int get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
491e2ce7255SDoug Thompson u64 *hole_offset, u64 *hole_size)
492e2ce7255SDoug Thompson {
493e2ce7255SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info;
494e2ce7255SDoug Thompson
495e2ce7255SDoug Thompson /* only revE and later have the DRAM Hole Address Register */
496a4b4bedcSBorislav Petkov if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) {
497956b9ba1SJoe Perches edac_dbg(1, " revision %d for node %d does not support DHAR\n",
498e2ce7255SDoug Thompson pvt->ext_model, pvt->mc_node_id);
499e2ce7255SDoug Thompson return 1;
500e2ce7255SDoug Thompson }
501e2ce7255SDoug Thompson
502bc21fa57SBorislav Petkov /* valid for Fam10h and above */
503a4b4bedcSBorislav Petkov if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
504956b9ba1SJoe Perches edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n");
505e2ce7255SDoug Thompson return 1;
506e2ce7255SDoug Thompson }
507e2ce7255SDoug Thompson
508c8e518d5SBorislav Petkov if (!dhar_valid(pvt)) {
509956b9ba1SJoe Perches edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n",
510e2ce7255SDoug Thompson pvt->mc_node_id);
511e2ce7255SDoug Thompson return 1;
512e2ce7255SDoug Thompson }
513e2ce7255SDoug Thompson
514e2ce7255SDoug Thompson /* This node has Memory Hoisting */
515e2ce7255SDoug Thompson
516e2ce7255SDoug Thompson /* +------------------+--------------------+--------------------+-----
517e2ce7255SDoug Thompson * | memory | DRAM hole | relocated |
518e2ce7255SDoug Thompson * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
519e2ce7255SDoug Thompson * | | | DRAM hole |
520e2ce7255SDoug Thompson * | | | [0x100000000, |
521e2ce7255SDoug Thompson * | | | (0x100000000+ |
522e2ce7255SDoug Thompson * | | | (0xffffffff-x))] |
523e2ce7255SDoug Thompson * +------------------+--------------------+--------------------+-----
524e2ce7255SDoug Thompson *
525e2ce7255SDoug Thompson * Above is a diagram of physical memory showing the DRAM hole and the
526e2ce7255SDoug Thompson * relocated addresses from the DRAM hole. As shown, the DRAM hole
527e2ce7255SDoug Thompson * starts at address x (the base address) and extends through address
528e2ce7255SDoug Thompson * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
529e2ce7255SDoug Thompson * addresses in the hole so that they start at 0x100000000.
530e2ce7255SDoug Thompson */
531e2ce7255SDoug Thompson
5321f31677eSBorislav Petkov *hole_base = dhar_base(pvt);
5331f31677eSBorislav Petkov *hole_size = (1ULL << 32) - *hole_base;
534e2ce7255SDoug Thompson
535a4b4bedcSBorislav Petkov *hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt)
536a4b4bedcSBorislav Petkov : k8_dhar_offset(pvt);
537e2ce7255SDoug Thompson
538956b9ba1SJoe Perches edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
539e2ce7255SDoug Thompson pvt->mc_node_id, (unsigned long)*hole_base,
540e2ce7255SDoug Thompson (unsigned long)*hole_offset, (unsigned long)*hole_size);
541e2ce7255SDoug Thompson
542e2ce7255SDoug Thompson return 0;
543e2ce7255SDoug Thompson }
5442a28ceefSBorislav Petkov
5452a28ceefSBorislav Petkov #ifdef CONFIG_EDAC_DEBUG
5462a28ceefSBorislav Petkov #define EDAC_DCT_ATTR_SHOW(reg) \
5472a28ceefSBorislav Petkov static ssize_t reg##_show(struct device *dev, \
5482a28ceefSBorislav Petkov struct device_attribute *mattr, char *data) \
5492a28ceefSBorislav Petkov { \
5502a28ceefSBorislav Petkov struct mem_ctl_info *mci = to_mci(dev); \
5512a28ceefSBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; \
5522a28ceefSBorislav Petkov \
5532a28ceefSBorislav Petkov return sprintf(data, "0x%016llx\n", (u64)pvt->reg); \
5542a28ceefSBorislav Petkov }
5552a28ceefSBorislav Petkov
5562a28ceefSBorislav Petkov EDAC_DCT_ATTR_SHOW(dhar);
5572a28ceefSBorislav Petkov EDAC_DCT_ATTR_SHOW(dbam0);
5582a28ceefSBorislav Petkov EDAC_DCT_ATTR_SHOW(top_mem);
5592a28ceefSBorislav Petkov EDAC_DCT_ATTR_SHOW(top_mem2);
5602a28ceefSBorislav Petkov
dram_hole_show(struct device * dev,struct device_attribute * mattr,char * data)561d19faf0eSDwaipayan Ray static ssize_t dram_hole_show(struct device *dev, struct device_attribute *mattr,
5622a28ceefSBorislav Petkov char *data)
5632a28ceefSBorislav Petkov {
5642a28ceefSBorislav Petkov struct mem_ctl_info *mci = to_mci(dev);
5652a28ceefSBorislav Petkov
5662a28ceefSBorislav Petkov u64 hole_base = 0;
5672a28ceefSBorislav Petkov u64 hole_offset = 0;
5682a28ceefSBorislav Petkov u64 hole_size = 0;
5692a28ceefSBorislav Petkov
5702a28ceefSBorislav Petkov get_dram_hole_info(mci, &hole_base, &hole_offset, &hole_size);
5712a28ceefSBorislav Petkov
5722a28ceefSBorislav Petkov return sprintf(data, "%llx %llx %llx\n", hole_base, hole_offset,
5732a28ceefSBorislav Petkov hole_size);
5742a28ceefSBorislav Petkov }
5752a28ceefSBorislav Petkov
5762a28ceefSBorislav Petkov /*
5772a28ceefSBorislav Petkov * update NUM_DBG_ATTRS in case you add new members
5782a28ceefSBorislav Petkov */
5792a28ceefSBorislav Petkov static DEVICE_ATTR(dhar, S_IRUGO, dhar_show, NULL);
5802a28ceefSBorislav Petkov static DEVICE_ATTR(dbam, S_IRUGO, dbam0_show, NULL);
5812a28ceefSBorislav Petkov static DEVICE_ATTR(topmem, S_IRUGO, top_mem_show, NULL);
5822a28ceefSBorislav Petkov static DEVICE_ATTR(topmem2, S_IRUGO, top_mem2_show, NULL);
583d19faf0eSDwaipayan Ray static DEVICE_ATTR_RO(dram_hole);
5842a28ceefSBorislav Petkov
5852a28ceefSBorislav Petkov static struct attribute *dbg_attrs[] = {
5862a28ceefSBorislav Petkov &dev_attr_dhar.attr,
5872a28ceefSBorislav Petkov &dev_attr_dbam.attr,
5882a28ceefSBorislav Petkov &dev_attr_topmem.attr,
5892a28ceefSBorislav Petkov &dev_attr_topmem2.attr,
5902a28ceefSBorislav Petkov &dev_attr_dram_hole.attr,
5912a28ceefSBorislav Petkov NULL
5922a28ceefSBorislav Petkov };
5932a28ceefSBorislav Petkov
5942a28ceefSBorislav Petkov static const struct attribute_group dbg_group = {
5952a28ceefSBorislav Petkov .attrs = dbg_attrs,
5962a28ceefSBorislav Petkov };
5972a28ceefSBorislav Petkov
inject_section_show(struct device * dev,struct device_attribute * mattr,char * buf)59861810096SBorislav Petkov static ssize_t inject_section_show(struct device *dev,
59961810096SBorislav Petkov struct device_attribute *mattr, char *buf)
60061810096SBorislav Petkov {
60161810096SBorislav Petkov struct mem_ctl_info *mci = to_mci(dev);
60261810096SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info;
60361810096SBorislav Petkov return sprintf(buf, "0x%x\n", pvt->injection.section);
60461810096SBorislav Petkov }
60561810096SBorislav Petkov
60661810096SBorislav Petkov /*
60761810096SBorislav Petkov * store error injection section value which refers to one of 4 16-byte sections
60861810096SBorislav Petkov * within a 64-byte cacheline
60961810096SBorislav Petkov *
61061810096SBorislav Petkov * range: 0..3
61161810096SBorislav Petkov */
inject_section_store(struct device * dev,struct device_attribute * mattr,const char * data,size_t count)61261810096SBorislav Petkov static ssize_t inject_section_store(struct device *dev,
61361810096SBorislav Petkov struct device_attribute *mattr,
61461810096SBorislav Petkov const char *data, size_t count)
61561810096SBorislav Petkov {
61661810096SBorislav Petkov struct mem_ctl_info *mci = to_mci(dev);
61761810096SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info;
61861810096SBorislav Petkov unsigned long value;
61961810096SBorislav Petkov int ret;
62061810096SBorislav Petkov
62161810096SBorislav Petkov ret = kstrtoul(data, 10, &value);
62261810096SBorislav Petkov if (ret < 0)
62361810096SBorislav Petkov return ret;
62461810096SBorislav Petkov
62561810096SBorislav Petkov if (value > 3) {
62661810096SBorislav Petkov amd64_warn("%s: invalid section 0x%lx\n", __func__, value);
62761810096SBorislav Petkov return -EINVAL;
62861810096SBorislav Petkov }
62961810096SBorislav Petkov
63061810096SBorislav Petkov pvt->injection.section = (u32) value;
63161810096SBorislav Petkov return count;
63261810096SBorislav Petkov }
63361810096SBorislav Petkov
inject_word_show(struct device * dev,struct device_attribute * mattr,char * buf)63461810096SBorislav Petkov static ssize_t inject_word_show(struct device *dev,
63561810096SBorislav Petkov struct device_attribute *mattr, char *buf)
63661810096SBorislav Petkov {
63761810096SBorislav Petkov struct mem_ctl_info *mci = to_mci(dev);
63861810096SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info;
63961810096SBorislav Petkov return sprintf(buf, "0x%x\n", pvt->injection.word);
64061810096SBorislav Petkov }
64161810096SBorislav Petkov
64261810096SBorislav Petkov /*
64361810096SBorislav Petkov * store error injection word value which refers to one of 9 16-bit word of the
64461810096SBorislav Petkov * 16-byte (128-bit + ECC bits) section
64561810096SBorislav Petkov *
64661810096SBorislav Petkov * range: 0..8
64761810096SBorislav Petkov */
inject_word_store(struct device * dev,struct device_attribute * mattr,const char * data,size_t count)64861810096SBorislav Petkov static ssize_t inject_word_store(struct device *dev,
64961810096SBorislav Petkov struct device_attribute *mattr,
65061810096SBorislav Petkov const char *data, size_t count)
65161810096SBorislav Petkov {
65261810096SBorislav Petkov struct mem_ctl_info *mci = to_mci(dev);
65361810096SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info;
65461810096SBorislav Petkov unsigned long value;
65561810096SBorislav Petkov int ret;
65661810096SBorislav Petkov
65761810096SBorislav Petkov ret = kstrtoul(data, 10, &value);
65861810096SBorislav Petkov if (ret < 0)
65961810096SBorislav Petkov return ret;
66061810096SBorislav Petkov
66161810096SBorislav Petkov if (value > 8) {
66261810096SBorislav Petkov amd64_warn("%s: invalid word 0x%lx\n", __func__, value);
66361810096SBorislav Petkov return -EINVAL;
66461810096SBorislav Petkov }
66561810096SBorislav Petkov
66661810096SBorislav Petkov pvt->injection.word = (u32) value;
66761810096SBorislav Petkov return count;
66861810096SBorislav Petkov }
66961810096SBorislav Petkov
inject_ecc_vector_show(struct device * dev,struct device_attribute * mattr,char * buf)67061810096SBorislav Petkov static ssize_t inject_ecc_vector_show(struct device *dev,
67161810096SBorislav Petkov struct device_attribute *mattr,
67261810096SBorislav Petkov char *buf)
67361810096SBorislav Petkov {
67461810096SBorislav Petkov struct mem_ctl_info *mci = to_mci(dev);
67561810096SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info;
67661810096SBorislav Petkov return sprintf(buf, "0x%x\n", pvt->injection.bit_map);
67761810096SBorislav Petkov }
67861810096SBorislav Petkov
67961810096SBorislav Petkov /*
68061810096SBorislav Petkov * store 16 bit error injection vector which enables injecting errors to the
68161810096SBorislav Petkov * corresponding bit within the error injection word above. When used during a
68261810096SBorislav Petkov * DRAM ECC read, it holds the contents of the of the DRAM ECC bits.
68361810096SBorislav Petkov */
inject_ecc_vector_store(struct device * dev,struct device_attribute * mattr,const char * data,size_t count)68461810096SBorislav Petkov static ssize_t inject_ecc_vector_store(struct device *dev,
68561810096SBorislav Petkov struct device_attribute *mattr,
68661810096SBorislav Petkov const char *data, size_t count)
68761810096SBorislav Petkov {
68861810096SBorislav Petkov struct mem_ctl_info *mci = to_mci(dev);
68961810096SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info;
69061810096SBorislav Petkov unsigned long value;
69161810096SBorislav Petkov int ret;
69261810096SBorislav Petkov
69361810096SBorislav Petkov ret = kstrtoul(data, 16, &value);
69461810096SBorislav Petkov if (ret < 0)
69561810096SBorislav Petkov return ret;
69661810096SBorislav Petkov
69761810096SBorislav Petkov if (value & 0xFFFF0000) {
69861810096SBorislav Petkov amd64_warn("%s: invalid EccVector: 0x%lx\n", __func__, value);
69961810096SBorislav Petkov return -EINVAL;
70061810096SBorislav Petkov }
70161810096SBorislav Petkov
70261810096SBorislav Petkov pvt->injection.bit_map = (u32) value;
70361810096SBorislav Petkov return count;
70461810096SBorislav Petkov }
70561810096SBorislav Petkov
70661810096SBorislav Petkov /*
70761810096SBorislav Petkov * Do a DRAM ECC read. Assemble staged values in the pvt area, format into
70861810096SBorislav Petkov * fields needed by the injection registers and read the NB Array Data Port.
70961810096SBorislav Petkov */
inject_read_store(struct device * dev,struct device_attribute * mattr,const char * data,size_t count)71061810096SBorislav Petkov static ssize_t inject_read_store(struct device *dev,
71161810096SBorislav Petkov struct device_attribute *mattr,
71261810096SBorislav Petkov const char *data, size_t count)
71361810096SBorislav Petkov {
71461810096SBorislav Petkov struct mem_ctl_info *mci = to_mci(dev);
71561810096SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info;
71661810096SBorislav Petkov unsigned long value;
71761810096SBorislav Petkov u32 section, word_bits;
71861810096SBorislav Petkov int ret;
71961810096SBorislav Petkov
72061810096SBorislav Petkov ret = kstrtoul(data, 10, &value);
72161810096SBorislav Petkov if (ret < 0)
72261810096SBorislav Petkov return ret;
72361810096SBorislav Petkov
72461810096SBorislav Petkov /* Form value to choose 16-byte section of cacheline */
72561810096SBorislav Petkov section = F10_NB_ARRAY_DRAM | SET_NB_ARRAY_ADDR(pvt->injection.section);
72661810096SBorislav Petkov
72761810096SBorislav Petkov amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_ADDR, section);
72861810096SBorislav Petkov
72961810096SBorislav Petkov word_bits = SET_NB_DRAM_INJECTION_READ(pvt->injection);
73061810096SBorislav Petkov
73161810096SBorislav Petkov /* Issue 'word' and 'bit' along with the READ request */
73261810096SBorislav Petkov amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, word_bits);
73361810096SBorislav Petkov
73461810096SBorislav Petkov edac_dbg(0, "section=0x%x word_bits=0x%x\n", section, word_bits);
73561810096SBorislav Petkov
73661810096SBorislav Petkov return count;
73761810096SBorislav Petkov }
73861810096SBorislav Petkov
73961810096SBorislav Petkov /*
74061810096SBorislav Petkov * Do a DRAM ECC write. Assemble staged values in the pvt area and format into
74161810096SBorislav Petkov * fields needed by the injection registers.
74261810096SBorislav Petkov */
inject_write_store(struct device * dev,struct device_attribute * mattr,const char * data,size_t count)74361810096SBorislav Petkov static ssize_t inject_write_store(struct device *dev,
74461810096SBorislav Petkov struct device_attribute *mattr,
74561810096SBorislav Petkov const char *data, size_t count)
74661810096SBorislav Petkov {
74761810096SBorislav Petkov struct mem_ctl_info *mci = to_mci(dev);
74861810096SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info;
74961810096SBorislav Petkov u32 section, word_bits, tmp;
75061810096SBorislav Petkov unsigned long value;
75161810096SBorislav Petkov int ret;
75261810096SBorislav Petkov
75361810096SBorislav Petkov ret = kstrtoul(data, 10, &value);
75461810096SBorislav Petkov if (ret < 0)
75561810096SBorislav Petkov return ret;
75661810096SBorislav Petkov
75761810096SBorislav Petkov /* Form value to choose 16-byte section of cacheline */
75861810096SBorislav Petkov section = F10_NB_ARRAY_DRAM | SET_NB_ARRAY_ADDR(pvt->injection.section);
75961810096SBorislav Petkov
76061810096SBorislav Petkov amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_ADDR, section);
76161810096SBorislav Petkov
76261810096SBorislav Petkov word_bits = SET_NB_DRAM_INJECTION_WRITE(pvt->injection);
76361810096SBorislav Petkov
76461810096SBorislav Petkov pr_notice_once("Don't forget to decrease MCE polling interval in\n"
76561810096SBorislav Petkov "/sys/bus/machinecheck/devices/machinecheck<CPUNUM>/check_interval\n"
76661810096SBorislav Petkov "so that you can get the error report faster.\n");
76761810096SBorislav Petkov
76861810096SBorislav Petkov on_each_cpu(disable_caches, NULL, 1);
76961810096SBorislav Petkov
77061810096SBorislav Petkov /* Issue 'word' and 'bit' along with the READ request */
77161810096SBorislav Petkov amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, word_bits);
77261810096SBorislav Petkov
77361810096SBorislav Petkov retry:
77461810096SBorislav Petkov /* wait until injection happens */
77561810096SBorislav Petkov amd64_read_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, &tmp);
77661810096SBorislav Petkov if (tmp & F10_NB_ARR_ECC_WR_REQ) {
77761810096SBorislav Petkov cpu_relax();
77861810096SBorislav Petkov goto retry;
77961810096SBorislav Petkov }
78061810096SBorislav Petkov
78161810096SBorislav Petkov on_each_cpu(enable_caches, NULL, 1);
78261810096SBorislav Petkov
78361810096SBorislav Petkov edac_dbg(0, "section=0x%x word_bits=0x%x\n", section, word_bits);
78461810096SBorislav Petkov
78561810096SBorislav Petkov return count;
78661810096SBorislav Petkov }
78761810096SBorislav Petkov
78861810096SBorislav Petkov /*
78961810096SBorislav Petkov * update NUM_INJ_ATTRS in case you add new members
79061810096SBorislav Petkov */
79161810096SBorislav Petkov
792d19faf0eSDwaipayan Ray static DEVICE_ATTR_RW(inject_section);
793d19faf0eSDwaipayan Ray static DEVICE_ATTR_RW(inject_word);
794d19faf0eSDwaipayan Ray static DEVICE_ATTR_RW(inject_ecc_vector);
795d19faf0eSDwaipayan Ray static DEVICE_ATTR_WO(inject_write);
796d19faf0eSDwaipayan Ray static DEVICE_ATTR_WO(inject_read);
79761810096SBorislav Petkov
79861810096SBorislav Petkov static struct attribute *inj_attrs[] = {
79961810096SBorislav Petkov &dev_attr_inject_section.attr,
80061810096SBorislav Petkov &dev_attr_inject_word.attr,
80161810096SBorislav Petkov &dev_attr_inject_ecc_vector.attr,
80261810096SBorislav Petkov &dev_attr_inject_write.attr,
80361810096SBorislav Petkov &dev_attr_inject_read.attr,
80461810096SBorislav Petkov NULL
80561810096SBorislav Petkov };
80661810096SBorislav Petkov
inj_is_visible(struct kobject * kobj,struct attribute * attr,int idx)80761810096SBorislav Petkov static umode_t inj_is_visible(struct kobject *kobj, struct attribute *attr, int idx)
80861810096SBorislav Petkov {
80961810096SBorislav Petkov struct device *dev = kobj_to_dev(kobj);
81061810096SBorislav Petkov struct mem_ctl_info *mci = container_of(dev, struct mem_ctl_info, dev);
81161810096SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info;
81261810096SBorislav Petkov
8131865bc71SBorislav Petkov /* Families which have that injection hw */
8141865bc71SBorislav Petkov if (pvt->fam >= 0x10 && pvt->fam <= 0x16)
81561810096SBorislav Petkov return attr->mode;
8161865bc71SBorislav Petkov
8171865bc71SBorislav Petkov return 0;
81861810096SBorislav Petkov }
81961810096SBorislav Petkov
82061810096SBorislav Petkov static const struct attribute_group inj_group = {
82161810096SBorislav Petkov .attrs = inj_attrs,
82261810096SBorislav Petkov .is_visible = inj_is_visible,
82361810096SBorislav Petkov };
82461810096SBorislav Petkov #endif /* CONFIG_EDAC_DEBUG */
825e2ce7255SDoug Thompson
82693c2df58SDoug Thompson /*
82793c2df58SDoug Thompson * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
82893c2df58SDoug Thompson * assumed that sys_addr maps to the node given by mci.
82993c2df58SDoug Thompson *
83093c2df58SDoug Thompson * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
83193c2df58SDoug Thompson * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
83293c2df58SDoug Thompson * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
83393c2df58SDoug Thompson * then it is also involved in translating a SysAddr to a DramAddr. Sections
83493c2df58SDoug Thompson * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
83593c2df58SDoug Thompson * These parts of the documentation are unclear. I interpret them as follows:
83693c2df58SDoug Thompson *
83793c2df58SDoug Thompson * When node n receives a SysAddr, it processes the SysAddr as follows:
83893c2df58SDoug Thompson *
83993c2df58SDoug Thompson * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
84093c2df58SDoug Thompson * Limit registers for node n. If the SysAddr is not within the range
84193c2df58SDoug Thompson * specified by the base and limit values, then node n ignores the Sysaddr
84293c2df58SDoug Thompson * (since it does not map to node n). Otherwise continue to step 2 below.
84393c2df58SDoug Thompson *
84493c2df58SDoug Thompson * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
84593c2df58SDoug Thompson * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
84693c2df58SDoug Thompson * the range of relocated addresses (starting at 0x100000000) from the DRAM
84793c2df58SDoug Thompson * hole. If not, skip to step 3 below. Else get the value of the
84893c2df58SDoug Thompson * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
84993c2df58SDoug Thompson * offset defined by this value from the SysAddr.
85093c2df58SDoug Thompson *
85193c2df58SDoug Thompson * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
85293c2df58SDoug Thompson * Base register for node n. To obtain the DramAddr, subtract the base
85393c2df58SDoug Thompson * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
85493c2df58SDoug Thompson */
sys_addr_to_dram_addr(struct mem_ctl_info * mci,u64 sys_addr)85593c2df58SDoug Thompson static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
85693c2df58SDoug Thompson {
8577f19bf75SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info;
85893c2df58SDoug Thompson u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
8591f31677eSBorislav Petkov int ret;
86093c2df58SDoug Thompson
8617f19bf75SBorislav Petkov dram_base = get_dram_base(pvt, pvt->mc_node_id);
86293c2df58SDoug Thompson
8632a28ceefSBorislav Petkov ret = get_dram_hole_info(mci, &hole_base, &hole_offset, &hole_size);
86493c2df58SDoug Thompson if (!ret) {
8651f31677eSBorislav Petkov if ((sys_addr >= (1ULL << 32)) &&
8661f31677eSBorislav Petkov (sys_addr < ((1ULL << 32) + hole_size))) {
86793c2df58SDoug Thompson /* use DHAR to translate SysAddr to DramAddr */
86893c2df58SDoug Thompson dram_addr = sys_addr - hole_offset;
86993c2df58SDoug Thompson
870956b9ba1SJoe Perches edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
87193c2df58SDoug Thompson (unsigned long)sys_addr,
87293c2df58SDoug Thompson (unsigned long)dram_addr);
87393c2df58SDoug Thompson
87493c2df58SDoug Thompson return dram_addr;
87593c2df58SDoug Thompson }
87693c2df58SDoug Thompson }
87793c2df58SDoug Thompson
87893c2df58SDoug Thompson /*
87993c2df58SDoug Thompson * Translate the SysAddr to a DramAddr as shown near the start of
88093c2df58SDoug Thompson * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
88193c2df58SDoug Thompson * only deals with 40-bit values. Therefore we discard bits 63-40 of
88293c2df58SDoug Thompson * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
88393c2df58SDoug Thompson * discard are all 1s. Otherwise the bits we discard are all 0s. See
88493c2df58SDoug Thompson * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
88593c2df58SDoug Thompson * Programmer's Manual Volume 1 Application Programming.
88693c2df58SDoug Thompson */
88710ef6b0dSChen, Gong dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base;
88893c2df58SDoug Thompson
889956b9ba1SJoe Perches edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
890956b9ba1SJoe Perches (unsigned long)sys_addr, (unsigned long)dram_addr);
89193c2df58SDoug Thompson return dram_addr;
89293c2df58SDoug Thompson }
89393c2df58SDoug Thompson
89493c2df58SDoug Thompson /*
89593c2df58SDoug Thompson * @intlv_en is the value of the IntlvEn field from a DRAM Base register
89693c2df58SDoug Thompson * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
89793c2df58SDoug Thompson * for node interleaving.
89893c2df58SDoug Thompson */
num_node_interleave_bits(unsigned intlv_en)89993c2df58SDoug Thompson static int num_node_interleave_bits(unsigned intlv_en)
90093c2df58SDoug Thompson {
90193c2df58SDoug Thompson static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
90293c2df58SDoug Thompson int n;
90393c2df58SDoug Thompson
90493c2df58SDoug Thompson BUG_ON(intlv_en > 7);
90593c2df58SDoug Thompson n = intlv_shift_table[intlv_en];
90693c2df58SDoug Thompson return n;
90793c2df58SDoug Thompson }
90893c2df58SDoug Thompson
90993c2df58SDoug Thompson /* Translate the DramAddr given by @dram_addr to an InputAddr. */
dram_addr_to_input_addr(struct mem_ctl_info * mci,u64 dram_addr)91093c2df58SDoug Thompson static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
91193c2df58SDoug Thompson {
91293c2df58SDoug Thompson struct amd64_pvt *pvt;
91393c2df58SDoug Thompson int intlv_shift;
91493c2df58SDoug Thompson u64 input_addr;
91593c2df58SDoug Thompson
91693c2df58SDoug Thompson pvt = mci->pvt_info;
91793c2df58SDoug Thompson
91893c2df58SDoug Thompson /*
91993c2df58SDoug Thompson * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
92093c2df58SDoug Thompson * concerning translating a DramAddr to an InputAddr.
92193c2df58SDoug Thompson */
9227f19bf75SBorislav Petkov intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
92310ef6b0dSChen, Gong input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) +
92493c2df58SDoug Thompson (dram_addr & 0xfff);
92593c2df58SDoug Thompson
926956b9ba1SJoe Perches edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
92793c2df58SDoug Thompson intlv_shift, (unsigned long)dram_addr,
92893c2df58SDoug Thompson (unsigned long)input_addr);
92993c2df58SDoug Thompson
93093c2df58SDoug Thompson return input_addr;
93193c2df58SDoug Thompson }
93293c2df58SDoug Thompson
93393c2df58SDoug Thompson /*
93493c2df58SDoug Thompson * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
93593c2df58SDoug Thompson * assumed that @sys_addr maps to the node given by mci.
93693c2df58SDoug Thompson */
sys_addr_to_input_addr(struct mem_ctl_info * mci,u64 sys_addr)93793c2df58SDoug Thompson static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
93893c2df58SDoug Thompson {
93993c2df58SDoug Thompson u64 input_addr;
94093c2df58SDoug Thompson
94193c2df58SDoug Thompson input_addr =
94293c2df58SDoug Thompson dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
94393c2df58SDoug Thompson
944c19ca6cbSMasanari Iida edac_dbg(2, "SysAddr 0x%lx translates to InputAddr 0x%lx\n",
94593c2df58SDoug Thompson (unsigned long)sys_addr, (unsigned long)input_addr);
94693c2df58SDoug Thompson
94793c2df58SDoug Thompson return input_addr;
94893c2df58SDoug Thompson }
94993c2df58SDoug Thompson
95093c2df58SDoug Thompson /* Map the Error address to a PAGE and PAGE OFFSET. */
error_address_to_page_and_offset(u64 error_address,struct err_info * err)95193c2df58SDoug Thompson static inline void error_address_to_page_and_offset(u64 error_address,
95233ca0643SBorislav Petkov struct err_info *err)
95393c2df58SDoug Thompson {
95433ca0643SBorislav Petkov err->page = (u32) (error_address >> PAGE_SHIFT);
95533ca0643SBorislav Petkov err->offset = ((u32) error_address) & ~PAGE_MASK;
95693c2df58SDoug Thompson }
95793c2df58SDoug Thompson
95893c2df58SDoug Thompson /*
95993c2df58SDoug Thompson * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
96093c2df58SDoug Thompson * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
96193c2df58SDoug Thompson * of a node that detected an ECC memory error. mci represents the node that
96293c2df58SDoug Thompson * the error address maps to (possibly different from the node that detected
96393c2df58SDoug Thompson * the error). Return the number of the csrow that sys_addr maps to, or -1 on
96493c2df58SDoug Thompson * error.
96593c2df58SDoug Thompson */
sys_addr_to_csrow(struct mem_ctl_info * mci,u64 sys_addr)96693c2df58SDoug Thompson static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
96793c2df58SDoug Thompson {
96893c2df58SDoug Thompson int csrow;
96993c2df58SDoug Thompson
97093c2df58SDoug Thompson csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
97193c2df58SDoug Thompson
97293c2df58SDoug Thompson if (csrow == -1)
97324f9a7feSBorislav Petkov amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
97493c2df58SDoug Thompson "address 0x%lx\n", (unsigned long)sys_addr);
97593c2df58SDoug Thompson return csrow;
97693c2df58SDoug Thompson }
977e2ce7255SDoug Thompson
9784251566eSYazen Ghannam /*
9794251566eSYazen Ghannam * See AMD PPR DF::LclNodeTypeMap
9804251566eSYazen Ghannam *
9814251566eSYazen Ghannam * This register gives information for nodes of the same type within a system.
9824251566eSYazen Ghannam *
9834251566eSYazen Ghannam * Reading this register from a GPU node will tell how many GPU nodes are in the
9844251566eSYazen Ghannam * system and what the lowest AMD Node ID value is for the GPU nodes. Use this
9854251566eSYazen Ghannam * info to fixup the Linux logical "Node ID" value set in the AMD NB code and EDAC.
9864251566eSYazen Ghannam */
9874251566eSYazen Ghannam static struct local_node_map {
9884251566eSYazen Ghannam u16 node_count;
9894251566eSYazen Ghannam u16 base_node_id;
9904251566eSYazen Ghannam } gpu_node_map;
9914251566eSYazen Ghannam
9924251566eSYazen Ghannam #define PCI_DEVICE_ID_AMD_MI200_DF_F1 0x14d1
9934251566eSYazen Ghannam #define REG_LOCAL_NODE_TYPE_MAP 0x144
9944251566eSYazen Ghannam
9954251566eSYazen Ghannam /* Local Node Type Map (LNTM) fields */
9964251566eSYazen Ghannam #define LNTM_NODE_COUNT GENMASK(27, 16)
9974251566eSYazen Ghannam #define LNTM_BASE_NODE_ID GENMASK(11, 0)
9984251566eSYazen Ghannam
gpu_get_node_map(void)9994251566eSYazen Ghannam static int gpu_get_node_map(void)
10004251566eSYazen Ghannam {
10014251566eSYazen Ghannam struct pci_dev *pdev;
10024251566eSYazen Ghannam int ret;
10034251566eSYazen Ghannam u32 tmp;
10044251566eSYazen Ghannam
10054251566eSYazen Ghannam /*
10064251566eSYazen Ghannam * Node ID 0 is reserved for CPUs.
10074251566eSYazen Ghannam * Therefore, a non-zero Node ID means we've already cached the values.
10084251566eSYazen Ghannam */
10094251566eSYazen Ghannam if (gpu_node_map.base_node_id)
10104251566eSYazen Ghannam return 0;
10114251566eSYazen Ghannam
10124251566eSYazen Ghannam pdev = pci_get_device(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_MI200_DF_F1, NULL);
10134251566eSYazen Ghannam if (!pdev) {
10144251566eSYazen Ghannam ret = -ENODEV;
10154251566eSYazen Ghannam goto out;
10164251566eSYazen Ghannam }
10174251566eSYazen Ghannam
10184251566eSYazen Ghannam ret = pci_read_config_dword(pdev, REG_LOCAL_NODE_TYPE_MAP, &tmp);
10198f84ae50SIlpo Järvinen if (ret) {
10208f84ae50SIlpo Järvinen ret = pcibios_err_to_errno(ret);
10214251566eSYazen Ghannam goto out;
10228f84ae50SIlpo Järvinen }
10234251566eSYazen Ghannam
10244251566eSYazen Ghannam gpu_node_map.node_count = FIELD_GET(LNTM_NODE_COUNT, tmp);
10254251566eSYazen Ghannam gpu_node_map.base_node_id = FIELD_GET(LNTM_BASE_NODE_ID, tmp);
10264251566eSYazen Ghannam
10274251566eSYazen Ghannam out:
10284251566eSYazen Ghannam pci_dev_put(pdev);
10294251566eSYazen Ghannam return ret;
10304251566eSYazen Ghannam }
10314251566eSYazen Ghannam
fixup_node_id(int node_id,struct mce * m)10324251566eSYazen Ghannam static int fixup_node_id(int node_id, struct mce *m)
10334251566eSYazen Ghannam {
10344251566eSYazen Ghannam /* MCA_IPID[InstanceIdHi] give the AMD Node ID for the bank. */
10354251566eSYazen Ghannam u8 nid = (m->ipid >> 44) & 0xF;
10364251566eSYazen Ghannam
10374251566eSYazen Ghannam if (smca_get_bank_type(m->extcpu, m->bank) != SMCA_UMC_V2)
10384251566eSYazen Ghannam return node_id;
10394251566eSYazen Ghannam
10404251566eSYazen Ghannam /* Nodes below the GPU base node are CPU nodes and don't need a fixup. */
10414251566eSYazen Ghannam if (nid < gpu_node_map.base_node_id)
10424251566eSYazen Ghannam return node_id;
10434251566eSYazen Ghannam
10444251566eSYazen Ghannam /* Convert the hardware-provided AMD Node ID to a Linux logical one. */
10454251566eSYazen Ghannam return nid - gpu_node_map.base_node_id + 1;
10464251566eSYazen Ghannam }
10474251566eSYazen Ghannam
1048b3218ae4SYazen Ghannam /* Protect the PCI config register pairs used for DF indirect access. */
1049b3218ae4SYazen Ghannam static DEFINE_MUTEX(df_indirect_mutex);
1050b3218ae4SYazen Ghannam
1051b3218ae4SYazen Ghannam /*
1052b3218ae4SYazen Ghannam * Data Fabric Indirect Access uses FICAA/FICAD.
1053b3218ae4SYazen Ghannam *
1054b3218ae4SYazen Ghannam * Fabric Indirect Configuration Access Address (FICAA): Constructed based
1055b3218ae4SYazen Ghannam * on the device's Instance Id and the PCI function and register offset of
1056b3218ae4SYazen Ghannam * the desired register.
1057b3218ae4SYazen Ghannam *
1058b3218ae4SYazen Ghannam * Fabric Indirect Configuration Access Data (FICAD): There are FICAD LO
1059b3218ae4SYazen Ghannam * and FICAD HI registers but so far we only need the LO register.
1060448c3d60SYazen Ghannam *
1061448c3d60SYazen Ghannam * Use Instance Id 0xFF to indicate a broadcast read.
1062b3218ae4SYazen Ghannam */
1063448c3d60SYazen Ghannam #define DF_BROADCAST 0xFF
__df_indirect_read(u16 node,u8 func,u16 reg,u8 instance_id,u32 * lo)1064448c3d60SYazen Ghannam static int __df_indirect_read(u16 node, u8 func, u16 reg, u8 instance_id, u32 *lo)
1065b3218ae4SYazen Ghannam {
1066b3218ae4SYazen Ghannam struct pci_dev *F4;
1067b3218ae4SYazen Ghannam u32 ficaa;
1068b3218ae4SYazen Ghannam int err = -ENODEV;
1069b3218ae4SYazen Ghannam
1070b3218ae4SYazen Ghannam if (node >= amd_nb_num())
1071b3218ae4SYazen Ghannam goto out;
1072b3218ae4SYazen Ghannam
1073b3218ae4SYazen Ghannam F4 = node_to_amd_nb(node)->link;
1074b3218ae4SYazen Ghannam if (!F4)
1075b3218ae4SYazen Ghannam goto out;
1076b3218ae4SYazen Ghannam
1077448c3d60SYazen Ghannam ficaa = (instance_id == DF_BROADCAST) ? 0 : 1;
1078b3218ae4SYazen Ghannam ficaa |= reg & 0x3FC;
1079b3218ae4SYazen Ghannam ficaa |= (func & 0x7) << 11;
1080b3218ae4SYazen Ghannam ficaa |= instance_id << 16;
1081b3218ae4SYazen Ghannam
1082b3218ae4SYazen Ghannam mutex_lock(&df_indirect_mutex);
1083b3218ae4SYazen Ghannam
1084b3218ae4SYazen Ghannam err = pci_write_config_dword(F4, 0x5C, ficaa);
1085b3218ae4SYazen Ghannam if (err) {
1086b3218ae4SYazen Ghannam pr_warn("Error writing DF Indirect FICAA, FICAA=0x%x\n", ficaa);
1087b3218ae4SYazen Ghannam goto out_unlock;
1088b3218ae4SYazen Ghannam }
1089b3218ae4SYazen Ghannam
1090b3218ae4SYazen Ghannam err = pci_read_config_dword(F4, 0x98, lo);
1091b3218ae4SYazen Ghannam if (err)
1092b3218ae4SYazen Ghannam pr_warn("Error reading DF Indirect FICAD LO, FICAA=0x%x.\n", ficaa);
1093b3218ae4SYazen Ghannam
1094b3218ae4SYazen Ghannam out_unlock:
1095b3218ae4SYazen Ghannam mutex_unlock(&df_indirect_mutex);
1096b3218ae4SYazen Ghannam
1097b3218ae4SYazen Ghannam out:
1098b3218ae4SYazen Ghannam return err;
1099b3218ae4SYazen Ghannam }
1100b3218ae4SYazen Ghannam
df_indirect_read_instance(u16 node,u8 func,u16 reg,u8 instance_id,u32 * lo)1101448c3d60SYazen Ghannam static int df_indirect_read_instance(u16 node, u8 func, u16 reg, u8 instance_id, u32 *lo)
1102448c3d60SYazen Ghannam {
1103448c3d60SYazen Ghannam return __df_indirect_read(node, func, reg, instance_id, lo);
1104448c3d60SYazen Ghannam }
1105448c3d60SYazen Ghannam
df_indirect_read_broadcast(u16 node,u8 func,u16 reg,u32 * lo)1106448c3d60SYazen Ghannam static int df_indirect_read_broadcast(u16 node, u8 func, u16 reg, u32 *lo)
1107448c3d60SYazen Ghannam {
1108448c3d60SYazen Ghannam return __df_indirect_read(node, func, reg, DF_BROADCAST, lo);
1109448c3d60SYazen Ghannam }
1110448c3d60SYazen Ghannam
111170aeb807SYazen Ghannam struct addr_ctx {
111270aeb807SYazen Ghannam u64 ret_addr;
111370aeb807SYazen Ghannam u32 tmp;
111470aeb807SYazen Ghannam u16 nid;
111570aeb807SYazen Ghannam u8 inst_id;
111670aeb807SYazen Ghannam };
111770aeb807SYazen Ghannam
umc_normaddr_to_sysaddr(u64 norm_addr,u16 nid,u8 umc,u64 * sys_addr)11180b746e8cSYazen Ghannam static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr)
11190b746e8cSYazen Ghannam {
11200b746e8cSYazen Ghannam u64 dram_base_addr, dram_limit_addr, dram_hole_base;
11210b746e8cSYazen Ghannam
11220b746e8cSYazen Ghannam u8 die_id_shift, die_id_mask, socket_id_shift, socket_id_mask;
11230b746e8cSYazen Ghannam u8 intlv_num_dies, intlv_num_chan, intlv_num_sockets;
11240b746e8cSYazen Ghannam u8 intlv_addr_sel, intlv_addr_bit;
11250b746e8cSYazen Ghannam u8 num_intlv_bits, hashed_bit;
11260b746e8cSYazen Ghannam u8 lgcy_mmio_hole_en, base = 0;
11270b746e8cSYazen Ghannam u8 cs_mask, cs_id = 0;
11280b746e8cSYazen Ghannam bool hash_enabled = false;
11290b746e8cSYazen Ghannam
113070aeb807SYazen Ghannam struct addr_ctx ctx;
113170aeb807SYazen Ghannam
113270aeb807SYazen Ghannam memset(&ctx, 0, sizeof(ctx));
113370aeb807SYazen Ghannam
113470aeb807SYazen Ghannam /* Start from the normalized address */
113570aeb807SYazen Ghannam ctx.ret_addr = norm_addr;
113670aeb807SYazen Ghannam
113770aeb807SYazen Ghannam ctx.nid = nid;
113870aeb807SYazen Ghannam ctx.inst_id = umc;
113970aeb807SYazen Ghannam
11400b746e8cSYazen Ghannam /* Read D18F0x1B4 (DramOffset), check if base 1 is used. */
114170aeb807SYazen Ghannam if (df_indirect_read_instance(nid, 0, 0x1B4, umc, &ctx.tmp))
11420b746e8cSYazen Ghannam goto out_err;
11430b746e8cSYazen Ghannam
11440b746e8cSYazen Ghannam /* Remove HiAddrOffset from normalized address, if enabled: */
114570aeb807SYazen Ghannam if (ctx.tmp & BIT(0)) {
114670aeb807SYazen Ghannam u64 hi_addr_offset = (ctx.tmp & GENMASK_ULL(31, 20)) << 8;
11470b746e8cSYazen Ghannam
11480b746e8cSYazen Ghannam if (norm_addr >= hi_addr_offset) {
114970aeb807SYazen Ghannam ctx.ret_addr -= hi_addr_offset;
11500b746e8cSYazen Ghannam base = 1;
11510b746e8cSYazen Ghannam }
11520b746e8cSYazen Ghannam }
11530b746e8cSYazen Ghannam
11540b746e8cSYazen Ghannam /* Read D18F0x110 (DramBaseAddress). */
115570aeb807SYazen Ghannam if (df_indirect_read_instance(nid, 0, 0x110 + (8 * base), umc, &ctx.tmp))
11560b746e8cSYazen Ghannam goto out_err;
11570b746e8cSYazen Ghannam
11580b746e8cSYazen Ghannam /* Check if address range is valid. */
115970aeb807SYazen Ghannam if (!(ctx.tmp & BIT(0))) {
11600b746e8cSYazen Ghannam pr_err("%s: Invalid DramBaseAddress range: 0x%x.\n",
116170aeb807SYazen Ghannam __func__, ctx.tmp);
11620b746e8cSYazen Ghannam goto out_err;
11630b746e8cSYazen Ghannam }
11640b746e8cSYazen Ghannam
116570aeb807SYazen Ghannam lgcy_mmio_hole_en = ctx.tmp & BIT(1);
116670aeb807SYazen Ghannam intlv_num_chan = (ctx.tmp >> 4) & 0xF;
116770aeb807SYazen Ghannam intlv_addr_sel = (ctx.tmp >> 8) & 0x7;
116870aeb807SYazen Ghannam dram_base_addr = (ctx.tmp & GENMASK_ULL(31, 12)) << 16;
11690b746e8cSYazen Ghannam
11700b746e8cSYazen Ghannam /* {0, 1, 2, 3} map to address bits {8, 9, 10, 11} respectively */
11710b746e8cSYazen Ghannam if (intlv_addr_sel > 3) {
11720b746e8cSYazen Ghannam pr_err("%s: Invalid interleave address select %d.\n",
11730b746e8cSYazen Ghannam __func__, intlv_addr_sel);
11740b746e8cSYazen Ghannam goto out_err;
11750b746e8cSYazen Ghannam }
11760b746e8cSYazen Ghannam
11770b746e8cSYazen Ghannam /* Read D18F0x114 (DramLimitAddress). */
117870aeb807SYazen Ghannam if (df_indirect_read_instance(nid, 0, 0x114 + (8 * base), umc, &ctx.tmp))
11790b746e8cSYazen Ghannam goto out_err;
11800b746e8cSYazen Ghannam
118170aeb807SYazen Ghannam intlv_num_sockets = (ctx.tmp >> 8) & 0x1;
118270aeb807SYazen Ghannam intlv_num_dies = (ctx.tmp >> 10) & 0x3;
118370aeb807SYazen Ghannam dram_limit_addr = ((ctx.tmp & GENMASK_ULL(31, 12)) << 16) | GENMASK_ULL(27, 0);
11840b746e8cSYazen Ghannam
11850b746e8cSYazen Ghannam intlv_addr_bit = intlv_addr_sel + 8;
11860b746e8cSYazen Ghannam
11870b746e8cSYazen Ghannam /* Re-use intlv_num_chan by setting it equal to log2(#channels) */
11880b746e8cSYazen Ghannam switch (intlv_num_chan) {
11890b746e8cSYazen Ghannam case 0: intlv_num_chan = 0; break;
11900b746e8cSYazen Ghannam case 1: intlv_num_chan = 1; break;
11910b746e8cSYazen Ghannam case 3: intlv_num_chan = 2; break;
11920b746e8cSYazen Ghannam case 5: intlv_num_chan = 3; break;
11930b746e8cSYazen Ghannam case 7: intlv_num_chan = 4; break;
11940b746e8cSYazen Ghannam
11950b746e8cSYazen Ghannam case 8: intlv_num_chan = 1;
11960b746e8cSYazen Ghannam hash_enabled = true;
11970b746e8cSYazen Ghannam break;
11980b746e8cSYazen Ghannam default:
11990b746e8cSYazen Ghannam pr_err("%s: Invalid number of interleaved channels %d.\n",
12000b746e8cSYazen Ghannam __func__, intlv_num_chan);
12010b746e8cSYazen Ghannam goto out_err;
12020b746e8cSYazen Ghannam }
12030b746e8cSYazen Ghannam
12040b746e8cSYazen Ghannam num_intlv_bits = intlv_num_chan;
12050b746e8cSYazen Ghannam
12060b746e8cSYazen Ghannam if (intlv_num_dies > 2) {
12070b746e8cSYazen Ghannam pr_err("%s: Invalid number of interleaved nodes/dies %d.\n",
12080b746e8cSYazen Ghannam __func__, intlv_num_dies);
12090b746e8cSYazen Ghannam goto out_err;
12100b746e8cSYazen Ghannam }
12110b746e8cSYazen Ghannam
12120b746e8cSYazen Ghannam num_intlv_bits += intlv_num_dies;
12130b746e8cSYazen Ghannam
12140b746e8cSYazen Ghannam /* Add a bit if sockets are interleaved. */
12150b746e8cSYazen Ghannam num_intlv_bits += intlv_num_sockets;
12160b746e8cSYazen Ghannam
12170b746e8cSYazen Ghannam /* Assert num_intlv_bits <= 4 */
12180b746e8cSYazen Ghannam if (num_intlv_bits > 4) {
12190b746e8cSYazen Ghannam pr_err("%s: Invalid interleave bits %d.\n",
12200b746e8cSYazen Ghannam __func__, num_intlv_bits);
12210b746e8cSYazen Ghannam goto out_err;
12220b746e8cSYazen Ghannam }
12230b746e8cSYazen Ghannam
12240b746e8cSYazen Ghannam if (num_intlv_bits > 0) {
12250b746e8cSYazen Ghannam u64 temp_addr_x, temp_addr_i, temp_addr_y;
12260b746e8cSYazen Ghannam u8 die_id_bit, sock_id_bit, cs_fabric_id;
12270b746e8cSYazen Ghannam
12280b746e8cSYazen Ghannam /*
12290b746e8cSYazen Ghannam * Read FabricBlockInstanceInformation3_CS[BlockFabricID].
12300b746e8cSYazen Ghannam * This is the fabric id for this coherent slave. Use
12310b746e8cSYazen Ghannam * umc/channel# as instance id of the coherent slave
12320b746e8cSYazen Ghannam * for FICAA.
12330b746e8cSYazen Ghannam */
123470aeb807SYazen Ghannam if (df_indirect_read_instance(nid, 0, 0x50, umc, &ctx.tmp))
12350b746e8cSYazen Ghannam goto out_err;
12360b746e8cSYazen Ghannam
123770aeb807SYazen Ghannam cs_fabric_id = (ctx.tmp >> 8) & 0xFF;
12380b746e8cSYazen Ghannam die_id_bit = 0;
12390b746e8cSYazen Ghannam
12400b746e8cSYazen Ghannam /* If interleaved over more than 1 channel: */
12410b746e8cSYazen Ghannam if (intlv_num_chan) {
12420b746e8cSYazen Ghannam die_id_bit = intlv_num_chan;
12430b746e8cSYazen Ghannam cs_mask = (1 << die_id_bit) - 1;
12440b746e8cSYazen Ghannam cs_id = cs_fabric_id & cs_mask;
12450b746e8cSYazen Ghannam }
12460b746e8cSYazen Ghannam
12470b746e8cSYazen Ghannam sock_id_bit = die_id_bit;
12480b746e8cSYazen Ghannam
12490b746e8cSYazen Ghannam /* Read D18F1x208 (SystemFabricIdMask). */
12500b746e8cSYazen Ghannam if (intlv_num_dies || intlv_num_sockets)
125170aeb807SYazen Ghannam if (df_indirect_read_broadcast(nid, 1, 0x208, &ctx.tmp))
12520b746e8cSYazen Ghannam goto out_err;
12530b746e8cSYazen Ghannam
12540b746e8cSYazen Ghannam /* If interleaved over more than 1 die. */
12550b746e8cSYazen Ghannam if (intlv_num_dies) {
12560b746e8cSYazen Ghannam sock_id_bit = die_id_bit + intlv_num_dies;
125770aeb807SYazen Ghannam die_id_shift = (ctx.tmp >> 24) & 0xF;
125870aeb807SYazen Ghannam die_id_mask = (ctx.tmp >> 8) & 0xFF;
12590b746e8cSYazen Ghannam
12600b746e8cSYazen Ghannam cs_id |= ((cs_fabric_id & die_id_mask) >> die_id_shift) << die_id_bit;
12610b746e8cSYazen Ghannam }
12620b746e8cSYazen Ghannam
12630b746e8cSYazen Ghannam /* If interleaved over more than 1 socket. */
12640b746e8cSYazen Ghannam if (intlv_num_sockets) {
126570aeb807SYazen Ghannam socket_id_shift = (ctx.tmp >> 28) & 0xF;
126670aeb807SYazen Ghannam socket_id_mask = (ctx.tmp >> 16) & 0xFF;
12670b746e8cSYazen Ghannam
12680b746e8cSYazen Ghannam cs_id |= ((cs_fabric_id & socket_id_mask) >> socket_id_shift) << sock_id_bit;
12690b746e8cSYazen Ghannam }
12700b746e8cSYazen Ghannam
12710b746e8cSYazen Ghannam /*
12720b746e8cSYazen Ghannam * The pre-interleaved address consists of XXXXXXIIIYYYYY
12730b746e8cSYazen Ghannam * where III is the ID for this CS, and XXXXXXYYYYY are the
12740b746e8cSYazen Ghannam * address bits from the post-interleaved address.
12750b746e8cSYazen Ghannam * "num_intlv_bits" has been calculated to tell us how many "I"
12760b746e8cSYazen Ghannam * bits there are. "intlv_addr_bit" tells us how many "Y" bits
12770b746e8cSYazen Ghannam * there are (where "I" starts).
12780b746e8cSYazen Ghannam */
127970aeb807SYazen Ghannam temp_addr_y = ctx.ret_addr & GENMASK_ULL(intlv_addr_bit - 1, 0);
12800b746e8cSYazen Ghannam temp_addr_i = (cs_id << intlv_addr_bit);
128170aeb807SYazen Ghannam temp_addr_x = (ctx.ret_addr & GENMASK_ULL(63, intlv_addr_bit)) << num_intlv_bits;
128270aeb807SYazen Ghannam ctx.ret_addr = temp_addr_x | temp_addr_i | temp_addr_y;
12830b746e8cSYazen Ghannam }
12840b746e8cSYazen Ghannam
12850b746e8cSYazen Ghannam /* Add dram base address */
128670aeb807SYazen Ghannam ctx.ret_addr += dram_base_addr;
12870b746e8cSYazen Ghannam
12880b746e8cSYazen Ghannam /* If legacy MMIO hole enabled */
12890b746e8cSYazen Ghannam if (lgcy_mmio_hole_en) {
129070aeb807SYazen Ghannam if (df_indirect_read_broadcast(nid, 0, 0x104, &ctx.tmp))
12910b746e8cSYazen Ghannam goto out_err;
12920b746e8cSYazen Ghannam
129370aeb807SYazen Ghannam dram_hole_base = ctx.tmp & GENMASK(31, 24);
129470aeb807SYazen Ghannam if (ctx.ret_addr >= dram_hole_base)
129570aeb807SYazen Ghannam ctx.ret_addr += (BIT_ULL(32) - dram_hole_base);
12960b746e8cSYazen Ghannam }
12970b746e8cSYazen Ghannam
12980b746e8cSYazen Ghannam if (hash_enabled) {
12990b746e8cSYazen Ghannam /* Save some parentheses and grab ls-bit at the end. */
130070aeb807SYazen Ghannam hashed_bit = (ctx.ret_addr >> 12) ^
130170aeb807SYazen Ghannam (ctx.ret_addr >> 18) ^
130270aeb807SYazen Ghannam (ctx.ret_addr >> 21) ^
130370aeb807SYazen Ghannam (ctx.ret_addr >> 30) ^
13040b746e8cSYazen Ghannam cs_id;
13050b746e8cSYazen Ghannam
13060b746e8cSYazen Ghannam hashed_bit &= BIT(0);
13070b746e8cSYazen Ghannam
130870aeb807SYazen Ghannam if (hashed_bit != ((ctx.ret_addr >> intlv_addr_bit) & BIT(0)))
130970aeb807SYazen Ghannam ctx.ret_addr ^= BIT(intlv_addr_bit);
13100b746e8cSYazen Ghannam }
13110b746e8cSYazen Ghannam
13120b746e8cSYazen Ghannam /* Is calculated system address is above DRAM limit address? */
131370aeb807SYazen Ghannam if (ctx.ret_addr > dram_limit_addr)
13140b746e8cSYazen Ghannam goto out_err;
13150b746e8cSYazen Ghannam
131670aeb807SYazen Ghannam *sys_addr = ctx.ret_addr;
13170b746e8cSYazen Ghannam return 0;
13180b746e8cSYazen Ghannam
13190b746e8cSYazen Ghannam out_err:
13200b746e8cSYazen Ghannam return -EINVAL;
13210b746e8cSYazen Ghannam }
13220b746e8cSYazen Ghannam
1323bfc04aecSBorislav Petkov static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
13242da11654SDoug Thompson
13252da11654SDoug Thompson /*
13262da11654SDoug Thompson * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
13272da11654SDoug Thompson * are ECC capable.
13282da11654SDoug Thompson */
dct_determine_edac_cap(struct amd64_pvt * pvt)1329f6a4b4a1SMuralidhara M K static unsigned long dct_determine_edac_cap(struct amd64_pvt *pvt)
13302da11654SDoug Thompson {
13311f6189edSDan Carpenter unsigned long edac_cap = EDAC_FLAG_NONE;
1332d27f3a34SYazen Ghannam u8 bit;
13332da11654SDoug Thompson
1334f6a4b4a1SMuralidhara M K bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F)
1335f6a4b4a1SMuralidhara M K ? 19
1336f6a4b4a1SMuralidhara M K : 17;
1337f6a4b4a1SMuralidhara M K
1338f6a4b4a1SMuralidhara M K if (pvt->dclr0 & BIT(bit))
1339f6a4b4a1SMuralidhara M K edac_cap = EDAC_FLAG_SECDED;
1340f6a4b4a1SMuralidhara M K
1341f6a4b4a1SMuralidhara M K return edac_cap;
1342f6a4b4a1SMuralidhara M K }
1343f6a4b4a1SMuralidhara M K
umc_determine_edac_cap(struct amd64_pvt * pvt)1344f6a4b4a1SMuralidhara M K static unsigned long umc_determine_edac_cap(struct amd64_pvt *pvt)
1345f6a4b4a1SMuralidhara M K {
1346d27f3a34SYazen Ghannam u8 i, umc_en_mask = 0, dimm_ecc_en_mask = 0;
1347f6a4b4a1SMuralidhara M K unsigned long edac_cap = EDAC_FLAG_NONE;
1348d27f3a34SYazen Ghannam
13494d30d2bcSYazen Ghannam for_each_umc(i) {
1350d27f3a34SYazen Ghannam if (!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT))
1351d27f3a34SYazen Ghannam continue;
1352d27f3a34SYazen Ghannam
1353d27f3a34SYazen Ghannam umc_en_mask |= BIT(i);
1354d27f3a34SYazen Ghannam
1355d27f3a34SYazen Ghannam /* UMC Configuration bit 12 (DimmEccEn) */
1356d27f3a34SYazen Ghannam if (pvt->umc[i].umc_cfg & BIT(12))
1357d27f3a34SYazen Ghannam dimm_ecc_en_mask |= BIT(i);
1358d27f3a34SYazen Ghannam }
1359d27f3a34SYazen Ghannam
1360d27f3a34SYazen Ghannam if (umc_en_mask == dimm_ecc_en_mask)
1361d27f3a34SYazen Ghannam edac_cap = EDAC_FLAG_SECDED;
13622da11654SDoug Thompson
13632da11654SDoug Thompson return edac_cap;
13642da11654SDoug Thompson }
13652da11654SDoug Thompson
136600e4feb8SYazen Ghannam /*
136700e4feb8SYazen Ghannam * debug routine to display the memory sizes of all logical DIMMs and its
136800e4feb8SYazen Ghannam * CSROWs
136900e4feb8SYazen Ghannam */
dct_debug_display_dimm_sizes(struct amd64_pvt * pvt,u8 ctrl)137000e4feb8SYazen Ghannam static void dct_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
137100e4feb8SYazen Ghannam {
137200e4feb8SYazen Ghannam u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
137300e4feb8SYazen Ghannam u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
137400e4feb8SYazen Ghannam int dimm, size0, size1;
137500e4feb8SYazen Ghannam
137600e4feb8SYazen Ghannam if (pvt->fam == 0xf) {
137700e4feb8SYazen Ghannam /* K8 families < revF not supported yet */
137800e4feb8SYazen Ghannam if (pvt->ext_model < K8_REV_F)
137900e4feb8SYazen Ghannam return;
138000e4feb8SYazen Ghannam
138100e4feb8SYazen Ghannam WARN_ON(ctrl != 0);
138200e4feb8SYazen Ghannam }
138300e4feb8SYazen Ghannam
138400e4feb8SYazen Ghannam if (pvt->fam == 0x10) {
138500e4feb8SYazen Ghannam dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1
138600e4feb8SYazen Ghannam : pvt->dbam0;
138700e4feb8SYazen Ghannam dcsb = (ctrl && !dct_ganging_enabled(pvt)) ?
138800e4feb8SYazen Ghannam pvt->csels[1].csbases :
138900e4feb8SYazen Ghannam pvt->csels[0].csbases;
139000e4feb8SYazen Ghannam } else if (ctrl) {
139100e4feb8SYazen Ghannam dbam = pvt->dbam0;
139200e4feb8SYazen Ghannam dcsb = pvt->csels[1].csbases;
139300e4feb8SYazen Ghannam }
139400e4feb8SYazen Ghannam edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
139500e4feb8SYazen Ghannam ctrl, dbam);
139600e4feb8SYazen Ghannam
139700e4feb8SYazen Ghannam edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
139800e4feb8SYazen Ghannam
139900e4feb8SYazen Ghannam /* Dump memory sizes for DIMM and its CSROWs */
140000e4feb8SYazen Ghannam for (dimm = 0; dimm < 4; dimm++) {
140100e4feb8SYazen Ghannam size0 = 0;
140200e4feb8SYazen Ghannam if (dcsb[dimm * 2] & DCSB_CS_ENABLE)
140300e4feb8SYazen Ghannam /*
140400e4feb8SYazen Ghannam * For F15m60h, we need multiplier for LRDIMM cs_size
140500e4feb8SYazen Ghannam * calculation. We pass dimm value to the dbam_to_cs
140600e4feb8SYazen Ghannam * mapper so we can find the multiplier from the
140700e4feb8SYazen Ghannam * corresponding DCSM.
140800e4feb8SYazen Ghannam */
140900e4feb8SYazen Ghannam size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
141000e4feb8SYazen Ghannam DBAM_DIMM(dimm, dbam),
141100e4feb8SYazen Ghannam dimm);
141200e4feb8SYazen Ghannam
141300e4feb8SYazen Ghannam size1 = 0;
141400e4feb8SYazen Ghannam if (dcsb[dimm * 2 + 1] & DCSB_CS_ENABLE)
141500e4feb8SYazen Ghannam size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
141600e4feb8SYazen Ghannam DBAM_DIMM(dimm, dbam),
141700e4feb8SYazen Ghannam dimm);
141800e4feb8SYazen Ghannam
141900e4feb8SYazen Ghannam amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
142000e4feb8SYazen Ghannam dimm * 2, size0,
142100e4feb8SYazen Ghannam dimm * 2 + 1, size1);
142200e4feb8SYazen Ghannam }
142300e4feb8SYazen Ghannam }
142400e4feb8SYazen Ghannam
14252da11654SDoug Thompson
debug_dump_dramcfg_low(struct amd64_pvt * pvt,u32 dclr,int chan)1426d1ea71cdSBorislav Petkov static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)
142768798e17SBorislav Petkov {
1428956b9ba1SJoe Perches edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
142968798e17SBorislav Petkov
1430a597d2a5SAravind Gopalakrishnan if (pvt->dram_type == MEM_LRDDR3) {
1431a597d2a5SAravind Gopalakrishnan u32 dcsm = pvt->csels[chan].csmasks[0];
1432a597d2a5SAravind Gopalakrishnan /*
1433a597d2a5SAravind Gopalakrishnan * It's assumed all LRDIMMs in a DCT are going to be of
1434a597d2a5SAravind Gopalakrishnan * same 'type' until proven otherwise. So, use a cs
1435a597d2a5SAravind Gopalakrishnan * value of '0' here to get dcsm value.
1436a597d2a5SAravind Gopalakrishnan */
1437a597d2a5SAravind Gopalakrishnan edac_dbg(1, " LRDIMM %dx rank multiply\n", (dcsm & 0x3));
1438a597d2a5SAravind Gopalakrishnan }
1439a597d2a5SAravind Gopalakrishnan
1440a597d2a5SAravind Gopalakrishnan edac_dbg(1, "All DIMMs support ECC:%s\n",
144168798e17SBorislav Petkov (dclr & BIT(19)) ? "yes" : "no");
144268798e17SBorislav Petkov
1443a597d2a5SAravind Gopalakrishnan
1444956b9ba1SJoe Perches edac_dbg(1, " PAR/ERR parity: %s\n",
144568798e17SBorislav Petkov (dclr & BIT(8)) ? "enabled" : "disabled");
144668798e17SBorislav Petkov
1447a4b4bedcSBorislav Petkov if (pvt->fam == 0x10)
1448956b9ba1SJoe Perches edac_dbg(1, " DCT 128bit mode width: %s\n",
144968798e17SBorislav Petkov (dclr & BIT(11)) ? "128b" : "64b");
145068798e17SBorislav Petkov
1451956b9ba1SJoe Perches edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
145268798e17SBorislav Petkov (dclr & BIT(12)) ? "yes" : "no",
145368798e17SBorislav Petkov (dclr & BIT(13)) ? "yes" : "no",
145468798e17SBorislav Petkov (dclr & BIT(14)) ? "yes" : "no",
145568798e17SBorislav Petkov (dclr & BIT(15)) ? "yes" : "no");
145668798e17SBorislav Petkov }
145768798e17SBorislav Petkov
1458e53a3b26SYazen Ghannam #define CS_EVEN_PRIMARY BIT(0)
1459e53a3b26SYazen Ghannam #define CS_ODD_PRIMARY BIT(1)
146081f5090dSYazen Ghannam #define CS_EVEN_SECONDARY BIT(2)
146181f5090dSYazen Ghannam #define CS_ODD_SECONDARY BIT(3)
14629f4873fbSYazen Ghannam #define CS_3R_INTERLEAVE BIT(4)
1463e53a3b26SYazen Ghannam
146481f5090dSYazen Ghannam #define CS_EVEN (CS_EVEN_PRIMARY | CS_EVEN_SECONDARY)
146581f5090dSYazen Ghannam #define CS_ODD (CS_ODD_PRIMARY | CS_ODD_SECONDARY)
1466e53a3b26SYazen Ghannam
umc_get_cs_mode(int dimm,u8 ctrl,struct amd64_pvt * pvt)1467c0984666SYazen Ghannam static int umc_get_cs_mode(int dimm, u8 ctrl, struct amd64_pvt *pvt)
1468fc00c6a4SYazen Ghannam {
14699f4873fbSYazen Ghannam u8 base, count = 0;
1470e53a3b26SYazen Ghannam int cs_mode = 0;
1471fc00c6a4SYazen Ghannam
1472e53a3b26SYazen Ghannam if (csrow_enabled(2 * dimm, ctrl, pvt))
1473e53a3b26SYazen Ghannam cs_mode |= CS_EVEN_PRIMARY;
1474fc00c6a4SYazen Ghannam
1475e53a3b26SYazen Ghannam if (csrow_enabled(2 * dimm + 1, ctrl, pvt))
1476e53a3b26SYazen Ghannam cs_mode |= CS_ODD_PRIMARY;
1477e53a3b26SYazen Ghannam
147881f5090dSYazen Ghannam /* Asymmetric dual-rank DIMM support. */
147981f5090dSYazen Ghannam if (csrow_sec_enabled(2 * dimm + 1, ctrl, pvt))
148081f5090dSYazen Ghannam cs_mode |= CS_ODD_SECONDARY;
148181f5090dSYazen Ghannam
14829f4873fbSYazen Ghannam /*
14839f4873fbSYazen Ghannam * 3 Rank inteleaving support.
14849f4873fbSYazen Ghannam * There should be only three bases enabled and their two masks should
14859f4873fbSYazen Ghannam * be equal.
14869f4873fbSYazen Ghannam */
14879f4873fbSYazen Ghannam for_each_chip_select(base, ctrl, pvt)
14889f4873fbSYazen Ghannam count += csrow_enabled(base, ctrl, pvt);
14899f4873fbSYazen Ghannam
14909f4873fbSYazen Ghannam if (count == 3 &&
14919f4873fbSYazen Ghannam pvt->csels[ctrl].csmasks[0] == pvt->csels[ctrl].csmasks[1]) {
14929f4873fbSYazen Ghannam edac_dbg(1, "3R interleaving in use.\n");
14939f4873fbSYazen Ghannam cs_mode |= CS_3R_INTERLEAVE;
14949f4873fbSYazen Ghannam }
14959f4873fbSYazen Ghannam
1496e53a3b26SYazen Ghannam return cs_mode;
1497fc00c6a4SYazen Ghannam }
1498fc00c6a4SYazen Ghannam
__addr_mask_to_cs_size(u32 addr_mask_orig,unsigned int cs_mode,int csrow_nr,int dimm)14999c42edd5SMuralidhara M K static int __addr_mask_to_cs_size(u32 addr_mask_orig, unsigned int cs_mode,
15009c42edd5SMuralidhara M K int csrow_nr, int dimm)
15019c42edd5SMuralidhara M K {
15029c42edd5SMuralidhara M K u32 msb, weight, num_zero_bits;
15039c42edd5SMuralidhara M K u32 addr_mask_deinterleaved;
15049c42edd5SMuralidhara M K int size = 0;
15059c42edd5SMuralidhara M K
15069c42edd5SMuralidhara M K /*
15079c42edd5SMuralidhara M K * The number of zero bits in the mask is equal to the number of bits
15089c42edd5SMuralidhara M K * in a full mask minus the number of bits in the current mask.
15099c42edd5SMuralidhara M K *
15109c42edd5SMuralidhara M K * The MSB is the number of bits in the full mask because BIT[0] is
15119c42edd5SMuralidhara M K * always 0.
15129c42edd5SMuralidhara M K *
15139c42edd5SMuralidhara M K * In the special 3 Rank interleaving case, a single bit is flipped
15149c42edd5SMuralidhara M K * without swapping with the most significant bit. This can be handled
15159c42edd5SMuralidhara M K * by keeping the MSB where it is and ignoring the single zero bit.
15169c42edd5SMuralidhara M K */
15179c42edd5SMuralidhara M K msb = fls(addr_mask_orig) - 1;
15189c42edd5SMuralidhara M K weight = hweight_long(addr_mask_orig);
15199c42edd5SMuralidhara M K num_zero_bits = msb - weight - !!(cs_mode & CS_3R_INTERLEAVE);
15209c42edd5SMuralidhara M K
15219c42edd5SMuralidhara M K /* Take the number of zero bits off from the top of the mask. */
15229c42edd5SMuralidhara M K addr_mask_deinterleaved = GENMASK_ULL(msb - num_zero_bits, 1);
15239c42edd5SMuralidhara M K
15249c42edd5SMuralidhara M K edac_dbg(1, "CS%d DIMM%d AddrMasks:\n", csrow_nr, dimm);
15259c42edd5SMuralidhara M K edac_dbg(1, " Original AddrMask: 0x%x\n", addr_mask_orig);
15269c42edd5SMuralidhara M K edac_dbg(1, " Deinterleaved AddrMask: 0x%x\n", addr_mask_deinterleaved);
15279c42edd5SMuralidhara M K
15289c42edd5SMuralidhara M K /* Register [31:1] = Address [39:9]. Size is in kBs here. */
15299c42edd5SMuralidhara M K size = (addr_mask_deinterleaved >> 2) + 1;
15309c42edd5SMuralidhara M K
15319c42edd5SMuralidhara M K /* Return size in MBs. */
15329c42edd5SMuralidhara M K return size >> 10;
15339c42edd5SMuralidhara M K }
15349c42edd5SMuralidhara M K
umc_addr_mask_to_cs_size(struct amd64_pvt * pvt,u8 umc,unsigned int cs_mode,int csrow_nr)1535a2e59ab8SYazen Ghannam static int umc_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc,
1536a2e59ab8SYazen Ghannam unsigned int cs_mode, int csrow_nr)
1537a2e59ab8SYazen Ghannam {
1538a2e59ab8SYazen Ghannam int cs_mask_nr = csrow_nr;
15399c42edd5SMuralidhara M K u32 addr_mask_orig;
1540a2e59ab8SYazen Ghannam int dimm, size = 0;
1541a2e59ab8SYazen Ghannam
1542a2e59ab8SYazen Ghannam /* No Chip Selects are enabled. */
1543a2e59ab8SYazen Ghannam if (!cs_mode)
1544a2e59ab8SYazen Ghannam return size;
1545a2e59ab8SYazen Ghannam
1546a2e59ab8SYazen Ghannam /* Requested size of an even CS but none are enabled. */
1547a2e59ab8SYazen Ghannam if (!(cs_mode & CS_EVEN) && !(csrow_nr & 1))
1548a2e59ab8SYazen Ghannam return size;
1549a2e59ab8SYazen Ghannam
1550a2e59ab8SYazen Ghannam /* Requested size of an odd CS but none are enabled. */
1551a2e59ab8SYazen Ghannam if (!(cs_mode & CS_ODD) && (csrow_nr & 1))
1552a2e59ab8SYazen Ghannam return size;
1553a2e59ab8SYazen Ghannam
1554a2e59ab8SYazen Ghannam /*
1555a2e59ab8SYazen Ghannam * Family 17h introduced systems with one mask per DIMM,
1556a2e59ab8SYazen Ghannam * and two Chip Selects per DIMM.
1557a2e59ab8SYazen Ghannam *
1558a2e59ab8SYazen Ghannam * CS0 and CS1 -> MASK0 / DIMM0
1559a2e59ab8SYazen Ghannam * CS2 and CS3 -> MASK1 / DIMM1
1560a2e59ab8SYazen Ghannam *
1561a2e59ab8SYazen Ghannam * Family 19h Model 10h introduced systems with one mask per Chip Select,
1562a2e59ab8SYazen Ghannam * and two Chip Selects per DIMM.
1563a2e59ab8SYazen Ghannam *
1564a2e59ab8SYazen Ghannam * CS0 -> MASK0 -> DIMM0
1565a2e59ab8SYazen Ghannam * CS1 -> MASK1 -> DIMM0
1566a2e59ab8SYazen Ghannam * CS2 -> MASK2 -> DIMM1
1567a2e59ab8SYazen Ghannam * CS3 -> MASK3 -> DIMM1
1568a2e59ab8SYazen Ghannam *
1569a2e59ab8SYazen Ghannam * Keep the mask number equal to the Chip Select number for newer systems,
1570a2e59ab8SYazen Ghannam * and shift the mask number for older systems.
1571a2e59ab8SYazen Ghannam */
1572a2e59ab8SYazen Ghannam dimm = csrow_nr >> 1;
1573a2e59ab8SYazen Ghannam
1574ed623d55SMuralidhara M K if (!pvt->flags.zn_regs_v2)
1575a2e59ab8SYazen Ghannam cs_mask_nr >>= 1;
1576a2e59ab8SYazen Ghannam
1577a2e59ab8SYazen Ghannam /* Asymmetric dual-rank DIMM support. */
1578a2e59ab8SYazen Ghannam if ((csrow_nr & 1) && (cs_mode & CS_ODD_SECONDARY))
1579a2e59ab8SYazen Ghannam addr_mask_orig = pvt->csels[umc].csmasks_sec[cs_mask_nr];
1580a2e59ab8SYazen Ghannam else
1581a2e59ab8SYazen Ghannam addr_mask_orig = pvt->csels[umc].csmasks[cs_mask_nr];
1582a2e59ab8SYazen Ghannam
15839c42edd5SMuralidhara M K return __addr_mask_to_cs_size(addr_mask_orig, cs_mode, csrow_nr, dimm);
1584a2e59ab8SYazen Ghannam }
1585a2e59ab8SYazen Ghannam
umc_debug_display_dimm_sizes(struct amd64_pvt * pvt,u8 ctrl)158600e4feb8SYazen Ghannam static void umc_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
158707ed82efSYazen Ghannam {
1588e53a3b26SYazen Ghannam int dimm, size0, size1, cs0, cs1, cs_mode;
158907ed82efSYazen Ghannam
159007ed82efSYazen Ghannam edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:\n", ctrl);
159107ed82efSYazen Ghannam
1592d971e28eSYazen Ghannam for (dimm = 0; dimm < 2; dimm++) {
1593eb77e6b8SYazen Ghannam cs0 = dimm * 2;
1594eb77e6b8SYazen Ghannam cs1 = dimm * 2 + 1;
1595eb77e6b8SYazen Ghannam
1596c0984666SYazen Ghannam cs_mode = umc_get_cs_mode(dimm, ctrl, pvt);
1597e53a3b26SYazen Ghannam
1598a2e59ab8SYazen Ghannam size0 = umc_addr_mask_to_cs_size(pvt, ctrl, cs_mode, cs0);
1599a2e59ab8SYazen Ghannam size1 = umc_addr_mask_to_cs_size(pvt, ctrl, cs_mode, cs1);
160007ed82efSYazen Ghannam
160107ed82efSYazen Ghannam amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1602eb77e6b8SYazen Ghannam cs0, size0,
1603eb77e6b8SYazen Ghannam cs1, size1);
160407ed82efSYazen Ghannam }
160507ed82efSYazen Ghannam }
160607ed82efSYazen Ghannam
umc_dump_misc_regs(struct amd64_pvt * pvt)1607f6f36382SMuralidhara M K static void umc_dump_misc_regs(struct amd64_pvt *pvt)
160807ed82efSYazen Ghannam {
160907ed82efSYazen Ghannam struct amd64_umc *umc;
161007ed82efSYazen Ghannam u32 i, tmp, umc_base;
161107ed82efSYazen Ghannam
16124d30d2bcSYazen Ghannam for_each_umc(i) {
161307ed82efSYazen Ghannam umc_base = get_umc_base(i);
161407ed82efSYazen Ghannam umc = &pvt->umc[i];
161507ed82efSYazen Ghannam
161607ed82efSYazen Ghannam edac_dbg(1, "UMC%d DIMM cfg: 0x%x\n", i, umc->dimm_cfg);
161707ed82efSYazen Ghannam edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i, umc->umc_cfg);
161807ed82efSYazen Ghannam edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i, umc->sdp_ctrl);
161907ed82efSYazen Ghannam edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i, umc->ecc_ctrl);
162007ed82efSYazen Ghannam
162107ed82efSYazen Ghannam amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ECC_BAD_SYMBOL, &tmp);
162207ed82efSYazen Ghannam edac_dbg(1, "UMC%d ECC bad symbol: 0x%x\n", i, tmp);
162307ed82efSYazen Ghannam
162407ed82efSYazen Ghannam amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_UMC_CAP, &tmp);
162507ed82efSYazen Ghannam edac_dbg(1, "UMC%d UMC cap: 0x%x\n", i, tmp);
162607ed82efSYazen Ghannam edac_dbg(1, "UMC%d UMC cap high: 0x%x\n", i, umc->umc_cap_hi);
162707ed82efSYazen Ghannam
162807ed82efSYazen Ghannam edac_dbg(1, "UMC%d ECC capable: %s, ChipKill ECC capable: %s\n",
162907ed82efSYazen Ghannam i, (umc->umc_cap_hi & BIT(30)) ? "yes" : "no",
163007ed82efSYazen Ghannam (umc->umc_cap_hi & BIT(31)) ? "yes" : "no");
163107ed82efSYazen Ghannam edac_dbg(1, "UMC%d All DIMMs support ECC: %s\n",
163207ed82efSYazen Ghannam i, (umc->umc_cfg & BIT(12)) ? "yes" : "no");
163307ed82efSYazen Ghannam edac_dbg(1, "UMC%d x4 DIMMs present: %s\n",
163407ed82efSYazen Ghannam i, (umc->dimm_cfg & BIT(6)) ? "yes" : "no");
163507ed82efSYazen Ghannam edac_dbg(1, "UMC%d x16 DIMMs present: %s\n",
163607ed82efSYazen Ghannam i, (umc->dimm_cfg & BIT(7)) ? "yes" : "no");
163707ed82efSYazen Ghannam
16382151c84eSYazen Ghannam if (umc->dram_type == MEM_LRDDR4 || umc->dram_type == MEM_LRDDR5) {
16392151c84eSYazen Ghannam amd_smn_read(pvt->mc_node_id,
1640ed623d55SMuralidhara M K umc_base + get_umc_reg(pvt, UMCCH_ADDR_CFG),
16412151c84eSYazen Ghannam &tmp);
164207ed82efSYazen Ghannam edac_dbg(1, "UMC%d LRDIMM %dx rank multiply\n",
164307ed82efSYazen Ghannam i, 1 << ((tmp >> 4) & 0x3));
164407ed82efSYazen Ghannam }
164507ed82efSYazen Ghannam
164600e4feb8SYazen Ghannam umc_debug_display_dimm_sizes(pvt, i);
164707ed82efSYazen Ghannam }
164807ed82efSYazen Ghannam }
164907ed82efSYazen Ghannam
dct_dump_misc_regs(struct amd64_pvt * pvt)1650f6f36382SMuralidhara M K static void dct_dump_misc_regs(struct amd64_pvt *pvt)
16512da11654SDoug Thompson {
1652956b9ba1SJoe Perches edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
16532da11654SDoug Thompson
1654956b9ba1SJoe Perches edac_dbg(1, " NB two channel DRAM capable: %s\n",
16555980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
165668798e17SBorislav Petkov
1657956b9ba1SJoe Perches edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n",
16585980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
16595980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
166068798e17SBorislav Petkov
1661d1ea71cdSBorislav Petkov debug_dump_dramcfg_low(pvt, pvt->dclr0, 0);
16622da11654SDoug Thompson
1663956b9ba1SJoe Perches edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
16642da11654SDoug Thompson
1665956b9ba1SJoe Perches edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n",
1666bc21fa57SBorislav Petkov pvt->dhar, dhar_base(pvt),
1667a4b4bedcSBorislav Petkov (pvt->fam == 0xf) ? k8_dhar_offset(pvt)
1668bc21fa57SBorislav Petkov : f10_dhar_offset(pvt));
16692da11654SDoug Thompson
167000e4feb8SYazen Ghannam dct_debug_display_dimm_sizes(pvt, 0);
16714d796364SBorislav Petkov
16724d796364SBorislav Petkov /* everything below this point is Fam10h and above */
1673a4b4bedcSBorislav Petkov if (pvt->fam == 0xf)
16742da11654SDoug Thompson return;
16754d796364SBorislav Petkov
167600e4feb8SYazen Ghannam dct_debug_display_dimm_sizes(pvt, 1);
16772da11654SDoug Thompson
16788de1d91eSBorislav Petkov /* Only if NOT ganged does dclr1 have valid info */
167968798e17SBorislav Petkov if (!dct_ganging_enabled(pvt))
1680d1ea71cdSBorislav Petkov debug_dump_dramcfg_low(pvt, pvt->dclr1, 1);
1681cf981562SYazen Ghannam
1682cf981562SYazen Ghannam edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
168307ed82efSYazen Ghannam
16847835961dSYazen Ghannam amd64_info("using x%u syndromes.\n", pvt->ecc_sym_sz);
168507ed82efSYazen Ghannam }
168607ed82efSYazen Ghannam
168794be4bffSDoug Thompson /*
168818b94f66SAravind Gopalakrishnan * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
168994be4bffSDoug Thompson */
dct_prep_chip_selects(struct amd64_pvt * pvt)1690637f60efSMuralidhara M K static void dct_prep_chip_selects(struct amd64_pvt *pvt)
169194be4bffSDoug Thompson {
169218b94f66SAravind Gopalakrishnan if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
169311c75eadSBorislav Petkov pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
169411c75eadSBorislav Petkov pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
1695a597d2a5SAravind Gopalakrishnan } else if (pvt->fam == 0x15 && pvt->model == 0x30) {
169618b94f66SAravind Gopalakrishnan pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4;
169718b94f66SAravind Gopalakrishnan pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2;
16989d858bb1SBorislav Petkov } else {
169911c75eadSBorislav Petkov pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
170011c75eadSBorislav Petkov pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
17019d858bb1SBorislav Petkov }
170294be4bffSDoug Thompson }
170394be4bffSDoug Thompson
umc_prep_chip_selects(struct amd64_pvt * pvt)1704637f60efSMuralidhara M K static void umc_prep_chip_selects(struct amd64_pvt *pvt)
1705637f60efSMuralidhara M K {
170694be4bffSDoug Thompson int umc;
170794be4bffSDoug Thompson
170894be4bffSDoug Thompson for_each_umc(umc) {
170994be4bffSDoug Thompson pvt->csels[umc].b_cnt = 4;
1710ed623d55SMuralidhara M K pvt->csels[umc].m_cnt = pvt->flags.zn_regs_v2 ? 4 : 2;
171194be4bffSDoug Thompson }
171294be4bffSDoug Thompson }
171394be4bffSDoug Thompson
umc_read_base_mask(struct amd64_pvt * pvt)1714b29dad9bSMuralidhara M K static void umc_read_base_mask(struct amd64_pvt *pvt)
1715d971e28eSYazen Ghannam {
17167574729eSYazen Ghannam u32 umc_base_reg, umc_base_reg_sec;
17177574729eSYazen Ghannam u32 umc_mask_reg, umc_mask_reg_sec;
17187574729eSYazen Ghannam u32 base_reg, base_reg_sec;
17197574729eSYazen Ghannam u32 mask_reg, mask_reg_sec;
17207574729eSYazen Ghannam u32 *base, *base_sec;
17217574729eSYazen Ghannam u32 *mask, *mask_sec;
1722d971e28eSYazen Ghannam int cs, umc;
1723d971e28eSYazen Ghannam
1724d971e28eSYazen Ghannam for_each_umc(umc) {
1725d971e28eSYazen Ghannam umc_base_reg = get_umc_base(umc) + UMCCH_BASE_ADDR;
17267574729eSYazen Ghannam umc_base_reg_sec = get_umc_base(umc) + UMCCH_BASE_ADDR_SEC;
1727d971e28eSYazen Ghannam
1728d971e28eSYazen Ghannam for_each_chip_select(cs, umc, pvt) {
1729d971e28eSYazen Ghannam base = &pvt->csels[umc].csbases[cs];
17307574729eSYazen Ghannam base_sec = &pvt->csels[umc].csbases_sec[cs];
1731d971e28eSYazen Ghannam
1732d971e28eSYazen Ghannam base_reg = umc_base_reg + (cs * 4);
17337574729eSYazen Ghannam base_reg_sec = umc_base_reg_sec + (cs * 4);
1734d971e28eSYazen Ghannam
1735d971e28eSYazen Ghannam if (!amd_smn_read(pvt->mc_node_id, base_reg, base))
1736d971e28eSYazen Ghannam edac_dbg(0, " DCSB%d[%d]=0x%08x reg: 0x%x\n",
1737d971e28eSYazen Ghannam umc, cs, *base, base_reg);
17387574729eSYazen Ghannam
17397574729eSYazen Ghannam if (!amd_smn_read(pvt->mc_node_id, base_reg_sec, base_sec))
17407574729eSYazen Ghannam edac_dbg(0, " DCSB_SEC%d[%d]=0x%08x reg: 0x%x\n",
17417574729eSYazen Ghannam umc, cs, *base_sec, base_reg_sec);
1742d971e28eSYazen Ghannam }
1743d971e28eSYazen Ghannam
1744d971e28eSYazen Ghannam umc_mask_reg = get_umc_base(umc) + UMCCH_ADDR_MASK;
1745ed623d55SMuralidhara M K umc_mask_reg_sec = get_umc_base(umc) + get_umc_reg(pvt, UMCCH_ADDR_MASK_SEC);
1746d971e28eSYazen Ghannam
1747d971e28eSYazen Ghannam for_each_chip_select_mask(cs, umc, pvt) {
1748d971e28eSYazen Ghannam mask = &pvt->csels[umc].csmasks[cs];
17497574729eSYazen Ghannam mask_sec = &pvt->csels[umc].csmasks_sec[cs];
1750d971e28eSYazen Ghannam
1751d971e28eSYazen Ghannam mask_reg = umc_mask_reg + (cs * 4);
17527574729eSYazen Ghannam mask_reg_sec = umc_mask_reg_sec + (cs * 4);
1753d971e28eSYazen Ghannam
1754d971e28eSYazen Ghannam if (!amd_smn_read(pvt->mc_node_id, mask_reg, mask))
1755d971e28eSYazen Ghannam edac_dbg(0, " DCSM%d[%d]=0x%08x reg: 0x%x\n",
1756d971e28eSYazen Ghannam umc, cs, *mask, mask_reg);
17577574729eSYazen Ghannam
17587574729eSYazen Ghannam if (!amd_smn_read(pvt->mc_node_id, mask_reg_sec, mask_sec))
17597574729eSYazen Ghannam edac_dbg(0, " DCSM_SEC%d[%d]=0x%08x reg: 0x%x\n",
17607574729eSYazen Ghannam umc, cs, *mask_sec, mask_reg_sec);
1761d971e28eSYazen Ghannam }
1762d971e28eSYazen Ghannam }
1763d971e28eSYazen Ghannam }
1764d971e28eSYazen Ghannam
176594be4bffSDoug Thompson /*
176611c75eadSBorislav Petkov * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
176794be4bffSDoug Thompson */
dct_read_base_mask(struct amd64_pvt * pvt)1768b29dad9bSMuralidhara M K static void dct_read_base_mask(struct amd64_pvt *pvt)
176994be4bffSDoug Thompson {
1770d971e28eSYazen Ghannam int cs;
177194be4bffSDoug Thompson
177211c75eadSBorislav Petkov for_each_chip_select(cs, 0, pvt) {
1773d971e28eSYazen Ghannam int reg0 = DCSB0 + (cs * 4);
1774d971e28eSYazen Ghannam int reg1 = DCSB1 + (cs * 4);
177511c75eadSBorislav Petkov u32 *base0 = &pvt->csels[0].csbases[cs];
177611c75eadSBorislav Petkov u32 *base1 = &pvt->csels[1].csbases[cs];
1777b2b0c605SBorislav Petkov
17787981a28fSAravind Gopalakrishnan if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0))
1779956b9ba1SJoe Perches edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n",
178011c75eadSBorislav Petkov cs, *base0, reg0);
178194be4bffSDoug Thompson
17827981a28fSAravind Gopalakrishnan if (pvt->fam == 0xf)
178311c75eadSBorislav Petkov continue;
1784b2b0c605SBorislav Petkov
17857981a28fSAravind Gopalakrishnan if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1))
1786956b9ba1SJoe Perches edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n",
17877981a28fSAravind Gopalakrishnan cs, *base1, (pvt->fam == 0x10) ? reg1
17887981a28fSAravind Gopalakrishnan : reg0);
178994be4bffSDoug Thompson }
179094be4bffSDoug Thompson
179111c75eadSBorislav Petkov for_each_chip_select_mask(cs, 0, pvt) {
1792d971e28eSYazen Ghannam int reg0 = DCSM0 + (cs * 4);
1793d971e28eSYazen Ghannam int reg1 = DCSM1 + (cs * 4);
179411c75eadSBorislav Petkov u32 *mask0 = &pvt->csels[0].csmasks[cs];
179511c75eadSBorislav Petkov u32 *mask1 = &pvt->csels[1].csmasks[cs];
1796b2b0c605SBorislav Petkov
17977981a28fSAravind Gopalakrishnan if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0))
1798956b9ba1SJoe Perches edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n",
179911c75eadSBorislav Petkov cs, *mask0, reg0);
180094be4bffSDoug Thompson
18017981a28fSAravind Gopalakrishnan if (pvt->fam == 0xf)
180211c75eadSBorislav Petkov continue;
1803b2b0c605SBorislav Petkov
18047981a28fSAravind Gopalakrishnan if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1))
1805956b9ba1SJoe Perches edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n",
18067981a28fSAravind Gopalakrishnan cs, *mask1, (pvt->fam == 0x10) ? reg1
18077981a28fSAravind Gopalakrishnan : reg0);
180894be4bffSDoug Thompson }
18096ba5dcdcSBorislav Petkov }
181094be4bffSDoug Thompson
umc_determine_memory_type(struct amd64_pvt * pvt)181178ec161aSMuralidhara M K static void umc_determine_memory_type(struct amd64_pvt *pvt)
181275aeaaf2SYazen Ghannam {
181375aeaaf2SYazen Ghannam struct amd64_umc *umc;
181475aeaaf2SYazen Ghannam u32 i;
181575aeaaf2SYazen Ghannam
181675aeaaf2SYazen Ghannam for_each_umc(i) {
181775aeaaf2SYazen Ghannam umc = &pvt->umc[i];
181875aeaaf2SYazen Ghannam
181975aeaaf2SYazen Ghannam if (!(umc->sdp_ctrl & UMC_SDP_INIT)) {
182075aeaaf2SYazen Ghannam umc->dram_type = MEM_EMPTY;
182175aeaaf2SYazen Ghannam continue;
182275aeaaf2SYazen Ghannam }
182375aeaaf2SYazen Ghannam
18242151c84eSYazen Ghannam /*
18252151c84eSYazen Ghannam * Check if the system supports the "DDR Type" field in UMC Config
18262151c84eSYazen Ghannam * and has DDR5 DIMMs in use.
18272151c84eSYazen Ghannam */
1828ed623d55SMuralidhara M K if (pvt->flags.zn_regs_v2 && ((umc->umc_cfg & GENMASK(2, 0)) == 0x1)) {
18292151c84eSYazen Ghannam if (umc->dimm_cfg & BIT(5))
18302151c84eSYazen Ghannam umc->dram_type = MEM_LRDDR5;
18312151c84eSYazen Ghannam else if (umc->dimm_cfg & BIT(4))
18322151c84eSYazen Ghannam umc->dram_type = MEM_RDDR5;
18332151c84eSYazen Ghannam else
18342151c84eSYazen Ghannam umc->dram_type = MEM_DDR5;
18352151c84eSYazen Ghannam } else {
183675aeaaf2SYazen Ghannam if (umc->dimm_cfg & BIT(5))
183775aeaaf2SYazen Ghannam umc->dram_type = MEM_LRDDR4;
183875aeaaf2SYazen Ghannam else if (umc->dimm_cfg & BIT(4))
183975aeaaf2SYazen Ghannam umc->dram_type = MEM_RDDR4;
184075aeaaf2SYazen Ghannam else
184175aeaaf2SYazen Ghannam umc->dram_type = MEM_DDR4;
18422151c84eSYazen Ghannam }
184375aeaaf2SYazen Ghannam
184475aeaaf2SYazen Ghannam edac_dbg(1, " UMC%d DIMM type: %s\n", i, edac_mem_types[umc->dram_type]);
184575aeaaf2SYazen Ghannam }
184675aeaaf2SYazen Ghannam }
184775aeaaf2SYazen Ghannam
dct_determine_memory_type(struct amd64_pvt * pvt)184878ec161aSMuralidhara M K static void dct_determine_memory_type(struct amd64_pvt *pvt)
184994be4bffSDoug Thompson {
1850a597d2a5SAravind Gopalakrishnan u32 dram_ctrl, dcsm;
185194be4bffSDoug Thompson
1852a597d2a5SAravind Gopalakrishnan switch (pvt->fam) {
1853a597d2a5SAravind Gopalakrishnan case 0xf:
1854a597d2a5SAravind Gopalakrishnan if (pvt->ext_model >= K8_REV_F)
1855a597d2a5SAravind Gopalakrishnan goto ddr3;
1856a597d2a5SAravind Gopalakrishnan
1857a597d2a5SAravind Gopalakrishnan pvt->dram_type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
1858a597d2a5SAravind Gopalakrishnan return;
1859a597d2a5SAravind Gopalakrishnan
1860a597d2a5SAravind Gopalakrishnan case 0x10:
18616b4c0bdeSBorislav Petkov if (pvt->dchr0 & DDR3_MODE)
1862a597d2a5SAravind Gopalakrishnan goto ddr3;
1863a597d2a5SAravind Gopalakrishnan
1864a597d2a5SAravind Gopalakrishnan pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
1865a597d2a5SAravind Gopalakrishnan return;
1866a597d2a5SAravind Gopalakrishnan
1867a597d2a5SAravind Gopalakrishnan case 0x15:
1868a597d2a5SAravind Gopalakrishnan if (pvt->model < 0x60)
1869a597d2a5SAravind Gopalakrishnan goto ddr3;
1870a597d2a5SAravind Gopalakrishnan
1871a597d2a5SAravind Gopalakrishnan /*
1872a597d2a5SAravind Gopalakrishnan * Model 0x60h needs special handling:
1873a597d2a5SAravind Gopalakrishnan *
1874a597d2a5SAravind Gopalakrishnan * We use a Chip Select value of '0' to obtain dcsm.
1875a597d2a5SAravind Gopalakrishnan * Theoretically, it is possible to populate LRDIMMs of different
1876a597d2a5SAravind Gopalakrishnan * 'Rank' value on a DCT. But this is not the common case. So,
1877a597d2a5SAravind Gopalakrishnan * it's reasonable to assume all DIMMs are going to be of same
1878a597d2a5SAravind Gopalakrishnan * 'type' until proven otherwise.
1879a597d2a5SAravind Gopalakrishnan */
1880a597d2a5SAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 0, DRAM_CONTROL, &dram_ctrl);
1881a597d2a5SAravind Gopalakrishnan dcsm = pvt->csels[0].csmasks[0];
1882a597d2a5SAravind Gopalakrishnan
1883a597d2a5SAravind Gopalakrishnan if (((dram_ctrl >> 8) & 0x7) == 0x2)
1884a597d2a5SAravind Gopalakrishnan pvt->dram_type = MEM_DDR4;
1885a597d2a5SAravind Gopalakrishnan else if (pvt->dclr0 & BIT(16))
1886a597d2a5SAravind Gopalakrishnan pvt->dram_type = MEM_DDR3;
1887a597d2a5SAravind Gopalakrishnan else if (dcsm & 0x3)
1888a597d2a5SAravind Gopalakrishnan pvt->dram_type = MEM_LRDDR3;
18896b4c0bdeSBorislav Petkov else
1890a597d2a5SAravind Gopalakrishnan pvt->dram_type = MEM_RDDR3;
1891a597d2a5SAravind Gopalakrishnan
1892a597d2a5SAravind Gopalakrishnan return;
1893a597d2a5SAravind Gopalakrishnan
1894a597d2a5SAravind Gopalakrishnan case 0x16:
1895a597d2a5SAravind Gopalakrishnan goto ddr3;
1896a597d2a5SAravind Gopalakrishnan
1897a597d2a5SAravind Gopalakrishnan default:
1898a597d2a5SAravind Gopalakrishnan WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam);
1899a597d2a5SAravind Gopalakrishnan pvt->dram_type = MEM_EMPTY;
190094be4bffSDoug Thompson }
190178ec161aSMuralidhara M K
190278ec161aSMuralidhara M K edac_dbg(1, " DIMM type: %s\n", edac_mem_types[pvt->dram_type]);
1903a597d2a5SAravind Gopalakrishnan return;
190494be4bffSDoug Thompson
1905a597d2a5SAravind Gopalakrishnan ddr3:
1906a597d2a5SAravind Gopalakrishnan pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
190794be4bffSDoug Thompson }
190894be4bffSDoug Thompson
190970046624SBorislav Petkov /* On F10h and later ErrAddr is MC4_ADDR[47:1] */
get_error_address(struct amd64_pvt * pvt,struct mce * m)1910a4b4bedcSBorislav Petkov static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m)
1911ddff876dSDoug Thompson {
1912db970bd2SYazen Ghannam u16 mce_nid = topology_die_id(m->extcpu);
19132ec591acSBorislav Petkov struct mem_ctl_info *mci;
191470046624SBorislav Petkov u8 start_bit = 1;
191570046624SBorislav Petkov u8 end_bit = 47;
19162ec591acSBorislav Petkov u64 addr;
19172ec591acSBorislav Petkov
19182ec591acSBorislav Petkov mci = edac_mc_find(mce_nid);
19192ec591acSBorislav Petkov if (!mci)
19202ec591acSBorislav Petkov return 0;
19212ec591acSBorislav Petkov
19222ec591acSBorislav Petkov pvt = mci->pvt_info;
192370046624SBorislav Petkov
1924a4b4bedcSBorislav Petkov if (pvt->fam == 0xf) {
192570046624SBorislav Petkov start_bit = 3;
192670046624SBorislav Petkov end_bit = 39;
192770046624SBorislav Petkov }
192870046624SBorislav Petkov
192910ef6b0dSChen, Gong addr = m->addr & GENMASK_ULL(end_bit, start_bit);
1930c1ae6830SBorislav Petkov
1931c1ae6830SBorislav Petkov /*
1932c1ae6830SBorislav Petkov * Erratum 637 workaround
1933c1ae6830SBorislav Petkov */
1934a4b4bedcSBorislav Petkov if (pvt->fam == 0x15) {
1935c1ae6830SBorislav Petkov u64 cc6_base, tmp_addr;
1936c1ae6830SBorislav Petkov u32 tmp;
19378b84c8dfSDaniel J Blueman u8 intlv_en;
1938c1ae6830SBorislav Petkov
193910ef6b0dSChen, Gong if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7)
1940c1ae6830SBorislav Petkov return addr;
1941c1ae6830SBorislav Petkov
1942c1ae6830SBorislav Petkov
1943c1ae6830SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
1944c1ae6830SBorislav Petkov intlv_en = tmp >> 21 & 0x7;
1945c1ae6830SBorislav Petkov
1946c1ae6830SBorislav Petkov /* add [47:27] + 3 trailing bits */
194710ef6b0dSChen, Gong cc6_base = (tmp & GENMASK_ULL(20, 0)) << 3;
1948c1ae6830SBorislav Petkov
1949c1ae6830SBorislav Petkov /* reverse and add DramIntlvEn */
1950c1ae6830SBorislav Petkov cc6_base |= intlv_en ^ 0x7;
1951c1ae6830SBorislav Petkov
1952c1ae6830SBorislav Petkov /* pin at [47:24] */
1953c1ae6830SBorislav Petkov cc6_base <<= 24;
1954c1ae6830SBorislav Petkov
1955c1ae6830SBorislav Petkov if (!intlv_en)
195610ef6b0dSChen, Gong return cc6_base | (addr & GENMASK_ULL(23, 0));
1957c1ae6830SBorislav Petkov
1958c1ae6830SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
1959c1ae6830SBorislav Petkov
1960c1ae6830SBorislav Petkov /* faster log2 */
196110ef6b0dSChen, Gong tmp_addr = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1);
1962c1ae6830SBorislav Petkov
1963c1ae6830SBorislav Petkov /* OR DramIntlvSel into bits [14:12] */
196410ef6b0dSChen, Gong tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9;
1965c1ae6830SBorislav Petkov
1966c1ae6830SBorislav Petkov /* add remaining [11:0] bits from original MC4_ADDR */
196710ef6b0dSChen, Gong tmp_addr |= addr & GENMASK_ULL(11, 0);
1968c1ae6830SBorislav Petkov
1969c1ae6830SBorislav Petkov return cc6_base | tmp_addr;
1970c1ae6830SBorislav Petkov }
1971c1ae6830SBorislav Petkov
1972c1ae6830SBorislav Petkov return addr;
1973ddff876dSDoug Thompson }
1974ddff876dSDoug Thompson
pci_get_related_function(unsigned int vendor,unsigned int device,struct pci_dev * related)1975e2c0bffeSDaniel J Blueman static struct pci_dev *pci_get_related_function(unsigned int vendor,
1976e2c0bffeSDaniel J Blueman unsigned int device,
1977e2c0bffeSDaniel J Blueman struct pci_dev *related)
1978e2c0bffeSDaniel J Blueman {
1979e2c0bffeSDaniel J Blueman struct pci_dev *dev = NULL;
1980e2c0bffeSDaniel J Blueman
1981e2c0bffeSDaniel J Blueman while ((dev = pci_get_device(vendor, device, dev))) {
1982e2c0bffeSDaniel J Blueman if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) &&
1983e2c0bffeSDaniel J Blueman (dev->bus->number == related->bus->number) &&
1984e2c0bffeSDaniel J Blueman (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1985e2c0bffeSDaniel J Blueman break;
1986e2c0bffeSDaniel J Blueman }
1987e2c0bffeSDaniel J Blueman
1988e2c0bffeSDaniel J Blueman return dev;
1989e2c0bffeSDaniel J Blueman }
1990e2c0bffeSDaniel J Blueman
read_dram_base_limit_regs(struct amd64_pvt * pvt,unsigned range)19917f19bf75SBorislav Petkov static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
1992ddff876dSDoug Thompson {
1993e2c0bffeSDaniel J Blueman struct amd_northbridge *nb;
199418b94f66SAravind Gopalakrishnan struct pci_dev *f1 = NULL;
199518b94f66SAravind Gopalakrishnan unsigned int pci_func;
199671d2a32eSBorislav Petkov int off = range << 3;
1997e2c0bffeSDaniel J Blueman u32 llim;
1998ddff876dSDoug Thompson
19997f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
20007f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
2001ddff876dSDoug Thompson
200218b94f66SAravind Gopalakrishnan if (pvt->fam == 0xf)
20037f19bf75SBorislav Petkov return;
2004ddff876dSDoug Thompson
20057f19bf75SBorislav Petkov if (!dram_rw(pvt, range))
20067f19bf75SBorislav Petkov return;
2007ddff876dSDoug Thompson
20087f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
20097f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
2010f08e457cSBorislav Petkov
2011e2c0bffeSDaniel J Blueman /* F15h: factor in CC6 save area by reading dst node's limit reg */
201218b94f66SAravind Gopalakrishnan if (pvt->fam != 0x15)
2013e2c0bffeSDaniel J Blueman return;
2014f08e457cSBorislav Petkov
2015e2c0bffeSDaniel J Blueman nb = node_to_amd_nb(dram_dst_node(pvt, range));
2016e2c0bffeSDaniel J Blueman if (WARN_ON(!nb))
2017e2c0bffeSDaniel J Blueman return;
2018e2c0bffeSDaniel J Blueman
2019a597d2a5SAravind Gopalakrishnan if (pvt->model == 0x60)
2020a597d2a5SAravind Gopalakrishnan pci_func = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1;
2021a597d2a5SAravind Gopalakrishnan else if (pvt->model == 0x30)
2022a597d2a5SAravind Gopalakrishnan pci_func = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1;
2023a597d2a5SAravind Gopalakrishnan else
2024a597d2a5SAravind Gopalakrishnan pci_func = PCI_DEVICE_ID_AMD_15H_NB_F1;
202518b94f66SAravind Gopalakrishnan
202618b94f66SAravind Gopalakrishnan f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc);
2027f08e457cSBorislav Petkov if (WARN_ON(!f1))
2028f08e457cSBorislav Petkov return;
2029f08e457cSBorislav Petkov
2030f08e457cSBorislav Petkov amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
2031f08e457cSBorislav Petkov
203210ef6b0dSChen, Gong pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0);
2033f08e457cSBorislav Petkov
2034f08e457cSBorislav Petkov /* {[39:27],111b} */
2035f08e457cSBorislav Petkov pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
2036f08e457cSBorislav Petkov
203710ef6b0dSChen, Gong pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0);
2038f08e457cSBorislav Petkov
2039f08e457cSBorislav Petkov /* [47:40] */
2040f08e457cSBorislav Petkov pvt->ranges[range].lim.hi |= llim >> 13;
2041f08e457cSBorislav Petkov
2042f08e457cSBorislav Petkov pci_dev_put(f1);
2043f08e457cSBorislav Petkov }
2044ddff876dSDoug Thompson
k8_map_sysaddr_to_csrow(struct mem_ctl_info * mci,u64 sys_addr,struct err_info * err)2045f192c7b1SBorislav Petkov static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
204633ca0643SBorislav Petkov struct err_info *err)
2047ddff876dSDoug Thompson {
2048f192c7b1SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info;
2049ddff876dSDoug Thompson
205033ca0643SBorislav Petkov error_address_to_page_and_offset(sys_addr, err);
2051ab5a503cSMauro Carvalho Chehab
2052ab5a503cSMauro Carvalho Chehab /*
2053ab5a503cSMauro Carvalho Chehab * Find out which node the error address belongs to. This may be
2054ab5a503cSMauro Carvalho Chehab * different from the node that detected the error.
2055ab5a503cSMauro Carvalho Chehab */
205633ca0643SBorislav Petkov err->src_mci = find_mc_by_sys_addr(mci, sys_addr);
205733ca0643SBorislav Petkov if (!err->src_mci) {
2058ab5a503cSMauro Carvalho Chehab amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
2059ab5a503cSMauro Carvalho Chehab (unsigned long)sys_addr);
206033ca0643SBorislav Petkov err->err_code = ERR_NODE;
2061ab5a503cSMauro Carvalho Chehab return;
2062ab5a503cSMauro Carvalho Chehab }
2063ab5a503cSMauro Carvalho Chehab
2064ab5a503cSMauro Carvalho Chehab /* Now map the sys_addr to a CSROW */
206533ca0643SBorislav Petkov err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr);
206633ca0643SBorislav Petkov if (err->csrow < 0) {
206733ca0643SBorislav Petkov err->err_code = ERR_CSROW;
2068ab5a503cSMauro Carvalho Chehab return;
2069ab5a503cSMauro Carvalho Chehab }
2070ab5a503cSMauro Carvalho Chehab
2071ddff876dSDoug Thompson /* CHIPKILL enabled */
2072f192c7b1SBorislav Petkov if (pvt->nbcfg & NBCFG_CHIPKILL) {
207333ca0643SBorislav Petkov err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
207433ca0643SBorislav Petkov if (err->channel < 0) {
2075ddff876dSDoug Thompson /*
2076ddff876dSDoug Thompson * Syndrome didn't map, so we don't know which of the
2077ddff876dSDoug Thompson * 2 DIMMs is in error. So we need to ID 'both' of them
2078ddff876dSDoug Thompson * as suspect.
2079ddff876dSDoug Thompson */
208033ca0643SBorislav Petkov amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - "
2081ab5a503cSMauro Carvalho Chehab "possible error reporting race\n",
208233ca0643SBorislav Petkov err->syndrome);
208333ca0643SBorislav Petkov err->err_code = ERR_CHANNEL;
2084ddff876dSDoug Thompson return;
2085ddff876dSDoug Thompson }
2086ddff876dSDoug Thompson } else {
2087ddff876dSDoug Thompson /*
2088ddff876dSDoug Thompson * non-chipkill ecc mode
2089ddff876dSDoug Thompson *
2090ddff876dSDoug Thompson * The k8 documentation is unclear about how to determine the
2091ddff876dSDoug Thompson * channel number when using non-chipkill memory. This method
2092ddff876dSDoug Thompson * was obtained from email communication with someone at AMD.
2093ddff876dSDoug Thompson * (Wish the email was placed in this comment - norsk)
2094ddff876dSDoug Thompson */
209533ca0643SBorislav Petkov err->channel = ((sys_addr & BIT(3)) != 0);
2096ddff876dSDoug Thompson }
2097ddff876dSDoug Thompson }
2098ddff876dSDoug Thompson
ddr2_cs_size(unsigned i,bool dct_width)209941d8bfabSBorislav Petkov static int ddr2_cs_size(unsigned i, bool dct_width)
2100ddff876dSDoug Thompson {
210141d8bfabSBorislav Petkov unsigned shift = 0;
2102ddff876dSDoug Thompson
210341d8bfabSBorislav Petkov if (i <= 2)
210441d8bfabSBorislav Petkov shift = i;
210541d8bfabSBorislav Petkov else if (!(i & 0x1))
210641d8bfabSBorislav Petkov shift = i >> 1;
21071433eb99SBorislav Petkov else
210841d8bfabSBorislav Petkov shift = (i + 1) >> 1;
2109ddff876dSDoug Thompson
211041d8bfabSBorislav Petkov return 128 << (shift + !!dct_width);
211141d8bfabSBorislav Petkov }
211241d8bfabSBorislav Petkov
k8_dbam_to_chip_select(struct amd64_pvt * pvt,u8 dct,unsigned cs_mode,int cs_mask_nr)211341d8bfabSBorislav Petkov static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
2114a597d2a5SAravind Gopalakrishnan unsigned cs_mode, int cs_mask_nr)
211541d8bfabSBorislav Petkov {
211641d8bfabSBorislav Petkov u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
211741d8bfabSBorislav Petkov
211841d8bfabSBorislav Petkov if (pvt->ext_model >= K8_REV_F) {
211941d8bfabSBorislav Petkov WARN_ON(cs_mode > 11);
212041d8bfabSBorislav Petkov return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
212141d8bfabSBorislav Petkov }
212241d8bfabSBorislav Petkov else if (pvt->ext_model >= K8_REV_D) {
212311b0a314SBorislav Petkov unsigned diff;
212441d8bfabSBorislav Petkov WARN_ON(cs_mode > 10);
212541d8bfabSBorislav Petkov
212611b0a314SBorislav Petkov /*
212711b0a314SBorislav Petkov * the below calculation, besides trying to win an obfuscated C
212811b0a314SBorislav Petkov * contest, maps cs_mode values to DIMM chip select sizes. The
212911b0a314SBorislav Petkov * mappings are:
213011b0a314SBorislav Petkov *
213111b0a314SBorislav Petkov * cs_mode CS size (mb)
213211b0a314SBorislav Petkov * ======= ============
213311b0a314SBorislav Petkov * 0 32
213411b0a314SBorislav Petkov * 1 64
213511b0a314SBorislav Petkov * 2 128
213611b0a314SBorislav Petkov * 3 128
213711b0a314SBorislav Petkov * 4 256
213811b0a314SBorislav Petkov * 5 512
213911b0a314SBorislav Petkov * 6 256
214011b0a314SBorislav Petkov * 7 512
214111b0a314SBorislav Petkov * 8 1024
214211b0a314SBorislav Petkov * 9 1024
214311b0a314SBorislav Petkov * 10 2048
214411b0a314SBorislav Petkov *
214511b0a314SBorislav Petkov * Basically, it calculates a value with which to shift the
214611b0a314SBorislav Petkov * smallest CS size of 32MB.
214711b0a314SBorislav Petkov *
214811b0a314SBorislav Petkov * ddr[23]_cs_size have a similar purpose.
214911b0a314SBorislav Petkov */
215011b0a314SBorislav Petkov diff = cs_mode/3 + (unsigned)(cs_mode > 5);
215111b0a314SBorislav Petkov
215211b0a314SBorislav Petkov return 32 << (cs_mode - diff);
215341d8bfabSBorislav Petkov }
215441d8bfabSBorislav Petkov else {
215541d8bfabSBorislav Petkov WARN_ON(cs_mode > 6);
215641d8bfabSBorislav Petkov return 32 << cs_mode;
215741d8bfabSBorislav Petkov }
2158ddff876dSDoug Thompson }
2159ddff876dSDoug Thompson
ddr3_cs_size(unsigned i,bool dct_width)216041d8bfabSBorislav Petkov static int ddr3_cs_size(unsigned i, bool dct_width)
21611afd3c98SDoug Thompson {
216241d8bfabSBorislav Petkov unsigned shift = 0;
216341d8bfabSBorislav Petkov int cs_size = 0;
216441d8bfabSBorislav Petkov
216541d8bfabSBorislav Petkov if (i == 0 || i == 3 || i == 4)
216641d8bfabSBorislav Petkov cs_size = -1;
216741d8bfabSBorislav Petkov else if (i <= 2)
216841d8bfabSBorislav Petkov shift = i;
216941d8bfabSBorislav Petkov else if (i == 12)
217041d8bfabSBorislav Petkov shift = 7;
217141d8bfabSBorislav Petkov else if (!(i & 0x1))
217241d8bfabSBorislav Petkov shift = i >> 1;
217341d8bfabSBorislav Petkov else
217441d8bfabSBorislav Petkov shift = (i + 1) >> 1;
217541d8bfabSBorislav Petkov
217641d8bfabSBorislav Petkov if (cs_size != -1)
217741d8bfabSBorislav Petkov cs_size = (128 * (1 << !!dct_width)) << shift;
217841d8bfabSBorislav Petkov
217941d8bfabSBorislav Petkov return cs_size;
218041d8bfabSBorislav Petkov }
218141d8bfabSBorislav Petkov
ddr3_lrdimm_cs_size(unsigned i,unsigned rank_multiply)2182a597d2a5SAravind Gopalakrishnan static int ddr3_lrdimm_cs_size(unsigned i, unsigned rank_multiply)
2183a597d2a5SAravind Gopalakrishnan {
2184a597d2a5SAravind Gopalakrishnan unsigned shift = 0;
2185a597d2a5SAravind Gopalakrishnan int cs_size = 0;
2186a597d2a5SAravind Gopalakrishnan
2187a597d2a5SAravind Gopalakrishnan if (i < 4 || i == 6)
2188a597d2a5SAravind Gopalakrishnan cs_size = -1;
2189a597d2a5SAravind Gopalakrishnan else if (i == 12)
2190a597d2a5SAravind Gopalakrishnan shift = 7;
2191a597d2a5SAravind Gopalakrishnan else if (!(i & 0x1))
2192a597d2a5SAravind Gopalakrishnan shift = i >> 1;
2193a597d2a5SAravind Gopalakrishnan else
2194a597d2a5SAravind Gopalakrishnan shift = (i + 1) >> 1;
2195a597d2a5SAravind Gopalakrishnan
2196a597d2a5SAravind Gopalakrishnan if (cs_size != -1)
2197a597d2a5SAravind Gopalakrishnan cs_size = rank_multiply * (128 << shift);
2198a597d2a5SAravind Gopalakrishnan
2199a597d2a5SAravind Gopalakrishnan return cs_size;
2200a597d2a5SAravind Gopalakrishnan }
2201a597d2a5SAravind Gopalakrishnan
ddr4_cs_size(unsigned i)2202a597d2a5SAravind Gopalakrishnan static int ddr4_cs_size(unsigned i)
2203a597d2a5SAravind Gopalakrishnan {
2204a597d2a5SAravind Gopalakrishnan int cs_size = 0;
2205a597d2a5SAravind Gopalakrishnan
2206a597d2a5SAravind Gopalakrishnan if (i == 0)
2207a597d2a5SAravind Gopalakrishnan cs_size = -1;
2208a597d2a5SAravind Gopalakrishnan else if (i == 1)
2209a597d2a5SAravind Gopalakrishnan cs_size = 1024;
2210a597d2a5SAravind Gopalakrishnan else
2211a597d2a5SAravind Gopalakrishnan /* Min cs_size = 1G */
2212a597d2a5SAravind Gopalakrishnan cs_size = 1024 * (1 << (i >> 1));
2213a597d2a5SAravind Gopalakrishnan
2214a597d2a5SAravind Gopalakrishnan return cs_size;
2215a597d2a5SAravind Gopalakrishnan }
2216a597d2a5SAravind Gopalakrishnan
f10_dbam_to_chip_select(struct amd64_pvt * pvt,u8 dct,unsigned cs_mode,int cs_mask_nr)221741d8bfabSBorislav Petkov static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
2218a597d2a5SAravind Gopalakrishnan unsigned cs_mode, int cs_mask_nr)
221941d8bfabSBorislav Petkov {
222041d8bfabSBorislav Petkov u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
222141d8bfabSBorislav Petkov
222241d8bfabSBorislav Petkov WARN_ON(cs_mode > 11);
22231433eb99SBorislav Petkov
22241433eb99SBorislav Petkov if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
222541d8bfabSBorislav Petkov return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
22261433eb99SBorislav Petkov else
222741d8bfabSBorislav Petkov return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
222841d8bfabSBorislav Petkov }
22291433eb99SBorislav Petkov
223041d8bfabSBorislav Petkov /*
223141d8bfabSBorislav Petkov * F15h supports only 64bit DCT interfaces
223241d8bfabSBorislav Petkov */
f15_dbam_to_chip_select(struct amd64_pvt * pvt,u8 dct,unsigned cs_mode,int cs_mask_nr)223341d8bfabSBorislav Petkov static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
2234a597d2a5SAravind Gopalakrishnan unsigned cs_mode, int cs_mask_nr)
223541d8bfabSBorislav Petkov {
223641d8bfabSBorislav Petkov WARN_ON(cs_mode > 12);
223741d8bfabSBorislav Petkov
223841d8bfabSBorislav Petkov return ddr3_cs_size(cs_mode, false);
22391afd3c98SDoug Thompson }
22401afd3c98SDoug Thompson
2241a597d2a5SAravind Gopalakrishnan /* F15h M60h supports DDR4 mapping as well.. */
f15_m60h_dbam_to_chip_select(struct amd64_pvt * pvt,u8 dct,unsigned cs_mode,int cs_mask_nr)2242a597d2a5SAravind Gopalakrishnan static int f15_m60h_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
2243a597d2a5SAravind Gopalakrishnan unsigned cs_mode, int cs_mask_nr)
2244a597d2a5SAravind Gopalakrishnan {
2245a597d2a5SAravind Gopalakrishnan int cs_size;
2246a597d2a5SAravind Gopalakrishnan u32 dcsm = pvt->csels[dct].csmasks[cs_mask_nr];
2247a597d2a5SAravind Gopalakrishnan
2248a597d2a5SAravind Gopalakrishnan WARN_ON(cs_mode > 12);
2249a597d2a5SAravind Gopalakrishnan
2250a597d2a5SAravind Gopalakrishnan if (pvt->dram_type == MEM_DDR4) {
2251a597d2a5SAravind Gopalakrishnan if (cs_mode > 9)
2252a597d2a5SAravind Gopalakrishnan return -1;
2253a597d2a5SAravind Gopalakrishnan
2254a597d2a5SAravind Gopalakrishnan cs_size = ddr4_cs_size(cs_mode);
2255a597d2a5SAravind Gopalakrishnan } else if (pvt->dram_type == MEM_LRDDR3) {
2256a597d2a5SAravind Gopalakrishnan unsigned rank_multiply = dcsm & 0xf;
2257a597d2a5SAravind Gopalakrishnan
2258a597d2a5SAravind Gopalakrishnan if (rank_multiply == 3)
2259a597d2a5SAravind Gopalakrishnan rank_multiply = 4;
2260a597d2a5SAravind Gopalakrishnan cs_size = ddr3_lrdimm_cs_size(cs_mode, rank_multiply);
2261a597d2a5SAravind Gopalakrishnan } else {
2262a597d2a5SAravind Gopalakrishnan /* Minimum cs size is 512mb for F15hM60h*/
2263a597d2a5SAravind Gopalakrishnan if (cs_mode == 0x1)
2264a597d2a5SAravind Gopalakrishnan return -1;
2265a597d2a5SAravind Gopalakrishnan
2266a597d2a5SAravind Gopalakrishnan cs_size = ddr3_cs_size(cs_mode, false);
2267a597d2a5SAravind Gopalakrishnan }
2268a597d2a5SAravind Gopalakrishnan
2269a597d2a5SAravind Gopalakrishnan return cs_size;
2270a597d2a5SAravind Gopalakrishnan }
2271a597d2a5SAravind Gopalakrishnan
227294c1acf2SAravind Gopalakrishnan /*
227318b94f66SAravind Gopalakrishnan * F16h and F15h model 30h have only limited cs_modes.
227494c1acf2SAravind Gopalakrishnan */
f16_dbam_to_chip_select(struct amd64_pvt * pvt,u8 dct,unsigned cs_mode,int cs_mask_nr)227594c1acf2SAravind Gopalakrishnan static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
2276a597d2a5SAravind Gopalakrishnan unsigned cs_mode, int cs_mask_nr)
227794c1acf2SAravind Gopalakrishnan {
227894c1acf2SAravind Gopalakrishnan WARN_ON(cs_mode > 12);
227994c1acf2SAravind Gopalakrishnan
228094c1acf2SAravind Gopalakrishnan if (cs_mode == 6 || cs_mode == 8 ||
228194c1acf2SAravind Gopalakrishnan cs_mode == 9 || cs_mode == 12)
228294c1acf2SAravind Gopalakrishnan return -1;
228394c1acf2SAravind Gopalakrishnan else
228494c1acf2SAravind Gopalakrishnan return ddr3_cs_size(cs_mode, false);
228594c1acf2SAravind Gopalakrishnan }
228694c1acf2SAravind Gopalakrishnan
read_dram_ctl_register(struct amd64_pvt * pvt)22875a5d2371SBorislav Petkov static void read_dram_ctl_register(struct amd64_pvt *pvt)
22886163b5d4SDoug Thompson {
22896163b5d4SDoug Thompson
2290a4b4bedcSBorislav Petkov if (pvt->fam == 0xf)
22915a5d2371SBorislav Petkov return;
22925a5d2371SBorislav Petkov
22937981a28fSAravind Gopalakrishnan if (!amd64_read_pci_cfg(pvt->F2, DCT_SEL_LO, &pvt->dct_sel_lo)) {
2294956b9ba1SJoe Perches edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
229578da121eSBorislav Petkov pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
22966163b5d4SDoug Thompson
2297956b9ba1SJoe Perches edac_dbg(0, " DCTs operate in %s mode\n",
22985a5d2371SBorislav Petkov (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
22996163b5d4SDoug Thompson
230072381bd5SBorislav Petkov if (!dct_ganging_enabled(pvt))
2301956b9ba1SJoe Perches edac_dbg(0, " Address range split per DCT: %s\n",
230272381bd5SBorislav Petkov (dct_high_range_enabled(pvt) ? "yes" : "no"));
230372381bd5SBorislav Petkov
2304956b9ba1SJoe Perches edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n",
230572381bd5SBorislav Petkov (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
230672381bd5SBorislav Petkov (dct_memory_cleared(pvt) ? "yes" : "no"));
230772381bd5SBorislav Petkov
2308956b9ba1SJoe Perches edac_dbg(0, " channel interleave: %s, "
230978da121eSBorislav Petkov "interleave bits selector: 0x%x\n",
231072381bd5SBorislav Petkov (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
23116163b5d4SDoug Thompson dct_sel_interleave_addr(pvt));
23126163b5d4SDoug Thompson }
23136163b5d4SDoug Thompson
23147981a28fSAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F2, DCT_SEL_HI, &pvt->dct_sel_hi);
23156163b5d4SDoug Thompson }
23166163b5d4SDoug Thompson
2317f71d0a05SDoug Thompson /*
231818b94f66SAravind Gopalakrishnan * Determine channel (DCT) based on the interleaving mode (see F15h M30h BKDG,
231918b94f66SAravind Gopalakrishnan * 2.10.12 Memory Interleaving Modes).
232018b94f66SAravind Gopalakrishnan */
f15_m30h_determine_channel(struct amd64_pvt * pvt,u64 sys_addr,u8 intlv_en,int num_dcts_intlv,u32 dct_sel)232118b94f66SAravind Gopalakrishnan static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
232218b94f66SAravind Gopalakrishnan u8 intlv_en, int num_dcts_intlv,
232318b94f66SAravind Gopalakrishnan u32 dct_sel)
232418b94f66SAravind Gopalakrishnan {
232518b94f66SAravind Gopalakrishnan u8 channel = 0;
232618b94f66SAravind Gopalakrishnan u8 select;
232718b94f66SAravind Gopalakrishnan
232818b94f66SAravind Gopalakrishnan if (!(intlv_en))
232918b94f66SAravind Gopalakrishnan return (u8)(dct_sel);
233018b94f66SAravind Gopalakrishnan
233118b94f66SAravind Gopalakrishnan if (num_dcts_intlv == 2) {
233218b94f66SAravind Gopalakrishnan select = (sys_addr >> 8) & 0x3;
233318b94f66SAravind Gopalakrishnan channel = select ? 0x3 : 0;
23349d0e8d83SAravind Gopalakrishnan } else if (num_dcts_intlv == 4) {
23359d0e8d83SAravind Gopalakrishnan u8 intlv_addr = dct_sel_interleave_addr(pvt);
23369d0e8d83SAravind Gopalakrishnan switch (intlv_addr) {
23379d0e8d83SAravind Gopalakrishnan case 0x4:
23389d0e8d83SAravind Gopalakrishnan channel = (sys_addr >> 8) & 0x3;
23399d0e8d83SAravind Gopalakrishnan break;
23409d0e8d83SAravind Gopalakrishnan case 0x5:
23419d0e8d83SAravind Gopalakrishnan channel = (sys_addr >> 9) & 0x3;
23429d0e8d83SAravind Gopalakrishnan break;
23439d0e8d83SAravind Gopalakrishnan }
23449d0e8d83SAravind Gopalakrishnan }
234518b94f66SAravind Gopalakrishnan return channel;
234618b94f66SAravind Gopalakrishnan }
234718b94f66SAravind Gopalakrishnan
234818b94f66SAravind Gopalakrishnan /*
2349229a7a11SBorislav Petkov * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
2350f71d0a05SDoug Thompson * Interleaving Modes.
2351f71d0a05SDoug Thompson */
f1x_determine_channel(struct amd64_pvt * pvt,u64 sys_addr,bool hi_range_sel,u8 intlv_en)2352b15f0fcaSBorislav Petkov static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
2353229a7a11SBorislav Petkov bool hi_range_sel, u8 intlv_en)
23546163b5d4SDoug Thompson {
2355151fa71cSBorislav Petkov u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
23566163b5d4SDoug Thompson
23576163b5d4SDoug Thompson if (dct_ganging_enabled(pvt))
2358229a7a11SBorislav Petkov return 0;
2359229a7a11SBorislav Petkov
2360229a7a11SBorislav Petkov if (hi_range_sel)
2361229a7a11SBorislav Petkov return dct_sel_high;
2362229a7a11SBorislav Petkov
2363f71d0a05SDoug Thompson /*
2364f71d0a05SDoug Thompson * see F2x110[DctSelIntLvAddr] - channel interleave mode
2365f71d0a05SDoug Thompson */
2366229a7a11SBorislav Petkov if (dct_interleave_enabled(pvt)) {
2367229a7a11SBorislav Petkov u8 intlv_addr = dct_sel_interleave_addr(pvt);
23686163b5d4SDoug Thompson
2369229a7a11SBorislav Petkov /* return DCT select function: 0=DCT0, 1=DCT1 */
2370229a7a11SBorislav Petkov if (!intlv_addr)
2371229a7a11SBorislav Petkov return sys_addr >> 6 & 1;
23726163b5d4SDoug Thompson
2373229a7a11SBorislav Petkov if (intlv_addr & 0x2) {
2374229a7a11SBorislav Petkov u8 shift = intlv_addr & 0x1 ? 9 : 6;
2375dc0a50a8SYazen Ghannam u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) & 1;
2376229a7a11SBorislav Petkov
2377229a7a11SBorislav Petkov return ((sys_addr >> shift) & 1) ^ temp;
23786163b5d4SDoug Thompson }
23796163b5d4SDoug Thompson
2380dc0a50a8SYazen Ghannam if (intlv_addr & 0x4) {
2381dc0a50a8SYazen Ghannam u8 shift = intlv_addr & 0x1 ? 9 : 8;
2382dc0a50a8SYazen Ghannam
2383dc0a50a8SYazen Ghannam return (sys_addr >> shift) & 1;
2384dc0a50a8SYazen Ghannam }
2385dc0a50a8SYazen Ghannam
2386229a7a11SBorislav Petkov return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
2387229a7a11SBorislav Petkov }
2388229a7a11SBorislav Petkov
2389229a7a11SBorislav Petkov if (dct_high_range_enabled(pvt))
2390229a7a11SBorislav Petkov return ~dct_sel_high & 1;
23916163b5d4SDoug Thompson
23926163b5d4SDoug Thompson return 0;
23936163b5d4SDoug Thompson }
23946163b5d4SDoug Thompson
2395c8e518d5SBorislav Petkov /* Convert the sys_addr to the normalized DCT address */
f1x_get_norm_dct_addr(struct amd64_pvt * pvt,u8 range,u64 sys_addr,bool hi_rng,u32 dct_sel_base_addr)2396c7e5301aSDaniel J Blueman static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range,
2397c8e518d5SBorislav Petkov u64 sys_addr, bool hi_rng,
2398c8e518d5SBorislav Petkov u32 dct_sel_base_addr)
23996163b5d4SDoug Thompson {
24006163b5d4SDoug Thompson u64 chan_off;
2401c8e518d5SBorislav Petkov u64 dram_base = get_dram_base(pvt, range);
2402c8e518d5SBorislav Petkov u64 hole_off = f10_dhar_offset(pvt);
24036f3508f6SDan Carpenter u64 dct_sel_base_off = (u64)(pvt->dct_sel_hi & 0xFFFFFC00) << 16;
24046163b5d4SDoug Thompson
2405c8e518d5SBorislav Petkov if (hi_rng) {
2406c8e518d5SBorislav Petkov /*
2407c8e518d5SBorislav Petkov * if
2408c8e518d5SBorislav Petkov * base address of high range is below 4Gb
2409c8e518d5SBorislav Petkov * (bits [47:27] at [31:11])
2410c8e518d5SBorislav Petkov * DRAM address space on this DCT is hoisted above 4Gb &&
2411c8e518d5SBorislav Petkov * sys_addr > 4Gb
2412c8e518d5SBorislav Petkov *
2413c8e518d5SBorislav Petkov * remove hole offset from sys_addr
2414c8e518d5SBorislav Petkov * else
2415c8e518d5SBorislav Petkov * remove high range offset from sys_addr
2416c8e518d5SBorislav Petkov */
2417c8e518d5SBorislav Petkov if ((!(dct_sel_base_addr >> 16) ||
2418c8e518d5SBorislav Petkov dct_sel_base_addr < dhar_base(pvt)) &&
2419972ea17aSBorislav Petkov dhar_valid(pvt) &&
2420c8e518d5SBorislav Petkov (sys_addr >= BIT_64(32)))
2421bc21fa57SBorislav Petkov chan_off = hole_off;
24226163b5d4SDoug Thompson else
24236163b5d4SDoug Thompson chan_off = dct_sel_base_off;
24246163b5d4SDoug Thompson } else {
2425c8e518d5SBorislav Petkov /*
2426c8e518d5SBorislav Petkov * if
2427c8e518d5SBorislav Petkov * we have a valid hole &&
2428c8e518d5SBorislav Petkov * sys_addr > 4Gb
2429c8e518d5SBorislav Petkov *
2430c8e518d5SBorislav Petkov * remove hole
2431c8e518d5SBorislav Petkov * else
2432c8e518d5SBorislav Petkov * remove dram base to normalize to DCT address
2433c8e518d5SBorislav Petkov */
2434972ea17aSBorislav Petkov if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
2435bc21fa57SBorislav Petkov chan_off = hole_off;
24366163b5d4SDoug Thompson else
2437c8e518d5SBorislav Petkov chan_off = dram_base;
24386163b5d4SDoug Thompson }
24396163b5d4SDoug Thompson
244010ef6b0dSChen, Gong return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23));
24416163b5d4SDoug Thompson }
24426163b5d4SDoug Thompson
24436163b5d4SDoug Thompson /*
24446163b5d4SDoug Thompson * checks if the csrow passed in is marked as SPARED, if so returns the new
24456163b5d4SDoug Thompson * spare row
24466163b5d4SDoug Thompson */
f10_process_possible_spare(struct amd64_pvt * pvt,u8 dct,int csrow)244711c75eadSBorislav Petkov static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
24486163b5d4SDoug Thompson {
2449614ec9d8SBorislav Petkov int tmp_cs;
24506163b5d4SDoug Thompson
2451614ec9d8SBorislav Petkov if (online_spare_swap_done(pvt, dct) &&
2452614ec9d8SBorislav Petkov csrow == online_spare_bad_dramcs(pvt, dct)) {
2453614ec9d8SBorislav Petkov
2454614ec9d8SBorislav Petkov for_each_chip_select(tmp_cs, dct, pvt) {
2455614ec9d8SBorislav Petkov if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
2456614ec9d8SBorislav Petkov csrow = tmp_cs;
2457614ec9d8SBorislav Petkov break;
2458614ec9d8SBorislav Petkov }
2459614ec9d8SBorislav Petkov }
24606163b5d4SDoug Thompson }
24616163b5d4SDoug Thompson return csrow;
24626163b5d4SDoug Thompson }
24636163b5d4SDoug Thompson
24646163b5d4SDoug Thompson /*
24656163b5d4SDoug Thompson * Iterate over the DRAM DCT "base" and "mask" registers looking for a
24666163b5d4SDoug Thompson * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
24676163b5d4SDoug Thompson *
24686163b5d4SDoug Thompson * Return:
24696163b5d4SDoug Thompson * -EINVAL: NOT FOUND
24706163b5d4SDoug Thompson * 0..csrow = Chip-Select Row
24716163b5d4SDoug Thompson */
f1x_lookup_addr_in_dct(u64 in_addr,u8 nid,u8 dct)2472c7e5301aSDaniel J Blueman static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct)
24736163b5d4SDoug Thompson {
24746163b5d4SDoug Thompson struct mem_ctl_info *mci;
24756163b5d4SDoug Thompson struct amd64_pvt *pvt;
247611c75eadSBorislav Petkov u64 cs_base, cs_mask;
24776163b5d4SDoug Thompson int cs_found = -EINVAL;
24786163b5d4SDoug Thompson int csrow;
24796163b5d4SDoug Thompson
24802ec591acSBorislav Petkov mci = edac_mc_find(nid);
24816163b5d4SDoug Thompson if (!mci)
24826163b5d4SDoug Thompson return cs_found;
24836163b5d4SDoug Thompson
24846163b5d4SDoug Thompson pvt = mci->pvt_info;
24856163b5d4SDoug Thompson
2486956b9ba1SJoe Perches edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct);
24876163b5d4SDoug Thompson
248811c75eadSBorislav Petkov for_each_chip_select(csrow, dct, pvt) {
248911c75eadSBorislav Petkov if (!csrow_enabled(csrow, dct, pvt))
24906163b5d4SDoug Thompson continue;
24916163b5d4SDoug Thompson
249211c75eadSBorislav Petkov get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
24936163b5d4SDoug Thompson
2494956b9ba1SJoe Perches edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
24956163b5d4SDoug Thompson csrow, cs_base, cs_mask);
24966163b5d4SDoug Thompson
249711c75eadSBorislav Petkov cs_mask = ~cs_mask;
24986163b5d4SDoug Thompson
2499956b9ba1SJoe Perches edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n",
250011c75eadSBorislav Petkov (in_addr & cs_mask), (cs_base & cs_mask));
25016163b5d4SDoug Thompson
250211c75eadSBorislav Petkov if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
250318b94f66SAravind Gopalakrishnan if (pvt->fam == 0x15 && pvt->model >= 0x30) {
250418b94f66SAravind Gopalakrishnan cs_found = csrow;
250518b94f66SAravind Gopalakrishnan break;
250618b94f66SAravind Gopalakrishnan }
250711c75eadSBorislav Petkov cs_found = f10_process_possible_spare(pvt, dct, csrow);
25086163b5d4SDoug Thompson
2509956b9ba1SJoe Perches edac_dbg(1, " MATCH csrow=%d\n", cs_found);
25106163b5d4SDoug Thompson break;
25116163b5d4SDoug Thompson }
25126163b5d4SDoug Thompson }
25136163b5d4SDoug Thompson return cs_found;
25146163b5d4SDoug Thompson }
25156163b5d4SDoug Thompson
251695b0ef55SBorislav Petkov /*
251795b0ef55SBorislav Petkov * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
251895b0ef55SBorislav Petkov * swapped with a region located at the bottom of memory so that the GPU can use
251995b0ef55SBorislav Petkov * the interleaved region and thus two channels.
252095b0ef55SBorislav Petkov */
f1x_swap_interleaved_region(struct amd64_pvt * pvt,u64 sys_addr)2521b15f0fcaSBorislav Petkov static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
252295b0ef55SBorislav Petkov {
252395b0ef55SBorislav Petkov u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
252495b0ef55SBorislav Petkov
2525a4b4bedcSBorislav Petkov if (pvt->fam == 0x10) {
252695b0ef55SBorislav Petkov /* only revC3 and revE have that feature */
2527a4b4bedcSBorislav Petkov if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3))
252895b0ef55SBorislav Petkov return sys_addr;
252995b0ef55SBorislav Petkov }
253095b0ef55SBorislav Petkov
25317981a28fSAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F2, SWAP_INTLV_REG, &swap_reg);
253295b0ef55SBorislav Petkov
253395b0ef55SBorislav Petkov if (!(swap_reg & 0x1))
253495b0ef55SBorislav Petkov return sys_addr;
253595b0ef55SBorislav Petkov
253695b0ef55SBorislav Petkov swap_base = (swap_reg >> 3) & 0x7f;
253795b0ef55SBorislav Petkov swap_limit = (swap_reg >> 11) & 0x7f;
253895b0ef55SBorislav Petkov rgn_size = (swap_reg >> 20) & 0x7f;
253995b0ef55SBorislav Petkov tmp_addr = sys_addr >> 27;
254095b0ef55SBorislav Petkov
254195b0ef55SBorislav Petkov if (!(sys_addr >> 34) &&
254295b0ef55SBorislav Petkov (((tmp_addr >= swap_base) &&
254395b0ef55SBorislav Petkov (tmp_addr <= swap_limit)) ||
254495b0ef55SBorislav Petkov (tmp_addr < rgn_size)))
254595b0ef55SBorislav Petkov return sys_addr ^ (u64)swap_base << 27;
254695b0ef55SBorislav Petkov
254795b0ef55SBorislav Petkov return sys_addr;
254895b0ef55SBorislav Petkov }
254995b0ef55SBorislav Petkov
2550f71d0a05SDoug Thompson /* For a given @dram_range, check if @sys_addr falls within it. */
f1x_match_to_this_node(struct amd64_pvt * pvt,unsigned range,u64 sys_addr,int * chan_sel)2551e761359aSBorislav Petkov static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
255233ca0643SBorislav Petkov u64 sys_addr, int *chan_sel)
2553f71d0a05SDoug Thompson {
2554229a7a11SBorislav Petkov int cs_found = -EINVAL;
2555c8e518d5SBorislav Petkov u64 chan_addr;
25565d4b58e8SBorislav Petkov u32 dct_sel_base;
255711c75eadSBorislav Petkov u8 channel;
2558229a7a11SBorislav Petkov bool high_range = false;
2559f71d0a05SDoug Thompson
25607f19bf75SBorislav Petkov u8 node_id = dram_dst_node(pvt, range);
2561229a7a11SBorislav Petkov u8 intlv_en = dram_intlv_en(pvt, range);
25627f19bf75SBorislav Petkov u32 intlv_sel = dram_intlv_sel(pvt, range);
2563f71d0a05SDoug Thompson
2564956b9ba1SJoe Perches edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
2565c8e518d5SBorislav Petkov range, sys_addr, get_dram_limit(pvt, range));
2566f71d0a05SDoug Thompson
2567355fba60SBorislav Petkov if (dhar_valid(pvt) &&
2568355fba60SBorislav Petkov dhar_base(pvt) <= sys_addr &&
2569355fba60SBorislav Petkov sys_addr < BIT_64(32)) {
2570355fba60SBorislav Petkov amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
2571355fba60SBorislav Petkov sys_addr);
2572f71d0a05SDoug Thompson return -EINVAL;
2573355fba60SBorislav Petkov }
2574355fba60SBorislav Petkov
2575f030ddfbSBorislav Petkov if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
2576355fba60SBorislav Petkov return -EINVAL;
2577f71d0a05SDoug Thompson
2578b15f0fcaSBorislav Petkov sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
257995b0ef55SBorislav Petkov
2580f71d0a05SDoug Thompson dct_sel_base = dct_sel_baseaddr(pvt);
2581f71d0a05SDoug Thompson
2582f71d0a05SDoug Thompson /*
2583f71d0a05SDoug Thompson * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
2584f71d0a05SDoug Thompson * select between DCT0 and DCT1.
2585f71d0a05SDoug Thompson */
2586f71d0a05SDoug Thompson if (dct_high_range_enabled(pvt) &&
2587f71d0a05SDoug Thompson !dct_ganging_enabled(pvt) &&
2588f71d0a05SDoug Thompson ((sys_addr >> 27) >= (dct_sel_base >> 11)))
2589229a7a11SBorislav Petkov high_range = true;
2590f71d0a05SDoug Thompson
2591b15f0fcaSBorislav Petkov channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
2592f71d0a05SDoug Thompson
2593b15f0fcaSBorislav Petkov chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
2594c8e518d5SBorislav Petkov high_range, dct_sel_base);
2595f71d0a05SDoug Thompson
2596e2f79dbdSBorislav Petkov /* Remove node interleaving, see F1x120 */
2597e2f79dbdSBorislav Petkov if (intlv_en)
2598e2f79dbdSBorislav Petkov chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
2599e2f79dbdSBorislav Petkov (chan_addr & 0xfff);
2600f71d0a05SDoug Thompson
26015d4b58e8SBorislav Petkov /* remove channel interleave */
2602f71d0a05SDoug Thompson if (dct_interleave_enabled(pvt) &&
2603f71d0a05SDoug Thompson !dct_high_range_enabled(pvt) &&
2604f71d0a05SDoug Thompson !dct_ganging_enabled(pvt)) {
26055d4b58e8SBorislav Petkov
26065d4b58e8SBorislav Petkov if (dct_sel_interleave_addr(pvt) != 1) {
26075d4b58e8SBorislav Petkov if (dct_sel_interleave_addr(pvt) == 0x3)
26085d4b58e8SBorislav Petkov /* hash 9 */
26095d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 10) << 9) |
26105d4b58e8SBorislav Petkov (chan_addr & 0x1ff);
26115d4b58e8SBorislav Petkov else
26125d4b58e8SBorislav Petkov /* A[6] or hash 6 */
26135d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 7) << 6) |
26145d4b58e8SBorislav Petkov (chan_addr & 0x3f);
26155d4b58e8SBorislav Petkov } else
26165d4b58e8SBorislav Petkov /* A[12] */
26175d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 13) << 12) |
26185d4b58e8SBorislav Petkov (chan_addr & 0xfff);
2619f71d0a05SDoug Thompson }
2620f71d0a05SDoug Thompson
2621956b9ba1SJoe Perches edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
2622f71d0a05SDoug Thompson
2623b15f0fcaSBorislav Petkov cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
2624f71d0a05SDoug Thompson
262533ca0643SBorislav Petkov if (cs_found >= 0)
2626f71d0a05SDoug Thompson *chan_sel = channel;
262733ca0643SBorislav Petkov
2628f71d0a05SDoug Thompson return cs_found;
2629f71d0a05SDoug Thompson }
2630f71d0a05SDoug Thompson
f15_m30h_match_to_this_node(struct amd64_pvt * pvt,unsigned range,u64 sys_addr,int * chan_sel)263118b94f66SAravind Gopalakrishnan static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
263218b94f66SAravind Gopalakrishnan u64 sys_addr, int *chan_sel)
263318b94f66SAravind Gopalakrishnan {
263418b94f66SAravind Gopalakrishnan int cs_found = -EINVAL;
263518b94f66SAravind Gopalakrishnan int num_dcts_intlv = 0;
263618b94f66SAravind Gopalakrishnan u64 chan_addr, chan_offset;
263718b94f66SAravind Gopalakrishnan u64 dct_base, dct_limit;
263818b94f66SAravind Gopalakrishnan u32 dct_cont_base_reg, dct_cont_limit_reg, tmp;
263918b94f66SAravind Gopalakrishnan u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en;
264018b94f66SAravind Gopalakrishnan
264118b94f66SAravind Gopalakrishnan u64 dhar_offset = f10_dhar_offset(pvt);
264218b94f66SAravind Gopalakrishnan u8 intlv_addr = dct_sel_interleave_addr(pvt);
264318b94f66SAravind Gopalakrishnan u8 node_id = dram_dst_node(pvt, range);
264418b94f66SAravind Gopalakrishnan u8 intlv_en = dram_intlv_en(pvt, range);
264518b94f66SAravind Gopalakrishnan
264618b94f66SAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg);
264718b94f66SAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg);
264818b94f66SAravind Gopalakrishnan
264918b94f66SAravind Gopalakrishnan dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0));
265018b94f66SAravind Gopalakrishnan dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7);
265118b94f66SAravind Gopalakrishnan
265218b94f66SAravind Gopalakrishnan edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
265318b94f66SAravind Gopalakrishnan range, sys_addr, get_dram_limit(pvt, range));
265418b94f66SAravind Gopalakrishnan
265518b94f66SAravind Gopalakrishnan if (!(get_dram_base(pvt, range) <= sys_addr) &&
265618b94f66SAravind Gopalakrishnan !(get_dram_limit(pvt, range) >= sys_addr))
265718b94f66SAravind Gopalakrishnan return -EINVAL;
265818b94f66SAravind Gopalakrishnan
265918b94f66SAravind Gopalakrishnan if (dhar_valid(pvt) &&
266018b94f66SAravind Gopalakrishnan dhar_base(pvt) <= sys_addr &&
266118b94f66SAravind Gopalakrishnan sys_addr < BIT_64(32)) {
266218b94f66SAravind Gopalakrishnan amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
266318b94f66SAravind Gopalakrishnan sys_addr);
266418b94f66SAravind Gopalakrishnan return -EINVAL;
266518b94f66SAravind Gopalakrishnan }
266618b94f66SAravind Gopalakrishnan
266718b94f66SAravind Gopalakrishnan /* Verify sys_addr is within DCT Range. */
26684fc06b31SAravind Gopalakrishnan dct_base = (u64) dct_sel_baseaddr(pvt);
26694fc06b31SAravind Gopalakrishnan dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF;
267018b94f66SAravind Gopalakrishnan
267118b94f66SAravind Gopalakrishnan if (!(dct_cont_base_reg & BIT(0)) &&
26724fc06b31SAravind Gopalakrishnan !(dct_base <= (sys_addr >> 27) &&
26734fc06b31SAravind Gopalakrishnan dct_limit >= (sys_addr >> 27)))
267418b94f66SAravind Gopalakrishnan return -EINVAL;
267518b94f66SAravind Gopalakrishnan
267618b94f66SAravind Gopalakrishnan /* Verify number of dct's that participate in channel interleaving. */
267718b94f66SAravind Gopalakrishnan num_dcts_intlv = (int) hweight8(intlv_en);
267818b94f66SAravind Gopalakrishnan
267918b94f66SAravind Gopalakrishnan if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4))
268018b94f66SAravind Gopalakrishnan return -EINVAL;
268118b94f66SAravind Gopalakrishnan
2682dc0a50a8SYazen Ghannam if (pvt->model >= 0x60)
2683dc0a50a8SYazen Ghannam channel = f1x_determine_channel(pvt, sys_addr, false, intlv_en);
2684dc0a50a8SYazen Ghannam else
268518b94f66SAravind Gopalakrishnan channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en,
268618b94f66SAravind Gopalakrishnan num_dcts_intlv, dct_sel);
268718b94f66SAravind Gopalakrishnan
268818b94f66SAravind Gopalakrishnan /* Verify we stay within the MAX number of channels allowed */
26897f3f5240SAravind Gopalakrishnan if (channel > 3)
269018b94f66SAravind Gopalakrishnan return -EINVAL;
269118b94f66SAravind Gopalakrishnan
269218b94f66SAravind Gopalakrishnan leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0));
269318b94f66SAravind Gopalakrishnan
269418b94f66SAravind Gopalakrishnan /* Get normalized DCT addr */
269518b94f66SAravind Gopalakrishnan if (leg_mmio_hole && (sys_addr >= BIT_64(32)))
269618b94f66SAravind Gopalakrishnan chan_offset = dhar_offset;
269718b94f66SAravind Gopalakrishnan else
26984fc06b31SAravind Gopalakrishnan chan_offset = dct_base << 27;
269918b94f66SAravind Gopalakrishnan
270018b94f66SAravind Gopalakrishnan chan_addr = sys_addr - chan_offset;
270118b94f66SAravind Gopalakrishnan
270218b94f66SAravind Gopalakrishnan /* remove channel interleave */
270318b94f66SAravind Gopalakrishnan if (num_dcts_intlv == 2) {
270418b94f66SAravind Gopalakrishnan if (intlv_addr == 0x4)
270518b94f66SAravind Gopalakrishnan chan_addr = ((chan_addr >> 9) << 8) |
270618b94f66SAravind Gopalakrishnan (chan_addr & 0xff);
270718b94f66SAravind Gopalakrishnan else if (intlv_addr == 0x5)
270818b94f66SAravind Gopalakrishnan chan_addr = ((chan_addr >> 10) << 9) |
270918b94f66SAravind Gopalakrishnan (chan_addr & 0x1ff);
271018b94f66SAravind Gopalakrishnan else
271118b94f66SAravind Gopalakrishnan return -EINVAL;
271218b94f66SAravind Gopalakrishnan
271318b94f66SAravind Gopalakrishnan } else if (num_dcts_intlv == 4) {
271418b94f66SAravind Gopalakrishnan if (intlv_addr == 0x4)
271518b94f66SAravind Gopalakrishnan chan_addr = ((chan_addr >> 10) << 8) |
271618b94f66SAravind Gopalakrishnan (chan_addr & 0xff);
271718b94f66SAravind Gopalakrishnan else if (intlv_addr == 0x5)
271818b94f66SAravind Gopalakrishnan chan_addr = ((chan_addr >> 11) << 9) |
271918b94f66SAravind Gopalakrishnan (chan_addr & 0x1ff);
272018b94f66SAravind Gopalakrishnan else
272118b94f66SAravind Gopalakrishnan return -EINVAL;
272218b94f66SAravind Gopalakrishnan }
272318b94f66SAravind Gopalakrishnan
272418b94f66SAravind Gopalakrishnan if (dct_offset_en) {
272518b94f66SAravind Gopalakrishnan amd64_read_pci_cfg(pvt->F1,
272618b94f66SAravind Gopalakrishnan DRAM_CONT_HIGH_OFF + (int) channel * 4,
272718b94f66SAravind Gopalakrishnan &tmp);
27284fc06b31SAravind Gopalakrishnan chan_addr += (u64) ((tmp >> 11) & 0xfff) << 27;
272918b94f66SAravind Gopalakrishnan }
273018b94f66SAravind Gopalakrishnan
273118b94f66SAravind Gopalakrishnan f15h_select_dct(pvt, channel);
273218b94f66SAravind Gopalakrishnan
273318b94f66SAravind Gopalakrishnan edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
273418b94f66SAravind Gopalakrishnan
273518b94f66SAravind Gopalakrishnan /*
273618b94f66SAravind Gopalakrishnan * Find Chip select:
273718b94f66SAravind Gopalakrishnan * if channel = 3, then alias it to 1. This is because, in F15 M30h,
273818b94f66SAravind Gopalakrishnan * there is support for 4 DCT's, but only 2 are currently functional.
273918b94f66SAravind Gopalakrishnan * They are DCT0 and DCT3. But we have read all registers of DCT3 into
274018b94f66SAravind Gopalakrishnan * pvt->csels[1]. So we need to use '1' here to get correct info.
274118b94f66SAravind Gopalakrishnan * Refer F15 M30h BKDG Section 2.10 and 2.10.3 for clarifications.
274218b94f66SAravind Gopalakrishnan */
274318b94f66SAravind Gopalakrishnan alias_channel = (channel == 3) ? 1 : channel;
274418b94f66SAravind Gopalakrishnan
274518b94f66SAravind Gopalakrishnan cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel);
274618b94f66SAravind Gopalakrishnan
274718b94f66SAravind Gopalakrishnan if (cs_found >= 0)
274818b94f66SAravind Gopalakrishnan *chan_sel = alias_channel;
274918b94f66SAravind Gopalakrishnan
275018b94f66SAravind Gopalakrishnan return cs_found;
275118b94f66SAravind Gopalakrishnan }
275218b94f66SAravind Gopalakrishnan
f1x_translate_sysaddr_to_cs(struct amd64_pvt * pvt,u64 sys_addr,int * chan_sel)275318b94f66SAravind Gopalakrishnan static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt,
275418b94f66SAravind Gopalakrishnan u64 sys_addr,
275533ca0643SBorislav Petkov int *chan_sel)
2756f71d0a05SDoug Thompson {
2757e761359aSBorislav Petkov int cs_found = -EINVAL;
2758e761359aSBorislav Petkov unsigned range;
2759f71d0a05SDoug Thompson
27607f19bf75SBorislav Petkov for (range = 0; range < DRAM_RANGES; range++) {
27617f19bf75SBorislav Petkov if (!dram_rw(pvt, range))
2762f71d0a05SDoug Thompson continue;
2763f71d0a05SDoug Thompson
276418b94f66SAravind Gopalakrishnan if (pvt->fam == 0x15 && pvt->model >= 0x30)
276518b94f66SAravind Gopalakrishnan cs_found = f15_m30h_match_to_this_node(pvt, range,
276618b94f66SAravind Gopalakrishnan sys_addr,
276718b94f66SAravind Gopalakrishnan chan_sel);
2768f71d0a05SDoug Thompson
276918b94f66SAravind Gopalakrishnan else if ((get_dram_base(pvt, range) <= sys_addr) &&
277018b94f66SAravind Gopalakrishnan (get_dram_limit(pvt, range) >= sys_addr)) {
2771b15f0fcaSBorislav Petkov cs_found = f1x_match_to_this_node(pvt, range,
277233ca0643SBorislav Petkov sys_addr, chan_sel);
2773f71d0a05SDoug Thompson if (cs_found >= 0)
2774f71d0a05SDoug Thompson break;
2775f71d0a05SDoug Thompson }
2776f71d0a05SDoug Thompson }
2777f71d0a05SDoug Thompson return cs_found;
2778f71d0a05SDoug Thompson }
2779f71d0a05SDoug Thompson
2780f71d0a05SDoug Thompson /*
2781bdc30a0cSBorislav Petkov * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
2782bdc30a0cSBorislav Petkov * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
2783f71d0a05SDoug Thompson *
2784bdc30a0cSBorislav Petkov * The @sys_addr is usually an error address received from the hardware
2785bdc30a0cSBorislav Petkov * (MCX_ADDR).
2786f71d0a05SDoug Thompson */
f1x_map_sysaddr_to_csrow(struct mem_ctl_info * mci,u64 sys_addr,struct err_info * err)2787b15f0fcaSBorislav Petkov static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
278833ca0643SBorislav Petkov struct err_info *err)
2789f71d0a05SDoug Thompson {
2790f71d0a05SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info;
2791f71d0a05SDoug Thompson
279233ca0643SBorislav Petkov error_address_to_page_and_offset(sys_addr, err);
2793ab5a503cSMauro Carvalho Chehab
279433ca0643SBorislav Petkov err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel);
279533ca0643SBorislav Petkov if (err->csrow < 0) {
279633ca0643SBorislav Petkov err->err_code = ERR_CSROW;
2797bdc30a0cSBorislav Petkov return;
2798bdc30a0cSBorislav Petkov }
2799bdc30a0cSBorislav Petkov
2800f71d0a05SDoug Thompson /*
2801bdc30a0cSBorislav Petkov * We need the syndromes for channel detection only when we're
2802bdc30a0cSBorislav Petkov * ganged. Otherwise @chan should already contain the channel at
2803bdc30a0cSBorislav Petkov * this point.
2804f71d0a05SDoug Thompson */
2805a97fa68eSBorislav Petkov if (dct_ganging_enabled(pvt))
280633ca0643SBorislav Petkov err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
2807f71d0a05SDoug Thompson }
2808f71d0a05SDoug Thompson
2809f71d0a05SDoug Thompson /*
2810bfc04aecSBorislav Petkov * These are tables of eigenvectors (one per line) which can be used for the
2811bfc04aecSBorislav Petkov * construction of the syndrome tables. The modified syndrome search algorithm
2812bfc04aecSBorislav Petkov * uses those to find the symbol in error and thus the DIMM.
2813b1289d6fSDoug Thompson *
2814bfc04aecSBorislav Petkov * Algorithm courtesy of Ross LaFetra from AMD.
2815b1289d6fSDoug Thompson */
2816c7e5301aSDaniel J Blueman static const u16 x4_vectors[] = {
2817bfc04aecSBorislav Petkov 0x2f57, 0x1afe, 0x66cc, 0xdd88,
2818bfc04aecSBorislav Petkov 0x11eb, 0x3396, 0x7f4c, 0xeac8,
2819bfc04aecSBorislav Petkov 0x0001, 0x0002, 0x0004, 0x0008,
2820bfc04aecSBorislav Petkov 0x1013, 0x3032, 0x4044, 0x8088,
2821bfc04aecSBorislav Petkov 0x106b, 0x30d6, 0x70fc, 0xe0a8,
2822bfc04aecSBorislav Petkov 0x4857, 0xc4fe, 0x13cc, 0x3288,
2823bfc04aecSBorislav Petkov 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
2824bfc04aecSBorislav Petkov 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
2825bfc04aecSBorislav Petkov 0x15c1, 0x2a42, 0x89ac, 0x4758,
2826bfc04aecSBorislav Petkov 0x2b03, 0x1602, 0x4f0c, 0xca08,
2827bfc04aecSBorislav Petkov 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
2828bfc04aecSBorislav Petkov 0x8ba7, 0x465e, 0x244c, 0x1cc8,
2829bfc04aecSBorislav Petkov 0x2b87, 0x164e, 0x642c, 0xdc18,
2830bfc04aecSBorislav Petkov 0x40b9, 0x80de, 0x1094, 0x20e8,
2831bfc04aecSBorislav Petkov 0x27db, 0x1eb6, 0x9dac, 0x7b58,
2832bfc04aecSBorislav Petkov 0x11c1, 0x2242, 0x84ac, 0x4c58,
2833bfc04aecSBorislav Petkov 0x1be5, 0x2d7a, 0x5e34, 0xa718,
2834bfc04aecSBorislav Petkov 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
2835bfc04aecSBorislav Petkov 0x4c97, 0xc87e, 0x11fc, 0x33a8,
2836bfc04aecSBorislav Petkov 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
2837bfc04aecSBorislav Petkov 0x16b3, 0x3d62, 0x4f34, 0x8518,
2838bfc04aecSBorislav Petkov 0x1e2f, 0x391a, 0x5cac, 0xf858,
2839bfc04aecSBorislav Petkov 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
2840bfc04aecSBorislav Petkov 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
2841bfc04aecSBorislav Petkov 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
2842bfc04aecSBorislav Petkov 0x4397, 0xc27e, 0x17fc, 0x3ea8,
2843bfc04aecSBorislav Petkov 0x1617, 0x3d3e, 0x6464, 0xb8b8,
2844bfc04aecSBorislav Petkov 0x23ff, 0x12aa, 0xab6c, 0x56d8,
2845bfc04aecSBorislav Petkov 0x2dfb, 0x1ba6, 0x913c, 0x7328,
2846bfc04aecSBorislav Petkov 0x185d, 0x2ca6, 0x7914, 0x9e28,
2847bfc04aecSBorislav Petkov 0x171b, 0x3e36, 0x7d7c, 0xebe8,
2848bfc04aecSBorislav Petkov 0x4199, 0x82ee, 0x19f4, 0x2e58,
2849bfc04aecSBorislav Petkov 0x4807, 0xc40e, 0x130c, 0x3208,
2850bfc04aecSBorislav Petkov 0x1905, 0x2e0a, 0x5804, 0xac08,
2851bfc04aecSBorislav Petkov 0x213f, 0x132a, 0xadfc, 0x5ba8,
2852bfc04aecSBorislav Petkov 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
2853b1289d6fSDoug Thompson };
2854b1289d6fSDoug Thompson
2855c7e5301aSDaniel J Blueman static const u16 x8_vectors[] = {
2856bfc04aecSBorislav Petkov 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
2857bfc04aecSBorislav Petkov 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
2858bfc04aecSBorislav Petkov 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
2859bfc04aecSBorislav Petkov 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
2860bfc04aecSBorislav Petkov 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
2861bfc04aecSBorislav Petkov 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
2862bfc04aecSBorislav Petkov 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
2863bfc04aecSBorislav Petkov 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
2864bfc04aecSBorislav Petkov 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
2865bfc04aecSBorislav Petkov 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
2866bfc04aecSBorislav Petkov 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
2867bfc04aecSBorislav Petkov 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
2868bfc04aecSBorislav Petkov 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
2869bfc04aecSBorislav Petkov 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
2870bfc04aecSBorislav Petkov 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
2871bfc04aecSBorislav Petkov 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
2872bfc04aecSBorislav Petkov 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
2873bfc04aecSBorislav Petkov 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
2874bfc04aecSBorislav Petkov 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
2875bfc04aecSBorislav Petkov };
2876bfc04aecSBorislav Petkov
decode_syndrome(u16 syndrome,const u16 * vectors,unsigned num_vecs,unsigned v_dim)2877c7e5301aSDaniel J Blueman static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs,
2878d34a6ecdSBorislav Petkov unsigned v_dim)
2879b1289d6fSDoug Thompson {
2880bfc04aecSBorislav Petkov unsigned int i, err_sym;
2881b1289d6fSDoug Thompson
2882bfc04aecSBorislav Petkov for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
2883bfc04aecSBorislav Petkov u16 s = syndrome;
2884d34a6ecdSBorislav Petkov unsigned v_idx = err_sym * v_dim;
2885d34a6ecdSBorislav Petkov unsigned v_end = (err_sym + 1) * v_dim;
2886b1289d6fSDoug Thompson
2887bfc04aecSBorislav Petkov /* walk over all 16 bits of the syndrome */
2888bfc04aecSBorislav Petkov for (i = 1; i < (1U << 16); i <<= 1) {
2889bfc04aecSBorislav Petkov
2890bfc04aecSBorislav Petkov /* if bit is set in that eigenvector... */
2891bfc04aecSBorislav Petkov if (v_idx < v_end && vectors[v_idx] & i) {
2892bfc04aecSBorislav Petkov u16 ev_comp = vectors[v_idx++];
2893bfc04aecSBorislav Petkov
2894bfc04aecSBorislav Petkov /* ... and bit set in the modified syndrome, */
2895bfc04aecSBorislav Petkov if (s & i) {
2896bfc04aecSBorislav Petkov /* remove it. */
2897bfc04aecSBorislav Petkov s ^= ev_comp;
2898bfc04aecSBorislav Petkov
2899bfc04aecSBorislav Petkov if (!s)
2900bfc04aecSBorislav Petkov return err_sym;
2901bfc04aecSBorislav Petkov }
2902bfc04aecSBorislav Petkov
2903bfc04aecSBorislav Petkov } else if (s & i)
2904bfc04aecSBorislav Petkov /* can't get to zero, move to next symbol */
2905bfc04aecSBorislav Petkov break;
2906bfc04aecSBorislav Petkov }
2907b1289d6fSDoug Thompson }
2908b1289d6fSDoug Thompson
2909956b9ba1SJoe Perches edac_dbg(0, "syndrome(%x) not found\n", syndrome);
2910b1289d6fSDoug Thompson return -1;
2911b1289d6fSDoug Thompson }
2912d27bf6faSDoug Thompson
map_err_sym_to_channel(int err_sym,int sym_size)2913bfc04aecSBorislav Petkov static int map_err_sym_to_channel(int err_sym, int sym_size)
2914bfc04aecSBorislav Petkov {
2915bfc04aecSBorislav Petkov if (sym_size == 4)
2916bfc04aecSBorislav Petkov switch (err_sym) {
2917bfc04aecSBorislav Petkov case 0x20:
2918bfc04aecSBorislav Petkov case 0x21:
2919bfc04aecSBorislav Petkov return 0;
2920bfc04aecSBorislav Petkov case 0x22:
2921bfc04aecSBorislav Petkov case 0x23:
2922bfc04aecSBorislav Petkov return 1;
2923bfc04aecSBorislav Petkov default:
2924bfc04aecSBorislav Petkov return err_sym >> 4;
2925bfc04aecSBorislav Petkov }
2926bfc04aecSBorislav Petkov /* x8 symbols */
2927bfc04aecSBorislav Petkov else
2928bfc04aecSBorislav Petkov switch (err_sym) {
2929bfc04aecSBorislav Petkov /* imaginary bits not in a DIMM */
2930bfc04aecSBorislav Petkov case 0x10:
2931bfc04aecSBorislav Petkov WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
2932bfc04aecSBorislav Petkov err_sym);
2933bfc04aecSBorislav Petkov return -1;
2934bfc04aecSBorislav Petkov case 0x11:
2935bfc04aecSBorislav Petkov return 0;
2936bfc04aecSBorislav Petkov case 0x12:
2937bfc04aecSBorislav Petkov return 1;
2938bfc04aecSBorislav Petkov default:
2939bfc04aecSBorislav Petkov return err_sym >> 3;
2940bfc04aecSBorislav Petkov }
2941bfc04aecSBorislav Petkov return -1;
2942bfc04aecSBorislav Petkov }
2943bfc04aecSBorislav Petkov
get_channel_from_ecc_syndrome(struct mem_ctl_info * mci,u16 syndrome)2944bfc04aecSBorislav Petkov static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
2945bfc04aecSBorislav Petkov {
2946bfc04aecSBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info;
2947ad6a32e9SBorislav Petkov int err_sym = -1;
2948bfc04aecSBorislav Petkov
2949a3b7db09SBorislav Petkov if (pvt->ecc_sym_sz == 8)
2950bfc04aecSBorislav Petkov err_sym = decode_syndrome(syndrome, x8_vectors,
2951ad6a32e9SBorislav Petkov ARRAY_SIZE(x8_vectors),
2952a3b7db09SBorislav Petkov pvt->ecc_sym_sz);
2953a3b7db09SBorislav Petkov else if (pvt->ecc_sym_sz == 4)
2954ad6a32e9SBorislav Petkov err_sym = decode_syndrome(syndrome, x4_vectors,
2955ad6a32e9SBorislav Petkov ARRAY_SIZE(x4_vectors),
2956a3b7db09SBorislav Petkov pvt->ecc_sym_sz);
2957ad6a32e9SBorislav Petkov else {
2958a3b7db09SBorislav Petkov amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
2959ad6a32e9SBorislav Petkov return err_sym;
2960bfc04aecSBorislav Petkov }
2961ad6a32e9SBorislav Petkov
2962a3b7db09SBorislav Petkov return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
296341c31044SBorislav Petkov }
2964bfc04aecSBorislav Petkov
__log_ecc_error(struct mem_ctl_info * mci,struct err_info * err,u8 ecc_type)2965e70984d9SYazen Ghannam static void __log_ecc_error(struct mem_ctl_info *mci, struct err_info *err,
296633ca0643SBorislav Petkov u8 ecc_type)
2967d27bf6faSDoug Thompson {
296833ca0643SBorislav Petkov enum hw_event_mc_err_type err_type;
296933ca0643SBorislav Petkov const char *string;
2970d27bf6faSDoug Thompson
297133ca0643SBorislav Petkov if (ecc_type == 2)
297233ca0643SBorislav Petkov err_type = HW_EVENT_ERR_CORRECTED;
297333ca0643SBorislav Petkov else if (ecc_type == 1)
297433ca0643SBorislav Petkov err_type = HW_EVENT_ERR_UNCORRECTED;
2975d12a969eSYazen Ghannam else if (ecc_type == 3)
2976d12a969eSYazen Ghannam err_type = HW_EVENT_ERR_DEFERRED;
297733ca0643SBorislav Petkov else {
297833ca0643SBorislav Petkov WARN(1, "Something is rotten in the state of Denmark.\n");
2979d27bf6faSDoug Thompson return;
2980d27bf6faSDoug Thompson }
2981d27bf6faSDoug Thompson
298233ca0643SBorislav Petkov switch (err->err_code) {
298333ca0643SBorislav Petkov case DECODE_OK:
298433ca0643SBorislav Petkov string = "";
298533ca0643SBorislav Petkov break;
298633ca0643SBorislav Petkov case ERR_NODE:
298733ca0643SBorislav Petkov string = "Failed to map error addr to a node";
298833ca0643SBorislav Petkov break;
298933ca0643SBorislav Petkov case ERR_CSROW:
299033ca0643SBorislav Petkov string = "Failed to map error addr to a csrow";
299133ca0643SBorislav Petkov break;
299233ca0643SBorislav Petkov case ERR_CHANNEL:
2993713ad546SYazen Ghannam string = "Unknown syndrome - possible error reporting race";
2994713ad546SYazen Ghannam break;
2995713ad546SYazen Ghannam case ERR_SYND:
2996713ad546SYazen Ghannam string = "MCA_SYND not valid - unknown syndrome and csrow";
2997713ad546SYazen Ghannam break;
2998713ad546SYazen Ghannam case ERR_NORM_ADDR:
2999713ad546SYazen Ghannam string = "Cannot decode normalized address";
300033ca0643SBorislav Petkov break;
300133ca0643SBorislav Petkov default:
300233ca0643SBorislav Petkov string = "WTF error";
300333ca0643SBorislav Petkov break;
3004d27bf6faSDoug Thompson }
300533ca0643SBorislav Petkov
300633ca0643SBorislav Petkov edac_mc_handle_error(err_type, mci, 1,
300733ca0643SBorislav Petkov err->page, err->offset, err->syndrome,
300833ca0643SBorislav Petkov err->csrow, err->channel, -1,
300933ca0643SBorislav Petkov string, "");
3010d27bf6faSDoug Thompson }
3011d27bf6faSDoug Thompson
decode_bus_error(int node_id,struct mce * m)3012df781d03SBorislav Petkov static inline void decode_bus_error(int node_id, struct mce *m)
3013d27bf6faSDoug Thompson {
30140c510cc8SDaniel J Blueman struct mem_ctl_info *mci;
30150c510cc8SDaniel J Blueman struct amd64_pvt *pvt;
3016f192c7b1SBorislav Petkov u8 ecc_type = (m->status >> 45) & 0x3;
301766fed2d4SBorislav Petkov u8 xec = XEC(m->status, 0x1f);
301866fed2d4SBorislav Petkov u16 ec = EC(m->status);
301933ca0643SBorislav Petkov u64 sys_addr;
302033ca0643SBorislav Petkov struct err_info err;
3021d27bf6faSDoug Thompson
30220c510cc8SDaniel J Blueman mci = edac_mc_find(node_id);
30230c510cc8SDaniel J Blueman if (!mci)
30240c510cc8SDaniel J Blueman return;
30250c510cc8SDaniel J Blueman
30260c510cc8SDaniel J Blueman pvt = mci->pvt_info;
30270c510cc8SDaniel J Blueman
302866fed2d4SBorislav Petkov /* Bail out early if this was an 'observed' error */
30295980bb9cSBorislav Petkov if (PP(ec) == NBSL_PP_OBS)
3030b70ef010SBorislav Petkov return;
3031d27bf6faSDoug Thompson
3032ecaf5606SBorislav Petkov /* Do only ECC errors */
3033ecaf5606SBorislav Petkov if (xec && xec != F10_NBSL_EXT_ERR_ECC)
3034d27bf6faSDoug Thompson return;
3035d27bf6faSDoug Thompson
303633ca0643SBorislav Petkov memset(&err, 0, sizeof(err));
303733ca0643SBorislav Petkov
3038a4b4bedcSBorislav Petkov sys_addr = get_error_address(pvt, m);
303933ca0643SBorislav Petkov
3040ecaf5606SBorislav Petkov if (ecc_type == 2)
304133ca0643SBorislav Petkov err.syndrome = extract_syndrome(m->status);
304233ca0643SBorislav Petkov
304333ca0643SBorislav Petkov pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err);
304433ca0643SBorislav Petkov
3045e70984d9SYazen Ghannam __log_ecc_error(mci, &err, ecc_type);
3046d27bf6faSDoug Thompson }
3047d27bf6faSDoug Thompson
30480ec449eeSDoug Thompson /*
3049713ad546SYazen Ghannam * To find the UMC channel represented by this bank we need to match on its
3050713ad546SYazen Ghannam * instance_id. The instance_id of a bank is held in the lower 32 bits of its
3051713ad546SYazen Ghannam * IPID.
3052bdcee774SYazen Ghannam *
3053bdcee774SYazen Ghannam * Currently, we can derive the channel number by looking at the 6th nibble in
3054bdcee774SYazen Ghannam * the instance_id. For example, instance_id=0xYXXXXX where Y is the channel
3055bdcee774SYazen Ghannam * number.
3056b3ece3a6SMuralidhara M K *
3057b3ece3a6SMuralidhara M K * For DRAM ECC errors, the Chip Select number is given in bits [2:0] of
3058b3ece3a6SMuralidhara M K * the MCA_SYND[ErrorInformation] field.
3059713ad546SYazen Ghannam */
umc_get_err_info(struct mce * m,struct err_info * err)3060b3ece3a6SMuralidhara M K static void umc_get_err_info(struct mce *m, struct err_info *err)
3061713ad546SYazen Ghannam {
3062b3ece3a6SMuralidhara M K err->channel = (m->ipid & GENMASK(31, 0)) >> 20;
3063b3ece3a6SMuralidhara M K err->csrow = m->synd & 0x7;
3064713ad546SYazen Ghannam }
3065713ad546SYazen Ghannam
decode_umc_error(int node_id,struct mce * m)3066713ad546SYazen Ghannam static void decode_umc_error(int node_id, struct mce *m)
3067713ad546SYazen Ghannam {
3068713ad546SYazen Ghannam u8 ecc_type = (m->status >> 45) & 0x3;
3069713ad546SYazen Ghannam struct mem_ctl_info *mci;
3070713ad546SYazen Ghannam struct amd64_pvt *pvt;
3071713ad546SYazen Ghannam struct err_info err;
3072713ad546SYazen Ghannam u64 sys_addr;
3073713ad546SYazen Ghannam
30744251566eSYazen Ghannam node_id = fixup_node_id(node_id, m);
30754251566eSYazen Ghannam
3076713ad546SYazen Ghannam mci = edac_mc_find(node_id);
3077713ad546SYazen Ghannam if (!mci)
3078713ad546SYazen Ghannam return;
3079713ad546SYazen Ghannam
3080713ad546SYazen Ghannam pvt = mci->pvt_info;
3081713ad546SYazen Ghannam
3082713ad546SYazen Ghannam memset(&err, 0, sizeof(err));
3083713ad546SYazen Ghannam
3084713ad546SYazen Ghannam if (m->status & MCI_STATUS_DEFERRED)
3085713ad546SYazen Ghannam ecc_type = 3;
3086713ad546SYazen Ghannam
3087713ad546SYazen Ghannam if (!(m->status & MCI_STATUS_SYNDV)) {
3088713ad546SYazen Ghannam err.err_code = ERR_SYND;
3089713ad546SYazen Ghannam goto log_error;
3090713ad546SYazen Ghannam }
3091713ad546SYazen Ghannam
3092713ad546SYazen Ghannam if (ecc_type == 2) {
3093713ad546SYazen Ghannam u8 length = (m->synd >> 18) & 0x3f;
3094713ad546SYazen Ghannam
3095713ad546SYazen Ghannam if (length)
3096713ad546SYazen Ghannam err.syndrome = (m->synd >> 32) & GENMASK(length - 1, 0);
3097713ad546SYazen Ghannam else
3098713ad546SYazen Ghannam err.err_code = ERR_CHANNEL;
3099713ad546SYazen Ghannam }
3100713ad546SYazen Ghannam
3101b3ece3a6SMuralidhara M K pvt->ops->get_err_info(m, &err);
3102713ad546SYazen Ghannam
31038a2eaab7SYazen Ghannam if (umc_normaddr_to_sysaddr(m->addr, pvt->mc_node_id, err.channel, &sys_addr)) {
31048a2eaab7SYazen Ghannam err.err_code = ERR_NORM_ADDR;
31058a2eaab7SYazen Ghannam goto log_error;
31068a2eaab7SYazen Ghannam }
31078a2eaab7SYazen Ghannam
31088a2eaab7SYazen Ghannam error_address_to_page_and_offset(sys_addr, &err);
31098a2eaab7SYazen Ghannam
3110713ad546SYazen Ghannam log_error:
3111713ad546SYazen Ghannam __log_ecc_error(mci, &err, ecc_type);
3112713ad546SYazen Ghannam }
3113713ad546SYazen Ghannam
3114713ad546SYazen Ghannam /*
31153f37a36bSBorislav Petkov * Use pvt->F3 which contains the F3 CPU PCI device to get the related
31163f37a36bSBorislav Petkov * F1 (AddrMap) and F2 (Dct) devices. Return negative value on error.
31170ec449eeSDoug Thompson */
3118936fc3afSYazen Ghannam static int
reserve_mc_sibling_devs(struct amd64_pvt * pvt,u16 pci_id1,u16 pci_id2)3119936fc3afSYazen Ghannam reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2)
31200ec449eeSDoug Thompson {
31210ec449eeSDoug Thompson /* Reserve the ADDRESS MAP Device */
3122936fc3afSYazen Ghannam pvt->F1 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
31238d5b5d9cSBorislav Petkov if (!pvt->F1) {
31246a4afe38SYazen Ghannam edac_dbg(1, "F1 not found: device 0x%x\n", pci_id1);
3125bbd0c1f6SBorislav Petkov return -ENODEV;
31260ec449eeSDoug Thompson }
31270ec449eeSDoug Thompson
31283f37a36bSBorislav Petkov /* Reserve the DCT Device */
3129936fc3afSYazen Ghannam pvt->F2 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
31303f37a36bSBorislav Petkov if (!pvt->F2) {
31318d5b5d9cSBorislav Petkov pci_dev_put(pvt->F1);
31328d5b5d9cSBorislav Petkov pvt->F1 = NULL;
31330ec449eeSDoug Thompson
31346a4afe38SYazen Ghannam edac_dbg(1, "F2 not found: device 0x%x\n", pci_id2);
3135bbd0c1f6SBorislav Petkov return -ENODEV;
31360ec449eeSDoug Thompson }
3137936fc3afSYazen Ghannam
3138706657b1SBorislav Petkov if (!pci_ctl_dev)
3139706657b1SBorislav Petkov pci_ctl_dev = &pvt->F2->dev;
3140706657b1SBorislav Petkov
3141956b9ba1SJoe Perches edac_dbg(1, "F1: %s\n", pci_name(pvt->F1));
3142956b9ba1SJoe Perches edac_dbg(1, "F2: %s\n", pci_name(pvt->F2));
3143956b9ba1SJoe Perches edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
31440ec449eeSDoug Thompson
31450ec449eeSDoug Thompson return 0;
31460ec449eeSDoug Thompson }
31470ec449eeSDoug Thompson
determine_ecc_sym_sz(struct amd64_pvt * pvt)3148b64ce7cdSYazen Ghannam static void determine_ecc_sym_sz(struct amd64_pvt *pvt)
3149b64ce7cdSYazen Ghannam {
3150b64ce7cdSYazen Ghannam pvt->ecc_sym_sz = 4;
3151b64ce7cdSYazen Ghannam
31525a1adb37SYazen Ghannam if (pvt->fam >= 0x10) {
3153b64ce7cdSYazen Ghannam u32 tmp;
3154b64ce7cdSYazen Ghannam
3155b64ce7cdSYazen Ghannam amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
3156b64ce7cdSYazen Ghannam /* F16h has only DCT0, so no need to read dbam1. */
3157b64ce7cdSYazen Ghannam if (pvt->fam != 0x16)
3158b64ce7cdSYazen Ghannam amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1);
3159b64ce7cdSYazen Ghannam
3160b64ce7cdSYazen Ghannam /* F10h, revD and later can do x8 ECC too. */
3161b64ce7cdSYazen Ghannam if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25))
3162b64ce7cdSYazen Ghannam pvt->ecc_sym_sz = 8;
3163b64ce7cdSYazen Ghannam }
3164b64ce7cdSYazen Ghannam }
3165b64ce7cdSYazen Ghannam
3166b64ce7cdSYazen Ghannam /*
3167b64ce7cdSYazen Ghannam * Retrieve the hardware registers of the memory controller.
3168b64ce7cdSYazen Ghannam */
umc_read_mc_regs(struct amd64_pvt * pvt)316932ecdf86SMuralidhara M K static void umc_read_mc_regs(struct amd64_pvt *pvt)
3170b64ce7cdSYazen Ghannam {
3171b64ce7cdSYazen Ghannam u8 nid = pvt->mc_node_id;
3172b64ce7cdSYazen Ghannam struct amd64_umc *umc;
3173b64ce7cdSYazen Ghannam u32 i, umc_base;
3174b64ce7cdSYazen Ghannam
3175b64ce7cdSYazen Ghannam /* Read registers from each UMC */
31764d30d2bcSYazen Ghannam for_each_umc(i) {
3177b64ce7cdSYazen Ghannam
3178b64ce7cdSYazen Ghannam umc_base = get_umc_base(i);
3179b64ce7cdSYazen Ghannam umc = &pvt->umc[i];
3180b64ce7cdSYazen Ghannam
3181ed623d55SMuralidhara M K amd_smn_read(nid, umc_base + get_umc_reg(pvt, UMCCH_DIMM_CFG), &umc->dimm_cfg);
318207ed82efSYazen Ghannam amd_smn_read(nid, umc_base + UMCCH_UMC_CFG, &umc->umc_cfg);
3183b64ce7cdSYazen Ghannam amd_smn_read(nid, umc_base + UMCCH_SDP_CTRL, &umc->sdp_ctrl);
3184b64ce7cdSYazen Ghannam amd_smn_read(nid, umc_base + UMCCH_ECC_CTRL, &umc->ecc_ctrl);
318507ed82efSYazen Ghannam amd_smn_read(nid, umc_base + UMCCH_UMC_CAP_HI, &umc->umc_cap_hi);
3186b64ce7cdSYazen Ghannam }
3187b64ce7cdSYazen Ghannam }
3188b64ce7cdSYazen Ghannam
31890ec449eeSDoug Thompson /*
31900ec449eeSDoug Thompson * Retrieve the hardware registers of the memory controller (this includes the
31910ec449eeSDoug Thompson * 'Address Map' and 'Misc' device regs)
31920ec449eeSDoug Thompson */
dct_read_mc_regs(struct amd64_pvt * pvt)319332ecdf86SMuralidhara M K static void dct_read_mc_regs(struct amd64_pvt *pvt)
31940ec449eeSDoug Thompson {
3195b64ce7cdSYazen Ghannam unsigned int range;
31960ec449eeSDoug Thompson u64 msr_val;
31970ec449eeSDoug Thompson
31980ec449eeSDoug Thompson /*
31990ec449eeSDoug Thompson * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
3200b64ce7cdSYazen Ghannam * those are Read-As-Zero.
32010ec449eeSDoug Thompson */
3202e97f8bb8SBorislav Petkov rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
3203956b9ba1SJoe Perches edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem);
32040ec449eeSDoug Thompson
3205b64ce7cdSYazen Ghannam /* Check first whether TOP_MEM2 is enabled: */
3206059e5c32SBrijesh Singh rdmsrl(MSR_AMD64_SYSCFG, msr_val);
3207b64ce7cdSYazen Ghannam if (msr_val & BIT(21)) {
3208e97f8bb8SBorislav Petkov rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
3209956b9ba1SJoe Perches edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
3210b64ce7cdSYazen Ghannam } else {
3211956b9ba1SJoe Perches edac_dbg(0, " TOP_MEM2 disabled\n");
3212b64ce7cdSYazen Ghannam }
3213b64ce7cdSYazen Ghannam
32145980bb9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
32150ec449eeSDoug Thompson
32165a5d2371SBorislav Petkov read_dram_ctl_register(pvt);
32170ec449eeSDoug Thompson
32187f19bf75SBorislav Petkov for (range = 0; range < DRAM_RANGES; range++) {
32197f19bf75SBorislav Petkov u8 rw;
32200ec449eeSDoug Thompson
32217f19bf75SBorislav Petkov /* read settings for this DRAM range */
32227f19bf75SBorislav Petkov read_dram_base_limit_regs(pvt, range);
3223e97f8bb8SBorislav Petkov
32247f19bf75SBorislav Petkov rw = dram_rw(pvt, range);
32257f19bf75SBorislav Petkov if (!rw)
32267f19bf75SBorislav Petkov continue;
32277f19bf75SBorislav Petkov
3228956b9ba1SJoe Perches edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
32297f19bf75SBorislav Petkov range,
32307f19bf75SBorislav Petkov get_dram_base(pvt, range),
32317f19bf75SBorislav Petkov get_dram_limit(pvt, range));
32327f19bf75SBorislav Petkov
3233956b9ba1SJoe Perches edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
32347f19bf75SBorislav Petkov dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
32357f19bf75SBorislav Petkov (rw & 0x1) ? "R" : "-",
32367f19bf75SBorislav Petkov (rw & 0x2) ? "W" : "-",
32377f19bf75SBorislav Petkov dram_intlv_sel(pvt, range),
32387f19bf75SBorislav Petkov dram_dst_node(pvt, range));
32390ec449eeSDoug Thompson }
32400ec449eeSDoug Thompson
3241bc21fa57SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
32427981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 0, DBAM0, &pvt->dbam0);
32430ec449eeSDoug Thompson
32448d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
32450ec449eeSDoug Thompson
32467981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 0, DCLR0, &pvt->dclr0);
32477981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 0, DCHR0, &pvt->dchr0);
32480ec449eeSDoug Thompson
324978da121eSBorislav Petkov if (!dct_ganging_enabled(pvt)) {
32507981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 1, DCLR0, &pvt->dclr1);
32517981a28fSAravind Gopalakrishnan amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1);
32520ec449eeSDoug Thompson }
3253b2b0c605SBorislav Petkov
3254b64ce7cdSYazen Ghannam determine_ecc_sym_sz(pvt);
32550ec449eeSDoug Thompson }
32560ec449eeSDoug Thompson
32570ec449eeSDoug Thompson /*
32580ec449eeSDoug Thompson * NOTE: CPU Revision Dependent code
32590ec449eeSDoug Thompson *
32600ec449eeSDoug Thompson * Input:
326111c75eadSBorislav Petkov * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
32620ec449eeSDoug Thompson * k8 private pointer to -->
32630ec449eeSDoug Thompson * DRAM Bank Address mapping register
32640ec449eeSDoug Thompson * node_id
32650ec449eeSDoug Thompson * DCL register where dual_channel_active is
32660ec449eeSDoug Thompson *
32670ec449eeSDoug Thompson * The DBAM register consists of 4 sets of 4 bits each definitions:
32680ec449eeSDoug Thompson *
32690ec449eeSDoug Thompson * Bits: CSROWs
32700ec449eeSDoug Thompson * 0-3 CSROWs 0 and 1
32710ec449eeSDoug Thompson * 4-7 CSROWs 2 and 3
32720ec449eeSDoug Thompson * 8-11 CSROWs 4 and 5
32730ec449eeSDoug Thompson * 12-15 CSROWs 6 and 7
32740ec449eeSDoug Thompson *
32750ec449eeSDoug Thompson * Values range from: 0 to 15
32760ec449eeSDoug Thompson * The meaning of the values depends on CPU revision and dual-channel state,
32770ec449eeSDoug Thompson * see relevant BKDG more info.
32780ec449eeSDoug Thompson *
32790ec449eeSDoug Thompson * The memory controller provides for total of only 8 CSROWs in its current
32800ec449eeSDoug Thompson * architecture. Each "pair" of CSROWs normally represents just one DIMM in
32810ec449eeSDoug Thompson * single channel or two (2) DIMMs in dual channel mode.
32820ec449eeSDoug Thompson *
32830ec449eeSDoug Thompson * The following code logic collapses the various tables for CSROW based on CPU
32840ec449eeSDoug Thompson * revision.
32850ec449eeSDoug Thompson *
32860ec449eeSDoug Thompson * Returns:
32870ec449eeSDoug Thompson * The number of PAGE_SIZE pages on the specified CSROW number it
32880ec449eeSDoug Thompson * encompasses
32890ec449eeSDoug Thompson *
32900ec449eeSDoug Thompson */
dct_get_csrow_nr_pages(struct amd64_pvt * pvt,u8 dct,int csrow_nr)3291c0984666SYazen Ghannam static u32 dct_get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
32920ec449eeSDoug Thompson {
3293f92cae45SAshish Shenoy u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
3294eb77e6b8SYazen Ghannam u32 cs_mode, nr_pages;
32950ec449eeSDoug Thompson
3296eb77e6b8SYazen Ghannam csrow_nr >>= 1;
3297eb77e6b8SYazen Ghannam cs_mode = DBAM_DIMM(csrow_nr, dbam);
32980ec449eeSDoug Thompson
3299eb77e6b8SYazen Ghannam nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, csrow_nr);
3300eb77e6b8SYazen Ghannam nr_pages <<= 20 - PAGE_SHIFT;
33010ec449eeSDoug Thompson
330210de6497SBorislav Petkov edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
3303c0984666SYazen Ghannam csrow_nr, dct, cs_mode);
3304c0984666SYazen Ghannam edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
3305c0984666SYazen Ghannam
3306c0984666SYazen Ghannam return nr_pages;
3307c0984666SYazen Ghannam }
3308c0984666SYazen Ghannam
umc_get_csrow_nr_pages(struct amd64_pvt * pvt,u8 dct,int csrow_nr_orig)3309c0984666SYazen Ghannam static u32 umc_get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr_orig)
3310c0984666SYazen Ghannam {
3311c0984666SYazen Ghannam int csrow_nr = csrow_nr_orig;
3312c0984666SYazen Ghannam u32 cs_mode, nr_pages;
3313c0984666SYazen Ghannam
3314c0984666SYazen Ghannam cs_mode = umc_get_cs_mode(csrow_nr >> 1, dct, pvt);
3315c0984666SYazen Ghannam
3316a2e59ab8SYazen Ghannam nr_pages = umc_addr_mask_to_cs_size(pvt, dct, cs_mode, csrow_nr);
3317c0984666SYazen Ghannam nr_pages <<= 20 - PAGE_SHIFT;
3318c0984666SYazen Ghannam
3319c0984666SYazen Ghannam edac_dbg(0, "csrow: %d, channel: %d, cs_mode %d\n",
3320eb77e6b8SYazen Ghannam csrow_nr_orig, dct, cs_mode);
332110de6497SBorislav Petkov edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
33220ec449eeSDoug Thompson
33230ec449eeSDoug Thompson return nr_pages;
33240ec449eeSDoug Thompson }
33250ec449eeSDoug Thompson
umc_init_csrows(struct mem_ctl_info * mci)33266fb8b5fbSMuralidhara M K static void umc_init_csrows(struct mem_ctl_info *mci)
3327353a1fcbSYazen Ghannam {
3328353a1fcbSYazen Ghannam struct amd64_pvt *pvt = mci->pvt_info;
3329353a1fcbSYazen Ghannam enum edac_type edac_mode = EDAC_NONE;
3330353a1fcbSYazen Ghannam enum dev_type dev_type = DEV_UNKNOWN;
3331353a1fcbSYazen Ghannam struct dimm_info *dimm;
3332353a1fcbSYazen Ghannam u8 umc, cs;
3333353a1fcbSYazen Ghannam
3334353a1fcbSYazen Ghannam if (mci->edac_ctl_cap & EDAC_FLAG_S16ECD16ED) {
3335353a1fcbSYazen Ghannam edac_mode = EDAC_S16ECD16ED;
3336353a1fcbSYazen Ghannam dev_type = DEV_X16;
3337353a1fcbSYazen Ghannam } else if (mci->edac_ctl_cap & EDAC_FLAG_S8ECD8ED) {
3338353a1fcbSYazen Ghannam edac_mode = EDAC_S8ECD8ED;
3339353a1fcbSYazen Ghannam dev_type = DEV_X8;
3340353a1fcbSYazen Ghannam } else if (mci->edac_ctl_cap & EDAC_FLAG_S4ECD4ED) {
3341353a1fcbSYazen Ghannam edac_mode = EDAC_S4ECD4ED;
3342353a1fcbSYazen Ghannam dev_type = DEV_X4;
3343353a1fcbSYazen Ghannam } else if (mci->edac_ctl_cap & EDAC_FLAG_SECDED) {
3344353a1fcbSYazen Ghannam edac_mode = EDAC_SECDED;
3345353a1fcbSYazen Ghannam }
3346353a1fcbSYazen Ghannam
3347353a1fcbSYazen Ghannam for_each_umc(umc) {
3348353a1fcbSYazen Ghannam for_each_chip_select(cs, umc, pvt) {
3349353a1fcbSYazen Ghannam if (!csrow_enabled(cs, umc, pvt))
3350353a1fcbSYazen Ghannam continue;
3351353a1fcbSYazen Ghannam
3352353a1fcbSYazen Ghannam dimm = mci->csrows[cs]->channels[umc]->dimm;
3353353a1fcbSYazen Ghannam
3354353a1fcbSYazen Ghannam edac_dbg(1, "MC node: %d, csrow: %d\n",
3355353a1fcbSYazen Ghannam pvt->mc_node_id, cs);
3356353a1fcbSYazen Ghannam
3357c0984666SYazen Ghannam dimm->nr_pages = umc_get_csrow_nr_pages(pvt, umc, cs);
335875aeaaf2SYazen Ghannam dimm->mtype = pvt->umc[umc].dram_type;
3359353a1fcbSYazen Ghannam dimm->edac_mode = edac_mode;
3360353a1fcbSYazen Ghannam dimm->dtype = dev_type;
3361466503d6SYazen Ghannam dimm->grain = 64;
3362353a1fcbSYazen Ghannam }
3363353a1fcbSYazen Ghannam }
3364353a1fcbSYazen Ghannam }
3365353a1fcbSYazen Ghannam
33660ec449eeSDoug Thompson /*
33670ec449eeSDoug Thompson * Initialize the array of csrow attribute instances, based on the values
33680ec449eeSDoug Thompson * from pci config hardware registers.
33690ec449eeSDoug Thompson */
dct_init_csrows(struct mem_ctl_info * mci)33706fb8b5fbSMuralidhara M K static void dct_init_csrows(struct mem_ctl_info *mci)
33710ec449eeSDoug Thompson {
337210de6497SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info;
33732d09d8f3SYazen Ghannam enum edac_type edac_mode = EDAC_NONE;
33740ec449eeSDoug Thompson struct csrow_info *csrow;
3375de3910ebSMauro Carvalho Chehab struct dimm_info *dimm;
3376a895bf8bSMauro Carvalho Chehab int nr_pages = 0;
33776fb8b5fbSMuralidhara M K int i, j;
337810de6497SBorislav Petkov u32 val;
33790ec449eeSDoug Thompson
3380a97fa68eSBorislav Petkov amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
33810ec449eeSDoug Thompson
33822299ef71SBorislav Petkov pvt->nbcfg = val;
33830ec449eeSDoug Thompson
3384956b9ba1SJoe Perches edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
33852299ef71SBorislav Petkov pvt->mc_node_id, val,
3386a97fa68eSBorislav Petkov !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
33870ec449eeSDoug Thompson
338810de6497SBorislav Petkov /*
338910de6497SBorislav Petkov * We iterate over DCT0 here but we look at DCT1 in parallel, if needed.
339010de6497SBorislav Petkov */
339111c75eadSBorislav Petkov for_each_chip_select(i, 0, pvt) {
339210de6497SBorislav Petkov bool row_dct0 = !!csrow_enabled(i, 0, pvt);
339310de6497SBorislav Petkov bool row_dct1 = false;
33940ec449eeSDoug Thompson
3395a4b4bedcSBorislav Petkov if (pvt->fam != 0xf)
339610de6497SBorislav Petkov row_dct1 = !!csrow_enabled(i, 1, pvt);
339710de6497SBorislav Petkov
339810de6497SBorislav Petkov if (!row_dct0 && !row_dct1)
33990ec449eeSDoug Thompson continue;
34000ec449eeSDoug Thompson
340110de6497SBorislav Petkov csrow = mci->csrows[i];
340211c75eadSBorislav Petkov
340310de6497SBorislav Petkov edac_dbg(1, "MC node: %d, csrow: %d\n",
340410de6497SBorislav Petkov pvt->mc_node_id, i);
340510de6497SBorislav Petkov
34061eef1282SMauro Carvalho Chehab if (row_dct0) {
3407c0984666SYazen Ghannam nr_pages = dct_get_csrow_nr_pages(pvt, 0, i);
34081eef1282SMauro Carvalho Chehab csrow->channels[0]->dimm->nr_pages = nr_pages;
34091eef1282SMauro Carvalho Chehab }
341010de6497SBorislav Petkov
341110de6497SBorislav Petkov /* K8 has only one DCT */
3412a4b4bedcSBorislav Petkov if (pvt->fam != 0xf && row_dct1) {
3413c0984666SYazen Ghannam int row_dct1_pages = dct_get_csrow_nr_pages(pvt, 1, i);
34141eef1282SMauro Carvalho Chehab
34151eef1282SMauro Carvalho Chehab csrow->channels[1]->dimm->nr_pages = row_dct1_pages;
34161eef1282SMauro Carvalho Chehab nr_pages += row_dct1_pages;
34171eef1282SMauro Carvalho Chehab }
34180ec449eeSDoug Thompson
341910de6497SBorislav Petkov edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
34200ec449eeSDoug Thompson
34212d09d8f3SYazen Ghannam /* Determine DIMM ECC mode: */
3422353a1fcbSYazen Ghannam if (pvt->nbcfg & NBCFG_ECC_ENABLE) {
34232d09d8f3SYazen Ghannam edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL)
34242d09d8f3SYazen Ghannam ? EDAC_S4ECD4ED
34252d09d8f3SYazen Ghannam : EDAC_SECDED;
34262d09d8f3SYazen Ghannam }
3427084a4fccSMauro Carvalho Chehab
3428ed623d55SMuralidhara M K for (j = 0; j < pvt->max_mcs; j++) {
3429de3910ebSMauro Carvalho Chehab dimm = csrow->channels[j]->dimm;
3430a597d2a5SAravind Gopalakrishnan dimm->mtype = pvt->dram_type;
3431de3910ebSMauro Carvalho Chehab dimm->edac_mode = edac_mode;
3432466503d6SYazen Ghannam dimm->grain = 64;
3433084a4fccSMauro Carvalho Chehab }
34340ec449eeSDoug Thompson }
34350ec449eeSDoug Thompson }
3436d27bf6faSDoug Thompson
343706724535SBorislav Petkov /* get all cores on this DCT */
get_cpus_on_this_dct_cpumask(struct cpumask * mask,u16 nid)34388b84c8dfSDaniel J Blueman static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid)
3439f9431992SDoug Thompson {
344006724535SBorislav Petkov int cpu;
3441f9431992SDoug Thompson
344206724535SBorislav Petkov for_each_online_cpu(cpu)
3443db970bd2SYazen Ghannam if (topology_die_id(cpu) == nid)
344406724535SBorislav Petkov cpumask_set_cpu(cpu, mask);
3445f9431992SDoug Thompson }
3446f9431992SDoug Thompson
3447f9431992SDoug Thompson /* check MCG_CTL on all the cpus on this node */
nb_mce_bank_enabled_on_node(u16 nid)3448d1ea71cdSBorislav Petkov static bool nb_mce_bank_enabled_on_node(u16 nid)
3449f9431992SDoug Thompson {
3450ba578cb3SRusty Russell cpumask_var_t mask;
345150542251SBorislav Petkov int cpu, nbe;
345206724535SBorislav Petkov bool ret = false;
3453f9431992SDoug Thompson
3454ba578cb3SRusty Russell if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
345524f9a7feSBorislav Petkov amd64_warn("%s: Error allocating mask\n", __func__);
345606724535SBorislav Petkov return false;
345706724535SBorislav Petkov }
345806724535SBorislav Petkov
3459ba578cb3SRusty Russell get_cpus_on_this_dct_cpumask(mask, nid);
346006724535SBorislav Petkov
3461ba578cb3SRusty Russell rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
3462ba578cb3SRusty Russell
3463ba578cb3SRusty Russell for_each_cpu(cpu, mask) {
346450542251SBorislav Petkov struct msr *reg = per_cpu_ptr(msrs, cpu);
34655980bb9cSBorislav Petkov nbe = reg->l & MSR_MCGCTL_NBE;
346606724535SBorislav Petkov
3467956b9ba1SJoe Perches edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
346850542251SBorislav Petkov cpu, reg->q,
346906724535SBorislav Petkov (nbe ? "enabled" : "disabled"));
347006724535SBorislav Petkov
347106724535SBorislav Petkov if (!nbe)
347206724535SBorislav Petkov goto out;
347306724535SBorislav Petkov }
347406724535SBorislav Petkov ret = true;
347506724535SBorislav Petkov
347606724535SBorislav Petkov out:
3477ba578cb3SRusty Russell free_cpumask_var(mask);
3478f9431992SDoug Thompson return ret;
3479f9431992SDoug Thompson }
3480f9431992SDoug Thompson
toggle_ecc_err_reporting(struct ecc_settings * s,u16 nid,bool on)3481c7e5301aSDaniel J Blueman static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on)
3482f6d6ae96SBorislav Petkov {
3483f6d6ae96SBorislav Petkov cpumask_var_t cmask;
348450542251SBorislav Petkov int cpu;
3485f6d6ae96SBorislav Petkov
3486f6d6ae96SBorislav Petkov if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
348724f9a7feSBorislav Petkov amd64_warn("%s: error allocating mask\n", __func__);
34880de27884SPan Bian return -ENOMEM;
3489f6d6ae96SBorislav Petkov }
3490f6d6ae96SBorislav Petkov
3491ae7bb7c6SBorislav Petkov get_cpus_on_this_dct_cpumask(cmask, nid);
3492f6d6ae96SBorislav Petkov
3493f6d6ae96SBorislav Petkov rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
3494f6d6ae96SBorislav Petkov
3495f6d6ae96SBorislav Petkov for_each_cpu(cpu, cmask) {
3496f6d6ae96SBorislav Petkov
349750542251SBorislav Petkov struct msr *reg = per_cpu_ptr(msrs, cpu);
349850542251SBorislav Petkov
3499f6d6ae96SBorislav Petkov if (on) {
35005980bb9cSBorislav Petkov if (reg->l & MSR_MCGCTL_NBE)
3501ae7bb7c6SBorislav Petkov s->flags.nb_mce_enable = 1;
3502f6d6ae96SBorislav Petkov
35035980bb9cSBorislav Petkov reg->l |= MSR_MCGCTL_NBE;
3504f6d6ae96SBorislav Petkov } else {
3505f6d6ae96SBorislav Petkov /*
3506d95cf4deSBorislav Petkov * Turn off NB MCE reporting only when it was off before
3507f6d6ae96SBorislav Petkov */
3508ae7bb7c6SBorislav Petkov if (!s->flags.nb_mce_enable)
35095980bb9cSBorislav Petkov reg->l &= ~MSR_MCGCTL_NBE;
3510f6d6ae96SBorislav Petkov }
3511f6d6ae96SBorislav Petkov }
3512f6d6ae96SBorislav Petkov wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
3513f6d6ae96SBorislav Petkov
3514f6d6ae96SBorislav Petkov free_cpumask_var(cmask);
3515f6d6ae96SBorislav Petkov
3516f6d6ae96SBorislav Petkov return 0;
3517f6d6ae96SBorislav Petkov }
3518f6d6ae96SBorislav Petkov
enable_ecc_error_reporting(struct ecc_settings * s,u16 nid,struct pci_dev * F3)3519c7e5301aSDaniel J Blueman static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid,
35202299ef71SBorislav Petkov struct pci_dev *F3)
3521f6d6ae96SBorislav Petkov {
35222299ef71SBorislav Petkov bool ret = true;
3523c9f4f26eSBorislav Petkov u32 value, mask = 0x3; /* UECC/CECC enable */
3524f6d6ae96SBorislav Petkov
35252299ef71SBorislav Petkov if (toggle_ecc_err_reporting(s, nid, ON)) {
35262299ef71SBorislav Petkov amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
35272299ef71SBorislav Petkov return false;
35282299ef71SBorislav Petkov }
35292299ef71SBorislav Petkov
3530c9f4f26eSBorislav Petkov amd64_read_pci_cfg(F3, NBCTL, &value);
3531f6d6ae96SBorislav Petkov
3532ae7bb7c6SBorislav Petkov s->old_nbctl = value & mask;
3533ae7bb7c6SBorislav Petkov s->nbctl_valid = true;
3534f6d6ae96SBorislav Petkov
3535f6d6ae96SBorislav Petkov value |= mask;
3536c9f4f26eSBorislav Petkov amd64_write_pci_cfg(F3, NBCTL, value);
3537f6d6ae96SBorislav Petkov
3538a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value);
3539f6d6ae96SBorislav Petkov
3540956b9ba1SJoe Perches edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
3541a97fa68eSBorislav Petkov nid, value, !!(value & NBCFG_ECC_ENABLE));
3542f6d6ae96SBorislav Petkov
3543a97fa68eSBorislav Petkov if (!(value & NBCFG_ECC_ENABLE)) {
354424f9a7feSBorislav Petkov amd64_warn("DRAM ECC disabled on this node, enabling...\n");
3545f6d6ae96SBorislav Petkov
3546ae7bb7c6SBorislav Petkov s->flags.nb_ecc_prev = 0;
3547d95cf4deSBorislav Petkov
3548f6d6ae96SBorislav Petkov /* Attempt to turn on DRAM ECC Enable */
3549a97fa68eSBorislav Petkov value |= NBCFG_ECC_ENABLE;
3550a97fa68eSBorislav Petkov amd64_write_pci_cfg(F3, NBCFG, value);
3551f6d6ae96SBorislav Petkov
3552a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value);
3553f6d6ae96SBorislav Petkov
3554a97fa68eSBorislav Petkov if (!(value & NBCFG_ECC_ENABLE)) {
355524f9a7feSBorislav Petkov amd64_warn("Hardware rejected DRAM ECC enable,"
355624f9a7feSBorislav Petkov "check memory DIMM configuration.\n");
35572299ef71SBorislav Petkov ret = false;
3558f6d6ae96SBorislav Petkov } else {
355924f9a7feSBorislav Petkov amd64_info("Hardware accepted DRAM ECC Enable\n");
3560f6d6ae96SBorislav Petkov }
3561d95cf4deSBorislav Petkov } else {
3562ae7bb7c6SBorislav Petkov s->flags.nb_ecc_prev = 1;
3563f6d6ae96SBorislav Petkov }
3564d95cf4deSBorislav Petkov
3565956b9ba1SJoe Perches edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
3566a97fa68eSBorislav Petkov nid, value, !!(value & NBCFG_ECC_ENABLE));
3567f6d6ae96SBorislav Petkov
35682299ef71SBorislav Petkov return ret;
3569f6d6ae96SBorislav Petkov }
3570f6d6ae96SBorislav Petkov
restore_ecc_error_reporting(struct ecc_settings * s,u16 nid,struct pci_dev * F3)3571c7e5301aSDaniel J Blueman static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
3572360b7f3cSBorislav Petkov struct pci_dev *F3)
3573f6d6ae96SBorislav Petkov {
3574c9f4f26eSBorislav Petkov u32 value, mask = 0x3; /* UECC/CECC enable */
3575c9f4f26eSBorislav Petkov
3576ae7bb7c6SBorislav Petkov if (!s->nbctl_valid)
3577f6d6ae96SBorislav Petkov return;
3578f6d6ae96SBorislav Petkov
3579c9f4f26eSBorislav Petkov amd64_read_pci_cfg(F3, NBCTL, &value);
3580f6d6ae96SBorislav Petkov value &= ~mask;
3581ae7bb7c6SBorislav Petkov value |= s->old_nbctl;
3582f6d6ae96SBorislav Petkov
3583c9f4f26eSBorislav Petkov amd64_write_pci_cfg(F3, NBCTL, value);
3584f6d6ae96SBorislav Petkov
3585ae7bb7c6SBorislav Petkov /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
3586ae7bb7c6SBorislav Petkov if (!s->flags.nb_ecc_prev) {
3587a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value);
3588a97fa68eSBorislav Petkov value &= ~NBCFG_ECC_ENABLE;
3589a97fa68eSBorislav Petkov amd64_write_pci_cfg(F3, NBCFG, value);
3590d95cf4deSBorislav Petkov }
3591d95cf4deSBorislav Petkov
3592d95cf4deSBorislav Petkov /* restore the NB Enable MCGCTL bit */
35932299ef71SBorislav Petkov if (toggle_ecc_err_reporting(s, nid, OFF))
359424f9a7feSBorislav Petkov amd64_warn("Error restoring NB MCGCTL settings!\n");
3595f6d6ae96SBorislav Petkov }
3596f6d6ae96SBorislav Petkov
dct_ecc_enabled(struct amd64_pvt * pvt)3597eb2bcdfcSMuralidhara M K static bool dct_ecc_enabled(struct amd64_pvt *pvt)
3598f9431992SDoug Thompson {
35991c9b08baSYazen Ghannam u16 nid = pvt->mc_node_id;
360006724535SBorislav Petkov bool nb_mce_en = false;
3601eb2bcdfcSMuralidhara M K u8 ecc_en = 0;
3602196b79fcSYazen Ghannam u32 value;
3603f9431992SDoug Thompson
3604eb2bcdfcSMuralidhara M K amd64_read_pci_cfg(pvt->F3, NBCFG, &value);
3605eb2bcdfcSMuralidhara M K
3606eb2bcdfcSMuralidhara M K ecc_en = !!(value & NBCFG_ECC_ENABLE);
3607eb2bcdfcSMuralidhara M K
3608eb2bcdfcSMuralidhara M K nb_mce_en = nb_mce_bank_enabled_on_node(nid);
3609eb2bcdfcSMuralidhara M K if (!nb_mce_en)
3610eb2bcdfcSMuralidhara M K edac_dbg(0, "NB MCE bank disabled, set MSR 0x%08x[4] on node %d to enable.\n",
3611eb2bcdfcSMuralidhara M K MSR_IA32_MCG_CTL, nid);
3612eb2bcdfcSMuralidhara M K
3613eb2bcdfcSMuralidhara M K edac_dbg(3, "Node %d: DRAM ECC %s.\n", nid, (ecc_en ? "enabled" : "disabled"));
3614eb2bcdfcSMuralidhara M K
3615eb2bcdfcSMuralidhara M K if (!ecc_en || !nb_mce_en)
3616eb2bcdfcSMuralidhara M K return false;
3617eb2bcdfcSMuralidhara M K else
3618eb2bcdfcSMuralidhara M K return true;
3619eb2bcdfcSMuralidhara M K }
3620eb2bcdfcSMuralidhara M K
umc_ecc_enabled(struct amd64_pvt * pvt)3621eb2bcdfcSMuralidhara M K static bool umc_ecc_enabled(struct amd64_pvt *pvt)
3622eb2bcdfcSMuralidhara M K {
3623196b79fcSYazen Ghannam u8 umc_en_mask = 0, ecc_en_mask = 0;
3624eb2bcdfcSMuralidhara M K u16 nid = pvt->mc_node_id;
36251c9b08baSYazen Ghannam struct amd64_umc *umc;
3626eb2bcdfcSMuralidhara M K u8 ecc_en = 0, i;
3627196b79fcSYazen Ghannam
36284d30d2bcSYazen Ghannam for_each_umc(i) {
36291c9b08baSYazen Ghannam umc = &pvt->umc[i];
3630196b79fcSYazen Ghannam
3631196b79fcSYazen Ghannam /* Only check enabled UMCs. */
36321c9b08baSYazen Ghannam if (!(umc->sdp_ctrl & UMC_SDP_INIT))
3633196b79fcSYazen Ghannam continue;
3634196b79fcSYazen Ghannam
3635196b79fcSYazen Ghannam umc_en_mask |= BIT(i);
3636196b79fcSYazen Ghannam
36371c9b08baSYazen Ghannam if (umc->umc_cap_hi & UMC_ECC_ENABLED)
3638196b79fcSYazen Ghannam ecc_en_mask |= BIT(i);
3639196b79fcSYazen Ghannam }
3640196b79fcSYazen Ghannam
3641196b79fcSYazen Ghannam /* Check whether at least one UMC is enabled: */
3642196b79fcSYazen Ghannam if (umc_en_mask)
3643196b79fcSYazen Ghannam ecc_en = umc_en_mask == ecc_en_mask;
364411ab1caeSYazen Ghannam else
364511ab1caeSYazen Ghannam edac_dbg(0, "Node %d: No enabled UMCs.\n", nid);
3646196b79fcSYazen Ghannam
36474cbcb73bSBorislav Petkov edac_dbg(3, "Node %d: DRAM ECC %s.\n", nid, (ecc_en ? "enabled" : "disabled"));
3648be3468e8SBorislav Petkov
3649eb2bcdfcSMuralidhara M K if (!ecc_en)
36502299ef71SBorislav Petkov return false;
36517fdfee92SBorislav Petkov else
36522299ef71SBorislav Petkov return true;
3653f9431992SDoug Thompson }
3654f9431992SDoug Thompson
36552d09d8f3SYazen Ghannam static inline void
umc_determine_edac_ctl_cap(struct mem_ctl_info * mci,struct amd64_pvt * pvt)36569369239eSYazen Ghannam umc_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt)
36572d09d8f3SYazen Ghannam {
3658f8be8e56SYazen Ghannam u8 i, ecc_en = 1, cpk_en = 1, dev_x4 = 1, dev_x16 = 1;
36592d09d8f3SYazen Ghannam
36604d30d2bcSYazen Ghannam for_each_umc(i) {
36612d09d8f3SYazen Ghannam if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
36622d09d8f3SYazen Ghannam ecc_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_ENABLED);
36632d09d8f3SYazen Ghannam cpk_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_CHIPKILL_CAP);
3664f8be8e56SYazen Ghannam
3665f8be8e56SYazen Ghannam dev_x4 &= !!(pvt->umc[i].dimm_cfg & BIT(6));
3666f8be8e56SYazen Ghannam dev_x16 &= !!(pvt->umc[i].dimm_cfg & BIT(7));
36672d09d8f3SYazen Ghannam }
36682d09d8f3SYazen Ghannam }
36692d09d8f3SYazen Ghannam
36702d09d8f3SYazen Ghannam /* Set chipkill only if ECC is enabled: */
36712d09d8f3SYazen Ghannam if (ecc_en) {
36722d09d8f3SYazen Ghannam mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
36732d09d8f3SYazen Ghannam
3674f8be8e56SYazen Ghannam if (!cpk_en)
3675f8be8e56SYazen Ghannam return;
3676f8be8e56SYazen Ghannam
3677f8be8e56SYazen Ghannam if (dev_x4)
36782d09d8f3SYazen Ghannam mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
3679f8be8e56SYazen Ghannam else if (dev_x16)
3680f8be8e56SYazen Ghannam mci->edac_ctl_cap |= EDAC_FLAG_S16ECD16ED;
3681f8be8e56SYazen Ghannam else
3682f8be8e56SYazen Ghannam mci->edac_ctl_cap |= EDAC_FLAG_S8ECD8ED;
36832d09d8f3SYazen Ghannam }
36842d09d8f3SYazen Ghannam }
36852d09d8f3SYazen Ghannam
dct_setup_mci_misc_attrs(struct mem_ctl_info * mci)36860a42a37fSMuralidhara M K static void dct_setup_mci_misc_attrs(struct mem_ctl_info *mci)
36877d6034d3SDoug Thompson {
36887d6034d3SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info;
36897d6034d3SDoug Thompson
36907d6034d3SDoug Thompson mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
36917d6034d3SDoug Thompson mci->edac_ctl_cap = EDAC_FLAG_NONE;
36927d6034d3SDoug Thompson
36935980bb9cSBorislav Petkov if (pvt->nbcap & NBCAP_SECDED)
36947d6034d3SDoug Thompson mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
36957d6034d3SDoug Thompson
36965980bb9cSBorislav Petkov if (pvt->nbcap & NBCAP_CHIPKILL)
36977d6034d3SDoug Thompson mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
36987d6034d3SDoug Thompson
3699f6a4b4a1SMuralidhara M K mci->edac_cap = dct_determine_edac_cap(pvt);
37007d6034d3SDoug Thompson mci->mod_name = EDAC_MOD_STR;
3701ed623d55SMuralidhara M K mci->ctl_name = pvt->ctl_name;
3702e7934b70SYazen Ghannam mci->dev_name = pci_name(pvt->F3);
37037d6034d3SDoug Thompson mci->ctl_page_to_phys = NULL;
37047d6034d3SDoug Thompson
37057d6034d3SDoug Thompson /* memory scrubber interface */
3706d1ea71cdSBorislav Petkov mci->set_sdram_scrub_rate = set_scrub_rate;
3707d1ea71cdSBorislav Petkov mci->get_sdram_scrub_rate = get_scrub_rate;
37086fb8b5fbSMuralidhara M K
37096fb8b5fbSMuralidhara M K dct_init_csrows(mci);
37107d6034d3SDoug Thompson }
37117d6034d3SDoug Thompson
umc_setup_mci_misc_attrs(struct mem_ctl_info * mci)37120a42a37fSMuralidhara M K static void umc_setup_mci_misc_attrs(struct mem_ctl_info *mci)
37130a42a37fSMuralidhara M K {
37140a42a37fSMuralidhara M K struct amd64_pvt *pvt = mci->pvt_info;
37150a42a37fSMuralidhara M K
37160a42a37fSMuralidhara M K mci->mtype_cap = MEM_FLAG_DDR4 | MEM_FLAG_RDDR4;
37170a42a37fSMuralidhara M K mci->edac_ctl_cap = EDAC_FLAG_NONE;
37180a42a37fSMuralidhara M K
37199369239eSYazen Ghannam umc_determine_edac_ctl_cap(mci, pvt);
37200a42a37fSMuralidhara M K
3721f6a4b4a1SMuralidhara M K mci->edac_cap = umc_determine_edac_cap(pvt);
37220a42a37fSMuralidhara M K mci->mod_name = EDAC_MOD_STR;
37230a42a37fSMuralidhara M K mci->ctl_name = pvt->ctl_name;
37240a42a37fSMuralidhara M K mci->dev_name = pci_name(pvt->F3);
37250a42a37fSMuralidhara M K mci->ctl_page_to_phys = NULL;
37266fb8b5fbSMuralidhara M K
37276fb8b5fbSMuralidhara M K umc_init_csrows(mci);
37280a42a37fSMuralidhara M K }
37290a42a37fSMuralidhara M K
dct_hw_info_get(struct amd64_pvt * pvt)37309a97a7f4SYazen Ghannam static int dct_hw_info_get(struct amd64_pvt *pvt)
37319a97a7f4SYazen Ghannam {
37329a97a7f4SYazen Ghannam int ret = reserve_mc_sibling_devs(pvt, pvt->f1_id, pvt->f2_id);
37339a97a7f4SYazen Ghannam
37349a97a7f4SYazen Ghannam if (ret)
37359a97a7f4SYazen Ghannam return ret;
37369a97a7f4SYazen Ghannam
3737637f60efSMuralidhara M K dct_prep_chip_selects(pvt);
3738b29dad9bSMuralidhara M K dct_read_base_mask(pvt);
373932ecdf86SMuralidhara M K dct_read_mc_regs(pvt);
374078ec161aSMuralidhara M K dct_determine_memory_type(pvt);
37419a97a7f4SYazen Ghannam
37429a97a7f4SYazen Ghannam return 0;
37439a97a7f4SYazen Ghannam }
37449a97a7f4SYazen Ghannam
umc_hw_info_get(struct amd64_pvt * pvt)37459a97a7f4SYazen Ghannam static int umc_hw_info_get(struct amd64_pvt *pvt)
37469a97a7f4SYazen Ghannam {
37479a97a7f4SYazen Ghannam pvt->umc = kcalloc(pvt->max_mcs, sizeof(struct amd64_umc), GFP_KERNEL);
37489a97a7f4SYazen Ghannam if (!pvt->umc)
37499a97a7f4SYazen Ghannam return -ENOMEM;
37509a97a7f4SYazen Ghannam
3751637f60efSMuralidhara M K umc_prep_chip_selects(pvt);
3752b29dad9bSMuralidhara M K umc_read_base_mask(pvt);
375332ecdf86SMuralidhara M K umc_read_mc_regs(pvt);
375478ec161aSMuralidhara M K umc_determine_memory_type(pvt);
37559a97a7f4SYazen Ghannam
37569a97a7f4SYazen Ghannam return 0;
37579a97a7f4SYazen Ghannam }
37589a97a7f4SYazen Ghannam
37599c42edd5SMuralidhara M K /*
37609c42edd5SMuralidhara M K * The CPUs have one channel per UMC, so UMC number is equivalent to a
37619c42edd5SMuralidhara M K * channel number. The GPUs have 8 channels per UMC, so the UMC number no
37629c42edd5SMuralidhara M K * longer works as a channel number.
37639c42edd5SMuralidhara M K *
37649c42edd5SMuralidhara M K * The channel number within a GPU UMC is given in MCA_IPID[15:12].
37659c42edd5SMuralidhara M K * However, the IDs are split such that two UMC values go to one UMC, and
37669c42edd5SMuralidhara M K * the channel numbers are split in two groups of four.
37679c42edd5SMuralidhara M K *
37689c42edd5SMuralidhara M K * Refer to comment on gpu_get_umc_base().
37699c42edd5SMuralidhara M K *
37709c42edd5SMuralidhara M K * For example,
37719c42edd5SMuralidhara M K * UMC0 CH[3:0] = 0x0005[3:0]000
37729c42edd5SMuralidhara M K * UMC0 CH[7:4] = 0x0015[3:0]000
37739c42edd5SMuralidhara M K * UMC1 CH[3:0] = 0x0025[3:0]000
37749c42edd5SMuralidhara M K * UMC1 CH[7:4] = 0x0035[3:0]000
37759c42edd5SMuralidhara M K */
gpu_get_err_info(struct mce * m,struct err_info * err)37769c42edd5SMuralidhara M K static void gpu_get_err_info(struct mce *m, struct err_info *err)
37779c42edd5SMuralidhara M K {
37789c42edd5SMuralidhara M K u8 ch = (m->ipid & GENMASK(31, 0)) >> 20;
37799c42edd5SMuralidhara M K u8 phy = ((m->ipid >> 12) & 0xf);
37809c42edd5SMuralidhara M K
37819c42edd5SMuralidhara M K err->channel = ch % 2 ? phy + 4 : phy;
37829c42edd5SMuralidhara M K err->csrow = phy;
37839c42edd5SMuralidhara M K }
37849c42edd5SMuralidhara M K
gpu_addr_mask_to_cs_size(struct amd64_pvt * pvt,u8 umc,unsigned int cs_mode,int csrow_nr)37859c42edd5SMuralidhara M K static int gpu_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc,
37869c42edd5SMuralidhara M K unsigned int cs_mode, int csrow_nr)
37879c42edd5SMuralidhara M K {
37889c42edd5SMuralidhara M K u32 addr_mask_orig = pvt->csels[umc].csmasks[csrow_nr];
37899c42edd5SMuralidhara M K
37909c42edd5SMuralidhara M K return __addr_mask_to_cs_size(addr_mask_orig, cs_mode, csrow_nr, csrow_nr >> 1);
37919c42edd5SMuralidhara M K }
37929c42edd5SMuralidhara M K
gpu_debug_display_dimm_sizes(struct amd64_pvt * pvt,u8 ctrl)37939c42edd5SMuralidhara M K static void gpu_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
37949c42edd5SMuralidhara M K {
37959c42edd5SMuralidhara M K int size, cs_mode, cs = 0;
37969c42edd5SMuralidhara M K
37979c42edd5SMuralidhara M K edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:\n", ctrl);
37989c42edd5SMuralidhara M K
37999c42edd5SMuralidhara M K cs_mode = CS_EVEN_PRIMARY | CS_ODD_PRIMARY;
38009c42edd5SMuralidhara M K
38019c42edd5SMuralidhara M K for_each_chip_select(cs, ctrl, pvt) {
38029c42edd5SMuralidhara M K size = gpu_addr_mask_to_cs_size(pvt, ctrl, cs_mode, cs);
38039c42edd5SMuralidhara M K amd64_info(EDAC_MC ": %d: %5dMB\n", cs, size);
38049c42edd5SMuralidhara M K }
38059c42edd5SMuralidhara M K }
38069c42edd5SMuralidhara M K
gpu_dump_misc_regs(struct amd64_pvt * pvt)38079c42edd5SMuralidhara M K static void gpu_dump_misc_regs(struct amd64_pvt *pvt)
38089c42edd5SMuralidhara M K {
38099c42edd5SMuralidhara M K struct amd64_umc *umc;
38109c42edd5SMuralidhara M K u32 i;
38119c42edd5SMuralidhara M K
38129c42edd5SMuralidhara M K for_each_umc(i) {
38139c42edd5SMuralidhara M K umc = &pvt->umc[i];
38149c42edd5SMuralidhara M K
38159c42edd5SMuralidhara M K edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i, umc->umc_cfg);
38169c42edd5SMuralidhara M K edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i, umc->sdp_ctrl);
38179c42edd5SMuralidhara M K edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i, umc->ecc_ctrl);
38189c42edd5SMuralidhara M K edac_dbg(1, "UMC%d All HBMs support ECC: yes\n", i);
38199c42edd5SMuralidhara M K
38209c42edd5SMuralidhara M K gpu_debug_display_dimm_sizes(pvt, i);
38219c42edd5SMuralidhara M K }
38229c42edd5SMuralidhara M K }
38239c42edd5SMuralidhara M K
gpu_get_csrow_nr_pages(struct amd64_pvt * pvt,u8 dct,int csrow_nr)38249c42edd5SMuralidhara M K static u32 gpu_get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
38259c42edd5SMuralidhara M K {
38269c42edd5SMuralidhara M K u32 nr_pages;
38279c42edd5SMuralidhara M K int cs_mode = CS_EVEN_PRIMARY | CS_ODD_PRIMARY;
38289c42edd5SMuralidhara M K
38299c42edd5SMuralidhara M K nr_pages = gpu_addr_mask_to_cs_size(pvt, dct, cs_mode, csrow_nr);
38309c42edd5SMuralidhara M K nr_pages <<= 20 - PAGE_SHIFT;
38319c42edd5SMuralidhara M K
38329c42edd5SMuralidhara M K edac_dbg(0, "csrow: %d, channel: %d\n", csrow_nr, dct);
38339c42edd5SMuralidhara M K edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
38349c42edd5SMuralidhara M K
38359c42edd5SMuralidhara M K return nr_pages;
38369c42edd5SMuralidhara M K }
38379c42edd5SMuralidhara M K
gpu_init_csrows(struct mem_ctl_info * mci)38389c42edd5SMuralidhara M K static void gpu_init_csrows(struct mem_ctl_info *mci)
38399c42edd5SMuralidhara M K {
38409c42edd5SMuralidhara M K struct amd64_pvt *pvt = mci->pvt_info;
38419c42edd5SMuralidhara M K struct dimm_info *dimm;
38429c42edd5SMuralidhara M K u8 umc, cs;
38439c42edd5SMuralidhara M K
38449c42edd5SMuralidhara M K for_each_umc(umc) {
38459c42edd5SMuralidhara M K for_each_chip_select(cs, umc, pvt) {
38469c42edd5SMuralidhara M K if (!csrow_enabled(cs, umc, pvt))
38479c42edd5SMuralidhara M K continue;
38489c42edd5SMuralidhara M K
38499c42edd5SMuralidhara M K dimm = mci->csrows[umc]->channels[cs]->dimm;
38509c42edd5SMuralidhara M K
38519c42edd5SMuralidhara M K edac_dbg(1, "MC node: %d, csrow: %d\n",
38529c42edd5SMuralidhara M K pvt->mc_node_id, cs);
38539c42edd5SMuralidhara M K
38549c42edd5SMuralidhara M K dimm->nr_pages = gpu_get_csrow_nr_pages(pvt, umc, cs);
38559c42edd5SMuralidhara M K dimm->edac_mode = EDAC_SECDED;
38569c42edd5SMuralidhara M K dimm->mtype = MEM_HBM2;
38579c42edd5SMuralidhara M K dimm->dtype = DEV_X16;
38589c42edd5SMuralidhara M K dimm->grain = 64;
38599c42edd5SMuralidhara M K }
38609c42edd5SMuralidhara M K }
38619c42edd5SMuralidhara M K }
38629c42edd5SMuralidhara M K
gpu_setup_mci_misc_attrs(struct mem_ctl_info * mci)38639c42edd5SMuralidhara M K static void gpu_setup_mci_misc_attrs(struct mem_ctl_info *mci)
38649c42edd5SMuralidhara M K {
38659c42edd5SMuralidhara M K struct amd64_pvt *pvt = mci->pvt_info;
38669c42edd5SMuralidhara M K
38679c42edd5SMuralidhara M K mci->mtype_cap = MEM_FLAG_HBM2;
38689c42edd5SMuralidhara M K mci->edac_ctl_cap = EDAC_FLAG_SECDED;
38699c42edd5SMuralidhara M K
38709c42edd5SMuralidhara M K mci->edac_cap = EDAC_FLAG_EC;
38719c42edd5SMuralidhara M K mci->mod_name = EDAC_MOD_STR;
38729c42edd5SMuralidhara M K mci->ctl_name = pvt->ctl_name;
38739c42edd5SMuralidhara M K mci->dev_name = pci_name(pvt->F3);
38749c42edd5SMuralidhara M K mci->ctl_page_to_phys = NULL;
38759c42edd5SMuralidhara M K
38769c42edd5SMuralidhara M K gpu_init_csrows(mci);
38779c42edd5SMuralidhara M K }
38789c42edd5SMuralidhara M K
38799c42edd5SMuralidhara M K /* ECC is enabled by default on GPU nodes */
gpu_ecc_enabled(struct amd64_pvt * pvt)38809c42edd5SMuralidhara M K static bool gpu_ecc_enabled(struct amd64_pvt *pvt)
38819c42edd5SMuralidhara M K {
38829c42edd5SMuralidhara M K return true;
38839c42edd5SMuralidhara M K }
38849c42edd5SMuralidhara M K
gpu_get_umc_base(u8 umc,u8 channel)38859c42edd5SMuralidhara M K static inline u32 gpu_get_umc_base(u8 umc, u8 channel)
38869c42edd5SMuralidhara M K {
38879c42edd5SMuralidhara M K /*
38889c42edd5SMuralidhara M K * On CPUs, there is one channel per UMC, so UMC numbering equals
38899c42edd5SMuralidhara M K * channel numbering. On GPUs, there are eight channels per UMC,
38909c42edd5SMuralidhara M K * so the channel numbering is different from UMC numbering.
38919c42edd5SMuralidhara M K *
38929c42edd5SMuralidhara M K * On CPU nodes channels are selected in 6th nibble
38939c42edd5SMuralidhara M K * UMC chY[3:0]= [(chY*2 + 1) : (chY*2)]50000;
38949c42edd5SMuralidhara M K *
38959c42edd5SMuralidhara M K * On GPU nodes channels are selected in 3rd nibble
38969c42edd5SMuralidhara M K * HBM chX[3:0]= [Y ]5X[3:0]000;
38979c42edd5SMuralidhara M K * HBM chX[7:4]= [Y+1]5X[3:0]000
38989c42edd5SMuralidhara M K */
38999c42edd5SMuralidhara M K umc *= 2;
39009c42edd5SMuralidhara M K
39019c42edd5SMuralidhara M K if (channel >= 4)
39029c42edd5SMuralidhara M K umc++;
39039c42edd5SMuralidhara M K
39049c42edd5SMuralidhara M K return 0x50000 + (umc << 20) + ((channel % 4) << 12);
39059c42edd5SMuralidhara M K }
39069c42edd5SMuralidhara M K
gpu_read_mc_regs(struct amd64_pvt * pvt)39079c42edd5SMuralidhara M K static void gpu_read_mc_regs(struct amd64_pvt *pvt)
39089c42edd5SMuralidhara M K {
39099c42edd5SMuralidhara M K u8 nid = pvt->mc_node_id;
39109c42edd5SMuralidhara M K struct amd64_umc *umc;
39119c42edd5SMuralidhara M K u32 i, umc_base;
39129c42edd5SMuralidhara M K
39139c42edd5SMuralidhara M K /* Read registers from each UMC */
39149c42edd5SMuralidhara M K for_each_umc(i) {
39159c42edd5SMuralidhara M K umc_base = gpu_get_umc_base(i, 0);
39169c42edd5SMuralidhara M K umc = &pvt->umc[i];
39179c42edd5SMuralidhara M K
39189c42edd5SMuralidhara M K amd_smn_read(nid, umc_base + UMCCH_UMC_CFG, &umc->umc_cfg);
39199c42edd5SMuralidhara M K amd_smn_read(nid, umc_base + UMCCH_SDP_CTRL, &umc->sdp_ctrl);
39209c42edd5SMuralidhara M K amd_smn_read(nid, umc_base + UMCCH_ECC_CTRL, &umc->ecc_ctrl);
39219c42edd5SMuralidhara M K }
39229c42edd5SMuralidhara M K }
39239c42edd5SMuralidhara M K
gpu_read_base_mask(struct amd64_pvt * pvt)39249c42edd5SMuralidhara M K static void gpu_read_base_mask(struct amd64_pvt *pvt)
39259c42edd5SMuralidhara M K {
39269c42edd5SMuralidhara M K u32 base_reg, mask_reg;
39279c42edd5SMuralidhara M K u32 *base, *mask;
39289c42edd5SMuralidhara M K int umc, cs;
39299c42edd5SMuralidhara M K
39309c42edd5SMuralidhara M K for_each_umc(umc) {
39319c42edd5SMuralidhara M K for_each_chip_select(cs, umc, pvt) {
39329c42edd5SMuralidhara M K base_reg = gpu_get_umc_base(umc, cs) + UMCCH_BASE_ADDR;
39339c42edd5SMuralidhara M K base = &pvt->csels[umc].csbases[cs];
39349c42edd5SMuralidhara M K
39359c42edd5SMuralidhara M K if (!amd_smn_read(pvt->mc_node_id, base_reg, base)) {
39369c42edd5SMuralidhara M K edac_dbg(0, " DCSB%d[%d]=0x%08x reg: 0x%x\n",
39379c42edd5SMuralidhara M K umc, cs, *base, base_reg);
39389c42edd5SMuralidhara M K }
39399c42edd5SMuralidhara M K
39409c42edd5SMuralidhara M K mask_reg = gpu_get_umc_base(umc, cs) + UMCCH_ADDR_MASK;
39419c42edd5SMuralidhara M K mask = &pvt->csels[umc].csmasks[cs];
39429c42edd5SMuralidhara M K
39439c42edd5SMuralidhara M K if (!amd_smn_read(pvt->mc_node_id, mask_reg, mask)) {
39449c42edd5SMuralidhara M K edac_dbg(0, " DCSM%d[%d]=0x%08x reg: 0x%x\n",
39459c42edd5SMuralidhara M K umc, cs, *mask, mask_reg);
39469c42edd5SMuralidhara M K }
39479c42edd5SMuralidhara M K }
39489c42edd5SMuralidhara M K }
39499c42edd5SMuralidhara M K }
39509c42edd5SMuralidhara M K
gpu_prep_chip_selects(struct amd64_pvt * pvt)39519c42edd5SMuralidhara M K static void gpu_prep_chip_selects(struct amd64_pvt *pvt)
39529c42edd5SMuralidhara M K {
39539c42edd5SMuralidhara M K int umc;
39549c42edd5SMuralidhara M K
39559c42edd5SMuralidhara M K for_each_umc(umc) {
39569c42edd5SMuralidhara M K pvt->csels[umc].b_cnt = 8;
39579c42edd5SMuralidhara M K pvt->csels[umc].m_cnt = 8;
39589c42edd5SMuralidhara M K }
39599c42edd5SMuralidhara M K }
39609c42edd5SMuralidhara M K
gpu_hw_info_get(struct amd64_pvt * pvt)39619c42edd5SMuralidhara M K static int gpu_hw_info_get(struct amd64_pvt *pvt)
39629c42edd5SMuralidhara M K {
39634251566eSYazen Ghannam int ret;
39644251566eSYazen Ghannam
39654251566eSYazen Ghannam ret = gpu_get_node_map();
39664251566eSYazen Ghannam if (ret)
39674251566eSYazen Ghannam return ret;
39684251566eSYazen Ghannam
39699c42edd5SMuralidhara M K pvt->umc = kcalloc(pvt->max_mcs, sizeof(struct amd64_umc), GFP_KERNEL);
39709c42edd5SMuralidhara M K if (!pvt->umc)
39719c42edd5SMuralidhara M K return -ENOMEM;
39729c42edd5SMuralidhara M K
39739c42edd5SMuralidhara M K gpu_prep_chip_selects(pvt);
39749c42edd5SMuralidhara M K gpu_read_base_mask(pvt);
39759c42edd5SMuralidhara M K gpu_read_mc_regs(pvt);
39769c42edd5SMuralidhara M K
39779c42edd5SMuralidhara M K return 0;
39789c42edd5SMuralidhara M K }
39799c42edd5SMuralidhara M K
hw_info_put(struct amd64_pvt * pvt)39809a97a7f4SYazen Ghannam static void hw_info_put(struct amd64_pvt *pvt)
39819a97a7f4SYazen Ghannam {
39829a97a7f4SYazen Ghannam pci_dev_put(pvt->F1);
39839a97a7f4SYazen Ghannam pci_dev_put(pvt->F2);
39849a97a7f4SYazen Ghannam kfree(pvt->umc);
39859a97a7f4SYazen Ghannam }
39869a97a7f4SYazen Ghannam
3987ed623d55SMuralidhara M K static struct low_ops umc_ops = {
39889a97a7f4SYazen Ghannam .hw_info_get = umc_hw_info_get,
3989eb2bcdfcSMuralidhara M K .ecc_enabled = umc_ecc_enabled,
39900a42a37fSMuralidhara M K .setup_mci_misc_attrs = umc_setup_mci_misc_attrs,
3991f6f36382SMuralidhara M K .dump_misc_regs = umc_dump_misc_regs,
3992b3ece3a6SMuralidhara M K .get_err_info = umc_get_err_info,
3993ed623d55SMuralidhara M K };
3994ed623d55SMuralidhara M K
39959c42edd5SMuralidhara M K static struct low_ops gpu_ops = {
39969c42edd5SMuralidhara M K .hw_info_get = gpu_hw_info_get,
39979c42edd5SMuralidhara M K .ecc_enabled = gpu_ecc_enabled,
39989c42edd5SMuralidhara M K .setup_mci_misc_attrs = gpu_setup_mci_misc_attrs,
39999c42edd5SMuralidhara M K .dump_misc_regs = gpu_dump_misc_regs,
40009c42edd5SMuralidhara M K .get_err_info = gpu_get_err_info,
40019c42edd5SMuralidhara M K };
40029c42edd5SMuralidhara M K
4003ed623d55SMuralidhara M K /* Use Family 16h versions for defaults and adjust as needed below. */
4004ed623d55SMuralidhara M K static struct low_ops dct_ops = {
4005ed623d55SMuralidhara M K .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
4006ed623d55SMuralidhara M K .dbam_to_cs = f16_dbam_to_chip_select,
40079a97a7f4SYazen Ghannam .hw_info_get = dct_hw_info_get,
4008eb2bcdfcSMuralidhara M K .ecc_enabled = dct_ecc_enabled,
40090a42a37fSMuralidhara M K .setup_mci_misc_attrs = dct_setup_mci_misc_attrs,
4010f6f36382SMuralidhara M K .dump_misc_regs = dct_dump_misc_regs,
4011ed623d55SMuralidhara M K };
4012ed623d55SMuralidhara M K
per_family_init(struct amd64_pvt * pvt)4013ed623d55SMuralidhara M K static int per_family_init(struct amd64_pvt *pvt)
4014395ae783SBorislav Petkov {
401518b94f66SAravind Gopalakrishnan pvt->ext_model = boot_cpu_data.x86_model >> 4;
4016b399151cSJia Zhang pvt->stepping = boot_cpu_data.x86_stepping;
401718b94f66SAravind Gopalakrishnan pvt->model = boot_cpu_data.x86_model;
401818b94f66SAravind Gopalakrishnan pvt->fam = boot_cpu_data.x86;
4019ed623d55SMuralidhara M K pvt->max_mcs = 2;
4020ed623d55SMuralidhara M K
4021ed623d55SMuralidhara M K /*
4022ed623d55SMuralidhara M K * Decide on which ops group to use here and do any family/model
4023ed623d55SMuralidhara M K * overrides below.
4024ed623d55SMuralidhara M K */
4025ed623d55SMuralidhara M K if (pvt->fam >= 0x17)
4026ed623d55SMuralidhara M K pvt->ops = &umc_ops;
4027ed623d55SMuralidhara M K else
4028ed623d55SMuralidhara M K pvt->ops = &dct_ops;
402918b94f66SAravind Gopalakrishnan
403018b94f66SAravind Gopalakrishnan switch (pvt->fam) {
4031395ae783SBorislav Petkov case 0xf:
4032ed623d55SMuralidhara M K pvt->ctl_name = (pvt->ext_model >= K8_REV_F) ?
4033ed623d55SMuralidhara M K "K8 revF or later" : "K8 revE or earlier";
4034ed623d55SMuralidhara M K pvt->f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP;
4035ed623d55SMuralidhara M K pvt->f2_id = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL;
4036ed623d55SMuralidhara M K pvt->ops->map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow;
4037ed623d55SMuralidhara M K pvt->ops->dbam_to_cs = k8_dbam_to_chip_select;
4038395ae783SBorislav Petkov break;
4039df71a053SBorislav Petkov
4040395ae783SBorislav Petkov case 0x10:
4041ed623d55SMuralidhara M K pvt->ctl_name = "F10h";
4042ed623d55SMuralidhara M K pvt->f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP;
4043ed623d55SMuralidhara M K pvt->f2_id = PCI_DEVICE_ID_AMD_10H_NB_DRAM;
4044ed623d55SMuralidhara M K pvt->ops->dbam_to_cs = f10_dbam_to_chip_select;
4045df71a053SBorislav Petkov break;
4046df71a053SBorislav Petkov
4047df71a053SBorislav Petkov case 0x15:
4048ed623d55SMuralidhara M K switch (pvt->model) {
4049ed623d55SMuralidhara M K case 0x30:
4050ed623d55SMuralidhara M K pvt->ctl_name = "F15h_M30h";
4051ed623d55SMuralidhara M K pvt->f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1;
4052ed623d55SMuralidhara M K pvt->f2_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2;
405318b94f66SAravind Gopalakrishnan break;
4054ed623d55SMuralidhara M K case 0x60:
4055ed623d55SMuralidhara M K pvt->ctl_name = "F15h_M60h";
4056ed623d55SMuralidhara M K pvt->f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1;
4057ed623d55SMuralidhara M K pvt->f2_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F2;
4058ed623d55SMuralidhara M K pvt->ops->dbam_to_cs = f15_m60h_dbam_to_chip_select;
4059a597d2a5SAravind Gopalakrishnan break;
4060ed623d55SMuralidhara M K case 0x13:
40616c13d7ffSBorislav Petkov /* Richland is only client */
4062ed623d55SMuralidhara M K return -ENODEV;
4063ed623d55SMuralidhara M K default:
4064ed623d55SMuralidhara M K pvt->ctl_name = "F15h";
4065ed623d55SMuralidhara M K pvt->f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1;
4066ed623d55SMuralidhara M K pvt->f2_id = PCI_DEVICE_ID_AMD_15H_NB_F2;
4067ed623d55SMuralidhara M K pvt->ops->dbam_to_cs = f15_dbam_to_chip_select;
4068ed623d55SMuralidhara M K break;
40696c13d7ffSBorislav Petkov }
4070395ae783SBorislav Petkov break;
4071395ae783SBorislav Petkov
407294c1acf2SAravind Gopalakrishnan case 0x16:
4073ed623d55SMuralidhara M K switch (pvt->model) {
4074ed623d55SMuralidhara M K case 0x30:
4075ed623d55SMuralidhara M K pvt->ctl_name = "F16h_M30h";
4076ed623d55SMuralidhara M K pvt->f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1;
4077ed623d55SMuralidhara M K pvt->f2_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2;
4078ed623d55SMuralidhara M K break;
4079ed623d55SMuralidhara M K default:
4080ed623d55SMuralidhara M K pvt->ctl_name = "F16h";
4081ed623d55SMuralidhara M K pvt->f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1;
4082ed623d55SMuralidhara M K pvt->f2_id = PCI_DEVICE_ID_AMD_16H_NB_F2;
408385a8885bSAravind Gopalakrishnan break;
408485a8885bSAravind Gopalakrishnan }
408594c1acf2SAravind Gopalakrishnan break;
408694c1acf2SAravind Gopalakrishnan
4087f1cbbec9SYazen Ghannam case 0x17:
4088ed623d55SMuralidhara M K switch (pvt->model) {
4089ed623d55SMuralidhara M K case 0x10 ... 0x2f:
4090ed623d55SMuralidhara M K pvt->ctl_name = "F17h_M10h";
40918960de4aSMichael Jin break;
4092ed623d55SMuralidhara M K case 0x30 ... 0x3f:
4093ed623d55SMuralidhara M K pvt->ctl_name = "F17h_M30h";
4094ed623d55SMuralidhara M K pvt->max_mcs = 8;
40956e846239SYazen Ghannam break;
4096ed623d55SMuralidhara M K case 0x60 ... 0x6f:
4097ed623d55SMuralidhara M K pvt->ctl_name = "F17h_M60h";
4098b6bea24dSAlexander Monakov break;
4099ed623d55SMuralidhara M K case 0x70 ... 0x7f:
4100ed623d55SMuralidhara M K pvt->ctl_name = "F17h_M70h";
4101ed623d55SMuralidhara M K break;
4102ed623d55SMuralidhara M K default:
4103ed623d55SMuralidhara M K pvt->ctl_name = "F17h";
41043e443eb3SIsaac Vaughn break;
41058960de4aSMichael Jin }
4106ed623d55SMuralidhara M K break;
4107c4a3e946SPu Wen
4108ed623d55SMuralidhara M K case 0x18:
4109ed623d55SMuralidhara M K pvt->ctl_name = "F18h";
4110f1cbbec9SYazen Ghannam break;
4111f1cbbec9SYazen Ghannam
41122eb61c91SYazen Ghannam case 0x19:
4113ed623d55SMuralidhara M K switch (pvt->model) {
4114ed623d55SMuralidhara M K case 0x00 ... 0x0f:
4115ed623d55SMuralidhara M K pvt->ctl_name = "F19h";
4116ed623d55SMuralidhara M K pvt->max_mcs = 8;
4117e2be5955SYazen Ghannam break;
4118ed623d55SMuralidhara M K case 0x10 ... 0x1f:
4119ed623d55SMuralidhara M K pvt->ctl_name = "F19h_M10h";
4120ed623d55SMuralidhara M K pvt->max_mcs = 12;
4121ed623d55SMuralidhara M K pvt->flags.zn_regs_v2 = 1;
4122b4210eabSYazen Ghannam break;
4123ed623d55SMuralidhara M K case 0x20 ... 0x2f:
4124ed623d55SMuralidhara M K pvt->ctl_name = "F19h_M20h";
41250b8bf9cbSMarc Bevand break;
41269c42edd5SMuralidhara M K case 0x30 ... 0x3f:
41279c42edd5SMuralidhara M K if (pvt->F3->device == PCI_DEVICE_ID_AMD_MI200_DF_F3) {
41289c42edd5SMuralidhara M K pvt->ctl_name = "MI200";
41299c42edd5SMuralidhara M K pvt->max_mcs = 4;
41309c42edd5SMuralidhara M K pvt->ops = &gpu_ops;
41319c42edd5SMuralidhara M K } else {
41329c42edd5SMuralidhara M K pvt->ctl_name = "F19h_M30h";
41339c42edd5SMuralidhara M K pvt->max_mcs = 8;
41349c42edd5SMuralidhara M K }
41359c42edd5SMuralidhara M K break;
4136ed623d55SMuralidhara M K case 0x50 ... 0x5f:
4137ed623d55SMuralidhara M K pvt->ctl_name = "F19h_M50h";
4138ed623d55SMuralidhara M K break;
41396c79e421SHristo Venev case 0x60 ... 0x6f:
41406c79e421SHristo Venev pvt->ctl_name = "F19h_M60h";
41416c79e421SHristo Venev pvt->flags.zn_regs_v2 = 1;
41426c79e421SHristo Venev break;
41436c79e421SHristo Venev case 0x70 ... 0x7f:
41446c79e421SHristo Venev pvt->ctl_name = "F19h_M70h";
41456c79e421SHristo Venev pvt->flags.zn_regs_v2 = 1;
41466c79e421SHristo Venev break;
4147ed623d55SMuralidhara M K case 0xa0 ... 0xaf:
4148ed623d55SMuralidhara M K pvt->ctl_name = "F19h_MA0h";
4149ed623d55SMuralidhara M K pvt->max_mcs = 12;
4150ed623d55SMuralidhara M K pvt->flags.zn_regs_v2 = 1;
4151e2be5955SYazen Ghannam break;
4152b4210eabSYazen Ghannam }
41532eb61c91SYazen Ghannam break;
41542eb61c91SYazen Ghannam
4155c4d07c37SAvadhut Naik case 0x1A:
4156c4d07c37SAvadhut Naik switch (pvt->model) {
4157c4d07c37SAvadhut Naik case 0x00 ... 0x1f:
4158c4d07c37SAvadhut Naik pvt->ctl_name = "F1Ah";
4159c4d07c37SAvadhut Naik pvt->max_mcs = 12;
4160c4d07c37SAvadhut Naik pvt->flags.zn_regs_v2 = 1;
4161c4d07c37SAvadhut Naik break;
4162c4d07c37SAvadhut Naik case 0x40 ... 0x4f:
4163c4d07c37SAvadhut Naik pvt->ctl_name = "F1Ah_M40h";
4164c4d07c37SAvadhut Naik pvt->flags.zn_regs_v2 = 1;
4165c4d07c37SAvadhut Naik break;
4166c4d07c37SAvadhut Naik }
4167c4d07c37SAvadhut Naik break;
4168c4d07c37SAvadhut Naik
4169395ae783SBorislav Petkov default:
417024f9a7feSBorislav Petkov amd64_err("Unsupported family!\n");
4171ed623d55SMuralidhara M K return -ENODEV;
4172395ae783SBorislav Petkov }
41730092b20dSBorislav Petkov
4174ed623d55SMuralidhara M K return 0;
4175395ae783SBorislav Petkov }
4176395ae783SBorislav Petkov
4177e339f1ecSTakashi Iwai static const struct attribute_group *amd64_edac_attr_groups[] = {
4178e339f1ecSTakashi Iwai #ifdef CONFIG_EDAC_DEBUG
41792a28ceefSBorislav Petkov &dbg_group,
418061810096SBorislav Petkov &inj_group,
4181e339f1ecSTakashi Iwai #endif
4182e339f1ecSTakashi Iwai NULL
4183e339f1ecSTakashi Iwai };
4184e339f1ecSTakashi Iwai
init_one_instance(struct amd64_pvt * pvt)418580355a3bSYazen Ghannam static int init_one_instance(struct amd64_pvt *pvt)
418680355a3bSYazen Ghannam {
418780355a3bSYazen Ghannam struct mem_ctl_info *mci = NULL;
418880355a3bSYazen Ghannam struct edac_mc_layer layers[2];
4189c4605bdeSYazen Ghannam int ret = -ENOMEM;
419080355a3bSYazen Ghannam
41919c42edd5SMuralidhara M K /*
41929c42edd5SMuralidhara M K * For Heterogeneous family EDAC CHIP_SELECT and CHANNEL layers should
41939c42edd5SMuralidhara M K * be swapped to fit into the layers.
41949c42edd5SMuralidhara M K */
4195ab5a503cSMauro Carvalho Chehab layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
41969c42edd5SMuralidhara M K layers[0].size = (pvt->F3->device == PCI_DEVICE_ID_AMD_MI200_DF_F3) ?
41979c42edd5SMuralidhara M K pvt->max_mcs : pvt->csels[0].b_cnt;
4198ab5a503cSMauro Carvalho Chehab layers[0].is_virt_csrow = true;
4199ab5a503cSMauro Carvalho Chehab layers[1].type = EDAC_MC_LAYER_CHANNEL;
42009c42edd5SMuralidhara M K layers[1].size = (pvt->F3->device == PCI_DEVICE_ID_AMD_MI200_DF_F3) ?
42019c42edd5SMuralidhara M K pvt->csels[0].b_cnt : pvt->max_mcs;
4202ab5a503cSMauro Carvalho Chehab layers[1].is_virt_csrow = false;
4203f0a56c48SBorislav Petkov
420480355a3bSYazen Ghannam mci = edac_mc_alloc(pvt->mc_node_id, ARRAY_SIZE(layers), layers, 0);
42057d6034d3SDoug Thompson if (!mci)
420680355a3bSYazen Ghannam return ret;
42077d6034d3SDoug Thompson
42087d6034d3SDoug Thompson mci->pvt_info = pvt;
42093f37a36bSBorislav Petkov mci->pdev = &pvt->F3->dev;
42107d6034d3SDoug Thompson
42110a42a37fSMuralidhara M K pvt->ops->setup_mci_misc_attrs(mci);
42127d6034d3SDoug Thompson
42137d6034d3SDoug Thompson ret = -ENODEV;
4214e339f1ecSTakashi Iwai if (edac_mc_add_mc_with_groups(mci, amd64_edac_attr_groups)) {
4215956b9ba1SJoe Perches edac_dbg(1, "failed edac_mc_add_mc()\n");
421680355a3bSYazen Ghannam edac_mc_free(mci);
421780355a3bSYazen Ghannam return ret;
42187d6034d3SDoug Thompson }
42197d6034d3SDoug Thompson
42207d6034d3SDoug Thompson return 0;
42217d6034d3SDoug Thompson }
42227d6034d3SDoug Thompson
instance_has_memory(struct amd64_pvt * pvt)4223582f94b5SYazen Ghannam static bool instance_has_memory(struct amd64_pvt *pvt)
4224582f94b5SYazen Ghannam {
4225582f94b5SYazen Ghannam bool cs_enabled = false;
4226582f94b5SYazen Ghannam int cs = 0, dct = 0;
4227582f94b5SYazen Ghannam
4228ed623d55SMuralidhara M K for (dct = 0; dct < pvt->max_mcs; dct++) {
4229582f94b5SYazen Ghannam for_each_chip_select(cs, dct, pvt)
4230582f94b5SYazen Ghannam cs_enabled |= csrow_enabled(cs, dct, pvt);
4231582f94b5SYazen Ghannam }
4232582f94b5SYazen Ghannam
4233582f94b5SYazen Ghannam return cs_enabled;
4234582f94b5SYazen Ghannam }
4235582f94b5SYazen Ghannam
probe_one_instance(unsigned int nid)42363f37a36bSBorislav Petkov static int probe_one_instance(unsigned int nid)
42377d6034d3SDoug Thompson {
42382299ef71SBorislav Petkov struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
423980355a3bSYazen Ghannam struct amd64_pvt *pvt = NULL;
4240ae7bb7c6SBorislav Petkov struct ecc_settings *s;
42413f37a36bSBorislav Petkov int ret;
4242b8cfa02fSBorislav Petkov
4243ae7bb7c6SBorislav Petkov ret = -ENOMEM;
4244ae7bb7c6SBorislav Petkov s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
4245ae7bb7c6SBorislav Petkov if (!s)
42462299ef71SBorislav Petkov goto err_out;
4247ae7bb7c6SBorislav Petkov
4248ae7bb7c6SBorislav Petkov ecc_stngs[nid] = s;
4249ae7bb7c6SBorislav Petkov
425080355a3bSYazen Ghannam pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
425180355a3bSYazen Ghannam if (!pvt)
425280355a3bSYazen Ghannam goto err_settings;
425380355a3bSYazen Ghannam
425480355a3bSYazen Ghannam pvt->mc_node_id = nid;
425580355a3bSYazen Ghannam pvt->F3 = F3;
425680355a3bSYazen Ghannam
4257ed623d55SMuralidhara M K ret = per_family_init(pvt);
4258ed623d55SMuralidhara M K if (ret < 0)
425980355a3bSYazen Ghannam goto err_enable;
426080355a3bSYazen Ghannam
42619a97a7f4SYazen Ghannam ret = pvt->ops->hw_info_get(pvt);
426280355a3bSYazen Ghannam if (ret < 0)
426380355a3bSYazen Ghannam goto err_enable;
426480355a3bSYazen Ghannam
42654688c9b4SYazen Ghannam ret = 0;
4266582f94b5SYazen Ghannam if (!instance_has_memory(pvt)) {
4267582f94b5SYazen Ghannam amd64_info("Node %d: No DIMMs detected.\n", nid);
4268582f94b5SYazen Ghannam goto err_enable;
4269582f94b5SYazen Ghannam }
4270582f94b5SYazen Ghannam
4271eb2bcdfcSMuralidhara M K if (!pvt->ops->ecc_enabled(pvt)) {
4272582f94b5SYazen Ghannam ret = -ENODEV;
42732299ef71SBorislav Petkov
42742299ef71SBorislav Petkov if (!ecc_enable_override)
42752299ef71SBorislav Petkov goto err_enable;
42762299ef71SBorislav Petkov
4277044e7a41SYazen Ghannam if (boot_cpu_data.x86 >= 0x17) {
4278044e7a41SYazen Ghannam amd64_warn("Forcing ECC on is not recommended on newer systems. Please enable ECC in BIOS.");
4279044e7a41SYazen Ghannam goto err_enable;
4280044e7a41SYazen Ghannam } else
42812299ef71SBorislav Petkov amd64_warn("Forcing ECC on!\n");
42822299ef71SBorislav Petkov
42832299ef71SBorislav Petkov if (!enable_ecc_error_reporting(s, nid, F3))
42842299ef71SBorislav Petkov goto err_enable;
42852299ef71SBorislav Petkov }
42862299ef71SBorislav Petkov
428780355a3bSYazen Ghannam ret = init_one_instance(pvt);
4288360b7f3cSBorislav Petkov if (ret < 0) {
4289ae7bb7c6SBorislav Petkov amd64_err("Error probing instance: %d\n", nid);
4290044e7a41SYazen Ghannam
4291044e7a41SYazen Ghannam if (boot_cpu_data.x86 < 0x17)
4292360b7f3cSBorislav Petkov restore_ecc_error_reporting(s, nid, F3);
42932b9b2c46SYazen Ghannam
42942b9b2c46SYazen Ghannam goto err_enable;
4295360b7f3cSBorislav Petkov }
42967d6034d3SDoug Thompson
4297ed623d55SMuralidhara M K amd64_info("%s detected (node %d).\n", pvt->ctl_name, pvt->mc_node_id);
42984cbcb73bSBorislav Petkov
4299f6f36382SMuralidhara M K /* Display and decode various registers for debug purposes. */
4300f6f36382SMuralidhara M K pvt->ops->dump_misc_regs(pvt);
4301582f94b5SYazen Ghannam
43027d6034d3SDoug Thompson return ret;
43032299ef71SBorislav Petkov
43042299ef71SBorislav Petkov err_enable:
430580355a3bSYazen Ghannam hw_info_put(pvt);
430680355a3bSYazen Ghannam kfree(pvt);
430780355a3bSYazen Ghannam
430880355a3bSYazen Ghannam err_settings:
43092299ef71SBorislav Petkov kfree(s);
43102299ef71SBorislav Petkov ecc_stngs[nid] = NULL;
43112299ef71SBorislav Petkov
43122299ef71SBorislav Petkov err_out:
43132299ef71SBorislav Petkov return ret;
43147d6034d3SDoug Thompson }
43157d6034d3SDoug Thompson
remove_one_instance(unsigned int nid)43163f37a36bSBorislav Petkov static void remove_one_instance(unsigned int nid)
43177d6034d3SDoug Thompson {
4318360b7f3cSBorislav Petkov struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
4319360b7f3cSBorislav Petkov struct ecc_settings *s = ecc_stngs[nid];
43203f37a36bSBorislav Petkov struct mem_ctl_info *mci;
43213f37a36bSBorislav Petkov struct amd64_pvt *pvt;
43227d6034d3SDoug Thompson
43237d6034d3SDoug Thompson /* Remove from EDAC CORE tracking list */
43243f37a36bSBorislav Petkov mci = edac_mc_del_mc(&F3->dev);
43257d6034d3SDoug Thompson if (!mci)
43267d6034d3SDoug Thompson return;
43277d6034d3SDoug Thompson
43287d6034d3SDoug Thompson pvt = mci->pvt_info;
43297d6034d3SDoug Thompson
4330360b7f3cSBorislav Petkov restore_ecc_error_reporting(s, nid, F3);
43317d6034d3SDoug Thompson
4332360b7f3cSBorislav Petkov kfree(ecc_stngs[nid]);
4333360b7f3cSBorislav Petkov ecc_stngs[nid] = NULL;
4334ae7bb7c6SBorislav Petkov
43357d6034d3SDoug Thompson /* Free the EDAC CORE resources */
43368f68ed97SBorislav Petkov mci->pvt_info = NULL;
43378f68ed97SBorislav Petkov
433880355a3bSYazen Ghannam hw_info_put(pvt);
43398f68ed97SBorislav Petkov kfree(pvt);
43407d6034d3SDoug Thompson edac_mc_free(mci);
43417d6034d3SDoug Thompson }
43427d6034d3SDoug Thompson
setup_pci_device(void)4343360b7f3cSBorislav Petkov static void setup_pci_device(void)
43447d6034d3SDoug Thompson {
4345d1ea71cdSBorislav Petkov if (pci_ctl)
43467d6034d3SDoug Thompson return;
43477d6034d3SDoug Thompson
4348706657b1SBorislav Petkov pci_ctl = edac_pci_create_generic_ctl(pci_ctl_dev, EDAC_MOD_STR);
4349d1ea71cdSBorislav Petkov if (!pci_ctl) {
4350d1ea71cdSBorislav Petkov pr_warn("%s(): Unable to create PCI control\n", __func__);
4351d1ea71cdSBorislav Petkov pr_warn("%s(): PCI error report via EDAC not set\n", __func__);
43527d6034d3SDoug Thompson }
43537d6034d3SDoug Thompson }
43547d6034d3SDoug Thompson
4355d6efab74SYazen Ghannam static const struct x86_cpu_id amd64_cpuids[] = {
435629842621SThomas Gleixner X86_MATCH_VENDOR_FAM(AMD, 0x0F, NULL),
435729842621SThomas Gleixner X86_MATCH_VENDOR_FAM(AMD, 0x10, NULL),
435829842621SThomas Gleixner X86_MATCH_VENDOR_FAM(AMD, 0x15, NULL),
435929842621SThomas Gleixner X86_MATCH_VENDOR_FAM(AMD, 0x16, NULL),
436029842621SThomas Gleixner X86_MATCH_VENDOR_FAM(AMD, 0x17, NULL),
436129842621SThomas Gleixner X86_MATCH_VENDOR_FAM(HYGON, 0x18, NULL),
436229842621SThomas Gleixner X86_MATCH_VENDOR_FAM(AMD, 0x19, NULL),
4363c4d07c37SAvadhut Naik X86_MATCH_VENDOR_FAM(AMD, 0x1A, NULL),
4364d6efab74SYazen Ghannam { }
4365d6efab74SYazen Ghannam };
4366d6efab74SYazen Ghannam MODULE_DEVICE_TABLE(x86cpu, amd64_cpuids);
4367d6efab74SYazen Ghannam
amd64_edac_init(void)43687d6034d3SDoug Thompson static int __init amd64_edac_init(void)
43697d6034d3SDoug Thompson {
4370301375e7SToshi Kani const char *owner;
4371360b7f3cSBorislav Petkov int err = -ENODEV;
43723f37a36bSBorislav Petkov int i;
43737d6034d3SDoug Thompson
4374315bada6SJia He if (ghes_get_devices())
4375315bada6SJia He return -EBUSY;
4376315bada6SJia He
4377301375e7SToshi Kani owner = edac_get_owner();
4378301375e7SToshi Kani if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
4379301375e7SToshi Kani return -EBUSY;
4380301375e7SToshi Kani
43811bd9900bSYazen Ghannam if (!x86_match_cpu(amd64_cpuids))
43821bd9900bSYazen Ghannam return -ENODEV;
43831bd9900bSYazen Ghannam
4384e1907d37SMuralidhara M K if (!amd_nb_num())
43851bd9900bSYazen Ghannam return -ENODEV;
43867d6034d3SDoug Thompson
43876ba92feaSBorislav Petkov opstate_init();
43886ba92feaSBorislav Petkov
4389cc4d8860SBorislav Petkov err = -ENOMEM;
43906396bb22SKees Cook ecc_stngs = kcalloc(amd_nb_num(), sizeof(ecc_stngs[0]), GFP_KERNEL);
43912ec591acSBorislav Petkov if (!ecc_stngs)
4392a9f0fbe2SBorislav Petkov goto err_free;
4393cc4d8860SBorislav Petkov
439450542251SBorislav Petkov msrs = msrs_alloc();
439556b34b91SBorislav Petkov if (!msrs)
4396360b7f3cSBorislav Petkov goto err_free;
439750542251SBorislav Petkov
43982287c636SYazen Ghannam for (i = 0; i < amd_nb_num(); i++) {
43992287c636SYazen Ghannam err = probe_one_instance(i);
44002287c636SYazen Ghannam if (err) {
44013f37a36bSBorislav Petkov /* unwind properly */
44023f37a36bSBorislav Petkov while (--i >= 0)
44033f37a36bSBorislav Petkov remove_one_instance(i);
44047d6034d3SDoug Thompson
44053f37a36bSBorislav Petkov goto err_pci;
44063f37a36bSBorislav Petkov }
44072287c636SYazen Ghannam }
44087d6034d3SDoug Thompson
44094688c9b4SYazen Ghannam if (!edac_has_mcs()) {
44104688c9b4SYazen Ghannam err = -ENODEV;
44114688c9b4SYazen Ghannam goto err_pci;
44124688c9b4SYazen Ghannam }
44134688c9b4SYazen Ghannam
4414234365f5SYazen Ghannam /* register stuff with EDAC MCE */
4415fdce765aSYazen Ghannam if (boot_cpu_data.x86 >= 0x17) {
4416234365f5SYazen Ghannam amd_register_ecc_decoder(decode_umc_error);
4417fdce765aSYazen Ghannam } else {
4418234365f5SYazen Ghannam amd_register_ecc_decoder(decode_bus_error);
4419360b7f3cSBorislav Petkov setup_pci_device();
4420fdce765aSYazen Ghannam }
4421f5b10c45STomasz Pala
4422f5b10c45STomasz Pala #ifdef CONFIG_X86_32
4423f5b10c45STomasz Pala amd64_err("%s on 32-bit is unsupported. USE AT YOUR OWN RISK!\n", EDAC_MOD_STR);
4424f5b10c45STomasz Pala #endif
4425f5b10c45STomasz Pala
44267d6034d3SDoug Thompson return 0;
44277d6034d3SDoug Thompson
442856b34b91SBorislav Petkov err_pci:
4429706657b1SBorislav Petkov pci_ctl_dev = NULL;
4430706657b1SBorislav Petkov
443156b34b91SBorislav Petkov msrs_free(msrs);
443256b34b91SBorislav Petkov msrs = NULL;
4433cc4d8860SBorislav Petkov
4434360b7f3cSBorislav Petkov err_free:
4435360b7f3cSBorislav Petkov kfree(ecc_stngs);
4436360b7f3cSBorislav Petkov ecc_stngs = NULL;
4437360b7f3cSBorislav Petkov
44387d6034d3SDoug Thompson return err;
44397d6034d3SDoug Thompson }
44407d6034d3SDoug Thompson
amd64_edac_exit(void)44417d6034d3SDoug Thompson static void __exit amd64_edac_exit(void)
44427d6034d3SDoug Thompson {
44433f37a36bSBorislav Petkov int i;
44443f37a36bSBorislav Petkov
4445d1ea71cdSBorislav Petkov if (pci_ctl)
4446d1ea71cdSBorislav Petkov edac_pci_release_generic_ctl(pci_ctl);
44477d6034d3SDoug Thompson
4448234365f5SYazen Ghannam /* unregister from EDAC MCE */
4449234365f5SYazen Ghannam if (boot_cpu_data.x86 >= 0x17)
4450234365f5SYazen Ghannam amd_unregister_ecc_decoder(decode_umc_error);
4451234365f5SYazen Ghannam else
4452234365f5SYazen Ghannam amd_unregister_ecc_decoder(decode_bus_error);
4453234365f5SYazen Ghannam
44543f37a36bSBorislav Petkov for (i = 0; i < amd_nb_num(); i++)
44553f37a36bSBorislav Petkov remove_one_instance(i);
445650542251SBorislav Petkov
4457ae7bb7c6SBorislav Petkov kfree(ecc_stngs);
4458ae7bb7c6SBorislav Petkov ecc_stngs = NULL;
4459ae7bb7c6SBorislav Petkov
4460706657b1SBorislav Petkov pci_ctl_dev = NULL;
4461706657b1SBorislav Petkov
446250542251SBorislav Petkov msrs_free(msrs);
446350542251SBorislav Petkov msrs = NULL;
44647d6034d3SDoug Thompson }
44657d6034d3SDoug Thompson
44667d6034d3SDoug Thompson module_init(amd64_edac_init);
44677d6034d3SDoug Thompson module_exit(amd64_edac_exit);
44687d6034d3SDoug Thompson
44697d6034d3SDoug Thompson MODULE_LICENSE("GPL");
4470371b27f2SBorislav Petkov (AMD) MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, Dave Peterson, Thayne Harbaugh; AMD");
4471b34348a0SYazen Ghannam MODULE_DESCRIPTION("MC support for AMD64 memory controllers");
44727d6034d3SDoug Thompson
44737d6034d3SDoug Thompson module_param(edac_op_state, int, 0444);
44747d6034d3SDoug Thompson MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
4475