12bc65418SDoug Thompson #include "amd64_edac.h" 223ac4ae8SAndreas Herrmann #include <asm/amd_nb.h> 32bc65418SDoug Thompson 42bc65418SDoug Thompson static struct edac_pci_ctl_info *amd64_ctl_pci; 52bc65418SDoug Thompson 62bc65418SDoug Thompson static int report_gart_errors; 72bc65418SDoug Thompson module_param(report_gart_errors, int, 0644); 82bc65418SDoug Thompson 92bc65418SDoug Thompson /* 102bc65418SDoug Thompson * Set by command line parameter. If BIOS has enabled the ECC, this override is 112bc65418SDoug Thompson * cleared to prevent re-enabling the hardware by this driver. 122bc65418SDoug Thompson */ 132bc65418SDoug Thompson static int ecc_enable_override; 142bc65418SDoug Thompson module_param(ecc_enable_override, int, 0644); 152bc65418SDoug Thompson 16a29d8b8eSTejun Heo static struct msr __percpu *msrs; 1750542251SBorislav Petkov 18360b7f3cSBorislav Petkov /* 19360b7f3cSBorislav Petkov * count successfully initialized driver instances for setup_pci_device() 20360b7f3cSBorislav Petkov */ 21360b7f3cSBorislav Petkov static atomic_t drv_instances = ATOMIC_INIT(0); 22360b7f3cSBorislav Petkov 23cc4d8860SBorislav Petkov /* Per-node driver instances */ 24cc4d8860SBorislav Petkov static struct mem_ctl_info **mcis; 25ae7bb7c6SBorislav Petkov static struct ecc_settings **ecc_stngs; 262bc65418SDoug Thompson 272bc65418SDoug Thompson /* 281433eb99SBorislav Petkov * Address to DRAM bank mapping: see F2x80 for K8 and F2x[1,0]80 for Fam10 and 291433eb99SBorislav Petkov * later. 30b70ef010SBorislav Petkov */ 311433eb99SBorislav Petkov static int ddr2_dbam_revCG[] = { 321433eb99SBorislav Petkov [0] = 32, 331433eb99SBorislav Petkov [1] = 64, 341433eb99SBorislav Petkov [2] = 128, 351433eb99SBorislav Petkov [3] = 256, 361433eb99SBorislav Petkov [4] = 512, 371433eb99SBorislav Petkov [5] = 1024, 381433eb99SBorislav Petkov [6] = 2048, 391433eb99SBorislav Petkov }; 401433eb99SBorislav Petkov 411433eb99SBorislav Petkov static int ddr2_dbam_revD[] = { 421433eb99SBorislav Petkov [0] = 32, 431433eb99SBorislav Petkov [1] = 64, 441433eb99SBorislav Petkov [2 ... 3] = 128, 451433eb99SBorislav Petkov [4] = 256, 461433eb99SBorislav Petkov [5] = 512, 471433eb99SBorislav Petkov [6] = 256, 481433eb99SBorislav Petkov [7] = 512, 491433eb99SBorislav Petkov [8 ... 9] = 1024, 501433eb99SBorislav Petkov [10] = 2048, 511433eb99SBorislav Petkov }; 521433eb99SBorislav Petkov 531433eb99SBorislav Petkov static int ddr2_dbam[] = { [0] = 128, 541433eb99SBorislav Petkov [1] = 256, 551433eb99SBorislav Petkov [2 ... 4] = 512, 561433eb99SBorislav Petkov [5 ... 6] = 1024, 571433eb99SBorislav Petkov [7 ... 8] = 2048, 581433eb99SBorislav Petkov [9 ... 10] = 4096, 591433eb99SBorislav Petkov [11] = 8192, 601433eb99SBorislav Petkov }; 611433eb99SBorislav Petkov 621433eb99SBorislav Petkov static int ddr3_dbam[] = { [0] = -1, 631433eb99SBorislav Petkov [1] = 256, 641433eb99SBorislav Petkov [2] = 512, 651433eb99SBorislav Petkov [3 ... 4] = -1, 661433eb99SBorislav Petkov [5 ... 6] = 1024, 671433eb99SBorislav Petkov [7 ... 8] = 2048, 681433eb99SBorislav Petkov [9 ... 10] = 4096, 691433eb99SBorislav Petkov [11] = 8192, 70b70ef010SBorislav Petkov }; 71b70ef010SBorislav Petkov 72b70ef010SBorislav Petkov /* 73b70ef010SBorislav Petkov * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing 74b70ef010SBorislav Petkov * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching- 75b70ef010SBorislav Petkov * or higher value'. 76b70ef010SBorislav Petkov * 77b70ef010SBorislav Petkov *FIXME: Produce a better mapping/linearisation. 78b70ef010SBorislav Petkov */ 79b70ef010SBorislav Petkov 80b70ef010SBorislav Petkov struct scrubrate scrubrates[] = { 81b70ef010SBorislav Petkov { 0x01, 1600000000UL}, 82b70ef010SBorislav Petkov { 0x02, 800000000UL}, 83b70ef010SBorislav Petkov { 0x03, 400000000UL}, 84b70ef010SBorislav Petkov { 0x04, 200000000UL}, 85b70ef010SBorislav Petkov { 0x05, 100000000UL}, 86b70ef010SBorislav Petkov { 0x06, 50000000UL}, 87b70ef010SBorislav Petkov { 0x07, 25000000UL}, 88b70ef010SBorislav Petkov { 0x08, 12284069UL}, 89b70ef010SBorislav Petkov { 0x09, 6274509UL}, 90b70ef010SBorislav Petkov { 0x0A, 3121951UL}, 91b70ef010SBorislav Petkov { 0x0B, 1560975UL}, 92b70ef010SBorislav Petkov { 0x0C, 781440UL}, 93b70ef010SBorislav Petkov { 0x0D, 390720UL}, 94b70ef010SBorislav Petkov { 0x0E, 195300UL}, 95b70ef010SBorislav Petkov { 0x0F, 97650UL}, 96b70ef010SBorislav Petkov { 0x10, 48854UL}, 97b70ef010SBorislav Petkov { 0x11, 24427UL}, 98b70ef010SBorislav Petkov { 0x12, 12213UL}, 99b70ef010SBorislav Petkov { 0x13, 6101UL}, 100b70ef010SBorislav Petkov { 0x14, 3051UL}, 101b70ef010SBorislav Petkov { 0x15, 1523UL}, 102b70ef010SBorislav Petkov { 0x16, 761UL}, 103b70ef010SBorislav Petkov { 0x00, 0UL}, /* scrubbing off */ 104b70ef010SBorislav Petkov }; 105b70ef010SBorislav Petkov 106b70ef010SBorislav Petkov /* 1072bc65418SDoug Thompson * Memory scrubber control interface. For K8, memory scrubbing is handled by 1082bc65418SDoug Thompson * hardware and can involve L2 cache, dcache as well as the main memory. With 1092bc65418SDoug Thompson * F10, this is extended to L3 cache scrubbing on CPU models sporting that 1102bc65418SDoug Thompson * functionality. 1112bc65418SDoug Thompson * 1122bc65418SDoug Thompson * This causes the "units" for the scrubbing speed to vary from 64 byte blocks 1132bc65418SDoug Thompson * (dram) over to cache lines. This is nasty, so we will use bandwidth in 1142bc65418SDoug Thompson * bytes/sec for the setting. 1152bc65418SDoug Thompson * 1162bc65418SDoug Thompson * Currently, we only do dram scrubbing. If the scrubbing is done in software on 1172bc65418SDoug Thompson * other archs, we might not have access to the caches directly. 1182bc65418SDoug Thompson */ 1192bc65418SDoug Thompson 1202bc65418SDoug Thompson /* 1212bc65418SDoug Thompson * scan the scrub rate mapping table for a close or matching bandwidth value to 1222bc65418SDoug Thompson * issue. If requested is too big, then use last maximum value found. 1232bc65418SDoug Thompson */ 124395ae783SBorislav Petkov static int __amd64_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate) 1252bc65418SDoug Thompson { 1262bc65418SDoug Thompson u32 scrubval; 1272bc65418SDoug Thompson int i; 1282bc65418SDoug Thompson 1292bc65418SDoug Thompson /* 1302bc65418SDoug Thompson * map the configured rate (new_bw) to a value specific to the AMD64 1312bc65418SDoug Thompson * memory controller and apply to register. Search for the first 1322bc65418SDoug Thompson * bandwidth entry that is greater or equal than the setting requested 1332bc65418SDoug Thompson * and program that. If at last entry, turn off DRAM scrubbing. 1342bc65418SDoug Thompson */ 1352bc65418SDoug Thompson for (i = 0; i < ARRAY_SIZE(scrubrates); i++) { 1362bc65418SDoug Thompson /* 1372bc65418SDoug Thompson * skip scrub rates which aren't recommended 1382bc65418SDoug Thompson * (see F10 BKDG, F3x58) 1392bc65418SDoug Thompson */ 140395ae783SBorislav Petkov if (scrubrates[i].scrubval < min_rate) 1412bc65418SDoug Thompson continue; 1422bc65418SDoug Thompson 1432bc65418SDoug Thompson if (scrubrates[i].bandwidth <= new_bw) 1442bc65418SDoug Thompson break; 1452bc65418SDoug Thompson 1462bc65418SDoug Thompson /* 1472bc65418SDoug Thompson * if no suitable bandwidth found, turn off DRAM scrubbing 1482bc65418SDoug Thompson * entirely by falling back to the last element in the 1492bc65418SDoug Thompson * scrubrates array. 1502bc65418SDoug Thompson */ 1512bc65418SDoug Thompson } 1522bc65418SDoug Thompson 1532bc65418SDoug Thompson scrubval = scrubrates[i].scrubval; 1542bc65418SDoug Thompson if (scrubval) 15524f9a7feSBorislav Petkov amd64_info("Setting scrub rate bandwidth: %u\n", 1562bc65418SDoug Thompson scrubrates[i].bandwidth); 1572bc65418SDoug Thompson else 15824f9a7feSBorislav Petkov amd64_info("Turning scrubbing off.\n"); 1592bc65418SDoug Thompson 1602bc65418SDoug Thompson pci_write_bits32(ctl, K8_SCRCTRL, scrubval, 0x001F); 1612bc65418SDoug Thompson 1622bc65418SDoug Thompson return 0; 1632bc65418SDoug Thompson } 1642bc65418SDoug Thompson 165395ae783SBorislav Petkov static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bw) 1662bc65418SDoug Thompson { 1672bc65418SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 1682bc65418SDoug Thompson 1698d5b5d9cSBorislav Petkov return __amd64_set_scrub_rate(pvt->F3, bw, pvt->min_scrubrate); 1702bc65418SDoug Thompson } 1712bc65418SDoug Thompson 1722bc65418SDoug Thompson static int amd64_get_scrub_rate(struct mem_ctl_info *mci, u32 *bw) 1732bc65418SDoug Thompson { 1742bc65418SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 1752bc65418SDoug Thompson u32 scrubval = 0; 1766ba5dcdcSBorislav Petkov int status = -1, i; 1772bc65418SDoug Thompson 1788d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, K8_SCRCTRL, &scrubval); 1792bc65418SDoug Thompson 1802bc65418SDoug Thompson scrubval = scrubval & 0x001F; 1812bc65418SDoug Thompson 18224f9a7feSBorislav Petkov amd64_debug("pci-read, sdram scrub control value: %d\n", scrubval); 1832bc65418SDoug Thompson 184926311fdSRoel Kluin for (i = 0; i < ARRAY_SIZE(scrubrates); i++) { 1852bc65418SDoug Thompson if (scrubrates[i].scrubval == scrubval) { 1862bc65418SDoug Thompson *bw = scrubrates[i].bandwidth; 1872bc65418SDoug Thompson status = 0; 1882bc65418SDoug Thompson break; 1892bc65418SDoug Thompson } 1902bc65418SDoug Thompson } 1912bc65418SDoug Thompson 1922bc65418SDoug Thompson return status; 1932bc65418SDoug Thompson } 1942bc65418SDoug Thompson 1956775763aSDoug Thompson /* Map from a CSROW entry to the mask entry that operates on it */ 1966775763aSDoug Thompson static inline u32 amd64_map_to_dcs_mask(struct amd64_pvt *pvt, int csrow) 1976775763aSDoug Thompson { 1981433eb99SBorislav Petkov if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) 1999d858bb1SBorislav Petkov return csrow; 2009d858bb1SBorislav Petkov else 2019d858bb1SBorislav Petkov return csrow >> 1; 2026775763aSDoug Thompson } 2036775763aSDoug Thompson 2046775763aSDoug Thompson /* return the 'base' address the i'th CS entry of the 'dct' DRAM controller */ 2056775763aSDoug Thompson static u32 amd64_get_dct_base(struct amd64_pvt *pvt, int dct, int csrow) 2066775763aSDoug Thompson { 2076775763aSDoug Thompson if (dct == 0) 2086775763aSDoug Thompson return pvt->dcsb0[csrow]; 2096775763aSDoug Thompson else 2106775763aSDoug Thompson return pvt->dcsb1[csrow]; 2116775763aSDoug Thompson } 2126775763aSDoug Thompson 2136775763aSDoug Thompson /* 2146775763aSDoug Thompson * Return the 'mask' address the i'th CS entry. This function is needed because 2156775763aSDoug Thompson * there number of DCSM registers on Rev E and prior vs Rev F and later is 2166775763aSDoug Thompson * different. 2176775763aSDoug Thompson */ 2186775763aSDoug Thompson static u32 amd64_get_dct_mask(struct amd64_pvt *pvt, int dct, int csrow) 2196775763aSDoug Thompson { 2206775763aSDoug Thompson if (dct == 0) 2216775763aSDoug Thompson return pvt->dcsm0[amd64_map_to_dcs_mask(pvt, csrow)]; 2226775763aSDoug Thompson else 2236775763aSDoug Thompson return pvt->dcsm1[amd64_map_to_dcs_mask(pvt, csrow)]; 2246775763aSDoug Thompson } 2256775763aSDoug Thompson 2266775763aSDoug Thompson 2276775763aSDoug Thompson /* 2286775763aSDoug Thompson * In *base and *limit, pass back the full 40-bit base and limit physical 2296775763aSDoug Thompson * addresses for the node given by node_id. This information is obtained from 2306775763aSDoug Thompson * DRAM Base (section 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers. The 2316775763aSDoug Thompson * base and limit addresses are of type SysAddr, as defined at the start of 2326775763aSDoug Thompson * section 3.4.4 (p. 70). They are the lowest and highest physical addresses 2336775763aSDoug Thompson * in the address range they represent. 2346775763aSDoug Thompson */ 2356775763aSDoug Thompson static void amd64_get_base_and_limit(struct amd64_pvt *pvt, int node_id, 2366775763aSDoug Thompson u64 *base, u64 *limit) 2376775763aSDoug Thompson { 2386775763aSDoug Thompson *base = pvt->dram_base[node_id]; 2396775763aSDoug Thompson *limit = pvt->dram_limit[node_id]; 2406775763aSDoug Thompson } 2416775763aSDoug Thompson 2426775763aSDoug Thompson /* 2436775763aSDoug Thompson * Return 1 if the SysAddr given by sys_addr matches the base/limit associated 2446775763aSDoug Thompson * with node_id 2456775763aSDoug Thompson */ 2466775763aSDoug Thompson static int amd64_base_limit_match(struct amd64_pvt *pvt, 2476775763aSDoug Thompson u64 sys_addr, int node_id) 2486775763aSDoug Thompson { 2496775763aSDoug Thompson u64 base, limit, addr; 2506775763aSDoug Thompson 2516775763aSDoug Thompson amd64_get_base_and_limit(pvt, node_id, &base, &limit); 2526775763aSDoug Thompson 2536775763aSDoug Thompson /* The K8 treats this as a 40-bit value. However, bits 63-40 will be 2546775763aSDoug Thompson * all ones if the most significant implemented address bit is 1. 2556775763aSDoug Thompson * Here we discard bits 63-40. See section 3.4.2 of AMD publication 2566775763aSDoug Thompson * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1 2576775763aSDoug Thompson * Application Programming. 2586775763aSDoug Thompson */ 2596775763aSDoug Thompson addr = sys_addr & 0x000000ffffffffffull; 2606775763aSDoug Thompson 2616775763aSDoug Thompson return (addr >= base) && (addr <= limit); 2626775763aSDoug Thompson } 2636775763aSDoug Thompson 2646775763aSDoug Thompson /* 2656775763aSDoug Thompson * Attempt to map a SysAddr to a node. On success, return a pointer to the 2666775763aSDoug Thompson * mem_ctl_info structure for the node that the SysAddr maps to. 2676775763aSDoug Thompson * 2686775763aSDoug Thompson * On failure, return NULL. 2696775763aSDoug Thompson */ 2706775763aSDoug Thompson static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci, 2716775763aSDoug Thompson u64 sys_addr) 2726775763aSDoug Thompson { 2736775763aSDoug Thompson struct amd64_pvt *pvt; 2746775763aSDoug Thompson int node_id; 2756775763aSDoug Thompson u32 intlv_en, bits; 2766775763aSDoug Thompson 2776775763aSDoug Thompson /* 2786775763aSDoug Thompson * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section 2796775763aSDoug Thompson * 3.4.4.2) registers to map the SysAddr to a node ID. 2806775763aSDoug Thompson */ 2816775763aSDoug Thompson pvt = mci->pvt_info; 2826775763aSDoug Thompson 2836775763aSDoug Thompson /* 2846775763aSDoug Thompson * The value of this field should be the same for all DRAM Base 2856775763aSDoug Thompson * registers. Therefore we arbitrarily choose to read it from the 2866775763aSDoug Thompson * register for node 0. 2876775763aSDoug Thompson */ 2886775763aSDoug Thompson intlv_en = pvt->dram_IntlvEn[0]; 2896775763aSDoug Thompson 2906775763aSDoug Thompson if (intlv_en == 0) { 2918edc5445SBorislav Petkov for (node_id = 0; node_id < DRAM_REG_COUNT; node_id++) { 2926775763aSDoug Thompson if (amd64_base_limit_match(pvt, sys_addr, node_id)) 2936775763aSDoug Thompson goto found; 2946775763aSDoug Thompson } 2958edc5445SBorislav Petkov goto err_no_match; 2968edc5445SBorislav Petkov } 2976775763aSDoug Thompson 29872f158feSBorislav Petkov if (unlikely((intlv_en != 0x01) && 29972f158feSBorislav Petkov (intlv_en != 0x03) && 30072f158feSBorislav Petkov (intlv_en != 0x07))) { 30124f9a7feSBorislav Petkov amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en); 3026775763aSDoug Thompson return NULL; 3036775763aSDoug Thompson } 3046775763aSDoug Thompson 3056775763aSDoug Thompson bits = (((u32) sys_addr) >> 12) & intlv_en; 3066775763aSDoug Thompson 3076775763aSDoug Thompson for (node_id = 0; ; ) { 3088edc5445SBorislav Petkov if ((pvt->dram_IntlvSel[node_id] & intlv_en) == bits) 3096775763aSDoug Thompson break; /* intlv_sel field matches */ 3106775763aSDoug Thompson 3116775763aSDoug Thompson if (++node_id >= DRAM_REG_COUNT) 3126775763aSDoug Thompson goto err_no_match; 3136775763aSDoug Thompson } 3146775763aSDoug Thompson 3156775763aSDoug Thompson /* sanity test for sys_addr */ 3166775763aSDoug Thompson if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) { 31724f9a7feSBorislav Petkov amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address" 31824f9a7feSBorislav Petkov "range for node %d with node interleaving enabled.\n", 3198edc5445SBorislav Petkov __func__, sys_addr, node_id); 3206775763aSDoug Thompson return NULL; 3216775763aSDoug Thompson } 3226775763aSDoug Thompson 3236775763aSDoug Thompson found: 3246775763aSDoug Thompson return edac_mc_find(node_id); 3256775763aSDoug Thompson 3266775763aSDoug Thompson err_no_match: 3276775763aSDoug Thompson debugf2("sys_addr 0x%lx doesn't match any node\n", 3286775763aSDoug Thompson (unsigned long)sys_addr); 3296775763aSDoug Thompson 3306775763aSDoug Thompson return NULL; 3316775763aSDoug Thompson } 332e2ce7255SDoug Thompson 333e2ce7255SDoug Thompson /* 334e2ce7255SDoug Thompson * Extract the DRAM CS base address from selected csrow register. 335e2ce7255SDoug Thompson */ 336e2ce7255SDoug Thompson static u64 base_from_dct_base(struct amd64_pvt *pvt, int csrow) 337e2ce7255SDoug Thompson { 338e2ce7255SDoug Thompson return ((u64) (amd64_get_dct_base(pvt, 0, csrow) & pvt->dcsb_base)) << 339e2ce7255SDoug Thompson pvt->dcs_shift; 340e2ce7255SDoug Thompson } 341e2ce7255SDoug Thompson 342e2ce7255SDoug Thompson /* 343e2ce7255SDoug Thompson * Extract the mask from the dcsb0[csrow] entry in a CPU revision-specific way. 344e2ce7255SDoug Thompson */ 345e2ce7255SDoug Thompson static u64 mask_from_dct_mask(struct amd64_pvt *pvt, int csrow) 346e2ce7255SDoug Thompson { 347e2ce7255SDoug Thompson u64 dcsm_bits, other_bits; 348e2ce7255SDoug Thompson u64 mask; 349e2ce7255SDoug Thompson 350e2ce7255SDoug Thompson /* Extract bits from DRAM CS Mask. */ 351e2ce7255SDoug Thompson dcsm_bits = amd64_get_dct_mask(pvt, 0, csrow) & pvt->dcsm_mask; 352e2ce7255SDoug Thompson 353e2ce7255SDoug Thompson other_bits = pvt->dcsm_mask; 354e2ce7255SDoug Thompson other_bits = ~(other_bits << pvt->dcs_shift); 355e2ce7255SDoug Thompson 356e2ce7255SDoug Thompson /* 357e2ce7255SDoug Thompson * The extracted bits from DCSM belong in the spaces represented by 358e2ce7255SDoug Thompson * the cleared bits in other_bits. 359e2ce7255SDoug Thompson */ 360e2ce7255SDoug Thompson mask = (dcsm_bits << pvt->dcs_shift) | other_bits; 361e2ce7255SDoug Thompson 362e2ce7255SDoug Thompson return mask; 363e2ce7255SDoug Thompson } 364e2ce7255SDoug Thompson 365e2ce7255SDoug Thompson /* 366e2ce7255SDoug Thompson * @input_addr is an InputAddr associated with the node given by mci. Return the 367e2ce7255SDoug Thompson * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr). 368e2ce7255SDoug Thompson */ 369e2ce7255SDoug Thompson static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr) 370e2ce7255SDoug Thompson { 371e2ce7255SDoug Thompson struct amd64_pvt *pvt; 372e2ce7255SDoug Thompson int csrow; 373e2ce7255SDoug Thompson u64 base, mask; 374e2ce7255SDoug Thompson 375e2ce7255SDoug Thompson pvt = mci->pvt_info; 376e2ce7255SDoug Thompson 377e2ce7255SDoug Thompson /* 378e2ce7255SDoug Thompson * Here we use the DRAM CS Base and DRAM CS Mask registers. For each CS 379e2ce7255SDoug Thompson * base/mask register pair, test the condition shown near the start of 380e2ce7255SDoug Thompson * section 3.5.4 (p. 84, BKDG #26094, K8, revA-E). 381e2ce7255SDoug Thompson */ 3829d858bb1SBorislav Petkov for (csrow = 0; csrow < pvt->cs_count; csrow++) { 383e2ce7255SDoug Thompson 384e2ce7255SDoug Thompson /* This DRAM chip select is disabled on this node */ 385e2ce7255SDoug Thompson if ((pvt->dcsb0[csrow] & K8_DCSB_CS_ENABLE) == 0) 386e2ce7255SDoug Thompson continue; 387e2ce7255SDoug Thompson 388e2ce7255SDoug Thompson base = base_from_dct_base(pvt, csrow); 389e2ce7255SDoug Thompson mask = ~mask_from_dct_mask(pvt, csrow); 390e2ce7255SDoug Thompson 391e2ce7255SDoug Thompson if ((input_addr & mask) == (base & mask)) { 392e2ce7255SDoug Thompson debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n", 393e2ce7255SDoug Thompson (unsigned long)input_addr, csrow, 394e2ce7255SDoug Thompson pvt->mc_node_id); 395e2ce7255SDoug Thompson 396e2ce7255SDoug Thompson return csrow; 397e2ce7255SDoug Thompson } 398e2ce7255SDoug Thompson } 399e2ce7255SDoug Thompson 400e2ce7255SDoug Thompson debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n", 401e2ce7255SDoug Thompson (unsigned long)input_addr, pvt->mc_node_id); 402e2ce7255SDoug Thompson 403e2ce7255SDoug Thompson return -1; 404e2ce7255SDoug Thompson } 405e2ce7255SDoug Thompson 406e2ce7255SDoug Thompson /* 407e2ce7255SDoug Thompson * Return the base value defined by the DRAM Base register for the node 408e2ce7255SDoug Thompson * represented by mci. This function returns the full 40-bit value despite the 409e2ce7255SDoug Thompson * fact that the register only stores bits 39-24 of the value. See section 410e2ce7255SDoug Thompson * 3.4.4.1 (BKDG #26094, K8, revA-E) 411e2ce7255SDoug Thompson */ 412e2ce7255SDoug Thompson static inline u64 get_dram_base(struct mem_ctl_info *mci) 413e2ce7255SDoug Thompson { 414e2ce7255SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 415e2ce7255SDoug Thompson 416e2ce7255SDoug Thompson return pvt->dram_base[pvt->mc_node_id]; 417e2ce7255SDoug Thompson } 418e2ce7255SDoug Thompson 419e2ce7255SDoug Thompson /* 420e2ce7255SDoug Thompson * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094) 421e2ce7255SDoug Thompson * for the node represented by mci. Info is passed back in *hole_base, 422e2ce7255SDoug Thompson * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if 423e2ce7255SDoug Thompson * info is invalid. Info may be invalid for either of the following reasons: 424e2ce7255SDoug Thompson * 425e2ce7255SDoug Thompson * - The revision of the node is not E or greater. In this case, the DRAM Hole 426e2ce7255SDoug Thompson * Address Register does not exist. 427e2ce7255SDoug Thompson * 428e2ce7255SDoug Thompson * - The DramHoleValid bit is cleared in the DRAM Hole Address Register, 429e2ce7255SDoug Thompson * indicating that its contents are not valid. 430e2ce7255SDoug Thompson * 431e2ce7255SDoug Thompson * The values passed back in *hole_base, *hole_offset, and *hole_size are 432e2ce7255SDoug Thompson * complete 32-bit values despite the fact that the bitfields in the DHAR 433e2ce7255SDoug Thompson * only represent bits 31-24 of the base and offset values. 434e2ce7255SDoug Thompson */ 435e2ce7255SDoug Thompson int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base, 436e2ce7255SDoug Thompson u64 *hole_offset, u64 *hole_size) 437e2ce7255SDoug Thompson { 438e2ce7255SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 439e2ce7255SDoug Thompson u64 base; 440e2ce7255SDoug Thompson 441e2ce7255SDoug Thompson /* only revE and later have the DRAM Hole Address Register */ 4421433eb99SBorislav Petkov if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) { 443e2ce7255SDoug Thompson debugf1(" revision %d for node %d does not support DHAR\n", 444e2ce7255SDoug Thompson pvt->ext_model, pvt->mc_node_id); 445e2ce7255SDoug Thompson return 1; 446e2ce7255SDoug Thompson } 447e2ce7255SDoug Thompson 448e2ce7255SDoug Thompson /* only valid for Fam10h */ 449e2ce7255SDoug Thompson if (boot_cpu_data.x86 == 0x10 && 450e2ce7255SDoug Thompson (pvt->dhar & F10_DRAM_MEM_HOIST_VALID) == 0) { 451e2ce7255SDoug Thompson debugf1(" Dram Memory Hoisting is DISABLED on this system\n"); 452e2ce7255SDoug Thompson return 1; 453e2ce7255SDoug Thompson } 454e2ce7255SDoug Thompson 455e2ce7255SDoug Thompson if ((pvt->dhar & DHAR_VALID) == 0) { 456e2ce7255SDoug Thompson debugf1(" Dram Memory Hoisting is DISABLED on this node %d\n", 457e2ce7255SDoug Thompson pvt->mc_node_id); 458e2ce7255SDoug Thompson return 1; 459e2ce7255SDoug Thompson } 460e2ce7255SDoug Thompson 461e2ce7255SDoug Thompson /* This node has Memory Hoisting */ 462e2ce7255SDoug Thompson 463e2ce7255SDoug Thompson /* +------------------+--------------------+--------------------+----- 464e2ce7255SDoug Thompson * | memory | DRAM hole | relocated | 465e2ce7255SDoug Thompson * | [0, (x - 1)] | [x, 0xffffffff] | addresses from | 466e2ce7255SDoug Thompson * | | | DRAM hole | 467e2ce7255SDoug Thompson * | | | [0x100000000, | 468e2ce7255SDoug Thompson * | | | (0x100000000+ | 469e2ce7255SDoug Thompson * | | | (0xffffffff-x))] | 470e2ce7255SDoug Thompson * +------------------+--------------------+--------------------+----- 471e2ce7255SDoug Thompson * 472e2ce7255SDoug Thompson * Above is a diagram of physical memory showing the DRAM hole and the 473e2ce7255SDoug Thompson * relocated addresses from the DRAM hole. As shown, the DRAM hole 474e2ce7255SDoug Thompson * starts at address x (the base address) and extends through address 475e2ce7255SDoug Thompson * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the 476e2ce7255SDoug Thompson * addresses in the hole so that they start at 0x100000000. 477e2ce7255SDoug Thompson */ 478e2ce7255SDoug Thompson 479e2ce7255SDoug Thompson base = dhar_base(pvt->dhar); 480e2ce7255SDoug Thompson 481e2ce7255SDoug Thompson *hole_base = base; 482e2ce7255SDoug Thompson *hole_size = (0x1ull << 32) - base; 483e2ce7255SDoug Thompson 484e2ce7255SDoug Thompson if (boot_cpu_data.x86 > 0xf) 485e2ce7255SDoug Thompson *hole_offset = f10_dhar_offset(pvt->dhar); 486e2ce7255SDoug Thompson else 487e2ce7255SDoug Thompson *hole_offset = k8_dhar_offset(pvt->dhar); 488e2ce7255SDoug Thompson 489e2ce7255SDoug Thompson debugf1(" DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n", 490e2ce7255SDoug Thompson pvt->mc_node_id, (unsigned long)*hole_base, 491e2ce7255SDoug Thompson (unsigned long)*hole_offset, (unsigned long)*hole_size); 492e2ce7255SDoug Thompson 493e2ce7255SDoug Thompson return 0; 494e2ce7255SDoug Thompson } 495e2ce7255SDoug Thompson EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info); 496e2ce7255SDoug Thompson 49793c2df58SDoug Thompson /* 49893c2df58SDoug Thompson * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is 49993c2df58SDoug Thompson * assumed that sys_addr maps to the node given by mci. 50093c2df58SDoug Thompson * 50193c2df58SDoug Thompson * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section 50293c2df58SDoug Thompson * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a 50393c2df58SDoug Thompson * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled, 50493c2df58SDoug Thompson * then it is also involved in translating a SysAddr to a DramAddr. Sections 50593c2df58SDoug Thompson * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting. 50693c2df58SDoug Thompson * These parts of the documentation are unclear. I interpret them as follows: 50793c2df58SDoug Thompson * 50893c2df58SDoug Thompson * When node n receives a SysAddr, it processes the SysAddr as follows: 50993c2df58SDoug Thompson * 51093c2df58SDoug Thompson * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM 51193c2df58SDoug Thompson * Limit registers for node n. If the SysAddr is not within the range 51293c2df58SDoug Thompson * specified by the base and limit values, then node n ignores the Sysaddr 51393c2df58SDoug Thompson * (since it does not map to node n). Otherwise continue to step 2 below. 51493c2df58SDoug Thompson * 51593c2df58SDoug Thompson * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is 51693c2df58SDoug Thompson * disabled so skip to step 3 below. Otherwise see if the SysAddr is within 51793c2df58SDoug Thompson * the range of relocated addresses (starting at 0x100000000) from the DRAM 51893c2df58SDoug Thompson * hole. If not, skip to step 3 below. Else get the value of the 51993c2df58SDoug Thompson * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the 52093c2df58SDoug Thompson * offset defined by this value from the SysAddr. 52193c2df58SDoug Thompson * 52293c2df58SDoug Thompson * 3. Obtain the base address for node n from the DRAMBase field of the DRAM 52393c2df58SDoug Thompson * Base register for node n. To obtain the DramAddr, subtract the base 52493c2df58SDoug Thompson * address from the SysAddr, as shown near the start of section 3.4.4 (p.70). 52593c2df58SDoug Thompson */ 52693c2df58SDoug Thompson static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr) 52793c2df58SDoug Thompson { 52893c2df58SDoug Thompson u64 dram_base, hole_base, hole_offset, hole_size, dram_addr; 52993c2df58SDoug Thompson int ret = 0; 53093c2df58SDoug Thompson 53193c2df58SDoug Thompson dram_base = get_dram_base(mci); 53293c2df58SDoug Thompson 53393c2df58SDoug Thompson ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset, 53493c2df58SDoug Thompson &hole_size); 53593c2df58SDoug Thompson if (!ret) { 53693c2df58SDoug Thompson if ((sys_addr >= (1ull << 32)) && 53793c2df58SDoug Thompson (sys_addr < ((1ull << 32) + hole_size))) { 53893c2df58SDoug Thompson /* use DHAR to translate SysAddr to DramAddr */ 53993c2df58SDoug Thompson dram_addr = sys_addr - hole_offset; 54093c2df58SDoug Thompson 54193c2df58SDoug Thompson debugf2("using DHAR to translate SysAddr 0x%lx to " 54293c2df58SDoug Thompson "DramAddr 0x%lx\n", 54393c2df58SDoug Thompson (unsigned long)sys_addr, 54493c2df58SDoug Thompson (unsigned long)dram_addr); 54593c2df58SDoug Thompson 54693c2df58SDoug Thompson return dram_addr; 54793c2df58SDoug Thompson } 54893c2df58SDoug Thompson } 54993c2df58SDoug Thompson 55093c2df58SDoug Thompson /* 55193c2df58SDoug Thompson * Translate the SysAddr to a DramAddr as shown near the start of 55293c2df58SDoug Thompson * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8 55393c2df58SDoug Thompson * only deals with 40-bit values. Therefore we discard bits 63-40 of 55493c2df58SDoug Thompson * sys_addr below. If bit 39 of sys_addr is 1 then the bits we 55593c2df58SDoug Thompson * discard are all 1s. Otherwise the bits we discard are all 0s. See 55693c2df58SDoug Thompson * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture 55793c2df58SDoug Thompson * Programmer's Manual Volume 1 Application Programming. 55893c2df58SDoug Thompson */ 55993c2df58SDoug Thompson dram_addr = (sys_addr & 0xffffffffffull) - dram_base; 56093c2df58SDoug Thompson 56193c2df58SDoug Thompson debugf2("using DRAM Base register to translate SysAddr 0x%lx to " 56293c2df58SDoug Thompson "DramAddr 0x%lx\n", (unsigned long)sys_addr, 56393c2df58SDoug Thompson (unsigned long)dram_addr); 56493c2df58SDoug Thompson return dram_addr; 56593c2df58SDoug Thompson } 56693c2df58SDoug Thompson 56793c2df58SDoug Thompson /* 56893c2df58SDoug Thompson * @intlv_en is the value of the IntlvEn field from a DRAM Base register 56993c2df58SDoug Thompson * (section 3.4.4.1). Return the number of bits from a SysAddr that are used 57093c2df58SDoug Thompson * for node interleaving. 57193c2df58SDoug Thompson */ 57293c2df58SDoug Thompson static int num_node_interleave_bits(unsigned intlv_en) 57393c2df58SDoug Thompson { 57493c2df58SDoug Thompson static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 }; 57593c2df58SDoug Thompson int n; 57693c2df58SDoug Thompson 57793c2df58SDoug Thompson BUG_ON(intlv_en > 7); 57893c2df58SDoug Thompson n = intlv_shift_table[intlv_en]; 57993c2df58SDoug Thompson return n; 58093c2df58SDoug Thompson } 58193c2df58SDoug Thompson 58293c2df58SDoug Thompson /* Translate the DramAddr given by @dram_addr to an InputAddr. */ 58393c2df58SDoug Thompson static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr) 58493c2df58SDoug Thompson { 58593c2df58SDoug Thompson struct amd64_pvt *pvt; 58693c2df58SDoug Thompson int intlv_shift; 58793c2df58SDoug Thompson u64 input_addr; 58893c2df58SDoug Thompson 58993c2df58SDoug Thompson pvt = mci->pvt_info; 59093c2df58SDoug Thompson 59193c2df58SDoug Thompson /* 59293c2df58SDoug Thompson * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E) 59393c2df58SDoug Thompson * concerning translating a DramAddr to an InputAddr. 59493c2df58SDoug Thompson */ 59593c2df58SDoug Thompson intlv_shift = num_node_interleave_bits(pvt->dram_IntlvEn[0]); 59693c2df58SDoug Thompson input_addr = ((dram_addr >> intlv_shift) & 0xffffff000ull) + 59793c2df58SDoug Thompson (dram_addr & 0xfff); 59893c2df58SDoug Thompson 59993c2df58SDoug Thompson debugf2(" Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n", 60093c2df58SDoug Thompson intlv_shift, (unsigned long)dram_addr, 60193c2df58SDoug Thompson (unsigned long)input_addr); 60293c2df58SDoug Thompson 60393c2df58SDoug Thompson return input_addr; 60493c2df58SDoug Thompson } 60593c2df58SDoug Thompson 60693c2df58SDoug Thompson /* 60793c2df58SDoug Thompson * Translate the SysAddr represented by @sys_addr to an InputAddr. It is 60893c2df58SDoug Thompson * assumed that @sys_addr maps to the node given by mci. 60993c2df58SDoug Thompson */ 61093c2df58SDoug Thompson static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr) 61193c2df58SDoug Thompson { 61293c2df58SDoug Thompson u64 input_addr; 61393c2df58SDoug Thompson 61493c2df58SDoug Thompson input_addr = 61593c2df58SDoug Thompson dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr)); 61693c2df58SDoug Thompson 61793c2df58SDoug Thompson debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n", 61893c2df58SDoug Thompson (unsigned long)sys_addr, (unsigned long)input_addr); 61993c2df58SDoug Thompson 62093c2df58SDoug Thompson return input_addr; 62193c2df58SDoug Thompson } 62293c2df58SDoug Thompson 62393c2df58SDoug Thompson 62493c2df58SDoug Thompson /* 62593c2df58SDoug Thompson * @input_addr is an InputAddr associated with the node represented by mci. 62693c2df58SDoug Thompson * Translate @input_addr to a DramAddr and return the result. 62793c2df58SDoug Thompson */ 62893c2df58SDoug Thompson static u64 input_addr_to_dram_addr(struct mem_ctl_info *mci, u64 input_addr) 62993c2df58SDoug Thompson { 63093c2df58SDoug Thompson struct amd64_pvt *pvt; 63193c2df58SDoug Thompson int node_id, intlv_shift; 63293c2df58SDoug Thompson u64 bits, dram_addr; 63393c2df58SDoug Thompson u32 intlv_sel; 63493c2df58SDoug Thompson 63593c2df58SDoug Thompson /* 63693c2df58SDoug Thompson * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E) 63793c2df58SDoug Thompson * shows how to translate a DramAddr to an InputAddr. Here we reverse 63893c2df58SDoug Thompson * this procedure. When translating from a DramAddr to an InputAddr, the 63993c2df58SDoug Thompson * bits used for node interleaving are discarded. Here we recover these 64093c2df58SDoug Thompson * bits from the IntlvSel field of the DRAM Limit register (section 64193c2df58SDoug Thompson * 3.4.4.2) for the node that input_addr is associated with. 64293c2df58SDoug Thompson */ 64393c2df58SDoug Thompson pvt = mci->pvt_info; 64493c2df58SDoug Thompson node_id = pvt->mc_node_id; 64593c2df58SDoug Thompson BUG_ON((node_id < 0) || (node_id > 7)); 64693c2df58SDoug Thompson 64793c2df58SDoug Thompson intlv_shift = num_node_interleave_bits(pvt->dram_IntlvEn[0]); 64893c2df58SDoug Thompson 64993c2df58SDoug Thompson if (intlv_shift == 0) { 65093c2df58SDoug Thompson debugf1(" InputAddr 0x%lx translates to DramAddr of " 65193c2df58SDoug Thompson "same value\n", (unsigned long)input_addr); 65293c2df58SDoug Thompson 65393c2df58SDoug Thompson return input_addr; 65493c2df58SDoug Thompson } 65593c2df58SDoug Thompson 65693c2df58SDoug Thompson bits = ((input_addr & 0xffffff000ull) << intlv_shift) + 65793c2df58SDoug Thompson (input_addr & 0xfff); 65893c2df58SDoug Thompson 65993c2df58SDoug Thompson intlv_sel = pvt->dram_IntlvSel[node_id] & ((1 << intlv_shift) - 1); 66093c2df58SDoug Thompson dram_addr = bits + (intlv_sel << 12); 66193c2df58SDoug Thompson 66293c2df58SDoug Thompson debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx " 66393c2df58SDoug Thompson "(%d node interleave bits)\n", (unsigned long)input_addr, 66493c2df58SDoug Thompson (unsigned long)dram_addr, intlv_shift); 66593c2df58SDoug Thompson 66693c2df58SDoug Thompson return dram_addr; 66793c2df58SDoug Thompson } 66893c2df58SDoug Thompson 66993c2df58SDoug Thompson /* 67093c2df58SDoug Thompson * @dram_addr is a DramAddr that maps to the node represented by mci. Convert 67193c2df58SDoug Thompson * @dram_addr to a SysAddr. 67293c2df58SDoug Thompson */ 67393c2df58SDoug Thompson static u64 dram_addr_to_sys_addr(struct mem_ctl_info *mci, u64 dram_addr) 67493c2df58SDoug Thompson { 67593c2df58SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 67693c2df58SDoug Thompson u64 hole_base, hole_offset, hole_size, base, limit, sys_addr; 67793c2df58SDoug Thompson int ret = 0; 67893c2df58SDoug Thompson 67993c2df58SDoug Thompson ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset, 68093c2df58SDoug Thompson &hole_size); 68193c2df58SDoug Thompson if (!ret) { 68293c2df58SDoug Thompson if ((dram_addr >= hole_base) && 68393c2df58SDoug Thompson (dram_addr < (hole_base + hole_size))) { 68493c2df58SDoug Thompson sys_addr = dram_addr + hole_offset; 68593c2df58SDoug Thompson 68693c2df58SDoug Thompson debugf1("using DHAR to translate DramAddr 0x%lx to " 68793c2df58SDoug Thompson "SysAddr 0x%lx\n", (unsigned long)dram_addr, 68893c2df58SDoug Thompson (unsigned long)sys_addr); 68993c2df58SDoug Thompson 69093c2df58SDoug Thompson return sys_addr; 69193c2df58SDoug Thompson } 69293c2df58SDoug Thompson } 69393c2df58SDoug Thompson 69493c2df58SDoug Thompson amd64_get_base_and_limit(pvt, pvt->mc_node_id, &base, &limit); 69593c2df58SDoug Thompson sys_addr = dram_addr + base; 69693c2df58SDoug Thompson 69793c2df58SDoug Thompson /* 69893c2df58SDoug Thompson * The sys_addr we have computed up to this point is a 40-bit value 69993c2df58SDoug Thompson * because the k8 deals with 40-bit values. However, the value we are 70093c2df58SDoug Thompson * supposed to return is a full 64-bit physical address. The AMD 70193c2df58SDoug Thompson * x86-64 architecture specifies that the most significant implemented 70293c2df58SDoug Thompson * address bit through bit 63 of a physical address must be either all 70393c2df58SDoug Thompson * 0s or all 1s. Therefore we sign-extend the 40-bit sys_addr to a 70493c2df58SDoug Thompson * 64-bit value below. See section 3.4.2 of AMD publication 24592: 70593c2df58SDoug Thompson * AMD x86-64 Architecture Programmer's Manual Volume 1 Application 70693c2df58SDoug Thompson * Programming. 70793c2df58SDoug Thompson */ 70893c2df58SDoug Thompson sys_addr |= ~((sys_addr & (1ull << 39)) - 1); 70993c2df58SDoug Thompson 71093c2df58SDoug Thompson debugf1(" Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n", 71193c2df58SDoug Thompson pvt->mc_node_id, (unsigned long)dram_addr, 71293c2df58SDoug Thompson (unsigned long)sys_addr); 71393c2df58SDoug Thompson 71493c2df58SDoug Thompson return sys_addr; 71593c2df58SDoug Thompson } 71693c2df58SDoug Thompson 71793c2df58SDoug Thompson /* 71893c2df58SDoug Thompson * @input_addr is an InputAddr associated with the node given by mci. Translate 71993c2df58SDoug Thompson * @input_addr to a SysAddr. 72093c2df58SDoug Thompson */ 72193c2df58SDoug Thompson static inline u64 input_addr_to_sys_addr(struct mem_ctl_info *mci, 72293c2df58SDoug Thompson u64 input_addr) 72393c2df58SDoug Thompson { 72493c2df58SDoug Thompson return dram_addr_to_sys_addr(mci, 72593c2df58SDoug Thompson input_addr_to_dram_addr(mci, input_addr)); 72693c2df58SDoug Thompson } 72793c2df58SDoug Thompson 72893c2df58SDoug Thompson /* 72993c2df58SDoug Thompson * Find the minimum and maximum InputAddr values that map to the given @csrow. 73093c2df58SDoug Thompson * Pass back these values in *input_addr_min and *input_addr_max. 73193c2df58SDoug Thompson */ 73293c2df58SDoug Thompson static void find_csrow_limits(struct mem_ctl_info *mci, int csrow, 73393c2df58SDoug Thompson u64 *input_addr_min, u64 *input_addr_max) 73493c2df58SDoug Thompson { 73593c2df58SDoug Thompson struct amd64_pvt *pvt; 73693c2df58SDoug Thompson u64 base, mask; 73793c2df58SDoug Thompson 73893c2df58SDoug Thompson pvt = mci->pvt_info; 7399d858bb1SBorislav Petkov BUG_ON((csrow < 0) || (csrow >= pvt->cs_count)); 74093c2df58SDoug Thompson 74193c2df58SDoug Thompson base = base_from_dct_base(pvt, csrow); 74293c2df58SDoug Thompson mask = mask_from_dct_mask(pvt, csrow); 74393c2df58SDoug Thompson 74493c2df58SDoug Thompson *input_addr_min = base & ~mask; 74593c2df58SDoug Thompson *input_addr_max = base | mask | pvt->dcs_mask_notused; 74693c2df58SDoug Thompson } 74793c2df58SDoug Thompson 74893c2df58SDoug Thompson /* Map the Error address to a PAGE and PAGE OFFSET. */ 74993c2df58SDoug Thompson static inline void error_address_to_page_and_offset(u64 error_address, 75093c2df58SDoug Thompson u32 *page, u32 *offset) 75193c2df58SDoug Thompson { 75293c2df58SDoug Thompson *page = (u32) (error_address >> PAGE_SHIFT); 75393c2df58SDoug Thompson *offset = ((u32) error_address) & ~PAGE_MASK; 75493c2df58SDoug Thompson } 75593c2df58SDoug Thompson 75693c2df58SDoug Thompson /* 75793c2df58SDoug Thompson * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address 75893c2df58SDoug Thompson * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers 75993c2df58SDoug Thompson * of a node that detected an ECC memory error. mci represents the node that 76093c2df58SDoug Thompson * the error address maps to (possibly different from the node that detected 76193c2df58SDoug Thompson * the error). Return the number of the csrow that sys_addr maps to, or -1 on 76293c2df58SDoug Thompson * error. 76393c2df58SDoug Thompson */ 76493c2df58SDoug Thompson static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr) 76593c2df58SDoug Thompson { 76693c2df58SDoug Thompson int csrow; 76793c2df58SDoug Thompson 76893c2df58SDoug Thompson csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr)); 76993c2df58SDoug Thompson 77093c2df58SDoug Thompson if (csrow == -1) 77124f9a7feSBorislav Petkov amd64_mc_err(mci, "Failed to translate InputAddr to csrow for " 77293c2df58SDoug Thompson "address 0x%lx\n", (unsigned long)sys_addr); 77393c2df58SDoug Thompson return csrow; 77493c2df58SDoug Thompson } 775e2ce7255SDoug Thompson 776bfc04aecSBorislav Petkov static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16); 7772da11654SDoug Thompson 778ad6a32e9SBorislav Petkov static u16 extract_syndrome(struct err_regs *err) 779ad6a32e9SBorislav Petkov { 780ad6a32e9SBorislav Petkov return ((err->nbsh >> 15) & 0xff) | ((err->nbsl >> 16) & 0xff00); 781ad6a32e9SBorislav Petkov } 782ad6a32e9SBorislav Petkov 7832da11654SDoug Thompson /* 7842da11654SDoug Thompson * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs 7852da11654SDoug Thompson * are ECC capable. 7862da11654SDoug Thompson */ 7872da11654SDoug Thompson static enum edac_type amd64_determine_edac_cap(struct amd64_pvt *pvt) 7882da11654SDoug Thompson { 7892da11654SDoug Thompson int bit; 790584fcff4SBorislav Petkov enum dev_type edac_cap = EDAC_FLAG_NONE; 7912da11654SDoug Thompson 7921433eb99SBorislav Petkov bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F) 7932da11654SDoug Thompson ? 19 7942da11654SDoug Thompson : 17; 7952da11654SDoug Thompson 796584fcff4SBorislav Petkov if (pvt->dclr0 & BIT(bit)) 7972da11654SDoug Thompson edac_cap = EDAC_FLAG_SECDED; 7982da11654SDoug Thompson 7992da11654SDoug Thompson return edac_cap; 8002da11654SDoug Thompson } 8012da11654SDoug Thompson 8022da11654SDoug Thompson 8038566c4dfSBorislav Petkov static void amd64_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt); 8042da11654SDoug Thompson 80568798e17SBorislav Petkov static void amd64_dump_dramcfg_low(u32 dclr, int chan) 80668798e17SBorislav Petkov { 80768798e17SBorislav Petkov debugf1("F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr); 80868798e17SBorislav Petkov 80968798e17SBorislav Petkov debugf1(" DIMM type: %sbuffered; all DIMMs support ECC: %s\n", 81068798e17SBorislav Petkov (dclr & BIT(16)) ? "un" : "", 81168798e17SBorislav Petkov (dclr & BIT(19)) ? "yes" : "no"); 81268798e17SBorislav Petkov 81368798e17SBorislav Petkov debugf1(" PAR/ERR parity: %s\n", 81468798e17SBorislav Petkov (dclr & BIT(8)) ? "enabled" : "disabled"); 81568798e17SBorislav Petkov 81668798e17SBorislav Petkov debugf1(" DCT 128bit mode width: %s\n", 81768798e17SBorislav Petkov (dclr & BIT(11)) ? "128b" : "64b"); 81868798e17SBorislav Petkov 81968798e17SBorislav Petkov debugf1(" x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n", 82068798e17SBorislav Petkov (dclr & BIT(12)) ? "yes" : "no", 82168798e17SBorislav Petkov (dclr & BIT(13)) ? "yes" : "no", 82268798e17SBorislav Petkov (dclr & BIT(14)) ? "yes" : "no", 82368798e17SBorislav Petkov (dclr & BIT(15)) ? "yes" : "no"); 82468798e17SBorislav Petkov } 82568798e17SBorislav Petkov 8262da11654SDoug Thompson /* Display and decode various NB registers for debug purposes. */ 8272da11654SDoug Thompson static void amd64_dump_misc_regs(struct amd64_pvt *pvt) 8282da11654SDoug Thompson { 8292da11654SDoug Thompson int ganged; 8302da11654SDoug Thompson 83168798e17SBorislav Petkov debugf1("F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap); 8322da11654SDoug Thompson 83368798e17SBorislav Petkov debugf1(" NB two channel DRAM capable: %s\n", 83468798e17SBorislav Petkov (pvt->nbcap & K8_NBCAP_DCT_DUAL) ? "yes" : "no"); 83568798e17SBorislav Petkov 83668798e17SBorislav Petkov debugf1(" ECC capable: %s, ChipKill ECC capable: %s\n", 83768798e17SBorislav Petkov (pvt->nbcap & K8_NBCAP_SECDED) ? "yes" : "no", 83868798e17SBorislav Petkov (pvt->nbcap & K8_NBCAP_CHIPKILL) ? "yes" : "no"); 83968798e17SBorislav Petkov 84068798e17SBorislav Petkov amd64_dump_dramcfg_low(pvt->dclr0, 0); 8412da11654SDoug Thompson 8428de1d91eSBorislav Petkov debugf1("F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare); 8432da11654SDoug Thompson 8448de1d91eSBorislav Petkov debugf1("F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, " 8458de1d91eSBorislav Petkov "offset: 0x%08x\n", 8468de1d91eSBorislav Petkov pvt->dhar, 8478de1d91eSBorislav Petkov dhar_base(pvt->dhar), 8488de1d91eSBorislav Petkov (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt->dhar) 8498de1d91eSBorislav Petkov : f10_dhar_offset(pvt->dhar)); 8502da11654SDoug Thompson 8518de1d91eSBorislav Petkov debugf1(" DramHoleValid: %s\n", 8528de1d91eSBorislav Petkov (pvt->dhar & DHAR_VALID) ? "yes" : "no"); 8532da11654SDoug Thompson 8542da11654SDoug Thompson /* everything below this point is Fam10h and above */ 8558566c4dfSBorislav Petkov if (boot_cpu_data.x86 == 0xf) { 8568566c4dfSBorislav Petkov amd64_debug_display_dimm_sizes(0, pvt); 8572da11654SDoug Thompson return; 8588566c4dfSBorislav Petkov } 8592da11654SDoug Thompson 86024f9a7feSBorislav Petkov amd64_info("using %s syndromes.\n", ((pvt->syn_type == 8) ? "x8" : "x4")); 861ad6a32e9SBorislav Petkov 8628de1d91eSBorislav Petkov /* Only if NOT ganged does dclr1 have valid info */ 86368798e17SBorislav Petkov if (!dct_ganging_enabled(pvt)) 86468798e17SBorislav Petkov amd64_dump_dramcfg_low(pvt->dclr1, 1); 8652da11654SDoug Thompson 8662da11654SDoug Thompson /* 8672da11654SDoug Thompson * Determine if ganged and then dump memory sizes for first controller, 8682da11654SDoug Thompson * and if NOT ganged dump info for 2nd controller. 8692da11654SDoug Thompson */ 8702da11654SDoug Thompson ganged = dct_ganging_enabled(pvt); 8712da11654SDoug Thompson 8728566c4dfSBorislav Petkov amd64_debug_display_dimm_sizes(0, pvt); 8732da11654SDoug Thompson 8742da11654SDoug Thompson if (!ganged) 8758566c4dfSBorislav Petkov amd64_debug_display_dimm_sizes(1, pvt); 8762da11654SDoug Thompson } 8772da11654SDoug Thompson 8782da11654SDoug Thompson /* Read in both of DBAM registers */ 8792da11654SDoug Thompson static void amd64_read_dbam_reg(struct amd64_pvt *pvt) 8802da11654SDoug Thompson { 8818d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F2, DBAM0, &pvt->dbam0); 8822da11654SDoug Thompson 8836ba5dcdcSBorislav Petkov if (boot_cpu_data.x86 >= 0x10) 8848d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F2, DBAM1, &pvt->dbam1); 8852da11654SDoug Thompson } 8862da11654SDoug Thompson 88794be4bffSDoug Thompson /* 88894be4bffSDoug Thompson * NOTE: CPU Revision Dependent code: Rev E and Rev F 88994be4bffSDoug Thompson * 89094be4bffSDoug Thompson * Set the DCSB and DCSM mask values depending on the CPU revision value. Also 89194be4bffSDoug Thompson * set the shift factor for the DCSB and DCSM values. 89294be4bffSDoug Thompson * 89394be4bffSDoug Thompson * ->dcs_mask_notused, RevE: 89494be4bffSDoug Thompson * 89594be4bffSDoug Thompson * To find the max InputAddr for the csrow, start with the base address and set 89694be4bffSDoug Thompson * all bits that are "don't care" bits in the test at the start of section 89794be4bffSDoug Thompson * 3.5.4 (p. 84). 89894be4bffSDoug Thompson * 89994be4bffSDoug Thompson * The "don't care" bits are all set bits in the mask and all bits in the gaps 90094be4bffSDoug Thompson * between bit ranges [35:25] and [19:13]. The value REV_E_DCS_NOTUSED_BITS 90194be4bffSDoug Thompson * represents bits [24:20] and [12:0], which are all bits in the above-mentioned 90294be4bffSDoug Thompson * gaps. 90394be4bffSDoug Thompson * 90494be4bffSDoug Thompson * ->dcs_mask_notused, RevF and later: 90594be4bffSDoug Thompson * 90694be4bffSDoug Thompson * To find the max InputAddr for the csrow, start with the base address and set 90794be4bffSDoug Thompson * all bits that are "don't care" bits in the test at the start of NPT section 90894be4bffSDoug Thompson * 4.5.4 (p. 87). 90994be4bffSDoug Thompson * 91094be4bffSDoug Thompson * The "don't care" bits are all set bits in the mask and all bits in the gaps 91194be4bffSDoug Thompson * between bit ranges [36:27] and [21:13]. 91294be4bffSDoug Thompson * 91394be4bffSDoug Thompson * The value REV_F_F1Xh_DCS_NOTUSED_BITS represents bits [26:22] and [12:0], 91494be4bffSDoug Thompson * which are all bits in the above-mentioned gaps. 91594be4bffSDoug Thompson */ 91694be4bffSDoug Thompson static void amd64_set_dct_base_and_mask(struct amd64_pvt *pvt) 91794be4bffSDoug Thompson { 9189d858bb1SBorislav Petkov 9191433eb99SBorislav Petkov if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) { 9209d858bb1SBorislav Petkov pvt->dcsb_base = REV_E_DCSB_BASE_BITS; 9219d858bb1SBorislav Petkov pvt->dcsm_mask = REV_E_DCSM_MASK_BITS; 9229d858bb1SBorislav Petkov pvt->dcs_mask_notused = REV_E_DCS_NOTUSED_BITS; 9239d858bb1SBorislav Petkov pvt->dcs_shift = REV_E_DCS_SHIFT; 9249d858bb1SBorislav Petkov pvt->cs_count = 8; 9259d858bb1SBorislav Petkov pvt->num_dcsm = 8; 9269d858bb1SBorislav Petkov } else { 92794be4bffSDoug Thompson pvt->dcsb_base = REV_F_F1Xh_DCSB_BASE_BITS; 92894be4bffSDoug Thompson pvt->dcsm_mask = REV_F_F1Xh_DCSM_MASK_BITS; 92994be4bffSDoug Thompson pvt->dcs_mask_notused = REV_F_F1Xh_DCS_NOTUSED_BITS; 93094be4bffSDoug Thompson pvt->dcs_shift = REV_F_F1Xh_DCS_SHIFT; 9319d858bb1SBorislav Petkov pvt->cs_count = 8; 9329d858bb1SBorislav Petkov pvt->num_dcsm = 4; 9339d858bb1SBorislav Petkov } 93494be4bffSDoug Thompson } 93594be4bffSDoug Thompson 93694be4bffSDoug Thompson /* 93794be4bffSDoug Thompson * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask hw registers 93894be4bffSDoug Thompson */ 93994be4bffSDoug Thompson static void amd64_read_dct_base_mask(struct amd64_pvt *pvt) 94094be4bffSDoug Thompson { 9416ba5dcdcSBorislav Petkov int cs, reg; 94294be4bffSDoug Thompson 94394be4bffSDoug Thompson amd64_set_dct_base_and_mask(pvt); 94494be4bffSDoug Thompson 9459d858bb1SBorislav Petkov for (cs = 0; cs < pvt->cs_count; cs++) { 94694be4bffSDoug Thompson reg = K8_DCSB0 + (cs * 4); 9478d5b5d9cSBorislav Petkov if (!amd64_read_pci_cfg(pvt->F2, reg, &pvt->dcsb0[cs])) 94894be4bffSDoug Thompson debugf0(" DCSB0[%d]=0x%08x reg: F2x%x\n", 94994be4bffSDoug Thompson cs, pvt->dcsb0[cs], reg); 95094be4bffSDoug Thompson 95194be4bffSDoug Thompson /* If DCT are NOT ganged, then read in DCT1's base */ 95294be4bffSDoug Thompson if (boot_cpu_data.x86 >= 0x10 && !dct_ganging_enabled(pvt)) { 95394be4bffSDoug Thompson reg = F10_DCSB1 + (cs * 4); 9548d5b5d9cSBorislav Petkov if (!amd64_read_pci_cfg(pvt->F2, reg, 9556ba5dcdcSBorislav Petkov &pvt->dcsb1[cs])) 95694be4bffSDoug Thompson debugf0(" DCSB1[%d]=0x%08x reg: F2x%x\n", 95794be4bffSDoug Thompson cs, pvt->dcsb1[cs], reg); 95894be4bffSDoug Thompson } else { 95994be4bffSDoug Thompson pvt->dcsb1[cs] = 0; 96094be4bffSDoug Thompson } 96194be4bffSDoug Thompson } 96294be4bffSDoug Thompson 96394be4bffSDoug Thompson for (cs = 0; cs < pvt->num_dcsm; cs++) { 9644afcd2dcSWan Wei reg = K8_DCSM0 + (cs * 4); 9658d5b5d9cSBorislav Petkov if (!amd64_read_pci_cfg(pvt->F2, reg, &pvt->dcsm0[cs])) 96694be4bffSDoug Thompson debugf0(" DCSM0[%d]=0x%08x reg: F2x%x\n", 96794be4bffSDoug Thompson cs, pvt->dcsm0[cs], reg); 96894be4bffSDoug Thompson 96994be4bffSDoug Thompson /* If DCT are NOT ganged, then read in DCT1's mask */ 97094be4bffSDoug Thompson if (boot_cpu_data.x86 >= 0x10 && !dct_ganging_enabled(pvt)) { 97194be4bffSDoug Thompson reg = F10_DCSM1 + (cs * 4); 9728d5b5d9cSBorislav Petkov if (!amd64_read_pci_cfg(pvt->F2, reg, 9736ba5dcdcSBorislav Petkov &pvt->dcsm1[cs])) 97494be4bffSDoug Thompson debugf0(" DCSM1[%d]=0x%08x reg: F2x%x\n", 97594be4bffSDoug Thompson cs, pvt->dcsm1[cs], reg); 9766ba5dcdcSBorislav Petkov } else { 97794be4bffSDoug Thompson pvt->dcsm1[cs] = 0; 97894be4bffSDoug Thompson } 97994be4bffSDoug Thompson } 9806ba5dcdcSBorislav Petkov } 98194be4bffSDoug Thompson 98224f9a7feSBorislav Petkov static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt, int cs) 98394be4bffSDoug Thompson { 98494be4bffSDoug Thompson enum mem_type type; 98594be4bffSDoug Thompson 9861433eb99SBorislav Petkov if (boot_cpu_data.x86 >= 0x10 || pvt->ext_model >= K8_REV_F) { 9876b4c0bdeSBorislav Petkov if (pvt->dchr0 & DDR3_MODE) 9886b4c0bdeSBorislav Petkov type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3; 9896b4c0bdeSBorislav Petkov else 99094be4bffSDoug Thompson type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2; 99194be4bffSDoug Thompson } else { 99294be4bffSDoug Thompson type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR; 99394be4bffSDoug Thompson } 99494be4bffSDoug Thompson 99524f9a7feSBorislav Petkov amd64_info("CS%d: %s\n", cs, edac_mem_types[type]); 99694be4bffSDoug Thompson 99794be4bffSDoug Thompson return type; 99894be4bffSDoug Thompson } 99994be4bffSDoug Thompson 1000ddff876dSDoug Thompson /* 1001ddff876dSDoug Thompson * Read the DRAM Configuration Low register. It differs between CG, D & E revs 1002ddff876dSDoug Thompson * and the later RevF memory controllers (DDR vs DDR2) 1003ddff876dSDoug Thompson * 1004ddff876dSDoug Thompson * Return: 1005ddff876dSDoug Thompson * number of memory channels in operation 1006ddff876dSDoug Thompson * Pass back: 1007ddff876dSDoug Thompson * contents of the DCL0_LOW register 1008ddff876dSDoug Thompson */ 1009ddff876dSDoug Thompson static int k8_early_channel_count(struct amd64_pvt *pvt) 1010ddff876dSDoug Thompson { 1011ddff876dSDoug Thompson int flag, err = 0; 1012ddff876dSDoug Thompson 10138d5b5d9cSBorislav Petkov err = amd64_read_pci_cfg(pvt->F2, F10_DCLR_0, &pvt->dclr0); 1014ddff876dSDoug Thompson if (err) 1015ddff876dSDoug Thompson return err; 1016ddff876dSDoug Thompson 10179f56da0eSBorislav Petkov if (pvt->ext_model >= K8_REV_F) 1018ddff876dSDoug Thompson /* RevF (NPT) and later */ 1019ddff876dSDoug Thompson flag = pvt->dclr0 & F10_WIDTH_128; 10209f56da0eSBorislav Petkov else 1021ddff876dSDoug Thompson /* RevE and earlier */ 1022ddff876dSDoug Thompson flag = pvt->dclr0 & REVE_WIDTH_128; 1023ddff876dSDoug Thompson 1024ddff876dSDoug Thompson /* not used */ 1025ddff876dSDoug Thompson pvt->dclr1 = 0; 1026ddff876dSDoug Thompson 1027ddff876dSDoug Thompson return (flag) ? 2 : 1; 1028ddff876dSDoug Thompson } 1029ddff876dSDoug Thompson 1030ddff876dSDoug Thompson /* extract the ERROR ADDRESS for the K8 CPUs */ 1031ddff876dSDoug Thompson static u64 k8_get_error_address(struct mem_ctl_info *mci, 1032ef44cc4cSBorislav Petkov struct err_regs *info) 1033ddff876dSDoug Thompson { 1034ddff876dSDoug Thompson return (((u64) (info->nbeah & 0xff)) << 32) + 1035ddff876dSDoug Thompson (info->nbeal & ~0x03); 1036ddff876dSDoug Thompson } 1037ddff876dSDoug Thompson 1038ddff876dSDoug Thompson /* 1039ddff876dSDoug Thompson * Read the Base and Limit registers for K8 based Memory controllers; extract 1040ddff876dSDoug Thompson * fields from the 'raw' reg into separate data fields 1041ddff876dSDoug Thompson * 1042ddff876dSDoug Thompson * Isolates: BASE, LIMIT, IntlvEn, IntlvSel, RW_EN 1043ddff876dSDoug Thompson */ 1044ddff876dSDoug Thompson static void k8_read_dram_base_limit(struct amd64_pvt *pvt, int dram) 1045ddff876dSDoug Thompson { 1046ddff876dSDoug Thompson u32 low; 1047ddff876dSDoug Thompson u32 off = dram << 3; /* 8 bytes between DRAM entries */ 1048ddff876dSDoug Thompson 10498d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F1, K8_DRAM_BASE_LOW + off, &low); 1050ddff876dSDoug Thompson 1051ddff876dSDoug Thompson /* Extract parts into separate data entries */ 10524997811eSBorislav Petkov pvt->dram_base[dram] = ((u64) low & 0xFFFF0000) << 8; 1053ddff876dSDoug Thompson pvt->dram_IntlvEn[dram] = (low >> 8) & 0x7; 1054ddff876dSDoug Thompson pvt->dram_rw_en[dram] = (low & 0x3); 1055ddff876dSDoug Thompson 10568d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F1, K8_DRAM_LIMIT_LOW + off, &low); 1057ddff876dSDoug Thompson 1058ddff876dSDoug Thompson /* 1059ddff876dSDoug Thompson * Extract parts into separate data entries. Limit is the HIGHEST memory 1060ddff876dSDoug Thompson * location of the region, so lower 24 bits need to be all ones 1061ddff876dSDoug Thompson */ 10624997811eSBorislav Petkov pvt->dram_limit[dram] = (((u64) low & 0xFFFF0000) << 8) | 0x00FFFFFF; 1063ddff876dSDoug Thompson pvt->dram_IntlvSel[dram] = (low >> 8) & 0x7; 1064ddff876dSDoug Thompson pvt->dram_DstNode[dram] = (low & 0x7); 1065ddff876dSDoug Thompson } 1066ddff876dSDoug Thompson 1067ddff876dSDoug Thompson static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, 1068ad6a32e9SBorislav Petkov struct err_regs *err_info, u64 sys_addr) 1069ddff876dSDoug Thompson { 1070ddff876dSDoug Thompson struct mem_ctl_info *src_mci; 1071ddff876dSDoug Thompson int channel, csrow; 1072ddff876dSDoug Thompson u32 page, offset; 1073ad6a32e9SBorislav Petkov u16 syndrome; 1074ddff876dSDoug Thompson 1075ad6a32e9SBorislav Petkov syndrome = extract_syndrome(err_info); 1076ddff876dSDoug Thompson 1077ddff876dSDoug Thompson /* CHIPKILL enabled */ 1078ad6a32e9SBorislav Petkov if (err_info->nbcfg & K8_NBCFG_CHIPKILL) { 1079bfc04aecSBorislav Petkov channel = get_channel_from_ecc_syndrome(mci, syndrome); 1080ddff876dSDoug Thompson if (channel < 0) { 1081ddff876dSDoug Thompson /* 1082ddff876dSDoug Thompson * Syndrome didn't map, so we don't know which of the 1083ddff876dSDoug Thompson * 2 DIMMs is in error. So we need to ID 'both' of them 1084ddff876dSDoug Thompson * as suspect. 1085ddff876dSDoug Thompson */ 108624f9a7feSBorislav Petkov amd64_mc_warn(mci, "unknown syndrome 0x%04x - possible " 1087ad6a32e9SBorislav Petkov "error reporting race\n", syndrome); 1088ddff876dSDoug Thompson edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1089ddff876dSDoug Thompson return; 1090ddff876dSDoug Thompson } 1091ddff876dSDoug Thompson } else { 1092ddff876dSDoug Thompson /* 1093ddff876dSDoug Thompson * non-chipkill ecc mode 1094ddff876dSDoug Thompson * 1095ddff876dSDoug Thompson * The k8 documentation is unclear about how to determine the 1096ddff876dSDoug Thompson * channel number when using non-chipkill memory. This method 1097ddff876dSDoug Thompson * was obtained from email communication with someone at AMD. 1098ddff876dSDoug Thompson * (Wish the email was placed in this comment - norsk) 1099ddff876dSDoug Thompson */ 110044e9e2eeSBorislav Petkov channel = ((sys_addr & BIT(3)) != 0); 1101ddff876dSDoug Thompson } 1102ddff876dSDoug Thompson 1103ddff876dSDoug Thompson /* 1104ddff876dSDoug Thompson * Find out which node the error address belongs to. This may be 1105ddff876dSDoug Thompson * different from the node that detected the error. 1106ddff876dSDoug Thompson */ 110744e9e2eeSBorislav Petkov src_mci = find_mc_by_sys_addr(mci, sys_addr); 11082cff18c2SKeith Mannthey if (!src_mci) { 110924f9a7feSBorislav Petkov amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n", 111044e9e2eeSBorislav Petkov (unsigned long)sys_addr); 1111ddff876dSDoug Thompson edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1112ddff876dSDoug Thompson return; 1113ddff876dSDoug Thompson } 1114ddff876dSDoug Thompson 111544e9e2eeSBorislav Petkov /* Now map the sys_addr to a CSROW */ 111644e9e2eeSBorislav Petkov csrow = sys_addr_to_csrow(src_mci, sys_addr); 1117ddff876dSDoug Thompson if (csrow < 0) { 1118ddff876dSDoug Thompson edac_mc_handle_ce_no_info(src_mci, EDAC_MOD_STR); 1119ddff876dSDoug Thompson } else { 112044e9e2eeSBorislav Petkov error_address_to_page_and_offset(sys_addr, &page, &offset); 1121ddff876dSDoug Thompson 1122ddff876dSDoug Thompson edac_mc_handle_ce(src_mci, page, offset, syndrome, csrow, 1123ddff876dSDoug Thompson channel, EDAC_MOD_STR); 1124ddff876dSDoug Thompson } 1125ddff876dSDoug Thompson } 1126ddff876dSDoug Thompson 11271433eb99SBorislav Petkov static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, int cs_mode) 1128ddff876dSDoug Thompson { 11291433eb99SBorislav Petkov int *dbam_map; 1130ddff876dSDoug Thompson 11311433eb99SBorislav Petkov if (pvt->ext_model >= K8_REV_F) 11321433eb99SBorislav Petkov dbam_map = ddr2_dbam; 11331433eb99SBorislav Petkov else if (pvt->ext_model >= K8_REV_D) 11341433eb99SBorislav Petkov dbam_map = ddr2_dbam_revD; 11351433eb99SBorislav Petkov else 11361433eb99SBorislav Petkov dbam_map = ddr2_dbam_revCG; 1137ddff876dSDoug Thompson 11381433eb99SBorislav Petkov return dbam_map[cs_mode]; 1139ddff876dSDoug Thompson } 1140ddff876dSDoug Thompson 11411afd3c98SDoug Thompson /* 11421afd3c98SDoug Thompson * Get the number of DCT channels in use. 11431afd3c98SDoug Thompson * 11441afd3c98SDoug Thompson * Return: 11451afd3c98SDoug Thompson * number of Memory Channels in operation 11461afd3c98SDoug Thompson * Pass back: 11471afd3c98SDoug Thompson * contents of the DCL0_LOW register 11481afd3c98SDoug Thompson */ 11491afd3c98SDoug Thompson static int f10_early_channel_count(struct amd64_pvt *pvt) 11501afd3c98SDoug Thompson { 115157a30854SWan Wei int dbams[] = { DBAM0, DBAM1 }; 11526ba5dcdcSBorislav Petkov int i, j, channels = 0; 11531afd3c98SDoug Thompson u32 dbam; 1154ddff876dSDoug Thompson 11551afd3c98SDoug Thompson /* If we are in 128 bit mode, then we are using 2 channels */ 11561afd3c98SDoug Thompson if (pvt->dclr0 & F10_WIDTH_128) { 11571afd3c98SDoug Thompson channels = 2; 11581afd3c98SDoug Thompson return channels; 11591afd3c98SDoug Thompson } 11601afd3c98SDoug Thompson 11611afd3c98SDoug Thompson /* 1162d16149e8SBorislav Petkov * Need to check if in unganged mode: In such, there are 2 channels, 1163d16149e8SBorislav Petkov * but they are not in 128 bit mode and thus the above 'dclr0' status 1164d16149e8SBorislav Petkov * bit will be OFF. 11651afd3c98SDoug Thompson * 11661afd3c98SDoug Thompson * Need to check DCT0[0] and DCT1[0] to see if only one of them has 11671afd3c98SDoug Thompson * their CSEnable bit on. If so, then SINGLE DIMM case. 11681afd3c98SDoug Thompson */ 1169d16149e8SBorislav Petkov debugf0("Data width is not 128 bits - need more decoding\n"); 11701afd3c98SDoug Thompson 11711afd3c98SDoug Thompson /* 11721afd3c98SDoug Thompson * Check DRAM Bank Address Mapping values for each DIMM to see if there 11731afd3c98SDoug Thompson * is more than just one DIMM present in unganged mode. Need to check 11741afd3c98SDoug Thompson * both controllers since DIMMs can be placed in either one. 11751afd3c98SDoug Thompson */ 117657a30854SWan Wei for (i = 0; i < ARRAY_SIZE(dbams); i++) { 11778d5b5d9cSBorislav Petkov if (amd64_read_pci_cfg(pvt->F2, dbams[i], &dbam)) 11781afd3c98SDoug Thompson goto err_reg; 11791afd3c98SDoug Thompson 118057a30854SWan Wei for (j = 0; j < 4; j++) { 118157a30854SWan Wei if (DBAM_DIMM(j, dbam) > 0) { 11821afd3c98SDoug Thompson channels++; 118357a30854SWan Wei break; 11841afd3c98SDoug Thompson } 118557a30854SWan Wei } 118657a30854SWan Wei } 11871afd3c98SDoug Thompson 1188d16149e8SBorislav Petkov if (channels > 2) 1189d16149e8SBorislav Petkov channels = 2; 1190d16149e8SBorislav Petkov 119124f9a7feSBorislav Petkov amd64_info("MCT channel count: %d\n", channels); 11921afd3c98SDoug Thompson 11931afd3c98SDoug Thompson return channels; 11941afd3c98SDoug Thompson 11951afd3c98SDoug Thompson err_reg: 11961afd3c98SDoug Thompson return -1; 11971afd3c98SDoug Thompson 11981afd3c98SDoug Thompson } 11991afd3c98SDoug Thompson 12001433eb99SBorislav Petkov static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, int cs_mode) 12011afd3c98SDoug Thompson { 12021433eb99SBorislav Petkov int *dbam_map; 12031433eb99SBorislav Petkov 12041433eb99SBorislav Petkov if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE) 12051433eb99SBorislav Petkov dbam_map = ddr3_dbam; 12061433eb99SBorislav Petkov else 12071433eb99SBorislav Petkov dbam_map = ddr2_dbam; 12081433eb99SBorislav Petkov 12091433eb99SBorislav Petkov return dbam_map[cs_mode]; 12101afd3c98SDoug Thompson } 12111afd3c98SDoug Thompson 12121afd3c98SDoug Thompson static u64 f10_get_error_address(struct mem_ctl_info *mci, 1213ef44cc4cSBorislav Petkov struct err_regs *info) 12141afd3c98SDoug Thompson { 12151afd3c98SDoug Thompson return (((u64) (info->nbeah & 0xffff)) << 32) + 12161afd3c98SDoug Thompson (info->nbeal & ~0x01); 12171afd3c98SDoug Thompson } 12181afd3c98SDoug Thompson 12191afd3c98SDoug Thompson /* 12201afd3c98SDoug Thompson * Read the Base and Limit registers for F10 based Memory controllers. Extract 12211afd3c98SDoug Thompson * fields from the 'raw' reg into separate data fields. 12221afd3c98SDoug Thompson * 12231afd3c98SDoug Thompson * Isolates: BASE, LIMIT, IntlvEn, IntlvSel, RW_EN. 12241afd3c98SDoug Thompson */ 12251afd3c98SDoug Thompson static void f10_read_dram_base_limit(struct amd64_pvt *pvt, int dram) 12261afd3c98SDoug Thompson { 12271afd3c98SDoug Thompson u32 high_offset, low_offset, high_base, low_base, high_limit, low_limit; 12281afd3c98SDoug Thompson 12291afd3c98SDoug Thompson low_offset = K8_DRAM_BASE_LOW + (dram << 3); 12301afd3c98SDoug Thompson high_offset = F10_DRAM_BASE_HIGH + (dram << 3); 12311afd3c98SDoug Thompson 12321afd3c98SDoug Thompson /* read the 'raw' DRAM BASE Address register */ 12338d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F1, low_offset, &low_base); 12348d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F1, high_offset, &high_base); 12351afd3c98SDoug Thompson 12361afd3c98SDoug Thompson /* Extract parts into separate data entries */ 12371afd3c98SDoug Thompson pvt->dram_rw_en[dram] = (low_base & 0x3); 12381afd3c98SDoug Thompson 12391afd3c98SDoug Thompson if (pvt->dram_rw_en[dram] == 0) 12401afd3c98SDoug Thompson return; 12411afd3c98SDoug Thompson 12421afd3c98SDoug Thompson pvt->dram_IntlvEn[dram] = (low_base >> 8) & 0x7; 12431afd3c98SDoug Thompson 124466216a7aSBorislav Petkov pvt->dram_base[dram] = (((u64)high_base & 0x000000FF) << 40) | 12454997811eSBorislav Petkov (((u64)low_base & 0xFFFF0000) << 8); 12461afd3c98SDoug Thompson 12471afd3c98SDoug Thompson low_offset = K8_DRAM_LIMIT_LOW + (dram << 3); 12481afd3c98SDoug Thompson high_offset = F10_DRAM_LIMIT_HIGH + (dram << 3); 12491afd3c98SDoug Thompson 12501afd3c98SDoug Thompson /* read the 'raw' LIMIT registers */ 12518d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F1, low_offset, &low_limit); 12528d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F1, high_offset, &high_limit); 12531afd3c98SDoug Thompson 12541afd3c98SDoug Thompson pvt->dram_DstNode[dram] = (low_limit & 0x7); 12551afd3c98SDoug Thompson pvt->dram_IntlvSel[dram] = (low_limit >> 8) & 0x7; 12561afd3c98SDoug Thompson 12571afd3c98SDoug Thompson /* 12581afd3c98SDoug Thompson * Extract address values and form a LIMIT address. Limit is the HIGHEST 12591afd3c98SDoug Thompson * memory location of the region, so low 24 bits need to be all ones. 12601afd3c98SDoug Thompson */ 126166216a7aSBorislav Petkov pvt->dram_limit[dram] = (((u64)high_limit & 0x000000FF) << 40) | 12624997811eSBorislav Petkov (((u64) low_limit & 0xFFFF0000) << 8) | 126366216a7aSBorislav Petkov 0x00FFFFFF; 12641afd3c98SDoug Thompson } 12656163b5d4SDoug Thompson 12666163b5d4SDoug Thompson static void f10_read_dram_ctl_register(struct amd64_pvt *pvt) 12676163b5d4SDoug Thompson { 12686163b5d4SDoug Thompson 12698d5b5d9cSBorislav Petkov if (!amd64_read_pci_cfg(pvt->F2, F10_DCTL_SEL_LOW, 12706ba5dcdcSBorislav Petkov &pvt->dram_ctl_select_low)) { 127172381bd5SBorislav Petkov debugf0("F2x110 (DCTL Sel. Low): 0x%08x, " 127272381bd5SBorislav Petkov "High range addresses at: 0x%x\n", 127372381bd5SBorislav Petkov pvt->dram_ctl_select_low, 127472381bd5SBorislav Petkov dct_sel_baseaddr(pvt)); 12756163b5d4SDoug Thompson 127672381bd5SBorislav Petkov debugf0(" DCT mode: %s, All DCTs on: %s\n", 127772381bd5SBorislav Petkov (dct_ganging_enabled(pvt) ? "ganged" : "unganged"), 127872381bd5SBorislav Petkov (dct_dram_enabled(pvt) ? "yes" : "no")); 12796163b5d4SDoug Thompson 128072381bd5SBorislav Petkov if (!dct_ganging_enabled(pvt)) 128172381bd5SBorislav Petkov debugf0(" Address range split per DCT: %s\n", 128272381bd5SBorislav Petkov (dct_high_range_enabled(pvt) ? "yes" : "no")); 128372381bd5SBorislav Petkov 128472381bd5SBorislav Petkov debugf0(" DCT data interleave for ECC: %s, " 128572381bd5SBorislav Petkov "DRAM cleared since last warm reset: %s\n", 128672381bd5SBorislav Petkov (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"), 128772381bd5SBorislav Petkov (dct_memory_cleared(pvt) ? "yes" : "no")); 128872381bd5SBorislav Petkov 128972381bd5SBorislav Petkov debugf0(" DCT channel interleave: %s, " 129072381bd5SBorislav Petkov "DCT interleave bits selector: 0x%x\n", 129172381bd5SBorislav Petkov (dct_interleave_enabled(pvt) ? "enabled" : "disabled"), 12926163b5d4SDoug Thompson dct_sel_interleave_addr(pvt)); 12936163b5d4SDoug Thompson } 12946163b5d4SDoug Thompson 12958d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F2, F10_DCTL_SEL_HIGH, 12966163b5d4SDoug Thompson &pvt->dram_ctl_select_high); 12976163b5d4SDoug Thompson } 12986163b5d4SDoug Thompson 1299f71d0a05SDoug Thompson /* 1300f71d0a05SDoug Thompson * determine channel based on the interleaving mode: F10h BKDG, 2.8.9 Memory 1301f71d0a05SDoug Thompson * Interleaving Modes. 1302f71d0a05SDoug Thompson */ 13036163b5d4SDoug Thompson static u32 f10_determine_channel(struct amd64_pvt *pvt, u64 sys_addr, 13046163b5d4SDoug Thompson int hi_range_sel, u32 intlv_en) 13056163b5d4SDoug Thompson { 13066163b5d4SDoug Thompson u32 cs, temp, dct_sel_high = (pvt->dram_ctl_select_low >> 1) & 1; 13076163b5d4SDoug Thompson 13086163b5d4SDoug Thompson if (dct_ganging_enabled(pvt)) 13096163b5d4SDoug Thompson cs = 0; 13106163b5d4SDoug Thompson else if (hi_range_sel) 13116163b5d4SDoug Thompson cs = dct_sel_high; 13126163b5d4SDoug Thompson else if (dct_interleave_enabled(pvt)) { 1313f71d0a05SDoug Thompson /* 1314f71d0a05SDoug Thompson * see F2x110[DctSelIntLvAddr] - channel interleave mode 1315f71d0a05SDoug Thompson */ 13166163b5d4SDoug Thompson if (dct_sel_interleave_addr(pvt) == 0) 13176163b5d4SDoug Thompson cs = sys_addr >> 6 & 1; 13186163b5d4SDoug Thompson else if ((dct_sel_interleave_addr(pvt) >> 1) & 1) { 13196163b5d4SDoug Thompson temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2; 13206163b5d4SDoug Thompson 13216163b5d4SDoug Thompson if (dct_sel_interleave_addr(pvt) & 1) 13226163b5d4SDoug Thompson cs = (sys_addr >> 9 & 1) ^ temp; 13236163b5d4SDoug Thompson else 13246163b5d4SDoug Thompson cs = (sys_addr >> 6 & 1) ^ temp; 13256163b5d4SDoug Thompson } else if (intlv_en & 4) 13266163b5d4SDoug Thompson cs = sys_addr >> 15 & 1; 13276163b5d4SDoug Thompson else if (intlv_en & 2) 13286163b5d4SDoug Thompson cs = sys_addr >> 14 & 1; 13296163b5d4SDoug Thompson else if (intlv_en & 1) 13306163b5d4SDoug Thompson cs = sys_addr >> 13 & 1; 13316163b5d4SDoug Thompson else 13326163b5d4SDoug Thompson cs = sys_addr >> 12 & 1; 13336163b5d4SDoug Thompson } else if (dct_high_range_enabled(pvt) && !dct_ganging_enabled(pvt)) 13346163b5d4SDoug Thompson cs = ~dct_sel_high & 1; 13356163b5d4SDoug Thompson else 13366163b5d4SDoug Thompson cs = 0; 13376163b5d4SDoug Thompson 13386163b5d4SDoug Thompson return cs; 13396163b5d4SDoug Thompson } 13406163b5d4SDoug Thompson 13416163b5d4SDoug Thompson static inline u32 f10_map_intlv_en_to_shift(u32 intlv_en) 13426163b5d4SDoug Thompson { 13436163b5d4SDoug Thompson if (intlv_en == 1) 13446163b5d4SDoug Thompson return 1; 13456163b5d4SDoug Thompson else if (intlv_en == 3) 13466163b5d4SDoug Thompson return 2; 13476163b5d4SDoug Thompson else if (intlv_en == 7) 13486163b5d4SDoug Thompson return 3; 13496163b5d4SDoug Thompson 13506163b5d4SDoug Thompson return 0; 13516163b5d4SDoug Thompson } 13526163b5d4SDoug Thompson 1353f71d0a05SDoug Thompson /* See F10h BKDG, 2.8.10.2 DctSelBaseOffset Programming */ 1354f71d0a05SDoug Thompson static inline u64 f10_get_base_addr_offset(u64 sys_addr, int hi_range_sel, 13556163b5d4SDoug Thompson u32 dct_sel_base_addr, 13566163b5d4SDoug Thompson u64 dct_sel_base_off, 1357f71d0a05SDoug Thompson u32 hole_valid, u32 hole_off, 13586163b5d4SDoug Thompson u64 dram_base) 13596163b5d4SDoug Thompson { 13606163b5d4SDoug Thompson u64 chan_off; 13616163b5d4SDoug Thompson 13626163b5d4SDoug Thompson if (hi_range_sel) { 13639975a5f2SBorislav Petkov if (!(dct_sel_base_addr & 0xFFFF0000) && 1364f71d0a05SDoug Thompson hole_valid && (sys_addr >= 0x100000000ULL)) 13656163b5d4SDoug Thompson chan_off = hole_off << 16; 13666163b5d4SDoug Thompson else 13676163b5d4SDoug Thompson chan_off = dct_sel_base_off; 13686163b5d4SDoug Thompson } else { 1369f71d0a05SDoug Thompson if (hole_valid && (sys_addr >= 0x100000000ULL)) 13706163b5d4SDoug Thompson chan_off = hole_off << 16; 13716163b5d4SDoug Thompson else 13726163b5d4SDoug Thompson chan_off = dram_base & 0xFFFFF8000000ULL; 13736163b5d4SDoug Thompson } 13746163b5d4SDoug Thompson 13756163b5d4SDoug Thompson return (sys_addr & 0x0000FFFFFFFFFFC0ULL) - 13766163b5d4SDoug Thompson (chan_off & 0x0000FFFFFF800000ULL); 13776163b5d4SDoug Thompson } 13786163b5d4SDoug Thompson 13796163b5d4SDoug Thompson /* Hack for the time being - Can we get this from BIOS?? */ 13806163b5d4SDoug Thompson #define CH0SPARE_RANK 0 13816163b5d4SDoug Thompson #define CH1SPARE_RANK 1 13826163b5d4SDoug Thompson 13836163b5d4SDoug Thompson /* 13846163b5d4SDoug Thompson * checks if the csrow passed in is marked as SPARED, if so returns the new 13856163b5d4SDoug Thompson * spare row 13866163b5d4SDoug Thompson */ 13876163b5d4SDoug Thompson static inline int f10_process_possible_spare(int csrow, 13886163b5d4SDoug Thompson u32 cs, struct amd64_pvt *pvt) 13896163b5d4SDoug Thompson { 13906163b5d4SDoug Thompson u32 swap_done; 13916163b5d4SDoug Thompson u32 bad_dram_cs; 13926163b5d4SDoug Thompson 13936163b5d4SDoug Thompson /* Depending on channel, isolate respective SPARING info */ 13946163b5d4SDoug Thompson if (cs) { 13956163b5d4SDoug Thompson swap_done = F10_ONLINE_SPARE_SWAPDONE1(pvt->online_spare); 13966163b5d4SDoug Thompson bad_dram_cs = F10_ONLINE_SPARE_BADDRAM_CS1(pvt->online_spare); 13976163b5d4SDoug Thompson if (swap_done && (csrow == bad_dram_cs)) 13986163b5d4SDoug Thompson csrow = CH1SPARE_RANK; 13996163b5d4SDoug Thompson } else { 14006163b5d4SDoug Thompson swap_done = F10_ONLINE_SPARE_SWAPDONE0(pvt->online_spare); 14016163b5d4SDoug Thompson bad_dram_cs = F10_ONLINE_SPARE_BADDRAM_CS0(pvt->online_spare); 14026163b5d4SDoug Thompson if (swap_done && (csrow == bad_dram_cs)) 14036163b5d4SDoug Thompson csrow = CH0SPARE_RANK; 14046163b5d4SDoug Thompson } 14056163b5d4SDoug Thompson return csrow; 14066163b5d4SDoug Thompson } 14076163b5d4SDoug Thompson 14086163b5d4SDoug Thompson /* 14096163b5d4SDoug Thompson * Iterate over the DRAM DCT "base" and "mask" registers looking for a 14106163b5d4SDoug Thompson * SystemAddr match on the specified 'ChannelSelect' and 'NodeID' 14116163b5d4SDoug Thompson * 14126163b5d4SDoug Thompson * Return: 14136163b5d4SDoug Thompson * -EINVAL: NOT FOUND 14146163b5d4SDoug Thompson * 0..csrow = Chip-Select Row 14156163b5d4SDoug Thompson */ 14166163b5d4SDoug Thompson static int f10_lookup_addr_in_dct(u32 in_addr, u32 nid, u32 cs) 14176163b5d4SDoug Thompson { 14186163b5d4SDoug Thompson struct mem_ctl_info *mci; 14196163b5d4SDoug Thompson struct amd64_pvt *pvt; 14206163b5d4SDoug Thompson u32 cs_base, cs_mask; 14216163b5d4SDoug Thompson int cs_found = -EINVAL; 14226163b5d4SDoug Thompson int csrow; 14236163b5d4SDoug Thompson 1424cc4d8860SBorislav Petkov mci = mcis[nid]; 14256163b5d4SDoug Thompson if (!mci) 14266163b5d4SDoug Thompson return cs_found; 14276163b5d4SDoug Thompson 14286163b5d4SDoug Thompson pvt = mci->pvt_info; 14296163b5d4SDoug Thompson 14306163b5d4SDoug Thompson debugf1("InputAddr=0x%x channelselect=%d\n", in_addr, cs); 14316163b5d4SDoug Thompson 14329d858bb1SBorislav Petkov for (csrow = 0; csrow < pvt->cs_count; csrow++) { 14336163b5d4SDoug Thompson 14346163b5d4SDoug Thompson cs_base = amd64_get_dct_base(pvt, cs, csrow); 14356163b5d4SDoug Thompson if (!(cs_base & K8_DCSB_CS_ENABLE)) 14366163b5d4SDoug Thompson continue; 14376163b5d4SDoug Thompson 14386163b5d4SDoug Thompson /* 14396163b5d4SDoug Thompson * We have an ENABLED CSROW, Isolate just the MASK bits of the 14406163b5d4SDoug Thompson * target: [28:19] and [13:5], which map to [36:27] and [21:13] 14416163b5d4SDoug Thompson * of the actual address. 14426163b5d4SDoug Thompson */ 14436163b5d4SDoug Thompson cs_base &= REV_F_F1Xh_DCSB_BASE_BITS; 14446163b5d4SDoug Thompson 14456163b5d4SDoug Thompson /* 14466163b5d4SDoug Thompson * Get the DCT Mask, and ENABLE the reserved bits: [18:16] and 14476163b5d4SDoug Thompson * [4:0] to become ON. Then mask off bits [28:0] ([36:8]) 14486163b5d4SDoug Thompson */ 14496163b5d4SDoug Thompson cs_mask = amd64_get_dct_mask(pvt, cs, csrow); 14506163b5d4SDoug Thompson 14516163b5d4SDoug Thompson debugf1(" CSROW=%d CSBase=0x%x RAW CSMask=0x%x\n", 14526163b5d4SDoug Thompson csrow, cs_base, cs_mask); 14536163b5d4SDoug Thompson 14546163b5d4SDoug Thompson cs_mask = (cs_mask | 0x0007C01F) & 0x1FFFFFFF; 14556163b5d4SDoug Thompson 14566163b5d4SDoug Thompson debugf1(" Final CSMask=0x%x\n", cs_mask); 14576163b5d4SDoug Thompson debugf1(" (InputAddr & ~CSMask)=0x%x " 14586163b5d4SDoug Thompson "(CSBase & ~CSMask)=0x%x\n", 14596163b5d4SDoug Thompson (in_addr & ~cs_mask), (cs_base & ~cs_mask)); 14606163b5d4SDoug Thompson 14616163b5d4SDoug Thompson if ((in_addr & ~cs_mask) == (cs_base & ~cs_mask)) { 14626163b5d4SDoug Thompson cs_found = f10_process_possible_spare(csrow, cs, pvt); 14636163b5d4SDoug Thompson 14646163b5d4SDoug Thompson debugf1(" MATCH csrow=%d\n", cs_found); 14656163b5d4SDoug Thompson break; 14666163b5d4SDoug Thompson } 14676163b5d4SDoug Thompson } 14686163b5d4SDoug Thompson return cs_found; 14696163b5d4SDoug Thompson } 14706163b5d4SDoug Thompson 1471f71d0a05SDoug Thompson /* For a given @dram_range, check if @sys_addr falls within it. */ 1472f71d0a05SDoug Thompson static int f10_match_to_this_node(struct amd64_pvt *pvt, int dram_range, 1473f71d0a05SDoug Thompson u64 sys_addr, int *nid, int *chan_sel) 1474f71d0a05SDoug Thompson { 1475f71d0a05SDoug Thompson int node_id, cs_found = -EINVAL, high_range = 0; 1476f71d0a05SDoug Thompson u32 intlv_en, intlv_sel, intlv_shift, hole_off; 1477f71d0a05SDoug Thompson u32 hole_valid, tmp, dct_sel_base, channel; 1478f71d0a05SDoug Thompson u64 dram_base, chan_addr, dct_sel_base_off; 1479f71d0a05SDoug Thompson 1480f71d0a05SDoug Thompson dram_base = pvt->dram_base[dram_range]; 1481f71d0a05SDoug Thompson intlv_en = pvt->dram_IntlvEn[dram_range]; 1482f71d0a05SDoug Thompson 1483f71d0a05SDoug Thompson node_id = pvt->dram_DstNode[dram_range]; 1484f71d0a05SDoug Thompson intlv_sel = pvt->dram_IntlvSel[dram_range]; 1485f71d0a05SDoug Thompson 1486f71d0a05SDoug Thompson debugf1("(dram=%d) Base=0x%llx SystemAddr= 0x%llx Limit=0x%llx\n", 1487f71d0a05SDoug Thompson dram_range, dram_base, sys_addr, pvt->dram_limit[dram_range]); 1488f71d0a05SDoug Thompson 1489f71d0a05SDoug Thompson /* 1490f71d0a05SDoug Thompson * This assumes that one node's DHAR is the same as all the other 1491f71d0a05SDoug Thompson * nodes' DHAR. 1492f71d0a05SDoug Thompson */ 1493f71d0a05SDoug Thompson hole_off = (pvt->dhar & 0x0000FF80); 1494f71d0a05SDoug Thompson hole_valid = (pvt->dhar & 0x1); 1495f71d0a05SDoug Thompson dct_sel_base_off = (pvt->dram_ctl_select_high & 0xFFFFFC00) << 16; 1496f71d0a05SDoug Thompson 1497f71d0a05SDoug Thompson debugf1(" HoleOffset=0x%x HoleValid=0x%x IntlvSel=0x%x\n", 1498f71d0a05SDoug Thompson hole_off, hole_valid, intlv_sel); 1499f71d0a05SDoug Thompson 1500e726f3c3SBorislav Petkov if (intlv_en && 1501f71d0a05SDoug Thompson (intlv_sel != ((sys_addr >> 12) & intlv_en))) 1502f71d0a05SDoug Thompson return -EINVAL; 1503f71d0a05SDoug Thompson 1504f71d0a05SDoug Thompson dct_sel_base = dct_sel_baseaddr(pvt); 1505f71d0a05SDoug Thompson 1506f71d0a05SDoug Thompson /* 1507f71d0a05SDoug Thompson * check whether addresses >= DctSelBaseAddr[47:27] are to be used to 1508f71d0a05SDoug Thompson * select between DCT0 and DCT1. 1509f71d0a05SDoug Thompson */ 1510f71d0a05SDoug Thompson if (dct_high_range_enabled(pvt) && 1511f71d0a05SDoug Thompson !dct_ganging_enabled(pvt) && 1512f71d0a05SDoug Thompson ((sys_addr >> 27) >= (dct_sel_base >> 11))) 1513f71d0a05SDoug Thompson high_range = 1; 1514f71d0a05SDoug Thompson 1515f71d0a05SDoug Thompson channel = f10_determine_channel(pvt, sys_addr, high_range, intlv_en); 1516f71d0a05SDoug Thompson 1517f71d0a05SDoug Thompson chan_addr = f10_get_base_addr_offset(sys_addr, high_range, dct_sel_base, 1518f71d0a05SDoug Thompson dct_sel_base_off, hole_valid, 1519f71d0a05SDoug Thompson hole_off, dram_base); 1520f71d0a05SDoug Thompson 1521f71d0a05SDoug Thompson intlv_shift = f10_map_intlv_en_to_shift(intlv_en); 1522f71d0a05SDoug Thompson 1523f71d0a05SDoug Thompson /* remove Node ID (in case of memory interleaving) */ 1524f71d0a05SDoug Thompson tmp = chan_addr & 0xFC0; 1525f71d0a05SDoug Thompson 1526f71d0a05SDoug Thompson chan_addr = ((chan_addr >> intlv_shift) & 0xFFFFFFFFF000ULL) | tmp; 1527f71d0a05SDoug Thompson 1528f71d0a05SDoug Thompson /* remove channel interleave and hash */ 1529f71d0a05SDoug Thompson if (dct_interleave_enabled(pvt) && 1530f71d0a05SDoug Thompson !dct_high_range_enabled(pvt) && 1531f71d0a05SDoug Thompson !dct_ganging_enabled(pvt)) { 1532f71d0a05SDoug Thompson if (dct_sel_interleave_addr(pvt) != 1) 1533f71d0a05SDoug Thompson chan_addr = (chan_addr >> 1) & 0xFFFFFFFFFFFFFFC0ULL; 1534f71d0a05SDoug Thompson else { 1535f71d0a05SDoug Thompson tmp = chan_addr & 0xFC0; 1536f71d0a05SDoug Thompson chan_addr = ((chan_addr & 0xFFFFFFFFFFFFC000ULL) >> 1) 1537f71d0a05SDoug Thompson | tmp; 1538f71d0a05SDoug Thompson } 1539f71d0a05SDoug Thompson } 1540f71d0a05SDoug Thompson 1541f71d0a05SDoug Thompson debugf1(" (ChannelAddrLong=0x%llx) >> 8 becomes InputAddr=0x%x\n", 1542f71d0a05SDoug Thompson chan_addr, (u32)(chan_addr >> 8)); 1543f71d0a05SDoug Thompson 1544f71d0a05SDoug Thompson cs_found = f10_lookup_addr_in_dct(chan_addr >> 8, node_id, channel); 1545f71d0a05SDoug Thompson 1546f71d0a05SDoug Thompson if (cs_found >= 0) { 1547f71d0a05SDoug Thompson *nid = node_id; 1548f71d0a05SDoug Thompson *chan_sel = channel; 1549f71d0a05SDoug Thompson } 1550f71d0a05SDoug Thompson return cs_found; 1551f71d0a05SDoug Thompson } 1552f71d0a05SDoug Thompson 1553f71d0a05SDoug Thompson static int f10_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr, 1554f71d0a05SDoug Thompson int *node, int *chan_sel) 1555f71d0a05SDoug Thompson { 1556f71d0a05SDoug Thompson int dram_range, cs_found = -EINVAL; 1557f71d0a05SDoug Thompson u64 dram_base, dram_limit; 1558f71d0a05SDoug Thompson 1559f71d0a05SDoug Thompson for (dram_range = 0; dram_range < DRAM_REG_COUNT; dram_range++) { 1560f71d0a05SDoug Thompson 1561f71d0a05SDoug Thompson if (!pvt->dram_rw_en[dram_range]) 1562f71d0a05SDoug Thompson continue; 1563f71d0a05SDoug Thompson 1564f71d0a05SDoug Thompson dram_base = pvt->dram_base[dram_range]; 1565f71d0a05SDoug Thompson dram_limit = pvt->dram_limit[dram_range]; 1566f71d0a05SDoug Thompson 1567f71d0a05SDoug Thompson if ((dram_base <= sys_addr) && (sys_addr <= dram_limit)) { 1568f71d0a05SDoug Thompson 1569f71d0a05SDoug Thompson cs_found = f10_match_to_this_node(pvt, dram_range, 1570f71d0a05SDoug Thompson sys_addr, node, 1571f71d0a05SDoug Thompson chan_sel); 1572f71d0a05SDoug Thompson if (cs_found >= 0) 1573f71d0a05SDoug Thompson break; 1574f71d0a05SDoug Thompson } 1575f71d0a05SDoug Thompson } 1576f71d0a05SDoug Thompson return cs_found; 1577f71d0a05SDoug Thompson } 1578f71d0a05SDoug Thompson 1579f71d0a05SDoug Thompson /* 1580bdc30a0cSBorislav Petkov * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps 1581bdc30a0cSBorislav Petkov * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW). 1582f71d0a05SDoug Thompson * 1583bdc30a0cSBorislav Petkov * The @sys_addr is usually an error address received from the hardware 1584bdc30a0cSBorislav Petkov * (MCX_ADDR). 1585f71d0a05SDoug Thompson */ 1586f71d0a05SDoug Thompson static void f10_map_sysaddr_to_csrow(struct mem_ctl_info *mci, 1587ad6a32e9SBorislav Petkov struct err_regs *err_info, 1588f71d0a05SDoug Thompson u64 sys_addr) 1589f71d0a05SDoug Thompson { 1590f71d0a05SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 1591f71d0a05SDoug Thompson u32 page, offset; 1592f71d0a05SDoug Thompson int nid, csrow, chan = 0; 1593ad6a32e9SBorislav Petkov u16 syndrome; 1594f71d0a05SDoug Thompson 1595f71d0a05SDoug Thompson csrow = f10_translate_sysaddr_to_cs(pvt, sys_addr, &nid, &chan); 1596f71d0a05SDoug Thompson 1597bdc30a0cSBorislav Petkov if (csrow < 0) { 1598bdc30a0cSBorislav Petkov edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1599bdc30a0cSBorislav Petkov return; 1600bdc30a0cSBorislav Petkov } 1601bdc30a0cSBorislav Petkov 1602f71d0a05SDoug Thompson error_address_to_page_and_offset(sys_addr, &page, &offset); 1603f71d0a05SDoug Thompson 1604ad6a32e9SBorislav Petkov syndrome = extract_syndrome(err_info); 1605f71d0a05SDoug Thompson 1606f71d0a05SDoug Thompson /* 1607bdc30a0cSBorislav Petkov * We need the syndromes for channel detection only when we're 1608bdc30a0cSBorislav Petkov * ganged. Otherwise @chan should already contain the channel at 1609bdc30a0cSBorislav Petkov * this point. 1610f71d0a05SDoug Thompson */ 1611962b70a1SBorislav Petkov if (dct_ganging_enabled(pvt) && (pvt->nbcfg & K8_NBCFG_CHIPKILL)) 1612bfc04aecSBorislav Petkov chan = get_channel_from_ecc_syndrome(mci, syndrome); 1613f71d0a05SDoug Thompson 1614bdc30a0cSBorislav Petkov if (chan >= 0) 1615bdc30a0cSBorislav Petkov edac_mc_handle_ce(mci, page, offset, syndrome, csrow, chan, 1616bdc30a0cSBorislav Petkov EDAC_MOD_STR); 1617bdc30a0cSBorislav Petkov else 1618bdc30a0cSBorislav Petkov /* 1619bdc30a0cSBorislav Petkov * Channel unknown, report all channels on this CSROW as failed. 1620bdc30a0cSBorislav Petkov */ 1621bdc30a0cSBorislav Petkov for (chan = 0; chan < mci->csrows[csrow].nr_channels; chan++) 1622f71d0a05SDoug Thompson edac_mc_handle_ce(mci, page, offset, syndrome, 1623f71d0a05SDoug Thompson csrow, chan, EDAC_MOD_STR); 1624f71d0a05SDoug Thompson } 1625f71d0a05SDoug Thompson 1626f71d0a05SDoug Thompson /* 16278566c4dfSBorislav Petkov * debug routine to display the memory sizes of all logical DIMMs and its 1628f71d0a05SDoug Thompson * CSROWs as well 1629f71d0a05SDoug Thompson */ 16308566c4dfSBorislav Petkov static void amd64_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt) 1631f71d0a05SDoug Thompson { 1632603adaf6SBorislav Petkov int dimm, size0, size1, factor = 0; 1633f71d0a05SDoug Thompson u32 dbam; 1634f71d0a05SDoug Thompson u32 *dcsb; 1635f71d0a05SDoug Thompson 16368566c4dfSBorislav Petkov if (boot_cpu_data.x86 == 0xf) { 1637603adaf6SBorislav Petkov if (pvt->dclr0 & F10_WIDTH_128) 1638603adaf6SBorislav Petkov factor = 1; 1639603adaf6SBorislav Petkov 16408566c4dfSBorislav Petkov /* K8 families < revF not supported yet */ 16411433eb99SBorislav Petkov if (pvt->ext_model < K8_REV_F) 16428566c4dfSBorislav Petkov return; 16438566c4dfSBorislav Petkov else 16448566c4dfSBorislav Petkov WARN_ON(ctrl != 0); 16458566c4dfSBorislav Petkov } 16468566c4dfSBorislav Petkov 16478566c4dfSBorislav Petkov debugf1("F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", 16488566c4dfSBorislav Petkov ctrl, ctrl ? pvt->dbam1 : pvt->dbam0); 1649f71d0a05SDoug Thompson 1650f71d0a05SDoug Thompson dbam = ctrl ? pvt->dbam1 : pvt->dbam0; 1651f71d0a05SDoug Thompson dcsb = ctrl ? pvt->dcsb1 : pvt->dcsb0; 1652f71d0a05SDoug Thompson 16538566c4dfSBorislav Petkov edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl); 16548566c4dfSBorislav Petkov 1655f71d0a05SDoug Thompson /* Dump memory sizes for DIMM and its CSROWs */ 1656f71d0a05SDoug Thompson for (dimm = 0; dimm < 4; dimm++) { 1657f71d0a05SDoug Thompson 1658f71d0a05SDoug Thompson size0 = 0; 1659f71d0a05SDoug Thompson if (dcsb[dimm*2] & K8_DCSB_CS_ENABLE) 16601433eb99SBorislav Petkov size0 = pvt->ops->dbam_to_cs(pvt, DBAM_DIMM(dimm, dbam)); 1661f71d0a05SDoug Thompson 1662f71d0a05SDoug Thompson size1 = 0; 1663f71d0a05SDoug Thompson if (dcsb[dimm*2 + 1] & K8_DCSB_CS_ENABLE) 16641433eb99SBorislav Petkov size1 = pvt->ops->dbam_to_cs(pvt, DBAM_DIMM(dimm, dbam)); 1665f71d0a05SDoug Thompson 166624f9a7feSBorislav Petkov amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n", 1667603adaf6SBorislav Petkov dimm * 2, size0 << factor, 1668603adaf6SBorislav Petkov dimm * 2 + 1, size1 << factor); 1669f71d0a05SDoug Thompson } 1670f71d0a05SDoug Thompson } 1671f71d0a05SDoug Thompson 16724d37607aSDoug Thompson static struct amd64_family_type amd64_family_types[] = { 16734d37607aSDoug Thompson [K8_CPUS] = { 16740092b20dSBorislav Petkov .ctl_name = "K8", 16758d5b5d9cSBorislav Petkov .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP, 16768d5b5d9cSBorislav Petkov .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC, 16774d37607aSDoug Thompson .ops = { 16784d37607aSDoug Thompson .early_channel_count = k8_early_channel_count, 16794d37607aSDoug Thompson .get_error_address = k8_get_error_address, 16804d37607aSDoug Thompson .read_dram_base_limit = k8_read_dram_base_limit, 16814d37607aSDoug Thompson .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow, 16821433eb99SBorislav Petkov .dbam_to_cs = k8_dbam_to_chip_select, 16834d37607aSDoug Thompson } 16844d37607aSDoug Thompson }, 16854d37607aSDoug Thompson [F10_CPUS] = { 16860092b20dSBorislav Petkov .ctl_name = "F10h", 16878d5b5d9cSBorislav Petkov .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP, 16888d5b5d9cSBorislav Petkov .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC, 16894d37607aSDoug Thompson .ops = { 16904d37607aSDoug Thompson .early_channel_count = f10_early_channel_count, 16914d37607aSDoug Thompson .get_error_address = f10_get_error_address, 16924d37607aSDoug Thompson .read_dram_base_limit = f10_read_dram_base_limit, 16934d37607aSDoug Thompson .read_dram_ctl_register = f10_read_dram_ctl_register, 16944d37607aSDoug Thompson .map_sysaddr_to_csrow = f10_map_sysaddr_to_csrow, 16951433eb99SBorislav Petkov .dbam_to_cs = f10_dbam_to_chip_select, 16964d37607aSDoug Thompson } 16974d37607aSDoug Thompson }, 16984d37607aSDoug Thompson }; 16994d37607aSDoug Thompson 17004d37607aSDoug Thompson static struct pci_dev *pci_get_related_function(unsigned int vendor, 17014d37607aSDoug Thompson unsigned int device, 17024d37607aSDoug Thompson struct pci_dev *related) 17034d37607aSDoug Thompson { 17044d37607aSDoug Thompson struct pci_dev *dev = NULL; 17054d37607aSDoug Thompson 17064d37607aSDoug Thompson dev = pci_get_device(vendor, device, dev); 17074d37607aSDoug Thompson while (dev) { 17084d37607aSDoug Thompson if ((dev->bus->number == related->bus->number) && 17094d37607aSDoug Thompson (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn))) 17104d37607aSDoug Thompson break; 17114d37607aSDoug Thompson dev = pci_get_device(vendor, device, dev); 17124d37607aSDoug Thompson } 17134d37607aSDoug Thompson 17144d37607aSDoug Thompson return dev; 17154d37607aSDoug Thompson } 17164d37607aSDoug Thompson 1717b1289d6fSDoug Thompson /* 1718bfc04aecSBorislav Petkov * These are tables of eigenvectors (one per line) which can be used for the 1719bfc04aecSBorislav Petkov * construction of the syndrome tables. The modified syndrome search algorithm 1720bfc04aecSBorislav Petkov * uses those to find the symbol in error and thus the DIMM. 1721b1289d6fSDoug Thompson * 1722bfc04aecSBorislav Petkov * Algorithm courtesy of Ross LaFetra from AMD. 1723b1289d6fSDoug Thompson */ 1724bfc04aecSBorislav Petkov static u16 x4_vectors[] = { 1725bfc04aecSBorislav Petkov 0x2f57, 0x1afe, 0x66cc, 0xdd88, 1726bfc04aecSBorislav Petkov 0x11eb, 0x3396, 0x7f4c, 0xeac8, 1727bfc04aecSBorislav Petkov 0x0001, 0x0002, 0x0004, 0x0008, 1728bfc04aecSBorislav Petkov 0x1013, 0x3032, 0x4044, 0x8088, 1729bfc04aecSBorislav Petkov 0x106b, 0x30d6, 0x70fc, 0xe0a8, 1730bfc04aecSBorislav Petkov 0x4857, 0xc4fe, 0x13cc, 0x3288, 1731bfc04aecSBorislav Petkov 0x1ac5, 0x2f4a, 0x5394, 0xa1e8, 1732bfc04aecSBorislav Petkov 0x1f39, 0x251e, 0xbd6c, 0x6bd8, 1733bfc04aecSBorislav Petkov 0x15c1, 0x2a42, 0x89ac, 0x4758, 1734bfc04aecSBorislav Petkov 0x2b03, 0x1602, 0x4f0c, 0xca08, 1735bfc04aecSBorislav Petkov 0x1f07, 0x3a0e, 0x6b04, 0xbd08, 1736bfc04aecSBorislav Petkov 0x8ba7, 0x465e, 0x244c, 0x1cc8, 1737bfc04aecSBorislav Petkov 0x2b87, 0x164e, 0x642c, 0xdc18, 1738bfc04aecSBorislav Petkov 0x40b9, 0x80de, 0x1094, 0x20e8, 1739bfc04aecSBorislav Petkov 0x27db, 0x1eb6, 0x9dac, 0x7b58, 1740bfc04aecSBorislav Petkov 0x11c1, 0x2242, 0x84ac, 0x4c58, 1741bfc04aecSBorislav Petkov 0x1be5, 0x2d7a, 0x5e34, 0xa718, 1742bfc04aecSBorislav Petkov 0x4b39, 0x8d1e, 0x14b4, 0x28d8, 1743bfc04aecSBorislav Petkov 0x4c97, 0xc87e, 0x11fc, 0x33a8, 1744bfc04aecSBorislav Petkov 0x8e97, 0x497e, 0x2ffc, 0x1aa8, 1745bfc04aecSBorislav Petkov 0x16b3, 0x3d62, 0x4f34, 0x8518, 1746bfc04aecSBorislav Petkov 0x1e2f, 0x391a, 0x5cac, 0xf858, 1747bfc04aecSBorislav Petkov 0x1d9f, 0x3b7a, 0x572c, 0xfe18, 1748bfc04aecSBorislav Petkov 0x15f5, 0x2a5a, 0x5264, 0xa3b8, 1749bfc04aecSBorislav Petkov 0x1dbb, 0x3b66, 0x715c, 0xe3f8, 1750bfc04aecSBorislav Petkov 0x4397, 0xc27e, 0x17fc, 0x3ea8, 1751bfc04aecSBorislav Petkov 0x1617, 0x3d3e, 0x6464, 0xb8b8, 1752bfc04aecSBorislav Petkov 0x23ff, 0x12aa, 0xab6c, 0x56d8, 1753bfc04aecSBorislav Petkov 0x2dfb, 0x1ba6, 0x913c, 0x7328, 1754bfc04aecSBorislav Petkov 0x185d, 0x2ca6, 0x7914, 0x9e28, 1755bfc04aecSBorislav Petkov 0x171b, 0x3e36, 0x7d7c, 0xebe8, 1756bfc04aecSBorislav Petkov 0x4199, 0x82ee, 0x19f4, 0x2e58, 1757bfc04aecSBorislav Petkov 0x4807, 0xc40e, 0x130c, 0x3208, 1758bfc04aecSBorislav Petkov 0x1905, 0x2e0a, 0x5804, 0xac08, 1759bfc04aecSBorislav Petkov 0x213f, 0x132a, 0xadfc, 0x5ba8, 1760bfc04aecSBorislav Petkov 0x19a9, 0x2efe, 0xb5cc, 0x6f88, 1761b1289d6fSDoug Thompson }; 1762b1289d6fSDoug Thompson 1763bfc04aecSBorislav Petkov static u16 x8_vectors[] = { 1764bfc04aecSBorislav Petkov 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480, 1765bfc04aecSBorislav Petkov 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80, 1766bfc04aecSBorislav Petkov 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80, 1767bfc04aecSBorislav Petkov 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80, 1768bfc04aecSBorislav Petkov 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780, 1769bfc04aecSBorislav Petkov 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080, 1770bfc04aecSBorislav Petkov 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080, 1771bfc04aecSBorislav Petkov 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080, 1772bfc04aecSBorislav Petkov 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80, 1773bfc04aecSBorislav Petkov 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580, 1774bfc04aecSBorislav Petkov 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880, 1775bfc04aecSBorislav Petkov 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280, 1776bfc04aecSBorislav Petkov 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180, 1777bfc04aecSBorislav Petkov 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580, 1778bfc04aecSBorislav Petkov 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280, 1779bfc04aecSBorislav Petkov 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180, 1780bfc04aecSBorislav Petkov 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080, 1781bfc04aecSBorislav Petkov 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 1782bfc04aecSBorislav Petkov 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000, 1783bfc04aecSBorislav Petkov }; 1784bfc04aecSBorislav Petkov 1785bfc04aecSBorislav Petkov static int decode_syndrome(u16 syndrome, u16 *vectors, int num_vecs, 1786bfc04aecSBorislav Petkov int v_dim) 1787b1289d6fSDoug Thompson { 1788bfc04aecSBorislav Petkov unsigned int i, err_sym; 1789b1289d6fSDoug Thompson 1790bfc04aecSBorislav Petkov for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) { 1791bfc04aecSBorislav Petkov u16 s = syndrome; 1792bfc04aecSBorislav Petkov int v_idx = err_sym * v_dim; 1793bfc04aecSBorislav Petkov int v_end = (err_sym + 1) * v_dim; 1794b1289d6fSDoug Thompson 1795bfc04aecSBorislav Petkov /* walk over all 16 bits of the syndrome */ 1796bfc04aecSBorislav Petkov for (i = 1; i < (1U << 16); i <<= 1) { 1797bfc04aecSBorislav Petkov 1798bfc04aecSBorislav Petkov /* if bit is set in that eigenvector... */ 1799bfc04aecSBorislav Petkov if (v_idx < v_end && vectors[v_idx] & i) { 1800bfc04aecSBorislav Petkov u16 ev_comp = vectors[v_idx++]; 1801bfc04aecSBorislav Petkov 1802bfc04aecSBorislav Petkov /* ... and bit set in the modified syndrome, */ 1803bfc04aecSBorislav Petkov if (s & i) { 1804bfc04aecSBorislav Petkov /* remove it. */ 1805bfc04aecSBorislav Petkov s ^= ev_comp; 1806bfc04aecSBorislav Petkov 1807bfc04aecSBorislav Petkov if (!s) 1808bfc04aecSBorislav Petkov return err_sym; 1809bfc04aecSBorislav Petkov } 1810bfc04aecSBorislav Petkov 1811bfc04aecSBorislav Petkov } else if (s & i) 1812bfc04aecSBorislav Petkov /* can't get to zero, move to next symbol */ 1813bfc04aecSBorislav Petkov break; 1814bfc04aecSBorislav Petkov } 1815b1289d6fSDoug Thompson } 1816b1289d6fSDoug Thompson 1817b1289d6fSDoug Thompson debugf0("syndrome(%x) not found\n", syndrome); 1818b1289d6fSDoug Thompson return -1; 1819b1289d6fSDoug Thompson } 1820d27bf6faSDoug Thompson 1821bfc04aecSBorislav Petkov static int map_err_sym_to_channel(int err_sym, int sym_size) 1822bfc04aecSBorislav Petkov { 1823bfc04aecSBorislav Petkov if (sym_size == 4) 1824bfc04aecSBorislav Petkov switch (err_sym) { 1825bfc04aecSBorislav Petkov case 0x20: 1826bfc04aecSBorislav Petkov case 0x21: 1827bfc04aecSBorislav Petkov return 0; 1828bfc04aecSBorislav Petkov break; 1829bfc04aecSBorislav Petkov case 0x22: 1830bfc04aecSBorislav Petkov case 0x23: 1831bfc04aecSBorislav Petkov return 1; 1832bfc04aecSBorislav Petkov break; 1833bfc04aecSBorislav Petkov default: 1834bfc04aecSBorislav Petkov return err_sym >> 4; 1835bfc04aecSBorislav Petkov break; 1836bfc04aecSBorislav Petkov } 1837bfc04aecSBorislav Petkov /* x8 symbols */ 1838bfc04aecSBorislav Petkov else 1839bfc04aecSBorislav Petkov switch (err_sym) { 1840bfc04aecSBorislav Petkov /* imaginary bits not in a DIMM */ 1841bfc04aecSBorislav Petkov case 0x10: 1842bfc04aecSBorislav Petkov WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n", 1843bfc04aecSBorislav Petkov err_sym); 1844bfc04aecSBorislav Petkov return -1; 1845bfc04aecSBorislav Petkov break; 1846bfc04aecSBorislav Petkov 1847bfc04aecSBorislav Petkov case 0x11: 1848bfc04aecSBorislav Petkov return 0; 1849bfc04aecSBorislav Petkov break; 1850bfc04aecSBorislav Petkov case 0x12: 1851bfc04aecSBorislav Petkov return 1; 1852bfc04aecSBorislav Petkov break; 1853bfc04aecSBorislav Petkov default: 1854bfc04aecSBorislav Petkov return err_sym >> 3; 1855bfc04aecSBorislav Petkov break; 1856bfc04aecSBorislav Petkov } 1857bfc04aecSBorislav Petkov return -1; 1858bfc04aecSBorislav Petkov } 1859bfc04aecSBorislav Petkov 1860bfc04aecSBorislav Petkov static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome) 1861bfc04aecSBorislav Petkov { 1862bfc04aecSBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 1863ad6a32e9SBorislav Petkov int err_sym = -1; 1864bfc04aecSBorislav Petkov 1865ad6a32e9SBorislav Petkov if (pvt->syn_type == 8) 1866bfc04aecSBorislav Petkov err_sym = decode_syndrome(syndrome, x8_vectors, 1867ad6a32e9SBorislav Petkov ARRAY_SIZE(x8_vectors), 1868ad6a32e9SBorislav Petkov pvt->syn_type); 1869ad6a32e9SBorislav Petkov else if (pvt->syn_type == 4) 1870ad6a32e9SBorislav Petkov err_sym = decode_syndrome(syndrome, x4_vectors, 1871ad6a32e9SBorislav Petkov ARRAY_SIZE(x4_vectors), 1872ad6a32e9SBorislav Petkov pvt->syn_type); 1873ad6a32e9SBorislav Petkov else { 187424f9a7feSBorislav Petkov amd64_warn("Illegal syndrome type: %u\n", pvt->syn_type); 1875ad6a32e9SBorislav Petkov return err_sym; 1876bfc04aecSBorislav Petkov } 1877ad6a32e9SBorislav Petkov 1878ad6a32e9SBorislav Petkov return map_err_sym_to_channel(err_sym, pvt->syn_type); 187941c31044SBorislav Petkov } 1880bfc04aecSBorislav Petkov 1881d27bf6faSDoug Thompson /* 1882d27bf6faSDoug Thompson * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR 1883d27bf6faSDoug Thompson * ADDRESS and process. 1884d27bf6faSDoug Thompson */ 1885d27bf6faSDoug Thompson static void amd64_handle_ce(struct mem_ctl_info *mci, 1886ef44cc4cSBorislav Petkov struct err_regs *info) 1887d27bf6faSDoug Thompson { 1888d27bf6faSDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 188944e9e2eeSBorislav Petkov u64 sys_addr; 1890d27bf6faSDoug Thompson 1891d27bf6faSDoug Thompson /* Ensure that the Error Address is VALID */ 189224f9a7feSBorislav Petkov if (!(info->nbsh & K8_NBSH_VALID_ERROR_ADDR)) { 189324f9a7feSBorislav Petkov amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n"); 1894d27bf6faSDoug Thompson edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1895d27bf6faSDoug Thompson return; 1896d27bf6faSDoug Thompson } 1897d27bf6faSDoug Thompson 18981f6bcee7SBorislav Petkov sys_addr = pvt->ops->get_error_address(mci, info); 1899d27bf6faSDoug Thompson 190024f9a7feSBorislav Petkov amd64_mc_err(mci, "CE ERROR_ADDRESS= 0x%llx\n", sys_addr); 1901d27bf6faSDoug Thompson 190244e9e2eeSBorislav Petkov pvt->ops->map_sysaddr_to_csrow(mci, info, sys_addr); 1903d27bf6faSDoug Thompson } 1904d27bf6faSDoug Thompson 1905d27bf6faSDoug Thompson /* Handle any Un-correctable Errors (UEs) */ 1906d27bf6faSDoug Thompson static void amd64_handle_ue(struct mem_ctl_info *mci, 1907ef44cc4cSBorislav Petkov struct err_regs *info) 1908d27bf6faSDoug Thompson { 19091f6bcee7SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 19101f6bcee7SBorislav Petkov struct mem_ctl_info *log_mci, *src_mci = NULL; 1911d27bf6faSDoug Thompson int csrow; 191244e9e2eeSBorislav Petkov u64 sys_addr; 1913d27bf6faSDoug Thompson u32 page, offset; 1914d27bf6faSDoug Thompson 1915d27bf6faSDoug Thompson log_mci = mci; 1916d27bf6faSDoug Thompson 191724f9a7feSBorislav Petkov if (!(info->nbsh & K8_NBSH_VALID_ERROR_ADDR)) { 191824f9a7feSBorislav Petkov amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n"); 1919d27bf6faSDoug Thompson edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); 1920d27bf6faSDoug Thompson return; 1921d27bf6faSDoug Thompson } 1922d27bf6faSDoug Thompson 19231f6bcee7SBorislav Petkov sys_addr = pvt->ops->get_error_address(mci, info); 1924d27bf6faSDoug Thompson 1925d27bf6faSDoug Thompson /* 1926d27bf6faSDoug Thompson * Find out which node the error address belongs to. This may be 1927d27bf6faSDoug Thompson * different from the node that detected the error. 1928d27bf6faSDoug Thompson */ 192944e9e2eeSBorislav Petkov src_mci = find_mc_by_sys_addr(mci, sys_addr); 1930d27bf6faSDoug Thompson if (!src_mci) { 193124f9a7feSBorislav Petkov amd64_mc_err(mci, "ERROR ADDRESS (0x%lx) NOT mapped to a MC\n", 193244e9e2eeSBorislav Petkov (unsigned long)sys_addr); 1933d27bf6faSDoug Thompson edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); 1934d27bf6faSDoug Thompson return; 1935d27bf6faSDoug Thompson } 1936d27bf6faSDoug Thompson 1937d27bf6faSDoug Thompson log_mci = src_mci; 1938d27bf6faSDoug Thompson 193944e9e2eeSBorislav Petkov csrow = sys_addr_to_csrow(log_mci, sys_addr); 1940d27bf6faSDoug Thompson if (csrow < 0) { 194124f9a7feSBorislav Petkov amd64_mc_err(mci, "ERROR_ADDRESS (0x%lx) NOT mapped to CS\n", 194244e9e2eeSBorislav Petkov (unsigned long)sys_addr); 1943d27bf6faSDoug Thompson edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); 1944d27bf6faSDoug Thompson } else { 194544e9e2eeSBorislav Petkov error_address_to_page_and_offset(sys_addr, &page, &offset); 1946d27bf6faSDoug Thompson edac_mc_handle_ue(log_mci, page, offset, csrow, EDAC_MOD_STR); 1947d27bf6faSDoug Thompson } 1948d27bf6faSDoug Thompson } 1949d27bf6faSDoug Thompson 1950549d042dSBorislav Petkov static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci, 1951b69b29deSBorislav Petkov struct err_regs *info) 1952d27bf6faSDoug Thompson { 1953b70ef010SBorislav Petkov u32 ec = ERROR_CODE(info->nbsl); 1954b70ef010SBorislav Petkov u32 xec = EXT_ERROR_CODE(info->nbsl); 195517adea01SBorislav Petkov int ecc_type = (info->nbsh >> 13) & 0x3; 1956d27bf6faSDoug Thompson 1957b70ef010SBorislav Petkov /* Bail early out if this was an 'observed' error */ 1958b70ef010SBorislav Petkov if (PP(ec) == K8_NBSL_PP_OBS) 1959b70ef010SBorislav Petkov return; 1960d27bf6faSDoug Thompson 1961ecaf5606SBorislav Petkov /* Do only ECC errors */ 1962ecaf5606SBorislav Petkov if (xec && xec != F10_NBSL_EXT_ERR_ECC) 1963d27bf6faSDoug Thompson return; 1964d27bf6faSDoug Thompson 1965ecaf5606SBorislav Petkov if (ecc_type == 2) 1966d27bf6faSDoug Thompson amd64_handle_ce(mci, info); 1967ecaf5606SBorislav Petkov else if (ecc_type == 1) 1968d27bf6faSDoug Thompson amd64_handle_ue(mci, info); 1969d27bf6faSDoug Thompson } 1970d27bf6faSDoug Thompson 19717cfd4a87SBorislav Petkov void amd64_decode_bus_error(int node_id, struct mce *m, u32 nbcfg) 1972d27bf6faSDoug Thompson { 1973cc4d8860SBorislav Petkov struct mem_ctl_info *mci = mcis[node_id]; 19747cfd4a87SBorislav Petkov struct err_regs regs; 1975d27bf6faSDoug Thompson 19767cfd4a87SBorislav Petkov regs.nbsl = (u32) m->status; 19777cfd4a87SBorislav Petkov regs.nbsh = (u32)(m->status >> 32); 19787cfd4a87SBorislav Petkov regs.nbeal = (u32) m->addr; 19797cfd4a87SBorislav Petkov regs.nbeah = (u32)(m->addr >> 32); 19807cfd4a87SBorislav Petkov regs.nbcfg = nbcfg; 19817cfd4a87SBorislav Petkov 19827cfd4a87SBorislav Petkov __amd64_decode_bus_error(mci, ®s); 1983d27bf6faSDoug Thompson 1984d27bf6faSDoug Thompson /* 1985d27bf6faSDoug Thompson * Check the UE bit of the NB status high register, if set generate some 1986d27bf6faSDoug Thompson * logs. If NOT a GART error, then process the event as a NO-INFO event. 1987d27bf6faSDoug Thompson * If it was a GART error, skip that process. 1988549d042dSBorislav Petkov * 1989549d042dSBorislav Petkov * FIXME: this should go somewhere else, if at all. 1990d27bf6faSDoug Thompson */ 19917cfd4a87SBorislav Petkov if (regs.nbsh & K8_NBSH_UC_ERR && !report_gart_errors) 19925110dbdeSBorislav Petkov edac_mc_handle_ue_no_info(mci, "UE bit is set"); 1993549d042dSBorislav Petkov 1994d27bf6faSDoug Thompson } 1995d27bf6faSDoug Thompson 19960ec449eeSDoug Thompson /* 19978d5b5d9cSBorislav Petkov * Use pvt->F2 which contains the F2 CPU PCI device to get the related 1998bbd0c1f6SBorislav Petkov * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error. 19990ec449eeSDoug Thompson */ 2000360b7f3cSBorislav Petkov static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id) 20010ec449eeSDoug Thompson { 20020ec449eeSDoug Thompson /* Reserve the ADDRESS MAP Device */ 20038d5b5d9cSBorislav Petkov pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2); 20048d5b5d9cSBorislav Petkov if (!pvt->F1) { 200524f9a7feSBorislav Petkov amd64_err("error address map device not found: " 20060ec449eeSDoug Thompson "vendor %x device 0x%x (broken BIOS?)\n", 2007bbd0c1f6SBorislav Petkov PCI_VENDOR_ID_AMD, f1_id); 2008bbd0c1f6SBorislav Petkov return -ENODEV; 20090ec449eeSDoug Thompson } 20100ec449eeSDoug Thompson 20110ec449eeSDoug Thompson /* Reserve the MISC Device */ 20128d5b5d9cSBorislav Petkov pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2); 20138d5b5d9cSBorislav Petkov if (!pvt->F3) { 20148d5b5d9cSBorislav Petkov pci_dev_put(pvt->F1); 20158d5b5d9cSBorislav Petkov pvt->F1 = NULL; 20160ec449eeSDoug Thompson 201724f9a7feSBorislav Petkov amd64_err("error F3 device not found: " 20180ec449eeSDoug Thompson "vendor %x device 0x%x (broken BIOS?)\n", 2019bbd0c1f6SBorislav Petkov PCI_VENDOR_ID_AMD, f3_id); 20208d5b5d9cSBorislav Petkov 2021bbd0c1f6SBorislav Petkov return -ENODEV; 20220ec449eeSDoug Thompson } 20238d5b5d9cSBorislav Petkov debugf1("F1: %s\n", pci_name(pvt->F1)); 20248d5b5d9cSBorislav Petkov debugf1("F2: %s\n", pci_name(pvt->F2)); 20258d5b5d9cSBorislav Petkov debugf1("F3: %s\n", pci_name(pvt->F3)); 20260ec449eeSDoug Thompson 20270ec449eeSDoug Thompson return 0; 20280ec449eeSDoug Thompson } 20290ec449eeSDoug Thompson 2030360b7f3cSBorislav Petkov static void free_mc_sibling_devs(struct amd64_pvt *pvt) 20310ec449eeSDoug Thompson { 20328d5b5d9cSBorislav Petkov pci_dev_put(pvt->F1); 20338d5b5d9cSBorislav Petkov pci_dev_put(pvt->F3); 20340ec449eeSDoug Thompson } 20350ec449eeSDoug Thompson 20360ec449eeSDoug Thompson /* 20370ec449eeSDoug Thompson * Retrieve the hardware registers of the memory controller (this includes the 20380ec449eeSDoug Thompson * 'Address Map' and 'Misc' device regs) 20390ec449eeSDoug Thompson */ 2040360b7f3cSBorislav Petkov static void read_mc_regs(struct amd64_pvt *pvt) 20410ec449eeSDoug Thompson { 20420ec449eeSDoug Thompson u64 msr_val; 2043ad6a32e9SBorislav Petkov u32 tmp; 20446ba5dcdcSBorislav Petkov int dram; 20450ec449eeSDoug Thompson 20460ec449eeSDoug Thompson /* 20470ec449eeSDoug Thompson * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since 20480ec449eeSDoug Thompson * those are Read-As-Zero 20490ec449eeSDoug Thompson */ 2050e97f8bb8SBorislav Petkov rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem); 2051e97f8bb8SBorislav Petkov debugf0(" TOP_MEM: 0x%016llx\n", pvt->top_mem); 20520ec449eeSDoug Thompson 20530ec449eeSDoug Thompson /* check first whether TOP_MEM2 is enabled */ 20540ec449eeSDoug Thompson rdmsrl(MSR_K8_SYSCFG, msr_val); 20550ec449eeSDoug Thompson if (msr_val & (1U << 21)) { 2056e97f8bb8SBorislav Petkov rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2); 2057e97f8bb8SBorislav Petkov debugf0(" TOP_MEM2: 0x%016llx\n", pvt->top_mem2); 20580ec449eeSDoug Thompson } else 20590ec449eeSDoug Thompson debugf0(" TOP_MEM2 disabled.\n"); 20600ec449eeSDoug Thompson 20618d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, K8_NBCAP, &pvt->nbcap); 20620ec449eeSDoug Thompson 20630ec449eeSDoug Thompson if (pvt->ops->read_dram_ctl_register) 20640ec449eeSDoug Thompson pvt->ops->read_dram_ctl_register(pvt); 20650ec449eeSDoug Thompson 20660ec449eeSDoug Thompson for (dram = 0; dram < DRAM_REG_COUNT; dram++) { 20670ec449eeSDoug Thompson /* 20680ec449eeSDoug Thompson * Call CPU specific READ function to get the DRAM Base and 20690ec449eeSDoug Thompson * Limit values from the DCT. 20700ec449eeSDoug Thompson */ 20710ec449eeSDoug Thompson pvt->ops->read_dram_base_limit(pvt, dram); 20720ec449eeSDoug Thompson 20730ec449eeSDoug Thompson /* 20740ec449eeSDoug Thompson * Only print out debug info on rows with both R and W Enabled. 20750ec449eeSDoug Thompson * Normal processing, compiler should optimize this whole 'if' 20760ec449eeSDoug Thompson * debug output block away. 20770ec449eeSDoug Thompson */ 20780ec449eeSDoug Thompson if (pvt->dram_rw_en[dram] != 0) { 2079e97f8bb8SBorislav Petkov debugf1(" DRAM-BASE[%d]: 0x%016llx " 2080e97f8bb8SBorislav Petkov "DRAM-LIMIT: 0x%016llx\n", 20810ec449eeSDoug Thompson dram, 2082e97f8bb8SBorislav Petkov pvt->dram_base[dram], 2083e97f8bb8SBorislav Petkov pvt->dram_limit[dram]); 2084e97f8bb8SBorislav Petkov 20850ec449eeSDoug Thompson debugf1(" IntlvEn=%s %s %s " 20860ec449eeSDoug Thompson "IntlvSel=%d DstNode=%d\n", 20870ec449eeSDoug Thompson pvt->dram_IntlvEn[dram] ? 20880ec449eeSDoug Thompson "Enabled" : "Disabled", 20890ec449eeSDoug Thompson (pvt->dram_rw_en[dram] & 0x2) ? "W" : "!W", 20900ec449eeSDoug Thompson (pvt->dram_rw_en[dram] & 0x1) ? "R" : "!R", 20910ec449eeSDoug Thompson pvt->dram_IntlvSel[dram], 20920ec449eeSDoug Thompson pvt->dram_DstNode[dram]); 20930ec449eeSDoug Thompson } 20940ec449eeSDoug Thompson } 20950ec449eeSDoug Thompson 20960ec449eeSDoug Thompson amd64_read_dct_base_mask(pvt); 20970ec449eeSDoug Thompson 20988d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F1, K8_DHAR, &pvt->dhar); 20990ec449eeSDoug Thompson amd64_read_dbam_reg(pvt); 21000ec449eeSDoug Thompson 21018d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare); 21020ec449eeSDoug Thompson 21038d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F2, F10_DCLR_0, &pvt->dclr0); 21048d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F2, F10_DCHR_0, &pvt->dchr0); 21050ec449eeSDoug Thompson 2106ad6a32e9SBorislav Petkov if (boot_cpu_data.x86 >= 0x10) { 2107ad6a32e9SBorislav Petkov if (!dct_ganging_enabled(pvt)) { 21088d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F2, F10_DCLR_1, &pvt->dclr1); 21098d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F2, F10_DCHR_1, &pvt->dchr1); 21100ec449eeSDoug Thompson } 21118d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp); 2112ad6a32e9SBorislav Petkov } 2113ad6a32e9SBorislav Petkov 2114ad6a32e9SBorislav Petkov if (boot_cpu_data.x86 == 0x10 && 2115ad6a32e9SBorislav Petkov boot_cpu_data.x86_model > 7 && 2116ad6a32e9SBorislav Petkov /* F3x180[EccSymbolSize]=1 => x8 symbols */ 2117ad6a32e9SBorislav Petkov tmp & BIT(25)) 2118ad6a32e9SBorislav Petkov pvt->syn_type = 8; 2119ad6a32e9SBorislav Petkov else 2120ad6a32e9SBorislav Petkov pvt->syn_type = 4; 2121ad6a32e9SBorislav Petkov 21220ec449eeSDoug Thompson amd64_dump_misc_regs(pvt); 21230ec449eeSDoug Thompson } 21240ec449eeSDoug Thompson 21250ec449eeSDoug Thompson /* 21260ec449eeSDoug Thompson * NOTE: CPU Revision Dependent code 21270ec449eeSDoug Thompson * 21280ec449eeSDoug Thompson * Input: 21299d858bb1SBorislav Petkov * @csrow_nr ChipSelect Row Number (0..pvt->cs_count-1) 21300ec449eeSDoug Thompson * k8 private pointer to --> 21310ec449eeSDoug Thompson * DRAM Bank Address mapping register 21320ec449eeSDoug Thompson * node_id 21330ec449eeSDoug Thompson * DCL register where dual_channel_active is 21340ec449eeSDoug Thompson * 21350ec449eeSDoug Thompson * The DBAM register consists of 4 sets of 4 bits each definitions: 21360ec449eeSDoug Thompson * 21370ec449eeSDoug Thompson * Bits: CSROWs 21380ec449eeSDoug Thompson * 0-3 CSROWs 0 and 1 21390ec449eeSDoug Thompson * 4-7 CSROWs 2 and 3 21400ec449eeSDoug Thompson * 8-11 CSROWs 4 and 5 21410ec449eeSDoug Thompson * 12-15 CSROWs 6 and 7 21420ec449eeSDoug Thompson * 21430ec449eeSDoug Thompson * Values range from: 0 to 15 21440ec449eeSDoug Thompson * The meaning of the values depends on CPU revision and dual-channel state, 21450ec449eeSDoug Thompson * see relevant BKDG more info. 21460ec449eeSDoug Thompson * 21470ec449eeSDoug Thompson * The memory controller provides for total of only 8 CSROWs in its current 21480ec449eeSDoug Thompson * architecture. Each "pair" of CSROWs normally represents just one DIMM in 21490ec449eeSDoug Thompson * single channel or two (2) DIMMs in dual channel mode. 21500ec449eeSDoug Thompson * 21510ec449eeSDoug Thompson * The following code logic collapses the various tables for CSROW based on CPU 21520ec449eeSDoug Thompson * revision. 21530ec449eeSDoug Thompson * 21540ec449eeSDoug Thompson * Returns: 21550ec449eeSDoug Thompson * The number of PAGE_SIZE pages on the specified CSROW number it 21560ec449eeSDoug Thompson * encompasses 21570ec449eeSDoug Thompson * 21580ec449eeSDoug Thompson */ 21590ec449eeSDoug Thompson static u32 amd64_csrow_nr_pages(int csrow_nr, struct amd64_pvt *pvt) 21600ec449eeSDoug Thompson { 21611433eb99SBorislav Petkov u32 cs_mode, nr_pages; 21620ec449eeSDoug Thompson 21630ec449eeSDoug Thompson /* 21640ec449eeSDoug Thompson * The math on this doesn't look right on the surface because x/2*4 can 21650ec449eeSDoug Thompson * be simplified to x*2 but this expression makes use of the fact that 21660ec449eeSDoug Thompson * it is integral math where 1/2=0. This intermediate value becomes the 21670ec449eeSDoug Thompson * number of bits to shift the DBAM register to extract the proper CSROW 21680ec449eeSDoug Thompson * field. 21690ec449eeSDoug Thompson */ 21701433eb99SBorislav Petkov cs_mode = (pvt->dbam0 >> ((csrow_nr / 2) * 4)) & 0xF; 21710ec449eeSDoug Thompson 21721433eb99SBorislav Petkov nr_pages = pvt->ops->dbam_to_cs(pvt, cs_mode) << (20 - PAGE_SHIFT); 21730ec449eeSDoug Thompson 21740ec449eeSDoug Thompson /* 21750ec449eeSDoug Thompson * If dual channel then double the memory size of single channel. 21760ec449eeSDoug Thompson * Channel count is 1 or 2 21770ec449eeSDoug Thompson */ 21780ec449eeSDoug Thompson nr_pages <<= (pvt->channel_count - 1); 21790ec449eeSDoug Thompson 21801433eb99SBorislav Petkov debugf0(" (csrow=%d) DBAM map index= %d\n", csrow_nr, cs_mode); 21810ec449eeSDoug Thompson debugf0(" nr_pages= %u channel-count = %d\n", 21820ec449eeSDoug Thompson nr_pages, pvt->channel_count); 21830ec449eeSDoug Thompson 21840ec449eeSDoug Thompson return nr_pages; 21850ec449eeSDoug Thompson } 21860ec449eeSDoug Thompson 21870ec449eeSDoug Thompson /* 21880ec449eeSDoug Thompson * Initialize the array of csrow attribute instances, based on the values 21890ec449eeSDoug Thompson * from pci config hardware registers. 21900ec449eeSDoug Thompson */ 2191360b7f3cSBorislav Petkov static int init_csrows(struct mem_ctl_info *mci) 21920ec449eeSDoug Thompson { 21930ec449eeSDoug Thompson struct csrow_info *csrow; 21942299ef71SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 21950ec449eeSDoug Thompson u64 input_addr_min, input_addr_max, sys_addr; 21962299ef71SBorislav Petkov u32 val; 21976ba5dcdcSBorislav Petkov int i, empty = 1; 21980ec449eeSDoug Thompson 21992299ef71SBorislav Petkov amd64_read_pci_cfg(pvt->F3, K8_NBCFG, &val); 22000ec449eeSDoug Thompson 22012299ef71SBorislav Petkov pvt->nbcfg = val; 22022299ef71SBorislav Petkov pvt->ctl_error_info.nbcfg = val; 22030ec449eeSDoug Thompson 22042299ef71SBorislav Petkov debugf0("node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n", 22052299ef71SBorislav Petkov pvt->mc_node_id, val, 22062299ef71SBorislav Petkov !!(val & K8_NBCFG_CHIPKILL), !!(val & K8_NBCFG_ECC_ENABLE)); 22070ec449eeSDoug Thompson 22089d858bb1SBorislav Petkov for (i = 0; i < pvt->cs_count; i++) { 22090ec449eeSDoug Thompson csrow = &mci->csrows[i]; 22100ec449eeSDoug Thompson 22110ec449eeSDoug Thompson if ((pvt->dcsb0[i] & K8_DCSB_CS_ENABLE) == 0) { 22120ec449eeSDoug Thompson debugf1("----CSROW %d EMPTY for node %d\n", i, 22130ec449eeSDoug Thompson pvt->mc_node_id); 22140ec449eeSDoug Thompson continue; 22150ec449eeSDoug Thompson } 22160ec449eeSDoug Thompson 22170ec449eeSDoug Thompson debugf1("----CSROW %d VALID for MC node %d\n", 22180ec449eeSDoug Thompson i, pvt->mc_node_id); 22190ec449eeSDoug Thompson 22200ec449eeSDoug Thompson empty = 0; 22210ec449eeSDoug Thompson csrow->nr_pages = amd64_csrow_nr_pages(i, pvt); 22220ec449eeSDoug Thompson find_csrow_limits(mci, i, &input_addr_min, &input_addr_max); 22230ec449eeSDoug Thompson sys_addr = input_addr_to_sys_addr(mci, input_addr_min); 22240ec449eeSDoug Thompson csrow->first_page = (u32) (sys_addr >> PAGE_SHIFT); 22250ec449eeSDoug Thompson sys_addr = input_addr_to_sys_addr(mci, input_addr_max); 22260ec449eeSDoug Thompson csrow->last_page = (u32) (sys_addr >> PAGE_SHIFT); 22270ec449eeSDoug Thompson csrow->page_mask = ~mask_from_dct_mask(pvt, i); 22280ec449eeSDoug Thompson /* 8 bytes of resolution */ 22290ec449eeSDoug Thompson 223024f9a7feSBorislav Petkov csrow->mtype = amd64_determine_memory_type(pvt, i); 22310ec449eeSDoug Thompson 22320ec449eeSDoug Thompson debugf1(" for MC node %d csrow %d:\n", pvt->mc_node_id, i); 22330ec449eeSDoug Thompson debugf1(" input_addr_min: 0x%lx input_addr_max: 0x%lx\n", 22340ec449eeSDoug Thompson (unsigned long)input_addr_min, 22350ec449eeSDoug Thompson (unsigned long)input_addr_max); 22360ec449eeSDoug Thompson debugf1(" sys_addr: 0x%lx page_mask: 0x%lx\n", 22370ec449eeSDoug Thompson (unsigned long)sys_addr, csrow->page_mask); 22380ec449eeSDoug Thompson debugf1(" nr_pages: %u first_page: 0x%lx " 22390ec449eeSDoug Thompson "last_page: 0x%lx\n", 22400ec449eeSDoug Thompson (unsigned)csrow->nr_pages, 22410ec449eeSDoug Thompson csrow->first_page, csrow->last_page); 22420ec449eeSDoug Thompson 22430ec449eeSDoug Thompson /* 22440ec449eeSDoug Thompson * determine whether CHIPKILL or JUST ECC or NO ECC is operating 22450ec449eeSDoug Thompson */ 22460ec449eeSDoug Thompson if (pvt->nbcfg & K8_NBCFG_ECC_ENABLE) 22470ec449eeSDoug Thompson csrow->edac_mode = 22480ec449eeSDoug Thompson (pvt->nbcfg & K8_NBCFG_CHIPKILL) ? 22490ec449eeSDoug Thompson EDAC_S4ECD4ED : EDAC_SECDED; 22500ec449eeSDoug Thompson else 22510ec449eeSDoug Thompson csrow->edac_mode = EDAC_NONE; 22520ec449eeSDoug Thompson } 22530ec449eeSDoug Thompson 22540ec449eeSDoug Thompson return empty; 22550ec449eeSDoug Thompson } 2256d27bf6faSDoug Thompson 225706724535SBorislav Petkov /* get all cores on this DCT */ 2258ba578cb3SRusty Russell static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, int nid) 2259f9431992SDoug Thompson { 226006724535SBorislav Petkov int cpu; 2261f9431992SDoug Thompson 226206724535SBorislav Petkov for_each_online_cpu(cpu) 226306724535SBorislav Petkov if (amd_get_nb_id(cpu) == nid) 226406724535SBorislav Petkov cpumask_set_cpu(cpu, mask); 2265f9431992SDoug Thompson } 2266f9431992SDoug Thompson 2267f9431992SDoug Thompson /* check MCG_CTL on all the cpus on this node */ 226806724535SBorislav Petkov static bool amd64_nb_mce_bank_enabled_on_node(int nid) 2269f9431992SDoug Thompson { 2270ba578cb3SRusty Russell cpumask_var_t mask; 227150542251SBorislav Petkov int cpu, nbe; 227206724535SBorislav Petkov bool ret = false; 2273f9431992SDoug Thompson 2274ba578cb3SRusty Russell if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { 227524f9a7feSBorislav Petkov amd64_warn("%s: Error allocating mask\n", __func__); 227606724535SBorislav Petkov return false; 227706724535SBorislav Petkov } 227806724535SBorislav Petkov 2279ba578cb3SRusty Russell get_cpus_on_this_dct_cpumask(mask, nid); 228006724535SBorislav Petkov 2281ba578cb3SRusty Russell rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs); 2282ba578cb3SRusty Russell 2283ba578cb3SRusty Russell for_each_cpu(cpu, mask) { 228450542251SBorislav Petkov struct msr *reg = per_cpu_ptr(msrs, cpu); 228550542251SBorislav Petkov nbe = reg->l & K8_MSR_MCGCTL_NBE; 228606724535SBorislav Petkov 228706724535SBorislav Petkov debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n", 228850542251SBorislav Petkov cpu, reg->q, 228906724535SBorislav Petkov (nbe ? "enabled" : "disabled")); 229006724535SBorislav Petkov 229106724535SBorislav Petkov if (!nbe) 229206724535SBorislav Petkov goto out; 229306724535SBorislav Petkov } 229406724535SBorislav Petkov ret = true; 229506724535SBorislav Petkov 229606724535SBorislav Petkov out: 2297ba578cb3SRusty Russell free_cpumask_var(mask); 2298f9431992SDoug Thompson return ret; 2299f9431992SDoug Thompson } 2300f9431992SDoug Thompson 23012299ef71SBorislav Petkov static int toggle_ecc_err_reporting(struct ecc_settings *s, u8 nid, bool on) 2302f6d6ae96SBorislav Petkov { 2303f6d6ae96SBorislav Petkov cpumask_var_t cmask; 230450542251SBorislav Petkov int cpu; 2305f6d6ae96SBorislav Petkov 2306f6d6ae96SBorislav Petkov if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) { 230724f9a7feSBorislav Petkov amd64_warn("%s: error allocating mask\n", __func__); 2308f6d6ae96SBorislav Petkov return false; 2309f6d6ae96SBorislav Petkov } 2310f6d6ae96SBorislav Petkov 2311ae7bb7c6SBorislav Petkov get_cpus_on_this_dct_cpumask(cmask, nid); 2312f6d6ae96SBorislav Petkov 2313f6d6ae96SBorislav Petkov rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); 2314f6d6ae96SBorislav Petkov 2315f6d6ae96SBorislav Petkov for_each_cpu(cpu, cmask) { 2316f6d6ae96SBorislav Petkov 231750542251SBorislav Petkov struct msr *reg = per_cpu_ptr(msrs, cpu); 231850542251SBorislav Petkov 2319f6d6ae96SBorislav Petkov if (on) { 232050542251SBorislav Petkov if (reg->l & K8_MSR_MCGCTL_NBE) 2321ae7bb7c6SBorislav Petkov s->flags.nb_mce_enable = 1; 2322f6d6ae96SBorislav Petkov 232350542251SBorislav Petkov reg->l |= K8_MSR_MCGCTL_NBE; 2324f6d6ae96SBorislav Petkov } else { 2325f6d6ae96SBorislav Petkov /* 2326d95cf4deSBorislav Petkov * Turn off NB MCE reporting only when it was off before 2327f6d6ae96SBorislav Petkov */ 2328ae7bb7c6SBorislav Petkov if (!s->flags.nb_mce_enable) 232950542251SBorislav Petkov reg->l &= ~K8_MSR_MCGCTL_NBE; 2330f6d6ae96SBorislav Petkov } 2331f6d6ae96SBorislav Petkov } 2332f6d6ae96SBorislav Petkov wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); 2333f6d6ae96SBorislav Petkov 2334f6d6ae96SBorislav Petkov free_cpumask_var(cmask); 2335f6d6ae96SBorislav Petkov 2336f6d6ae96SBorislav Petkov return 0; 2337f6d6ae96SBorislav Petkov } 2338f6d6ae96SBorislav Petkov 23392299ef71SBorislav Petkov static bool enable_ecc_error_reporting(struct ecc_settings *s, u8 nid, 23402299ef71SBorislav Petkov struct pci_dev *F3) 2341f6d6ae96SBorislav Petkov { 23422299ef71SBorislav Petkov bool ret = true; 2343f6d6ae96SBorislav Petkov u32 value, mask = K8_NBCTL_CECCEn | K8_NBCTL_UECCEn; 2344f6d6ae96SBorislav Petkov 23452299ef71SBorislav Petkov if (toggle_ecc_err_reporting(s, nid, ON)) { 23462299ef71SBorislav Petkov amd64_warn("Error enabling ECC reporting over MCGCTL!\n"); 23472299ef71SBorislav Petkov return false; 23482299ef71SBorislav Petkov } 23492299ef71SBorislav Petkov 23502299ef71SBorislav Petkov amd64_read_pci_cfg(F3, K8_NBCTL, &value); 2351f6d6ae96SBorislav Petkov 2352ae7bb7c6SBorislav Petkov /* turn on UECCEn and CECCEn bits */ 2353ae7bb7c6SBorislav Petkov s->old_nbctl = value & mask; 2354ae7bb7c6SBorislav Petkov s->nbctl_valid = true; 2355f6d6ae96SBorislav Petkov 2356f6d6ae96SBorislav Petkov value |= mask; 23572299ef71SBorislav Petkov pci_write_config_dword(F3, K8_NBCTL, value); 2358f6d6ae96SBorislav Petkov 23592299ef71SBorislav Petkov amd64_read_pci_cfg(F3, K8_NBCFG, &value); 2360f6d6ae96SBorislav Petkov 23612299ef71SBorislav Petkov debugf0("1: node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n", 23622299ef71SBorislav Petkov nid, value, 23632299ef71SBorislav Petkov !!(value & K8_NBCFG_CHIPKILL), !!(value & K8_NBCFG_ECC_ENABLE)); 2364f6d6ae96SBorislav Petkov 2365f6d6ae96SBorislav Petkov if (!(value & K8_NBCFG_ECC_ENABLE)) { 236624f9a7feSBorislav Petkov amd64_warn("DRAM ECC disabled on this node, enabling...\n"); 2367f6d6ae96SBorislav Petkov 2368ae7bb7c6SBorislav Petkov s->flags.nb_ecc_prev = 0; 2369d95cf4deSBorislav Petkov 2370f6d6ae96SBorislav Petkov /* Attempt to turn on DRAM ECC Enable */ 2371f6d6ae96SBorislav Petkov value |= K8_NBCFG_ECC_ENABLE; 23722299ef71SBorislav Petkov pci_write_config_dword(F3, K8_NBCFG, value); 2373f6d6ae96SBorislav Petkov 23742299ef71SBorislav Petkov amd64_read_pci_cfg(F3, K8_NBCFG, &value); 2375f6d6ae96SBorislav Petkov 2376f6d6ae96SBorislav Petkov if (!(value & K8_NBCFG_ECC_ENABLE)) { 237724f9a7feSBorislav Petkov amd64_warn("Hardware rejected DRAM ECC enable," 237824f9a7feSBorislav Petkov "check memory DIMM configuration.\n"); 23792299ef71SBorislav Petkov ret = false; 2380f6d6ae96SBorislav Petkov } else { 238124f9a7feSBorislav Petkov amd64_info("Hardware accepted DRAM ECC Enable\n"); 2382f6d6ae96SBorislav Petkov } 2383d95cf4deSBorislav Petkov } else { 2384ae7bb7c6SBorislav Petkov s->flags.nb_ecc_prev = 1; 2385f6d6ae96SBorislav Petkov } 2386d95cf4deSBorislav Petkov 23872299ef71SBorislav Petkov debugf0("2: node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n", 23882299ef71SBorislav Petkov nid, value, 23892299ef71SBorislav Petkov !!(value & K8_NBCFG_CHIPKILL), !!(value & K8_NBCFG_ECC_ENABLE)); 2390f6d6ae96SBorislav Petkov 23912299ef71SBorislav Petkov return ret; 2392f6d6ae96SBorislav Petkov } 2393f6d6ae96SBorislav Petkov 2394360b7f3cSBorislav Petkov static void restore_ecc_error_reporting(struct ecc_settings *s, u8 nid, 2395360b7f3cSBorislav Petkov struct pci_dev *F3) 2396f6d6ae96SBorislav Petkov { 2397f6d6ae96SBorislav Petkov u32 value, mask = K8_NBCTL_CECCEn | K8_NBCTL_UECCEn; 2398f6d6ae96SBorislav Petkov 2399ae7bb7c6SBorislav Petkov if (!s->nbctl_valid) 2400f6d6ae96SBorislav Petkov return; 2401f6d6ae96SBorislav Petkov 2402360b7f3cSBorislav Petkov amd64_read_pci_cfg(F3, K8_NBCTL, &value); 2403f6d6ae96SBorislav Petkov value &= ~mask; 2404ae7bb7c6SBorislav Petkov value |= s->old_nbctl; 2405f6d6ae96SBorislav Petkov 2406360b7f3cSBorislav Petkov pci_write_config_dword(F3, K8_NBCTL, value); 2407f6d6ae96SBorislav Petkov 2408ae7bb7c6SBorislav Petkov /* restore previous BIOS DRAM ECC "off" setting we force-enabled */ 2409ae7bb7c6SBorislav Petkov if (!s->flags.nb_ecc_prev) { 2410360b7f3cSBorislav Petkov amd64_read_pci_cfg(F3, K8_NBCFG, &value); 2411d95cf4deSBorislav Petkov value &= ~K8_NBCFG_ECC_ENABLE; 2412360b7f3cSBorislav Petkov pci_write_config_dword(F3, K8_NBCFG, value); 2413d95cf4deSBorislav Petkov } 2414d95cf4deSBorislav Petkov 2415d95cf4deSBorislav Petkov /* restore the NB Enable MCGCTL bit */ 24162299ef71SBorislav Petkov if (toggle_ecc_err_reporting(s, nid, OFF)) 241724f9a7feSBorislav Petkov amd64_warn("Error restoring NB MCGCTL settings!\n"); 2418f6d6ae96SBorislav Petkov } 2419f6d6ae96SBorislav Petkov 2420f9431992SDoug Thompson /* 24212299ef71SBorislav Petkov * EDAC requires that the BIOS have ECC enabled before 24222299ef71SBorislav Petkov * taking over the processing of ECC errors. A command line 24232299ef71SBorislav Petkov * option allows to force-enable hardware ECC later in 24242299ef71SBorislav Petkov * enable_ecc_error_reporting(). 2425f9431992SDoug Thompson */ 2426cab4d277SBorislav Petkov static const char *ecc_msg = 2427cab4d277SBorislav Petkov "ECC disabled in the BIOS or no ECC capability, module will not load.\n" 2428cab4d277SBorislav Petkov " Either enable ECC checking or force module loading by setting " 2429cab4d277SBorislav Petkov "'ecc_enable_override'.\n" 2430cab4d277SBorislav Petkov " (Note that use of the override may cause unknown side effects.)\n"; 2431be3468e8SBorislav Petkov 24322299ef71SBorislav Petkov static bool ecc_enabled(struct pci_dev *F3, u8 nid) 2433f9431992SDoug Thompson { 2434f9431992SDoug Thompson u32 value; 24352299ef71SBorislav Petkov u8 ecc_en = 0; 243606724535SBorislav Petkov bool nb_mce_en = false; 2437f9431992SDoug Thompson 24382299ef71SBorislav Petkov amd64_read_pci_cfg(F3, K8_NBCFG, &value); 2439f9431992SDoug Thompson 24402299ef71SBorislav Petkov ecc_en = !!(value & K8_NBCFG_ECC_ENABLE); 24412299ef71SBorislav Petkov amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled")); 2442be3468e8SBorislav Petkov 24432299ef71SBorislav Petkov nb_mce_en = amd64_nb_mce_bank_enabled_on_node(nid); 244406724535SBorislav Petkov if (!nb_mce_en) 24452299ef71SBorislav Petkov amd64_notice("NB MCE bank disabled, set MSR " 24462299ef71SBorislav Petkov "0x%08x[4] on node %d to enable.\n", 24472299ef71SBorislav Petkov MSR_IA32_MCG_CTL, nid); 2448be3468e8SBorislav Petkov 24492299ef71SBorislav Petkov if (!ecc_en || !nb_mce_en) { 245024f9a7feSBorislav Petkov amd64_notice("%s", ecc_msg); 24512299ef71SBorislav Petkov return false; 2452be3468e8SBorislav Petkov } 24532299ef71SBorislav Petkov return true; 2454f9431992SDoug Thompson } 2455f9431992SDoug Thompson 24567d6034d3SDoug Thompson struct mcidev_sysfs_attribute sysfs_attrs[ARRAY_SIZE(amd64_dbg_attrs) + 24577d6034d3SDoug Thompson ARRAY_SIZE(amd64_inj_attrs) + 24587d6034d3SDoug Thompson 1]; 24597d6034d3SDoug Thompson 24607d6034d3SDoug Thompson struct mcidev_sysfs_attribute terminator = { .attr = { .name = NULL } }; 24617d6034d3SDoug Thompson 2462360b7f3cSBorislav Petkov static void set_mc_sysfs_attrs(struct mem_ctl_info *mci) 24637d6034d3SDoug Thompson { 24647d6034d3SDoug Thompson unsigned int i = 0, j = 0; 24657d6034d3SDoug Thompson 24667d6034d3SDoug Thompson for (; i < ARRAY_SIZE(amd64_dbg_attrs); i++) 24677d6034d3SDoug Thompson sysfs_attrs[i] = amd64_dbg_attrs[i]; 24687d6034d3SDoug Thompson 24697d6034d3SDoug Thompson for (j = 0; j < ARRAY_SIZE(amd64_inj_attrs); j++, i++) 24707d6034d3SDoug Thompson sysfs_attrs[i] = amd64_inj_attrs[j]; 24717d6034d3SDoug Thompson 24727d6034d3SDoug Thompson sysfs_attrs[i] = terminator; 24737d6034d3SDoug Thompson 24747d6034d3SDoug Thompson mci->mc_driver_sysfs_attributes = sysfs_attrs; 24757d6034d3SDoug Thompson } 24767d6034d3SDoug Thompson 2477360b7f3cSBorislav Petkov static void setup_mci_misc_attrs(struct mem_ctl_info *mci) 24787d6034d3SDoug Thompson { 24797d6034d3SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 24807d6034d3SDoug Thompson 24817d6034d3SDoug Thompson mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2; 24827d6034d3SDoug Thompson mci->edac_ctl_cap = EDAC_FLAG_NONE; 24837d6034d3SDoug Thompson 24847d6034d3SDoug Thompson if (pvt->nbcap & K8_NBCAP_SECDED) 24857d6034d3SDoug Thompson mci->edac_ctl_cap |= EDAC_FLAG_SECDED; 24867d6034d3SDoug Thompson 24877d6034d3SDoug Thompson if (pvt->nbcap & K8_NBCAP_CHIPKILL) 24887d6034d3SDoug Thompson mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED; 24897d6034d3SDoug Thompson 24907d6034d3SDoug Thompson mci->edac_cap = amd64_determine_edac_cap(pvt); 24917d6034d3SDoug Thompson mci->mod_name = EDAC_MOD_STR; 24927d6034d3SDoug Thompson mci->mod_ver = EDAC_AMD64_VERSION; 24930092b20dSBorislav Petkov mci->ctl_name = pvt->ctl_name; 24948d5b5d9cSBorislav Petkov mci->dev_name = pci_name(pvt->F2); 24957d6034d3SDoug Thompson mci->ctl_page_to_phys = NULL; 24967d6034d3SDoug Thompson 24977d6034d3SDoug Thompson /* memory scrubber interface */ 24987d6034d3SDoug Thompson mci->set_sdram_scrub_rate = amd64_set_scrub_rate; 24997d6034d3SDoug Thompson mci->get_sdram_scrub_rate = amd64_get_scrub_rate; 25007d6034d3SDoug Thompson } 25017d6034d3SDoug Thompson 25020092b20dSBorislav Petkov /* 25030092b20dSBorislav Petkov * returns a pointer to the family descriptor on success, NULL otherwise. 25040092b20dSBorislav Petkov */ 25050092b20dSBorislav Petkov static struct amd64_family_type *amd64_per_family_init(struct amd64_pvt *pvt) 2506395ae783SBorislav Petkov { 25070092b20dSBorislav Petkov u8 fam = boot_cpu_data.x86; 25080092b20dSBorislav Petkov struct amd64_family_type *fam_type = NULL; 25090092b20dSBorislav Petkov 25100092b20dSBorislav Petkov switch (fam) { 2511395ae783SBorislav Petkov case 0xf: 25120092b20dSBorislav Petkov fam_type = &amd64_family_types[K8_CPUS]; 2513b8cfa02fSBorislav Petkov pvt->ops = &amd64_family_types[K8_CPUS].ops; 25140092b20dSBorislav Petkov pvt->ctl_name = fam_type->ctl_name; 2515395ae783SBorislav Petkov pvt->min_scrubrate = K8_MIN_SCRUB_RATE_BITS; 2516395ae783SBorislav Petkov break; 2517395ae783SBorislav Petkov case 0x10: 25180092b20dSBorislav Petkov fam_type = &amd64_family_types[F10_CPUS]; 2519b8cfa02fSBorislav Petkov pvt->ops = &amd64_family_types[F10_CPUS].ops; 25200092b20dSBorislav Petkov pvt->ctl_name = fam_type->ctl_name; 2521395ae783SBorislav Petkov pvt->min_scrubrate = F10_MIN_SCRUB_RATE_BITS; 2522395ae783SBorislav Petkov break; 2523395ae783SBorislav Petkov 2524395ae783SBorislav Petkov default: 252524f9a7feSBorislav Petkov amd64_err("Unsupported family!\n"); 25260092b20dSBorislav Petkov return NULL; 2527395ae783SBorislav Petkov } 25280092b20dSBorislav Petkov 2529b8cfa02fSBorislav Petkov pvt->ext_model = boot_cpu_data.x86_model >> 4; 2530b8cfa02fSBorislav Petkov 253124f9a7feSBorislav Petkov amd64_info("%s %sdetected (node %d).\n", pvt->ctl_name, 25320092b20dSBorislav Petkov (fam == 0xf ? 25330092b20dSBorislav Petkov (pvt->ext_model >= K8_REV_F ? "revF or later " 25340092b20dSBorislav Petkov : "revE or earlier ") 253524f9a7feSBorislav Petkov : ""), pvt->mc_node_id); 25360092b20dSBorislav Petkov return fam_type; 2537395ae783SBorislav Petkov } 2538395ae783SBorislav Petkov 25392299ef71SBorislav Petkov static int amd64_init_one_instance(struct pci_dev *F2) 25407d6034d3SDoug Thompson { 25417d6034d3SDoug Thompson struct amd64_pvt *pvt = NULL; 25420092b20dSBorislav Petkov struct amd64_family_type *fam_type = NULL; 2543360b7f3cSBorislav Petkov struct mem_ctl_info *mci = NULL; 25447d6034d3SDoug Thompson int err = 0, ret; 2545360b7f3cSBorislav Petkov u8 nid = get_node_id(F2); 25467d6034d3SDoug Thompson 25477d6034d3SDoug Thompson ret = -ENOMEM; 25487d6034d3SDoug Thompson pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL); 25497d6034d3SDoug Thompson if (!pvt) 2550360b7f3cSBorislav Petkov goto err_ret; 25517d6034d3SDoug Thompson 2552360b7f3cSBorislav Petkov pvt->mc_node_id = nid; 25538d5b5d9cSBorislav Petkov pvt->F2 = F2; 25547d6034d3SDoug Thompson 2555395ae783SBorislav Petkov ret = -EINVAL; 25560092b20dSBorislav Petkov fam_type = amd64_per_family_init(pvt); 25570092b20dSBorislav Petkov if (!fam_type) 2558395ae783SBorislav Petkov goto err_free; 2559395ae783SBorislav Petkov 25607d6034d3SDoug Thompson ret = -ENODEV; 2561360b7f3cSBorislav Petkov err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id); 25627d6034d3SDoug Thompson if (err) 25637d6034d3SDoug Thompson goto err_free; 25647d6034d3SDoug Thompson 2565360b7f3cSBorislav Petkov read_mc_regs(pvt); 25667d6034d3SDoug Thompson 25677d6034d3SDoug Thompson /* 25687d6034d3SDoug Thompson * We need to determine how many memory channels there are. Then use 25697d6034d3SDoug Thompson * that information for calculating the size of the dynamic instance 2570360b7f3cSBorislav Petkov * tables in the 'mci' structure. 25717d6034d3SDoug Thompson */ 2572360b7f3cSBorislav Petkov ret = -EINVAL; 25737d6034d3SDoug Thompson pvt->channel_count = pvt->ops->early_channel_count(pvt); 25747d6034d3SDoug Thompson if (pvt->channel_count < 0) 2575360b7f3cSBorislav Petkov goto err_siblings; 25767d6034d3SDoug Thompson 25777d6034d3SDoug Thompson ret = -ENOMEM; 2578360b7f3cSBorislav Petkov mci = edac_mc_alloc(0, pvt->cs_count, pvt->channel_count, nid); 25797d6034d3SDoug Thompson if (!mci) 2580360b7f3cSBorislav Petkov goto err_siblings; 25817d6034d3SDoug Thompson 25827d6034d3SDoug Thompson mci->pvt_info = pvt; 25838d5b5d9cSBorislav Petkov mci->dev = &pvt->F2->dev; 25847d6034d3SDoug Thompson 2585360b7f3cSBorislav Petkov setup_mci_misc_attrs(mci); 2586360b7f3cSBorislav Petkov 2587360b7f3cSBorislav Petkov if (init_csrows(mci)) 25887d6034d3SDoug Thompson mci->edac_cap = EDAC_FLAG_NONE; 25897d6034d3SDoug Thompson 2590360b7f3cSBorislav Petkov set_mc_sysfs_attrs(mci); 25917d6034d3SDoug Thompson 25927d6034d3SDoug Thompson ret = -ENODEV; 25937d6034d3SDoug Thompson if (edac_mc_add_mc(mci)) { 25947d6034d3SDoug Thompson debugf1("failed edac_mc_add_mc()\n"); 25957d6034d3SDoug Thompson goto err_add_mc; 25967d6034d3SDoug Thompson } 25977d6034d3SDoug Thompson 2598549d042dSBorislav Petkov /* register stuff with EDAC MCE */ 2599549d042dSBorislav Petkov if (report_gart_errors) 2600549d042dSBorislav Petkov amd_report_gart_errors(true); 2601549d042dSBorislav Petkov 2602549d042dSBorislav Petkov amd_register_ecc_decoder(amd64_decode_bus_error); 2603549d042dSBorislav Petkov 2604360b7f3cSBorislav Petkov mcis[nid] = mci; 2605360b7f3cSBorislav Petkov 2606360b7f3cSBorislav Petkov atomic_inc(&drv_instances); 2607360b7f3cSBorislav Petkov 26087d6034d3SDoug Thompson return 0; 26097d6034d3SDoug Thompson 26107d6034d3SDoug Thompson err_add_mc: 26117d6034d3SDoug Thompson edac_mc_free(mci); 26127d6034d3SDoug Thompson 2613360b7f3cSBorislav Petkov err_siblings: 2614360b7f3cSBorislav Petkov free_mc_sibling_devs(pvt); 26157d6034d3SDoug Thompson 2616360b7f3cSBorislav Petkov err_free: 2617360b7f3cSBorislav Petkov kfree(pvt); 26187d6034d3SDoug Thompson 2619360b7f3cSBorislav Petkov err_ret: 26207d6034d3SDoug Thompson return ret; 26217d6034d3SDoug Thompson } 26227d6034d3SDoug Thompson 26232299ef71SBorislav Petkov static int __devinit amd64_probe_one_instance(struct pci_dev *pdev, 26247d6034d3SDoug Thompson const struct pci_device_id *mc_type) 26257d6034d3SDoug Thompson { 2626ae7bb7c6SBorislav Petkov u8 nid = get_node_id(pdev); 26272299ef71SBorislav Petkov struct pci_dev *F3 = node_to_amd_nb(nid)->misc; 2628ae7bb7c6SBorislav Petkov struct ecc_settings *s; 26292299ef71SBorislav Petkov int ret = 0; 26307d6034d3SDoug Thompson 26317d6034d3SDoug Thompson ret = pci_enable_device(pdev); 2632b8cfa02fSBorislav Petkov if (ret < 0) { 26337d6034d3SDoug Thompson debugf0("ret=%d\n", ret); 2634b8cfa02fSBorislav Petkov return -EIO; 2635b8cfa02fSBorislav Petkov } 2636b8cfa02fSBorislav Petkov 2637ae7bb7c6SBorislav Petkov ret = -ENOMEM; 2638ae7bb7c6SBorislav Petkov s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL); 2639ae7bb7c6SBorislav Petkov if (!s) 26402299ef71SBorislav Petkov goto err_out; 2641ae7bb7c6SBorislav Petkov 2642ae7bb7c6SBorislav Petkov ecc_stngs[nid] = s; 2643ae7bb7c6SBorislav Petkov 26442299ef71SBorislav Petkov if (!ecc_enabled(F3, nid)) { 26452299ef71SBorislav Petkov ret = -ENODEV; 26462299ef71SBorislav Petkov 26472299ef71SBorislav Petkov if (!ecc_enable_override) 26482299ef71SBorislav Petkov goto err_enable; 26492299ef71SBorislav Petkov 26502299ef71SBorislav Petkov amd64_warn("Forcing ECC on!\n"); 26512299ef71SBorislav Petkov 26522299ef71SBorislav Petkov if (!enable_ecc_error_reporting(s, nid, F3)) 26532299ef71SBorislav Petkov goto err_enable; 26542299ef71SBorislav Petkov } 26552299ef71SBorislav Petkov 26562299ef71SBorislav Petkov ret = amd64_init_one_instance(pdev); 2657360b7f3cSBorislav Petkov if (ret < 0) { 2658ae7bb7c6SBorislav Petkov amd64_err("Error probing instance: %d\n", nid); 2659360b7f3cSBorislav Petkov restore_ecc_error_reporting(s, nid, F3); 2660360b7f3cSBorislav Petkov } 26617d6034d3SDoug Thompson 26627d6034d3SDoug Thompson return ret; 26632299ef71SBorislav Petkov 26642299ef71SBorislav Petkov err_enable: 26652299ef71SBorislav Petkov kfree(s); 26662299ef71SBorislav Petkov ecc_stngs[nid] = NULL; 26672299ef71SBorislav Petkov 26682299ef71SBorislav Petkov err_out: 26692299ef71SBorislav Petkov return ret; 26707d6034d3SDoug Thompson } 26717d6034d3SDoug Thompson 26727d6034d3SDoug Thompson static void __devexit amd64_remove_one_instance(struct pci_dev *pdev) 26737d6034d3SDoug Thompson { 26747d6034d3SDoug Thompson struct mem_ctl_info *mci; 26757d6034d3SDoug Thompson struct amd64_pvt *pvt; 2676360b7f3cSBorislav Petkov u8 nid = get_node_id(pdev); 2677360b7f3cSBorislav Petkov struct pci_dev *F3 = node_to_amd_nb(nid)->misc; 2678360b7f3cSBorislav Petkov struct ecc_settings *s = ecc_stngs[nid]; 26797d6034d3SDoug Thompson 26807d6034d3SDoug Thompson /* Remove from EDAC CORE tracking list */ 26817d6034d3SDoug Thompson mci = edac_mc_del_mc(&pdev->dev); 26827d6034d3SDoug Thompson if (!mci) 26837d6034d3SDoug Thompson return; 26847d6034d3SDoug Thompson 26857d6034d3SDoug Thompson pvt = mci->pvt_info; 26867d6034d3SDoug Thompson 2687360b7f3cSBorislav Petkov restore_ecc_error_reporting(s, nid, F3); 26887d6034d3SDoug Thompson 2689360b7f3cSBorislav Petkov free_mc_sibling_devs(pvt); 26907d6034d3SDoug Thompson 2691549d042dSBorislav Petkov /* unregister from EDAC MCE */ 2692549d042dSBorislav Petkov amd_report_gart_errors(false); 2693549d042dSBorislav Petkov amd_unregister_ecc_decoder(amd64_decode_bus_error); 2694549d042dSBorislav Petkov 2695360b7f3cSBorislav Petkov kfree(ecc_stngs[nid]); 2696360b7f3cSBorislav Petkov ecc_stngs[nid] = NULL; 2697ae7bb7c6SBorislav Petkov 26987d6034d3SDoug Thompson /* Free the EDAC CORE resources */ 26998f68ed97SBorislav Petkov mci->pvt_info = NULL; 2700360b7f3cSBorislav Petkov mcis[nid] = NULL; 27018f68ed97SBorislav Petkov 27028f68ed97SBorislav Petkov kfree(pvt); 27037d6034d3SDoug Thompson edac_mc_free(mci); 27047d6034d3SDoug Thompson } 27057d6034d3SDoug Thompson 27067d6034d3SDoug Thompson /* 27077d6034d3SDoug Thompson * This table is part of the interface for loading drivers for PCI devices. The 27087d6034d3SDoug Thompson * PCI core identifies what devices are on a system during boot, and then 27097d6034d3SDoug Thompson * inquiry this table to see if this driver is for a given device found. 27107d6034d3SDoug Thompson */ 27117d6034d3SDoug Thompson static const struct pci_device_id amd64_pci_table[] __devinitdata = { 27127d6034d3SDoug Thompson { 27137d6034d3SDoug Thompson .vendor = PCI_VENDOR_ID_AMD, 27147d6034d3SDoug Thompson .device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL, 27157d6034d3SDoug Thompson .subvendor = PCI_ANY_ID, 27167d6034d3SDoug Thompson .subdevice = PCI_ANY_ID, 27177d6034d3SDoug Thompson .class = 0, 27187d6034d3SDoug Thompson .class_mask = 0, 27197d6034d3SDoug Thompson }, 27207d6034d3SDoug Thompson { 27217d6034d3SDoug Thompson .vendor = PCI_VENDOR_ID_AMD, 27227d6034d3SDoug Thompson .device = PCI_DEVICE_ID_AMD_10H_NB_DRAM, 27237d6034d3SDoug Thompson .subvendor = PCI_ANY_ID, 27247d6034d3SDoug Thompson .subdevice = PCI_ANY_ID, 27257d6034d3SDoug Thompson .class = 0, 27267d6034d3SDoug Thompson .class_mask = 0, 27277d6034d3SDoug Thompson }, 27287d6034d3SDoug Thompson {0, } 27297d6034d3SDoug Thompson }; 27307d6034d3SDoug Thompson MODULE_DEVICE_TABLE(pci, amd64_pci_table); 27317d6034d3SDoug Thompson 27327d6034d3SDoug Thompson static struct pci_driver amd64_pci_driver = { 27337d6034d3SDoug Thompson .name = EDAC_MOD_STR, 27342299ef71SBorislav Petkov .probe = amd64_probe_one_instance, 27357d6034d3SDoug Thompson .remove = __devexit_p(amd64_remove_one_instance), 27367d6034d3SDoug Thompson .id_table = amd64_pci_table, 27377d6034d3SDoug Thompson }; 27387d6034d3SDoug Thompson 2739360b7f3cSBorislav Petkov static void setup_pci_device(void) 27407d6034d3SDoug Thompson { 27417d6034d3SDoug Thompson struct mem_ctl_info *mci; 27427d6034d3SDoug Thompson struct amd64_pvt *pvt; 27437d6034d3SDoug Thompson 27447d6034d3SDoug Thompson if (amd64_ctl_pci) 27457d6034d3SDoug Thompson return; 27467d6034d3SDoug Thompson 2747cc4d8860SBorislav Petkov mci = mcis[0]; 27487d6034d3SDoug Thompson if (mci) { 27497d6034d3SDoug Thompson 27507d6034d3SDoug Thompson pvt = mci->pvt_info; 27517d6034d3SDoug Thompson amd64_ctl_pci = 27528d5b5d9cSBorislav Petkov edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR); 27537d6034d3SDoug Thompson 27547d6034d3SDoug Thompson if (!amd64_ctl_pci) { 27557d6034d3SDoug Thompson pr_warning("%s(): Unable to create PCI control\n", 27567d6034d3SDoug Thompson __func__); 27577d6034d3SDoug Thompson 27587d6034d3SDoug Thompson pr_warning("%s(): PCI error report via EDAC not set\n", 27597d6034d3SDoug Thompson __func__); 27607d6034d3SDoug Thompson } 27617d6034d3SDoug Thompson } 27627d6034d3SDoug Thompson } 27637d6034d3SDoug Thompson 27647d6034d3SDoug Thompson static int __init amd64_edac_init(void) 27657d6034d3SDoug Thompson { 2766360b7f3cSBorislav Petkov int err = -ENODEV; 27677d6034d3SDoug Thompson 27687d6034d3SDoug Thompson edac_printk(KERN_INFO, EDAC_MOD_STR, EDAC_AMD64_VERSION "\n"); 27697d6034d3SDoug Thompson 27707d6034d3SDoug Thompson opstate_init(); 27717d6034d3SDoug Thompson 27729653a5c7SHans Rosenfeld if (amd_cache_northbridges() < 0) 277356b34b91SBorislav Petkov goto err_ret; 27747d6034d3SDoug Thompson 2775cc4d8860SBorislav Petkov err = -ENOMEM; 2776cc4d8860SBorislav Petkov mcis = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL); 2777ae7bb7c6SBorislav Petkov ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL); 2778360b7f3cSBorislav Petkov if (!(mcis && ecc_stngs)) 2779cc4d8860SBorislav Petkov goto err_ret; 2780cc4d8860SBorislav Petkov 278150542251SBorislav Petkov msrs = msrs_alloc(); 278256b34b91SBorislav Petkov if (!msrs) 2783360b7f3cSBorislav Petkov goto err_free; 278450542251SBorislav Petkov 27857d6034d3SDoug Thompson err = pci_register_driver(&amd64_pci_driver); 27867d6034d3SDoug Thompson if (err) 278756b34b91SBorislav Petkov goto err_pci; 27887d6034d3SDoug Thompson 278956b34b91SBorislav Petkov err = -ENODEV; 2790360b7f3cSBorislav Petkov if (!atomic_read(&drv_instances)) 2791360b7f3cSBorislav Petkov goto err_no_instances; 27927d6034d3SDoug Thompson 2793360b7f3cSBorislav Petkov setup_pci_device(); 27947d6034d3SDoug Thompson return 0; 27957d6034d3SDoug Thompson 2796360b7f3cSBorislav Petkov err_no_instances: 27977d6034d3SDoug Thompson pci_unregister_driver(&amd64_pci_driver); 2798cc4d8860SBorislav Petkov 279956b34b91SBorislav Petkov err_pci: 280056b34b91SBorislav Petkov msrs_free(msrs); 280156b34b91SBorislav Petkov msrs = NULL; 2802cc4d8860SBorislav Petkov 2803360b7f3cSBorislav Petkov err_free: 2804360b7f3cSBorislav Petkov kfree(mcis); 2805360b7f3cSBorislav Petkov mcis = NULL; 2806360b7f3cSBorislav Petkov 2807360b7f3cSBorislav Petkov kfree(ecc_stngs); 2808360b7f3cSBorislav Petkov ecc_stngs = NULL; 2809360b7f3cSBorislav Petkov 281056b34b91SBorislav Petkov err_ret: 28117d6034d3SDoug Thompson return err; 28127d6034d3SDoug Thompson } 28137d6034d3SDoug Thompson 28147d6034d3SDoug Thompson static void __exit amd64_edac_exit(void) 28157d6034d3SDoug Thompson { 28167d6034d3SDoug Thompson if (amd64_ctl_pci) 28177d6034d3SDoug Thompson edac_pci_release_generic_ctl(amd64_ctl_pci); 28187d6034d3SDoug Thompson 28197d6034d3SDoug Thompson pci_unregister_driver(&amd64_pci_driver); 282050542251SBorislav Petkov 2821ae7bb7c6SBorislav Petkov kfree(ecc_stngs); 2822ae7bb7c6SBorislav Petkov ecc_stngs = NULL; 2823ae7bb7c6SBorislav Petkov 2824cc4d8860SBorislav Petkov kfree(mcis); 2825cc4d8860SBorislav Petkov mcis = NULL; 2826cc4d8860SBorislav Petkov 282750542251SBorislav Petkov msrs_free(msrs); 282850542251SBorislav Petkov msrs = NULL; 28297d6034d3SDoug Thompson } 28307d6034d3SDoug Thompson 28317d6034d3SDoug Thompson module_init(amd64_edac_init); 28327d6034d3SDoug Thompson module_exit(amd64_edac_exit); 28337d6034d3SDoug Thompson 28347d6034d3SDoug Thompson MODULE_LICENSE("GPL"); 28357d6034d3SDoug Thompson MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, " 28367d6034d3SDoug Thompson "Dave Peterson, Thayne Harbaugh"); 28377d6034d3SDoug Thompson MODULE_DESCRIPTION("MC support for AMD64 memory controllers - " 28387d6034d3SDoug Thompson EDAC_AMD64_VERSION); 28397d6034d3SDoug Thompson 28407d6034d3SDoug Thompson module_param(edac_op_state, int, 0444); 28417d6034d3SDoug Thompson MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI"); 2842