1 #include "amd64_edac.h" 2 #include <asm/amd_nb.h> 3 4 static struct edac_pci_ctl_info *amd64_ctl_pci; 5 6 static int report_gart_errors; 7 module_param(report_gart_errors, int, 0644); 8 9 /* 10 * Set by command line parameter. If BIOS has enabled the ECC, this override is 11 * cleared to prevent re-enabling the hardware by this driver. 12 */ 13 static int ecc_enable_override; 14 module_param(ecc_enable_override, int, 0644); 15 16 static struct msr __percpu *msrs; 17 18 /* 19 * count successfully initialized driver instances for setup_pci_device() 20 */ 21 static atomic_t drv_instances = ATOMIC_INIT(0); 22 23 /* Per-node driver instances */ 24 static struct mem_ctl_info **mcis; 25 static struct ecc_settings **ecc_stngs; 26 27 /* 28 * Address to DRAM bank mapping: see F2x80 for K8 and F2x[1,0]80 for Fam10 and 29 * later. 30 */ 31 static int ddr2_dbam_revCG[] = { 32 [0] = 32, 33 [1] = 64, 34 [2] = 128, 35 [3] = 256, 36 [4] = 512, 37 [5] = 1024, 38 [6] = 2048, 39 }; 40 41 static int ddr2_dbam_revD[] = { 42 [0] = 32, 43 [1] = 64, 44 [2 ... 3] = 128, 45 [4] = 256, 46 [5] = 512, 47 [6] = 256, 48 [7] = 512, 49 [8 ... 9] = 1024, 50 [10] = 2048, 51 }; 52 53 static int ddr2_dbam[] = { [0] = 128, 54 [1] = 256, 55 [2 ... 4] = 512, 56 [5 ... 6] = 1024, 57 [7 ... 8] = 2048, 58 [9 ... 10] = 4096, 59 [11] = 8192, 60 }; 61 62 static int ddr3_dbam[] = { [0] = -1, 63 [1] = 256, 64 [2] = 512, 65 [3 ... 4] = -1, 66 [5 ... 6] = 1024, 67 [7 ... 8] = 2048, 68 [9 ... 10] = 4096, 69 [11] = 8192, 70 }; 71 72 /* 73 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing 74 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching- 75 * or higher value'. 76 * 77 *FIXME: Produce a better mapping/linearisation. 78 */ 79 80 81 struct scrubrate { 82 u32 scrubval; /* bit pattern for scrub rate */ 83 u32 bandwidth; /* bandwidth consumed (bytes/sec) */ 84 } scrubrates[] = { 85 { 0x01, 1600000000UL}, 86 { 0x02, 800000000UL}, 87 { 0x03, 400000000UL}, 88 { 0x04, 200000000UL}, 89 { 0x05, 100000000UL}, 90 { 0x06, 50000000UL}, 91 { 0x07, 25000000UL}, 92 { 0x08, 12284069UL}, 93 { 0x09, 6274509UL}, 94 { 0x0A, 3121951UL}, 95 { 0x0B, 1560975UL}, 96 { 0x0C, 781440UL}, 97 { 0x0D, 390720UL}, 98 { 0x0E, 195300UL}, 99 { 0x0F, 97650UL}, 100 { 0x10, 48854UL}, 101 { 0x11, 24427UL}, 102 { 0x12, 12213UL}, 103 { 0x13, 6101UL}, 104 { 0x14, 3051UL}, 105 { 0x15, 1523UL}, 106 { 0x16, 761UL}, 107 { 0x00, 0UL}, /* scrubbing off */ 108 }; 109 110 /* 111 * Memory scrubber control interface. For K8, memory scrubbing is handled by 112 * hardware and can involve L2 cache, dcache as well as the main memory. With 113 * F10, this is extended to L3 cache scrubbing on CPU models sporting that 114 * functionality. 115 * 116 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks 117 * (dram) over to cache lines. This is nasty, so we will use bandwidth in 118 * bytes/sec for the setting. 119 * 120 * Currently, we only do dram scrubbing. If the scrubbing is done in software on 121 * other archs, we might not have access to the caches directly. 122 */ 123 124 /* 125 * scan the scrub rate mapping table for a close or matching bandwidth value to 126 * issue. If requested is too big, then use last maximum value found. 127 */ 128 static int __amd64_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate) 129 { 130 u32 scrubval; 131 int i; 132 133 /* 134 * map the configured rate (new_bw) to a value specific to the AMD64 135 * memory controller and apply to register. Search for the first 136 * bandwidth entry that is greater or equal than the setting requested 137 * and program that. If at last entry, turn off DRAM scrubbing. 138 */ 139 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) { 140 /* 141 * skip scrub rates which aren't recommended 142 * (see F10 BKDG, F3x58) 143 */ 144 if (scrubrates[i].scrubval < min_rate) 145 continue; 146 147 if (scrubrates[i].bandwidth <= new_bw) 148 break; 149 150 /* 151 * if no suitable bandwidth found, turn off DRAM scrubbing 152 * entirely by falling back to the last element in the 153 * scrubrates array. 154 */ 155 } 156 157 scrubval = scrubrates[i].scrubval; 158 159 pci_write_bits32(ctl, K8_SCRCTRL, scrubval, 0x001F); 160 161 if (scrubval) 162 return scrubrates[i].bandwidth; 163 164 return 0; 165 } 166 167 static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bw) 168 { 169 struct amd64_pvt *pvt = mci->pvt_info; 170 171 return __amd64_set_scrub_rate(pvt->F3, bw, pvt->min_scrubrate); 172 } 173 174 static int amd64_get_scrub_rate(struct mem_ctl_info *mci) 175 { 176 struct amd64_pvt *pvt = mci->pvt_info; 177 u32 scrubval = 0; 178 int i, retval = -EINVAL; 179 180 amd64_read_pci_cfg(pvt->F3, K8_SCRCTRL, &scrubval); 181 182 scrubval = scrubval & 0x001F; 183 184 amd64_debug("pci-read, sdram scrub control value: %d\n", scrubval); 185 186 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) { 187 if (scrubrates[i].scrubval == scrubval) { 188 retval = scrubrates[i].bandwidth; 189 break; 190 } 191 } 192 return retval; 193 } 194 195 /* Map from a CSROW entry to the mask entry that operates on it */ 196 static inline u32 amd64_map_to_dcs_mask(struct amd64_pvt *pvt, int csrow) 197 { 198 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) 199 return csrow; 200 else 201 return csrow >> 1; 202 } 203 204 /* return the 'base' address the i'th CS entry of the 'dct' DRAM controller */ 205 static u32 amd64_get_dct_base(struct amd64_pvt *pvt, int dct, int csrow) 206 { 207 if (dct == 0) 208 return pvt->dcsb0[csrow]; 209 else 210 return pvt->dcsb1[csrow]; 211 } 212 213 /* 214 * Return the 'mask' address the i'th CS entry. This function is needed because 215 * there number of DCSM registers on Rev E and prior vs Rev F and later is 216 * different. 217 */ 218 static u32 amd64_get_dct_mask(struct amd64_pvt *pvt, int dct, int csrow) 219 { 220 if (dct == 0) 221 return pvt->dcsm0[amd64_map_to_dcs_mask(pvt, csrow)]; 222 else 223 return pvt->dcsm1[amd64_map_to_dcs_mask(pvt, csrow)]; 224 } 225 226 227 /* 228 * In *base and *limit, pass back the full 40-bit base and limit physical 229 * addresses for the node given by node_id. This information is obtained from 230 * DRAM Base (section 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers. The 231 * base and limit addresses are of type SysAddr, as defined at the start of 232 * section 3.4.4 (p. 70). They are the lowest and highest physical addresses 233 * in the address range they represent. 234 */ 235 static void amd64_get_base_and_limit(struct amd64_pvt *pvt, int node_id, 236 u64 *base, u64 *limit) 237 { 238 *base = pvt->dram_base[node_id]; 239 *limit = pvt->dram_limit[node_id]; 240 } 241 242 /* 243 * Return 1 if the SysAddr given by sys_addr matches the base/limit associated 244 * with node_id 245 */ 246 static int amd64_base_limit_match(struct amd64_pvt *pvt, 247 u64 sys_addr, int node_id) 248 { 249 u64 base, limit, addr; 250 251 amd64_get_base_and_limit(pvt, node_id, &base, &limit); 252 253 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be 254 * all ones if the most significant implemented address bit is 1. 255 * Here we discard bits 63-40. See section 3.4.2 of AMD publication 256 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1 257 * Application Programming. 258 */ 259 addr = sys_addr & 0x000000ffffffffffull; 260 261 return (addr >= base) && (addr <= limit); 262 } 263 264 /* 265 * Attempt to map a SysAddr to a node. On success, return a pointer to the 266 * mem_ctl_info structure for the node that the SysAddr maps to. 267 * 268 * On failure, return NULL. 269 */ 270 static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci, 271 u64 sys_addr) 272 { 273 struct amd64_pvt *pvt; 274 int node_id; 275 u32 intlv_en, bits; 276 277 /* 278 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section 279 * 3.4.4.2) registers to map the SysAddr to a node ID. 280 */ 281 pvt = mci->pvt_info; 282 283 /* 284 * The value of this field should be the same for all DRAM Base 285 * registers. Therefore we arbitrarily choose to read it from the 286 * register for node 0. 287 */ 288 intlv_en = pvt->dram_IntlvEn[0]; 289 290 if (intlv_en == 0) { 291 for (node_id = 0; node_id < DRAM_REG_COUNT; node_id++) { 292 if (amd64_base_limit_match(pvt, sys_addr, node_id)) 293 goto found; 294 } 295 goto err_no_match; 296 } 297 298 if (unlikely((intlv_en != 0x01) && 299 (intlv_en != 0x03) && 300 (intlv_en != 0x07))) { 301 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en); 302 return NULL; 303 } 304 305 bits = (((u32) sys_addr) >> 12) & intlv_en; 306 307 for (node_id = 0; ; ) { 308 if ((pvt->dram_IntlvSel[node_id] & intlv_en) == bits) 309 break; /* intlv_sel field matches */ 310 311 if (++node_id >= DRAM_REG_COUNT) 312 goto err_no_match; 313 } 314 315 /* sanity test for sys_addr */ 316 if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) { 317 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address" 318 "range for node %d with node interleaving enabled.\n", 319 __func__, sys_addr, node_id); 320 return NULL; 321 } 322 323 found: 324 return edac_mc_find(node_id); 325 326 err_no_match: 327 debugf2("sys_addr 0x%lx doesn't match any node\n", 328 (unsigned long)sys_addr); 329 330 return NULL; 331 } 332 333 /* 334 * Extract the DRAM CS base address from selected csrow register. 335 */ 336 static u64 base_from_dct_base(struct amd64_pvt *pvt, int csrow) 337 { 338 return ((u64) (amd64_get_dct_base(pvt, 0, csrow) & pvt->dcsb_base)) << 339 pvt->dcs_shift; 340 } 341 342 /* 343 * Extract the mask from the dcsb0[csrow] entry in a CPU revision-specific way. 344 */ 345 static u64 mask_from_dct_mask(struct amd64_pvt *pvt, int csrow) 346 { 347 u64 dcsm_bits, other_bits; 348 u64 mask; 349 350 /* Extract bits from DRAM CS Mask. */ 351 dcsm_bits = amd64_get_dct_mask(pvt, 0, csrow) & pvt->dcsm_mask; 352 353 other_bits = pvt->dcsm_mask; 354 other_bits = ~(other_bits << pvt->dcs_shift); 355 356 /* 357 * The extracted bits from DCSM belong in the spaces represented by 358 * the cleared bits in other_bits. 359 */ 360 mask = (dcsm_bits << pvt->dcs_shift) | other_bits; 361 362 return mask; 363 } 364 365 /* 366 * @input_addr is an InputAddr associated with the node given by mci. Return the 367 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr). 368 */ 369 static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr) 370 { 371 struct amd64_pvt *pvt; 372 int csrow; 373 u64 base, mask; 374 375 pvt = mci->pvt_info; 376 377 /* 378 * Here we use the DRAM CS Base and DRAM CS Mask registers. For each CS 379 * base/mask register pair, test the condition shown near the start of 380 * section 3.5.4 (p. 84, BKDG #26094, K8, revA-E). 381 */ 382 for (csrow = 0; csrow < pvt->cs_count; csrow++) { 383 384 /* This DRAM chip select is disabled on this node */ 385 if ((pvt->dcsb0[csrow] & K8_DCSB_CS_ENABLE) == 0) 386 continue; 387 388 base = base_from_dct_base(pvt, csrow); 389 mask = ~mask_from_dct_mask(pvt, csrow); 390 391 if ((input_addr & mask) == (base & mask)) { 392 debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n", 393 (unsigned long)input_addr, csrow, 394 pvt->mc_node_id); 395 396 return csrow; 397 } 398 } 399 400 debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n", 401 (unsigned long)input_addr, pvt->mc_node_id); 402 403 return -1; 404 } 405 406 /* 407 * Return the base value defined by the DRAM Base register for the node 408 * represented by mci. This function returns the full 40-bit value despite the 409 * fact that the register only stores bits 39-24 of the value. See section 410 * 3.4.4.1 (BKDG #26094, K8, revA-E) 411 */ 412 static inline u64 get_dram_base(struct mem_ctl_info *mci) 413 { 414 struct amd64_pvt *pvt = mci->pvt_info; 415 416 return pvt->dram_base[pvt->mc_node_id]; 417 } 418 419 /* 420 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094) 421 * for the node represented by mci. Info is passed back in *hole_base, 422 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if 423 * info is invalid. Info may be invalid for either of the following reasons: 424 * 425 * - The revision of the node is not E or greater. In this case, the DRAM Hole 426 * Address Register does not exist. 427 * 428 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register, 429 * indicating that its contents are not valid. 430 * 431 * The values passed back in *hole_base, *hole_offset, and *hole_size are 432 * complete 32-bit values despite the fact that the bitfields in the DHAR 433 * only represent bits 31-24 of the base and offset values. 434 */ 435 int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base, 436 u64 *hole_offset, u64 *hole_size) 437 { 438 struct amd64_pvt *pvt = mci->pvt_info; 439 u64 base; 440 441 /* only revE and later have the DRAM Hole Address Register */ 442 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) { 443 debugf1(" revision %d for node %d does not support DHAR\n", 444 pvt->ext_model, pvt->mc_node_id); 445 return 1; 446 } 447 448 /* only valid for Fam10h */ 449 if (boot_cpu_data.x86 == 0x10 && 450 (pvt->dhar & F10_DRAM_MEM_HOIST_VALID) == 0) { 451 debugf1(" Dram Memory Hoisting is DISABLED on this system\n"); 452 return 1; 453 } 454 455 if ((pvt->dhar & DHAR_VALID) == 0) { 456 debugf1(" Dram Memory Hoisting is DISABLED on this node %d\n", 457 pvt->mc_node_id); 458 return 1; 459 } 460 461 /* This node has Memory Hoisting */ 462 463 /* +------------------+--------------------+--------------------+----- 464 * | memory | DRAM hole | relocated | 465 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from | 466 * | | | DRAM hole | 467 * | | | [0x100000000, | 468 * | | | (0x100000000+ | 469 * | | | (0xffffffff-x))] | 470 * +------------------+--------------------+--------------------+----- 471 * 472 * Above is a diagram of physical memory showing the DRAM hole and the 473 * relocated addresses from the DRAM hole. As shown, the DRAM hole 474 * starts at address x (the base address) and extends through address 475 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the 476 * addresses in the hole so that they start at 0x100000000. 477 */ 478 479 base = dhar_base(pvt->dhar); 480 481 *hole_base = base; 482 *hole_size = (0x1ull << 32) - base; 483 484 if (boot_cpu_data.x86 > 0xf) 485 *hole_offset = f10_dhar_offset(pvt->dhar); 486 else 487 *hole_offset = k8_dhar_offset(pvt->dhar); 488 489 debugf1(" DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n", 490 pvt->mc_node_id, (unsigned long)*hole_base, 491 (unsigned long)*hole_offset, (unsigned long)*hole_size); 492 493 return 0; 494 } 495 EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info); 496 497 /* 498 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is 499 * assumed that sys_addr maps to the node given by mci. 500 * 501 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section 502 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a 503 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled, 504 * then it is also involved in translating a SysAddr to a DramAddr. Sections 505 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting. 506 * These parts of the documentation are unclear. I interpret them as follows: 507 * 508 * When node n receives a SysAddr, it processes the SysAddr as follows: 509 * 510 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM 511 * Limit registers for node n. If the SysAddr is not within the range 512 * specified by the base and limit values, then node n ignores the Sysaddr 513 * (since it does not map to node n). Otherwise continue to step 2 below. 514 * 515 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is 516 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within 517 * the range of relocated addresses (starting at 0x100000000) from the DRAM 518 * hole. If not, skip to step 3 below. Else get the value of the 519 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the 520 * offset defined by this value from the SysAddr. 521 * 522 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM 523 * Base register for node n. To obtain the DramAddr, subtract the base 524 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70). 525 */ 526 static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr) 527 { 528 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr; 529 int ret = 0; 530 531 dram_base = get_dram_base(mci); 532 533 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset, 534 &hole_size); 535 if (!ret) { 536 if ((sys_addr >= (1ull << 32)) && 537 (sys_addr < ((1ull << 32) + hole_size))) { 538 /* use DHAR to translate SysAddr to DramAddr */ 539 dram_addr = sys_addr - hole_offset; 540 541 debugf2("using DHAR to translate SysAddr 0x%lx to " 542 "DramAddr 0x%lx\n", 543 (unsigned long)sys_addr, 544 (unsigned long)dram_addr); 545 546 return dram_addr; 547 } 548 } 549 550 /* 551 * Translate the SysAddr to a DramAddr as shown near the start of 552 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8 553 * only deals with 40-bit values. Therefore we discard bits 63-40 of 554 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we 555 * discard are all 1s. Otherwise the bits we discard are all 0s. See 556 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture 557 * Programmer's Manual Volume 1 Application Programming. 558 */ 559 dram_addr = (sys_addr & 0xffffffffffull) - dram_base; 560 561 debugf2("using DRAM Base register to translate SysAddr 0x%lx to " 562 "DramAddr 0x%lx\n", (unsigned long)sys_addr, 563 (unsigned long)dram_addr); 564 return dram_addr; 565 } 566 567 /* 568 * @intlv_en is the value of the IntlvEn field from a DRAM Base register 569 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used 570 * for node interleaving. 571 */ 572 static int num_node_interleave_bits(unsigned intlv_en) 573 { 574 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 }; 575 int n; 576 577 BUG_ON(intlv_en > 7); 578 n = intlv_shift_table[intlv_en]; 579 return n; 580 } 581 582 /* Translate the DramAddr given by @dram_addr to an InputAddr. */ 583 static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr) 584 { 585 struct amd64_pvt *pvt; 586 int intlv_shift; 587 u64 input_addr; 588 589 pvt = mci->pvt_info; 590 591 /* 592 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E) 593 * concerning translating a DramAddr to an InputAddr. 594 */ 595 intlv_shift = num_node_interleave_bits(pvt->dram_IntlvEn[0]); 596 input_addr = ((dram_addr >> intlv_shift) & 0xffffff000ull) + 597 (dram_addr & 0xfff); 598 599 debugf2(" Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n", 600 intlv_shift, (unsigned long)dram_addr, 601 (unsigned long)input_addr); 602 603 return input_addr; 604 } 605 606 /* 607 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is 608 * assumed that @sys_addr maps to the node given by mci. 609 */ 610 static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr) 611 { 612 u64 input_addr; 613 614 input_addr = 615 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr)); 616 617 debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n", 618 (unsigned long)sys_addr, (unsigned long)input_addr); 619 620 return input_addr; 621 } 622 623 624 /* 625 * @input_addr is an InputAddr associated with the node represented by mci. 626 * Translate @input_addr to a DramAddr and return the result. 627 */ 628 static u64 input_addr_to_dram_addr(struct mem_ctl_info *mci, u64 input_addr) 629 { 630 struct amd64_pvt *pvt; 631 int node_id, intlv_shift; 632 u64 bits, dram_addr; 633 u32 intlv_sel; 634 635 /* 636 * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E) 637 * shows how to translate a DramAddr to an InputAddr. Here we reverse 638 * this procedure. When translating from a DramAddr to an InputAddr, the 639 * bits used for node interleaving are discarded. Here we recover these 640 * bits from the IntlvSel field of the DRAM Limit register (section 641 * 3.4.4.2) for the node that input_addr is associated with. 642 */ 643 pvt = mci->pvt_info; 644 node_id = pvt->mc_node_id; 645 BUG_ON((node_id < 0) || (node_id > 7)); 646 647 intlv_shift = num_node_interleave_bits(pvt->dram_IntlvEn[0]); 648 649 if (intlv_shift == 0) { 650 debugf1(" InputAddr 0x%lx translates to DramAddr of " 651 "same value\n", (unsigned long)input_addr); 652 653 return input_addr; 654 } 655 656 bits = ((input_addr & 0xffffff000ull) << intlv_shift) + 657 (input_addr & 0xfff); 658 659 intlv_sel = pvt->dram_IntlvSel[node_id] & ((1 << intlv_shift) - 1); 660 dram_addr = bits + (intlv_sel << 12); 661 662 debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx " 663 "(%d node interleave bits)\n", (unsigned long)input_addr, 664 (unsigned long)dram_addr, intlv_shift); 665 666 return dram_addr; 667 } 668 669 /* 670 * @dram_addr is a DramAddr that maps to the node represented by mci. Convert 671 * @dram_addr to a SysAddr. 672 */ 673 static u64 dram_addr_to_sys_addr(struct mem_ctl_info *mci, u64 dram_addr) 674 { 675 struct amd64_pvt *pvt = mci->pvt_info; 676 u64 hole_base, hole_offset, hole_size, base, limit, sys_addr; 677 int ret = 0; 678 679 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset, 680 &hole_size); 681 if (!ret) { 682 if ((dram_addr >= hole_base) && 683 (dram_addr < (hole_base + hole_size))) { 684 sys_addr = dram_addr + hole_offset; 685 686 debugf1("using DHAR to translate DramAddr 0x%lx to " 687 "SysAddr 0x%lx\n", (unsigned long)dram_addr, 688 (unsigned long)sys_addr); 689 690 return sys_addr; 691 } 692 } 693 694 amd64_get_base_and_limit(pvt, pvt->mc_node_id, &base, &limit); 695 sys_addr = dram_addr + base; 696 697 /* 698 * The sys_addr we have computed up to this point is a 40-bit value 699 * because the k8 deals with 40-bit values. However, the value we are 700 * supposed to return is a full 64-bit physical address. The AMD 701 * x86-64 architecture specifies that the most significant implemented 702 * address bit through bit 63 of a physical address must be either all 703 * 0s or all 1s. Therefore we sign-extend the 40-bit sys_addr to a 704 * 64-bit value below. See section 3.4.2 of AMD publication 24592: 705 * AMD x86-64 Architecture Programmer's Manual Volume 1 Application 706 * Programming. 707 */ 708 sys_addr |= ~((sys_addr & (1ull << 39)) - 1); 709 710 debugf1(" Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n", 711 pvt->mc_node_id, (unsigned long)dram_addr, 712 (unsigned long)sys_addr); 713 714 return sys_addr; 715 } 716 717 /* 718 * @input_addr is an InputAddr associated with the node given by mci. Translate 719 * @input_addr to a SysAddr. 720 */ 721 static inline u64 input_addr_to_sys_addr(struct mem_ctl_info *mci, 722 u64 input_addr) 723 { 724 return dram_addr_to_sys_addr(mci, 725 input_addr_to_dram_addr(mci, input_addr)); 726 } 727 728 /* 729 * Find the minimum and maximum InputAddr values that map to the given @csrow. 730 * Pass back these values in *input_addr_min and *input_addr_max. 731 */ 732 static void find_csrow_limits(struct mem_ctl_info *mci, int csrow, 733 u64 *input_addr_min, u64 *input_addr_max) 734 { 735 struct amd64_pvt *pvt; 736 u64 base, mask; 737 738 pvt = mci->pvt_info; 739 BUG_ON((csrow < 0) || (csrow >= pvt->cs_count)); 740 741 base = base_from_dct_base(pvt, csrow); 742 mask = mask_from_dct_mask(pvt, csrow); 743 744 *input_addr_min = base & ~mask; 745 *input_addr_max = base | mask | pvt->dcs_mask_notused; 746 } 747 748 /* Map the Error address to a PAGE and PAGE OFFSET. */ 749 static inline void error_address_to_page_and_offset(u64 error_address, 750 u32 *page, u32 *offset) 751 { 752 *page = (u32) (error_address >> PAGE_SHIFT); 753 *offset = ((u32) error_address) & ~PAGE_MASK; 754 } 755 756 /* 757 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address 758 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers 759 * of a node that detected an ECC memory error. mci represents the node that 760 * the error address maps to (possibly different from the node that detected 761 * the error). Return the number of the csrow that sys_addr maps to, or -1 on 762 * error. 763 */ 764 static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr) 765 { 766 int csrow; 767 768 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr)); 769 770 if (csrow == -1) 771 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for " 772 "address 0x%lx\n", (unsigned long)sys_addr); 773 return csrow; 774 } 775 776 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16); 777 778 static u16 extract_syndrome(struct err_regs *err) 779 { 780 return ((err->nbsh >> 15) & 0xff) | ((err->nbsl >> 16) & 0xff00); 781 } 782 783 /* 784 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs 785 * are ECC capable. 786 */ 787 static enum edac_type amd64_determine_edac_cap(struct amd64_pvt *pvt) 788 { 789 int bit; 790 enum dev_type edac_cap = EDAC_FLAG_NONE; 791 792 bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F) 793 ? 19 794 : 17; 795 796 if (pvt->dclr0 & BIT(bit)) 797 edac_cap = EDAC_FLAG_SECDED; 798 799 return edac_cap; 800 } 801 802 803 static void amd64_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt); 804 805 static void amd64_dump_dramcfg_low(u32 dclr, int chan) 806 { 807 debugf1("F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr); 808 809 debugf1(" DIMM type: %sbuffered; all DIMMs support ECC: %s\n", 810 (dclr & BIT(16)) ? "un" : "", 811 (dclr & BIT(19)) ? "yes" : "no"); 812 813 debugf1(" PAR/ERR parity: %s\n", 814 (dclr & BIT(8)) ? "enabled" : "disabled"); 815 816 debugf1(" DCT 128bit mode width: %s\n", 817 (dclr & BIT(11)) ? "128b" : "64b"); 818 819 debugf1(" x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n", 820 (dclr & BIT(12)) ? "yes" : "no", 821 (dclr & BIT(13)) ? "yes" : "no", 822 (dclr & BIT(14)) ? "yes" : "no", 823 (dclr & BIT(15)) ? "yes" : "no"); 824 } 825 826 /* Display and decode various NB registers for debug purposes. */ 827 static void amd64_dump_misc_regs(struct amd64_pvt *pvt) 828 { 829 debugf1("F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap); 830 831 debugf1(" NB two channel DRAM capable: %s\n", 832 (pvt->nbcap & K8_NBCAP_DCT_DUAL) ? "yes" : "no"); 833 834 debugf1(" ECC capable: %s, ChipKill ECC capable: %s\n", 835 (pvt->nbcap & K8_NBCAP_SECDED) ? "yes" : "no", 836 (pvt->nbcap & K8_NBCAP_CHIPKILL) ? "yes" : "no"); 837 838 amd64_dump_dramcfg_low(pvt->dclr0, 0); 839 840 debugf1("F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare); 841 842 debugf1("F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, " 843 "offset: 0x%08x\n", 844 pvt->dhar, 845 dhar_base(pvt->dhar), 846 (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt->dhar) 847 : f10_dhar_offset(pvt->dhar)); 848 849 debugf1(" DramHoleValid: %s\n", 850 (pvt->dhar & DHAR_VALID) ? "yes" : "no"); 851 852 amd64_debug_display_dimm_sizes(0, pvt); 853 854 /* everything below this point is Fam10h and above */ 855 if (boot_cpu_data.x86 == 0xf) 856 return; 857 858 amd64_debug_display_dimm_sizes(1, pvt); 859 860 amd64_info("using %s syndromes.\n", ((pvt->syn_type == 8) ? "x8" : "x4")); 861 862 /* Only if NOT ganged does dclr1 have valid info */ 863 if (!dct_ganging_enabled(pvt)) 864 amd64_dump_dramcfg_low(pvt->dclr1, 1); 865 } 866 867 /* Read in both of DBAM registers */ 868 static void amd64_read_dbam_reg(struct amd64_pvt *pvt) 869 { 870 amd64_read_pci_cfg(pvt->F2, DBAM0, &pvt->dbam0); 871 872 if (boot_cpu_data.x86 >= 0x10) 873 amd64_read_pci_cfg(pvt->F2, DBAM1, &pvt->dbam1); 874 } 875 876 /* 877 * NOTE: CPU Revision Dependent code: Rev E and Rev F 878 * 879 * Set the DCSB and DCSM mask values depending on the CPU revision value. Also 880 * set the shift factor for the DCSB and DCSM values. 881 * 882 * ->dcs_mask_notused, RevE: 883 * 884 * To find the max InputAddr for the csrow, start with the base address and set 885 * all bits that are "don't care" bits in the test at the start of section 886 * 3.5.4 (p. 84). 887 * 888 * The "don't care" bits are all set bits in the mask and all bits in the gaps 889 * between bit ranges [35:25] and [19:13]. The value REV_E_DCS_NOTUSED_BITS 890 * represents bits [24:20] and [12:0], which are all bits in the above-mentioned 891 * gaps. 892 * 893 * ->dcs_mask_notused, RevF and later: 894 * 895 * To find the max InputAddr for the csrow, start with the base address and set 896 * all bits that are "don't care" bits in the test at the start of NPT section 897 * 4.5.4 (p. 87). 898 * 899 * The "don't care" bits are all set bits in the mask and all bits in the gaps 900 * between bit ranges [36:27] and [21:13]. 901 * 902 * The value REV_F_F1Xh_DCS_NOTUSED_BITS represents bits [26:22] and [12:0], 903 * which are all bits in the above-mentioned gaps. 904 */ 905 static void amd64_set_dct_base_and_mask(struct amd64_pvt *pvt) 906 { 907 908 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) { 909 pvt->dcsb_base = REV_E_DCSB_BASE_BITS; 910 pvt->dcsm_mask = REV_E_DCSM_MASK_BITS; 911 pvt->dcs_mask_notused = REV_E_DCS_NOTUSED_BITS; 912 pvt->dcs_shift = REV_E_DCS_SHIFT; 913 pvt->cs_count = 8; 914 pvt->num_dcsm = 8; 915 } else { 916 pvt->dcsb_base = REV_F_F1Xh_DCSB_BASE_BITS; 917 pvt->dcsm_mask = REV_F_F1Xh_DCSM_MASK_BITS; 918 pvt->dcs_mask_notused = REV_F_F1Xh_DCS_NOTUSED_BITS; 919 pvt->dcs_shift = REV_F_F1Xh_DCS_SHIFT; 920 pvt->cs_count = 8; 921 pvt->num_dcsm = 4; 922 } 923 } 924 925 /* 926 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask hw registers 927 */ 928 static void amd64_read_dct_base_mask(struct amd64_pvt *pvt) 929 { 930 int cs, reg; 931 932 amd64_set_dct_base_and_mask(pvt); 933 934 for (cs = 0; cs < pvt->cs_count; cs++) { 935 reg = K8_DCSB0 + (cs * 4); 936 if (!amd64_read_pci_cfg(pvt->F2, reg, &pvt->dcsb0[cs])) 937 debugf0(" DCSB0[%d]=0x%08x reg: F2x%x\n", 938 cs, pvt->dcsb0[cs], reg); 939 940 /* If DCT are NOT ganged, then read in DCT1's base */ 941 if (boot_cpu_data.x86 >= 0x10 && !dct_ganging_enabled(pvt)) { 942 reg = F10_DCSB1 + (cs * 4); 943 if (!amd64_read_pci_cfg(pvt->F2, reg, 944 &pvt->dcsb1[cs])) 945 debugf0(" DCSB1[%d]=0x%08x reg: F2x%x\n", 946 cs, pvt->dcsb1[cs], reg); 947 } else { 948 pvt->dcsb1[cs] = 0; 949 } 950 } 951 952 for (cs = 0; cs < pvt->num_dcsm; cs++) { 953 reg = K8_DCSM0 + (cs * 4); 954 if (!amd64_read_pci_cfg(pvt->F2, reg, &pvt->dcsm0[cs])) 955 debugf0(" DCSM0[%d]=0x%08x reg: F2x%x\n", 956 cs, pvt->dcsm0[cs], reg); 957 958 /* If DCT are NOT ganged, then read in DCT1's mask */ 959 if (boot_cpu_data.x86 >= 0x10 && !dct_ganging_enabled(pvt)) { 960 reg = F10_DCSM1 + (cs * 4); 961 if (!amd64_read_pci_cfg(pvt->F2, reg, 962 &pvt->dcsm1[cs])) 963 debugf0(" DCSM1[%d]=0x%08x reg: F2x%x\n", 964 cs, pvt->dcsm1[cs], reg); 965 } else { 966 pvt->dcsm1[cs] = 0; 967 } 968 } 969 } 970 971 static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt, int cs) 972 { 973 enum mem_type type; 974 975 if (boot_cpu_data.x86 >= 0x10 || pvt->ext_model >= K8_REV_F) { 976 if (pvt->dchr0 & DDR3_MODE) 977 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3; 978 else 979 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2; 980 } else { 981 type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR; 982 } 983 984 amd64_info("CS%d: %s\n", cs, edac_mem_types[type]); 985 986 return type; 987 } 988 989 /* 990 * Read the DRAM Configuration Low register. It differs between CG, D & E revs 991 * and the later RevF memory controllers (DDR vs DDR2) 992 * 993 * Return: 994 * number of memory channels in operation 995 * Pass back: 996 * contents of the DCL0_LOW register 997 */ 998 static int k8_early_channel_count(struct amd64_pvt *pvt) 999 { 1000 int flag, err = 0; 1001 1002 err = amd64_read_pci_cfg(pvt->F2, F10_DCLR_0, &pvt->dclr0); 1003 if (err) 1004 return err; 1005 1006 if (pvt->ext_model >= K8_REV_F) 1007 /* RevF (NPT) and later */ 1008 flag = pvt->dclr0 & F10_WIDTH_128; 1009 else 1010 /* RevE and earlier */ 1011 flag = pvt->dclr0 & REVE_WIDTH_128; 1012 1013 /* not used */ 1014 pvt->dclr1 = 0; 1015 1016 return (flag) ? 2 : 1; 1017 } 1018 1019 /* extract the ERROR ADDRESS for the K8 CPUs */ 1020 static u64 k8_get_error_address(struct mem_ctl_info *mci, 1021 struct err_regs *info) 1022 { 1023 return (((u64) (info->nbeah & 0xff)) << 32) + 1024 (info->nbeal & ~0x03); 1025 } 1026 1027 /* 1028 * Read the Base and Limit registers for K8 based Memory controllers; extract 1029 * fields from the 'raw' reg into separate data fields 1030 * 1031 * Isolates: BASE, LIMIT, IntlvEn, IntlvSel, RW_EN 1032 */ 1033 static void k8_read_dram_base_limit(struct amd64_pvt *pvt, int dram) 1034 { 1035 u32 low; 1036 u32 off = dram << 3; /* 8 bytes between DRAM entries */ 1037 1038 amd64_read_pci_cfg(pvt->F1, K8_DRAM_BASE_LOW + off, &low); 1039 1040 /* Extract parts into separate data entries */ 1041 pvt->dram_base[dram] = ((u64) low & 0xFFFF0000) << 8; 1042 pvt->dram_IntlvEn[dram] = (low >> 8) & 0x7; 1043 pvt->dram_rw_en[dram] = (low & 0x3); 1044 1045 amd64_read_pci_cfg(pvt->F1, K8_DRAM_LIMIT_LOW + off, &low); 1046 1047 /* 1048 * Extract parts into separate data entries. Limit is the HIGHEST memory 1049 * location of the region, so lower 24 bits need to be all ones 1050 */ 1051 pvt->dram_limit[dram] = (((u64) low & 0xFFFF0000) << 8) | 0x00FFFFFF; 1052 pvt->dram_IntlvSel[dram] = (low >> 8) & 0x7; 1053 pvt->dram_DstNode[dram] = (low & 0x7); 1054 } 1055 1056 static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, 1057 struct err_regs *err_info, u64 sys_addr) 1058 { 1059 struct mem_ctl_info *src_mci; 1060 int channel, csrow; 1061 u32 page, offset; 1062 u16 syndrome; 1063 1064 syndrome = extract_syndrome(err_info); 1065 1066 /* CHIPKILL enabled */ 1067 if (err_info->nbcfg & K8_NBCFG_CHIPKILL) { 1068 channel = get_channel_from_ecc_syndrome(mci, syndrome); 1069 if (channel < 0) { 1070 /* 1071 * Syndrome didn't map, so we don't know which of the 1072 * 2 DIMMs is in error. So we need to ID 'both' of them 1073 * as suspect. 1074 */ 1075 amd64_mc_warn(mci, "unknown syndrome 0x%04x - possible " 1076 "error reporting race\n", syndrome); 1077 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1078 return; 1079 } 1080 } else { 1081 /* 1082 * non-chipkill ecc mode 1083 * 1084 * The k8 documentation is unclear about how to determine the 1085 * channel number when using non-chipkill memory. This method 1086 * was obtained from email communication with someone at AMD. 1087 * (Wish the email was placed in this comment - norsk) 1088 */ 1089 channel = ((sys_addr & BIT(3)) != 0); 1090 } 1091 1092 /* 1093 * Find out which node the error address belongs to. This may be 1094 * different from the node that detected the error. 1095 */ 1096 src_mci = find_mc_by_sys_addr(mci, sys_addr); 1097 if (!src_mci) { 1098 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n", 1099 (unsigned long)sys_addr); 1100 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1101 return; 1102 } 1103 1104 /* Now map the sys_addr to a CSROW */ 1105 csrow = sys_addr_to_csrow(src_mci, sys_addr); 1106 if (csrow < 0) { 1107 edac_mc_handle_ce_no_info(src_mci, EDAC_MOD_STR); 1108 } else { 1109 error_address_to_page_and_offset(sys_addr, &page, &offset); 1110 1111 edac_mc_handle_ce(src_mci, page, offset, syndrome, csrow, 1112 channel, EDAC_MOD_STR); 1113 } 1114 } 1115 1116 static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, int cs_mode) 1117 { 1118 int *dbam_map; 1119 1120 if (pvt->ext_model >= K8_REV_F) 1121 dbam_map = ddr2_dbam; 1122 else if (pvt->ext_model >= K8_REV_D) 1123 dbam_map = ddr2_dbam_revD; 1124 else 1125 dbam_map = ddr2_dbam_revCG; 1126 1127 return dbam_map[cs_mode]; 1128 } 1129 1130 /* 1131 * Get the number of DCT channels in use. 1132 * 1133 * Return: 1134 * number of Memory Channels in operation 1135 * Pass back: 1136 * contents of the DCL0_LOW register 1137 */ 1138 static int f10_early_channel_count(struct amd64_pvt *pvt) 1139 { 1140 int dbams[] = { DBAM0, DBAM1 }; 1141 int i, j, channels = 0; 1142 u32 dbam; 1143 1144 /* If we are in 128 bit mode, then we are using 2 channels */ 1145 if (pvt->dclr0 & F10_WIDTH_128) { 1146 channels = 2; 1147 return channels; 1148 } 1149 1150 /* 1151 * Need to check if in unganged mode: In such, there are 2 channels, 1152 * but they are not in 128 bit mode and thus the above 'dclr0' status 1153 * bit will be OFF. 1154 * 1155 * Need to check DCT0[0] and DCT1[0] to see if only one of them has 1156 * their CSEnable bit on. If so, then SINGLE DIMM case. 1157 */ 1158 debugf0("Data width is not 128 bits - need more decoding\n"); 1159 1160 /* 1161 * Check DRAM Bank Address Mapping values for each DIMM to see if there 1162 * is more than just one DIMM present in unganged mode. Need to check 1163 * both controllers since DIMMs can be placed in either one. 1164 */ 1165 for (i = 0; i < ARRAY_SIZE(dbams); i++) { 1166 if (amd64_read_pci_cfg(pvt->F2, dbams[i], &dbam)) 1167 goto err_reg; 1168 1169 for (j = 0; j < 4; j++) { 1170 if (DBAM_DIMM(j, dbam) > 0) { 1171 channels++; 1172 break; 1173 } 1174 } 1175 } 1176 1177 if (channels > 2) 1178 channels = 2; 1179 1180 amd64_info("MCT channel count: %d\n", channels); 1181 1182 return channels; 1183 1184 err_reg: 1185 return -1; 1186 1187 } 1188 1189 static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, int cs_mode) 1190 { 1191 int *dbam_map; 1192 1193 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE) 1194 dbam_map = ddr3_dbam; 1195 else 1196 dbam_map = ddr2_dbam; 1197 1198 return dbam_map[cs_mode]; 1199 } 1200 1201 static u64 f10_get_error_address(struct mem_ctl_info *mci, 1202 struct err_regs *info) 1203 { 1204 return (((u64) (info->nbeah & 0xffff)) << 32) + 1205 (info->nbeal & ~0x01); 1206 } 1207 1208 /* 1209 * Read the Base and Limit registers for F10 based Memory controllers. Extract 1210 * fields from the 'raw' reg into separate data fields. 1211 * 1212 * Isolates: BASE, LIMIT, IntlvEn, IntlvSel, RW_EN. 1213 */ 1214 static void f10_read_dram_base_limit(struct amd64_pvt *pvt, int dram) 1215 { 1216 u32 high_offset, low_offset, high_base, low_base, high_limit, low_limit; 1217 1218 low_offset = K8_DRAM_BASE_LOW + (dram << 3); 1219 high_offset = F10_DRAM_BASE_HIGH + (dram << 3); 1220 1221 /* read the 'raw' DRAM BASE Address register */ 1222 amd64_read_pci_cfg(pvt->F1, low_offset, &low_base); 1223 amd64_read_pci_cfg(pvt->F1, high_offset, &high_base); 1224 1225 /* Extract parts into separate data entries */ 1226 pvt->dram_rw_en[dram] = (low_base & 0x3); 1227 1228 if (pvt->dram_rw_en[dram] == 0) 1229 return; 1230 1231 pvt->dram_IntlvEn[dram] = (low_base >> 8) & 0x7; 1232 1233 pvt->dram_base[dram] = (((u64)high_base & 0x000000FF) << 40) | 1234 (((u64)low_base & 0xFFFF0000) << 8); 1235 1236 low_offset = K8_DRAM_LIMIT_LOW + (dram << 3); 1237 high_offset = F10_DRAM_LIMIT_HIGH + (dram << 3); 1238 1239 /* read the 'raw' LIMIT registers */ 1240 amd64_read_pci_cfg(pvt->F1, low_offset, &low_limit); 1241 amd64_read_pci_cfg(pvt->F1, high_offset, &high_limit); 1242 1243 pvt->dram_DstNode[dram] = (low_limit & 0x7); 1244 pvt->dram_IntlvSel[dram] = (low_limit >> 8) & 0x7; 1245 1246 /* 1247 * Extract address values and form a LIMIT address. Limit is the HIGHEST 1248 * memory location of the region, so low 24 bits need to be all ones. 1249 */ 1250 pvt->dram_limit[dram] = (((u64)high_limit & 0x000000FF) << 40) | 1251 (((u64) low_limit & 0xFFFF0000) << 8) | 1252 0x00FFFFFF; 1253 } 1254 1255 static void f10_read_dram_ctl_register(struct amd64_pvt *pvt) 1256 { 1257 1258 if (!amd64_read_pci_cfg(pvt->F2, F10_DCTL_SEL_LOW, 1259 &pvt->dram_ctl_select_low)) { 1260 debugf0("F2x110 (DCTL Sel. Low): 0x%08x, " 1261 "High range addresses at: 0x%x\n", 1262 pvt->dram_ctl_select_low, 1263 dct_sel_baseaddr(pvt)); 1264 1265 debugf0(" DCT mode: %s, All DCTs on: %s\n", 1266 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"), 1267 (dct_dram_enabled(pvt) ? "yes" : "no")); 1268 1269 if (!dct_ganging_enabled(pvt)) 1270 debugf0(" Address range split per DCT: %s\n", 1271 (dct_high_range_enabled(pvt) ? "yes" : "no")); 1272 1273 debugf0(" DCT data interleave for ECC: %s, " 1274 "DRAM cleared since last warm reset: %s\n", 1275 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"), 1276 (dct_memory_cleared(pvt) ? "yes" : "no")); 1277 1278 debugf0(" DCT channel interleave: %s, " 1279 "DCT interleave bits selector: 0x%x\n", 1280 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"), 1281 dct_sel_interleave_addr(pvt)); 1282 } 1283 1284 amd64_read_pci_cfg(pvt->F2, F10_DCTL_SEL_HIGH, 1285 &pvt->dram_ctl_select_high); 1286 } 1287 1288 /* 1289 * determine channel based on the interleaving mode: F10h BKDG, 2.8.9 Memory 1290 * Interleaving Modes. 1291 */ 1292 static u32 f10_determine_channel(struct amd64_pvt *pvt, u64 sys_addr, 1293 int hi_range_sel, u32 intlv_en) 1294 { 1295 u32 cs, temp, dct_sel_high = (pvt->dram_ctl_select_low >> 1) & 1; 1296 1297 if (dct_ganging_enabled(pvt)) 1298 cs = 0; 1299 else if (hi_range_sel) 1300 cs = dct_sel_high; 1301 else if (dct_interleave_enabled(pvt)) { 1302 /* 1303 * see F2x110[DctSelIntLvAddr] - channel interleave mode 1304 */ 1305 if (dct_sel_interleave_addr(pvt) == 0) 1306 cs = sys_addr >> 6 & 1; 1307 else if ((dct_sel_interleave_addr(pvt) >> 1) & 1) { 1308 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2; 1309 1310 if (dct_sel_interleave_addr(pvt) & 1) 1311 cs = (sys_addr >> 9 & 1) ^ temp; 1312 else 1313 cs = (sys_addr >> 6 & 1) ^ temp; 1314 } else if (intlv_en & 4) 1315 cs = sys_addr >> 15 & 1; 1316 else if (intlv_en & 2) 1317 cs = sys_addr >> 14 & 1; 1318 else if (intlv_en & 1) 1319 cs = sys_addr >> 13 & 1; 1320 else 1321 cs = sys_addr >> 12 & 1; 1322 } else if (dct_high_range_enabled(pvt) && !dct_ganging_enabled(pvt)) 1323 cs = ~dct_sel_high & 1; 1324 else 1325 cs = 0; 1326 1327 return cs; 1328 } 1329 1330 static inline u32 f10_map_intlv_en_to_shift(u32 intlv_en) 1331 { 1332 if (intlv_en == 1) 1333 return 1; 1334 else if (intlv_en == 3) 1335 return 2; 1336 else if (intlv_en == 7) 1337 return 3; 1338 1339 return 0; 1340 } 1341 1342 /* See F10h BKDG, 2.8.10.2 DctSelBaseOffset Programming */ 1343 static inline u64 f10_get_base_addr_offset(u64 sys_addr, int hi_range_sel, 1344 u32 dct_sel_base_addr, 1345 u64 dct_sel_base_off, 1346 u32 hole_valid, u32 hole_off, 1347 u64 dram_base) 1348 { 1349 u64 chan_off; 1350 1351 if (hi_range_sel) { 1352 if (!(dct_sel_base_addr & 0xFFFF0000) && 1353 hole_valid && (sys_addr >= 0x100000000ULL)) 1354 chan_off = hole_off << 16; 1355 else 1356 chan_off = dct_sel_base_off; 1357 } else { 1358 if (hole_valid && (sys_addr >= 0x100000000ULL)) 1359 chan_off = hole_off << 16; 1360 else 1361 chan_off = dram_base & 0xFFFFF8000000ULL; 1362 } 1363 1364 return (sys_addr & 0x0000FFFFFFFFFFC0ULL) - 1365 (chan_off & 0x0000FFFFFF800000ULL); 1366 } 1367 1368 /* Hack for the time being - Can we get this from BIOS?? */ 1369 #define CH0SPARE_RANK 0 1370 #define CH1SPARE_RANK 1 1371 1372 /* 1373 * checks if the csrow passed in is marked as SPARED, if so returns the new 1374 * spare row 1375 */ 1376 static inline int f10_process_possible_spare(int csrow, 1377 u32 cs, struct amd64_pvt *pvt) 1378 { 1379 u32 swap_done; 1380 u32 bad_dram_cs; 1381 1382 /* Depending on channel, isolate respective SPARING info */ 1383 if (cs) { 1384 swap_done = F10_ONLINE_SPARE_SWAPDONE1(pvt->online_spare); 1385 bad_dram_cs = F10_ONLINE_SPARE_BADDRAM_CS1(pvt->online_spare); 1386 if (swap_done && (csrow == bad_dram_cs)) 1387 csrow = CH1SPARE_RANK; 1388 } else { 1389 swap_done = F10_ONLINE_SPARE_SWAPDONE0(pvt->online_spare); 1390 bad_dram_cs = F10_ONLINE_SPARE_BADDRAM_CS0(pvt->online_spare); 1391 if (swap_done && (csrow == bad_dram_cs)) 1392 csrow = CH0SPARE_RANK; 1393 } 1394 return csrow; 1395 } 1396 1397 /* 1398 * Iterate over the DRAM DCT "base" and "mask" registers looking for a 1399 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID' 1400 * 1401 * Return: 1402 * -EINVAL: NOT FOUND 1403 * 0..csrow = Chip-Select Row 1404 */ 1405 static int f10_lookup_addr_in_dct(u32 in_addr, u32 nid, u32 cs) 1406 { 1407 struct mem_ctl_info *mci; 1408 struct amd64_pvt *pvt; 1409 u32 cs_base, cs_mask; 1410 int cs_found = -EINVAL; 1411 int csrow; 1412 1413 mci = mcis[nid]; 1414 if (!mci) 1415 return cs_found; 1416 1417 pvt = mci->pvt_info; 1418 1419 debugf1("InputAddr=0x%x channelselect=%d\n", in_addr, cs); 1420 1421 for (csrow = 0; csrow < pvt->cs_count; csrow++) { 1422 1423 cs_base = amd64_get_dct_base(pvt, cs, csrow); 1424 if (!(cs_base & K8_DCSB_CS_ENABLE)) 1425 continue; 1426 1427 /* 1428 * We have an ENABLED CSROW, Isolate just the MASK bits of the 1429 * target: [28:19] and [13:5], which map to [36:27] and [21:13] 1430 * of the actual address. 1431 */ 1432 cs_base &= REV_F_F1Xh_DCSB_BASE_BITS; 1433 1434 /* 1435 * Get the DCT Mask, and ENABLE the reserved bits: [18:16] and 1436 * [4:0] to become ON. Then mask off bits [28:0] ([36:8]) 1437 */ 1438 cs_mask = amd64_get_dct_mask(pvt, cs, csrow); 1439 1440 debugf1(" CSROW=%d CSBase=0x%x RAW CSMask=0x%x\n", 1441 csrow, cs_base, cs_mask); 1442 1443 cs_mask = (cs_mask | 0x0007C01F) & 0x1FFFFFFF; 1444 1445 debugf1(" Final CSMask=0x%x\n", cs_mask); 1446 debugf1(" (InputAddr & ~CSMask)=0x%x " 1447 "(CSBase & ~CSMask)=0x%x\n", 1448 (in_addr & ~cs_mask), (cs_base & ~cs_mask)); 1449 1450 if ((in_addr & ~cs_mask) == (cs_base & ~cs_mask)) { 1451 cs_found = f10_process_possible_spare(csrow, cs, pvt); 1452 1453 debugf1(" MATCH csrow=%d\n", cs_found); 1454 break; 1455 } 1456 } 1457 return cs_found; 1458 } 1459 1460 /* For a given @dram_range, check if @sys_addr falls within it. */ 1461 static int f10_match_to_this_node(struct amd64_pvt *pvt, int dram_range, 1462 u64 sys_addr, int *nid, int *chan_sel) 1463 { 1464 int node_id, cs_found = -EINVAL, high_range = 0; 1465 u32 intlv_en, intlv_sel, intlv_shift, hole_off; 1466 u32 hole_valid, tmp, dct_sel_base, channel; 1467 u64 dram_base, chan_addr, dct_sel_base_off; 1468 1469 dram_base = pvt->dram_base[dram_range]; 1470 intlv_en = pvt->dram_IntlvEn[dram_range]; 1471 1472 node_id = pvt->dram_DstNode[dram_range]; 1473 intlv_sel = pvt->dram_IntlvSel[dram_range]; 1474 1475 debugf1("(dram=%d) Base=0x%llx SystemAddr= 0x%llx Limit=0x%llx\n", 1476 dram_range, dram_base, sys_addr, pvt->dram_limit[dram_range]); 1477 1478 /* 1479 * This assumes that one node's DHAR is the same as all the other 1480 * nodes' DHAR. 1481 */ 1482 hole_off = (pvt->dhar & 0x0000FF80); 1483 hole_valid = (pvt->dhar & 0x1); 1484 dct_sel_base_off = (pvt->dram_ctl_select_high & 0xFFFFFC00) << 16; 1485 1486 debugf1(" HoleOffset=0x%x HoleValid=0x%x IntlvSel=0x%x\n", 1487 hole_off, hole_valid, intlv_sel); 1488 1489 if (intlv_en && 1490 (intlv_sel != ((sys_addr >> 12) & intlv_en))) 1491 return -EINVAL; 1492 1493 dct_sel_base = dct_sel_baseaddr(pvt); 1494 1495 /* 1496 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to 1497 * select between DCT0 and DCT1. 1498 */ 1499 if (dct_high_range_enabled(pvt) && 1500 !dct_ganging_enabled(pvt) && 1501 ((sys_addr >> 27) >= (dct_sel_base >> 11))) 1502 high_range = 1; 1503 1504 channel = f10_determine_channel(pvt, sys_addr, high_range, intlv_en); 1505 1506 chan_addr = f10_get_base_addr_offset(sys_addr, high_range, dct_sel_base, 1507 dct_sel_base_off, hole_valid, 1508 hole_off, dram_base); 1509 1510 intlv_shift = f10_map_intlv_en_to_shift(intlv_en); 1511 1512 /* remove Node ID (in case of memory interleaving) */ 1513 tmp = chan_addr & 0xFC0; 1514 1515 chan_addr = ((chan_addr >> intlv_shift) & 0xFFFFFFFFF000ULL) | tmp; 1516 1517 /* remove channel interleave and hash */ 1518 if (dct_interleave_enabled(pvt) && 1519 !dct_high_range_enabled(pvt) && 1520 !dct_ganging_enabled(pvt)) { 1521 if (dct_sel_interleave_addr(pvt) != 1) 1522 chan_addr = (chan_addr >> 1) & 0xFFFFFFFFFFFFFFC0ULL; 1523 else { 1524 tmp = chan_addr & 0xFC0; 1525 chan_addr = ((chan_addr & 0xFFFFFFFFFFFFC000ULL) >> 1) 1526 | tmp; 1527 } 1528 } 1529 1530 debugf1(" (ChannelAddrLong=0x%llx) >> 8 becomes InputAddr=0x%x\n", 1531 chan_addr, (u32)(chan_addr >> 8)); 1532 1533 cs_found = f10_lookup_addr_in_dct(chan_addr >> 8, node_id, channel); 1534 1535 if (cs_found >= 0) { 1536 *nid = node_id; 1537 *chan_sel = channel; 1538 } 1539 return cs_found; 1540 } 1541 1542 static int f10_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr, 1543 int *node, int *chan_sel) 1544 { 1545 int dram_range, cs_found = -EINVAL; 1546 u64 dram_base, dram_limit; 1547 1548 for (dram_range = 0; dram_range < DRAM_REG_COUNT; dram_range++) { 1549 1550 if (!pvt->dram_rw_en[dram_range]) 1551 continue; 1552 1553 dram_base = pvt->dram_base[dram_range]; 1554 dram_limit = pvt->dram_limit[dram_range]; 1555 1556 if ((dram_base <= sys_addr) && (sys_addr <= dram_limit)) { 1557 1558 cs_found = f10_match_to_this_node(pvt, dram_range, 1559 sys_addr, node, 1560 chan_sel); 1561 if (cs_found >= 0) 1562 break; 1563 } 1564 } 1565 return cs_found; 1566 } 1567 1568 /* 1569 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps 1570 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW). 1571 * 1572 * The @sys_addr is usually an error address received from the hardware 1573 * (MCX_ADDR). 1574 */ 1575 static void f10_map_sysaddr_to_csrow(struct mem_ctl_info *mci, 1576 struct err_regs *err_info, 1577 u64 sys_addr) 1578 { 1579 struct amd64_pvt *pvt = mci->pvt_info; 1580 u32 page, offset; 1581 int nid, csrow, chan = 0; 1582 u16 syndrome; 1583 1584 csrow = f10_translate_sysaddr_to_cs(pvt, sys_addr, &nid, &chan); 1585 1586 if (csrow < 0) { 1587 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1588 return; 1589 } 1590 1591 error_address_to_page_and_offset(sys_addr, &page, &offset); 1592 1593 syndrome = extract_syndrome(err_info); 1594 1595 /* 1596 * We need the syndromes for channel detection only when we're 1597 * ganged. Otherwise @chan should already contain the channel at 1598 * this point. 1599 */ 1600 if (dct_ganging_enabled(pvt) && (pvt->nbcfg & K8_NBCFG_CHIPKILL)) 1601 chan = get_channel_from_ecc_syndrome(mci, syndrome); 1602 1603 if (chan >= 0) 1604 edac_mc_handle_ce(mci, page, offset, syndrome, csrow, chan, 1605 EDAC_MOD_STR); 1606 else 1607 /* 1608 * Channel unknown, report all channels on this CSROW as failed. 1609 */ 1610 for (chan = 0; chan < mci->csrows[csrow].nr_channels; chan++) 1611 edac_mc_handle_ce(mci, page, offset, syndrome, 1612 csrow, chan, EDAC_MOD_STR); 1613 } 1614 1615 /* 1616 * debug routine to display the memory sizes of all logical DIMMs and its 1617 * CSROWs as well 1618 */ 1619 static void amd64_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt) 1620 { 1621 int dimm, size0, size1, factor = 0; 1622 u32 dbam; 1623 u32 *dcsb; 1624 1625 if (boot_cpu_data.x86 == 0xf) { 1626 if (pvt->dclr0 & F10_WIDTH_128) 1627 factor = 1; 1628 1629 /* K8 families < revF not supported yet */ 1630 if (pvt->ext_model < K8_REV_F) 1631 return; 1632 else 1633 WARN_ON(ctrl != 0); 1634 } 1635 1636 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 : pvt->dbam0; 1637 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dcsb1 : pvt->dcsb0; 1638 1639 debugf1("F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", ctrl, dbam); 1640 1641 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl); 1642 1643 /* Dump memory sizes for DIMM and its CSROWs */ 1644 for (dimm = 0; dimm < 4; dimm++) { 1645 1646 size0 = 0; 1647 if (dcsb[dimm*2] & K8_DCSB_CS_ENABLE) 1648 size0 = pvt->ops->dbam_to_cs(pvt, DBAM_DIMM(dimm, dbam)); 1649 1650 size1 = 0; 1651 if (dcsb[dimm*2 + 1] & K8_DCSB_CS_ENABLE) 1652 size1 = pvt->ops->dbam_to_cs(pvt, DBAM_DIMM(dimm, dbam)); 1653 1654 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n", 1655 dimm * 2, size0 << factor, 1656 dimm * 2 + 1, size1 << factor); 1657 } 1658 } 1659 1660 static struct amd64_family_type amd64_family_types[] = { 1661 [K8_CPUS] = { 1662 .ctl_name = "K8", 1663 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP, 1664 .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC, 1665 .ops = { 1666 .early_channel_count = k8_early_channel_count, 1667 .get_error_address = k8_get_error_address, 1668 .read_dram_base_limit = k8_read_dram_base_limit, 1669 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow, 1670 .dbam_to_cs = k8_dbam_to_chip_select, 1671 } 1672 }, 1673 [F10_CPUS] = { 1674 .ctl_name = "F10h", 1675 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP, 1676 .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC, 1677 .ops = { 1678 .early_channel_count = f10_early_channel_count, 1679 .get_error_address = f10_get_error_address, 1680 .read_dram_base_limit = f10_read_dram_base_limit, 1681 .read_dram_ctl_register = f10_read_dram_ctl_register, 1682 .map_sysaddr_to_csrow = f10_map_sysaddr_to_csrow, 1683 .dbam_to_cs = f10_dbam_to_chip_select, 1684 } 1685 }, 1686 }; 1687 1688 static struct pci_dev *pci_get_related_function(unsigned int vendor, 1689 unsigned int device, 1690 struct pci_dev *related) 1691 { 1692 struct pci_dev *dev = NULL; 1693 1694 dev = pci_get_device(vendor, device, dev); 1695 while (dev) { 1696 if ((dev->bus->number == related->bus->number) && 1697 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn))) 1698 break; 1699 dev = pci_get_device(vendor, device, dev); 1700 } 1701 1702 return dev; 1703 } 1704 1705 /* 1706 * These are tables of eigenvectors (one per line) which can be used for the 1707 * construction of the syndrome tables. The modified syndrome search algorithm 1708 * uses those to find the symbol in error and thus the DIMM. 1709 * 1710 * Algorithm courtesy of Ross LaFetra from AMD. 1711 */ 1712 static u16 x4_vectors[] = { 1713 0x2f57, 0x1afe, 0x66cc, 0xdd88, 1714 0x11eb, 0x3396, 0x7f4c, 0xeac8, 1715 0x0001, 0x0002, 0x0004, 0x0008, 1716 0x1013, 0x3032, 0x4044, 0x8088, 1717 0x106b, 0x30d6, 0x70fc, 0xe0a8, 1718 0x4857, 0xc4fe, 0x13cc, 0x3288, 1719 0x1ac5, 0x2f4a, 0x5394, 0xa1e8, 1720 0x1f39, 0x251e, 0xbd6c, 0x6bd8, 1721 0x15c1, 0x2a42, 0x89ac, 0x4758, 1722 0x2b03, 0x1602, 0x4f0c, 0xca08, 1723 0x1f07, 0x3a0e, 0x6b04, 0xbd08, 1724 0x8ba7, 0x465e, 0x244c, 0x1cc8, 1725 0x2b87, 0x164e, 0x642c, 0xdc18, 1726 0x40b9, 0x80de, 0x1094, 0x20e8, 1727 0x27db, 0x1eb6, 0x9dac, 0x7b58, 1728 0x11c1, 0x2242, 0x84ac, 0x4c58, 1729 0x1be5, 0x2d7a, 0x5e34, 0xa718, 1730 0x4b39, 0x8d1e, 0x14b4, 0x28d8, 1731 0x4c97, 0xc87e, 0x11fc, 0x33a8, 1732 0x8e97, 0x497e, 0x2ffc, 0x1aa8, 1733 0x16b3, 0x3d62, 0x4f34, 0x8518, 1734 0x1e2f, 0x391a, 0x5cac, 0xf858, 1735 0x1d9f, 0x3b7a, 0x572c, 0xfe18, 1736 0x15f5, 0x2a5a, 0x5264, 0xa3b8, 1737 0x1dbb, 0x3b66, 0x715c, 0xe3f8, 1738 0x4397, 0xc27e, 0x17fc, 0x3ea8, 1739 0x1617, 0x3d3e, 0x6464, 0xb8b8, 1740 0x23ff, 0x12aa, 0xab6c, 0x56d8, 1741 0x2dfb, 0x1ba6, 0x913c, 0x7328, 1742 0x185d, 0x2ca6, 0x7914, 0x9e28, 1743 0x171b, 0x3e36, 0x7d7c, 0xebe8, 1744 0x4199, 0x82ee, 0x19f4, 0x2e58, 1745 0x4807, 0xc40e, 0x130c, 0x3208, 1746 0x1905, 0x2e0a, 0x5804, 0xac08, 1747 0x213f, 0x132a, 0xadfc, 0x5ba8, 1748 0x19a9, 0x2efe, 0xb5cc, 0x6f88, 1749 }; 1750 1751 static u16 x8_vectors[] = { 1752 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480, 1753 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80, 1754 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80, 1755 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80, 1756 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780, 1757 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080, 1758 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080, 1759 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080, 1760 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80, 1761 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580, 1762 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880, 1763 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280, 1764 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180, 1765 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580, 1766 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280, 1767 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180, 1768 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080, 1769 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 1770 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000, 1771 }; 1772 1773 static int decode_syndrome(u16 syndrome, u16 *vectors, int num_vecs, 1774 int v_dim) 1775 { 1776 unsigned int i, err_sym; 1777 1778 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) { 1779 u16 s = syndrome; 1780 int v_idx = err_sym * v_dim; 1781 int v_end = (err_sym + 1) * v_dim; 1782 1783 /* walk over all 16 bits of the syndrome */ 1784 for (i = 1; i < (1U << 16); i <<= 1) { 1785 1786 /* if bit is set in that eigenvector... */ 1787 if (v_idx < v_end && vectors[v_idx] & i) { 1788 u16 ev_comp = vectors[v_idx++]; 1789 1790 /* ... and bit set in the modified syndrome, */ 1791 if (s & i) { 1792 /* remove it. */ 1793 s ^= ev_comp; 1794 1795 if (!s) 1796 return err_sym; 1797 } 1798 1799 } else if (s & i) 1800 /* can't get to zero, move to next symbol */ 1801 break; 1802 } 1803 } 1804 1805 debugf0("syndrome(%x) not found\n", syndrome); 1806 return -1; 1807 } 1808 1809 static int map_err_sym_to_channel(int err_sym, int sym_size) 1810 { 1811 if (sym_size == 4) 1812 switch (err_sym) { 1813 case 0x20: 1814 case 0x21: 1815 return 0; 1816 break; 1817 case 0x22: 1818 case 0x23: 1819 return 1; 1820 break; 1821 default: 1822 return err_sym >> 4; 1823 break; 1824 } 1825 /* x8 symbols */ 1826 else 1827 switch (err_sym) { 1828 /* imaginary bits not in a DIMM */ 1829 case 0x10: 1830 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n", 1831 err_sym); 1832 return -1; 1833 break; 1834 1835 case 0x11: 1836 return 0; 1837 break; 1838 case 0x12: 1839 return 1; 1840 break; 1841 default: 1842 return err_sym >> 3; 1843 break; 1844 } 1845 return -1; 1846 } 1847 1848 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome) 1849 { 1850 struct amd64_pvt *pvt = mci->pvt_info; 1851 int err_sym = -1; 1852 1853 if (pvt->syn_type == 8) 1854 err_sym = decode_syndrome(syndrome, x8_vectors, 1855 ARRAY_SIZE(x8_vectors), 1856 pvt->syn_type); 1857 else if (pvt->syn_type == 4) 1858 err_sym = decode_syndrome(syndrome, x4_vectors, 1859 ARRAY_SIZE(x4_vectors), 1860 pvt->syn_type); 1861 else { 1862 amd64_warn("Illegal syndrome type: %u\n", pvt->syn_type); 1863 return err_sym; 1864 } 1865 1866 return map_err_sym_to_channel(err_sym, pvt->syn_type); 1867 } 1868 1869 /* 1870 * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR 1871 * ADDRESS and process. 1872 */ 1873 static void amd64_handle_ce(struct mem_ctl_info *mci, 1874 struct err_regs *info) 1875 { 1876 struct amd64_pvt *pvt = mci->pvt_info; 1877 u64 sys_addr; 1878 1879 /* Ensure that the Error Address is VALID */ 1880 if (!(info->nbsh & K8_NBSH_VALID_ERROR_ADDR)) { 1881 amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n"); 1882 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1883 return; 1884 } 1885 1886 sys_addr = pvt->ops->get_error_address(mci, info); 1887 1888 amd64_mc_err(mci, "CE ERROR_ADDRESS= 0x%llx\n", sys_addr); 1889 1890 pvt->ops->map_sysaddr_to_csrow(mci, info, sys_addr); 1891 } 1892 1893 /* Handle any Un-correctable Errors (UEs) */ 1894 static void amd64_handle_ue(struct mem_ctl_info *mci, 1895 struct err_regs *info) 1896 { 1897 struct amd64_pvt *pvt = mci->pvt_info; 1898 struct mem_ctl_info *log_mci, *src_mci = NULL; 1899 int csrow; 1900 u64 sys_addr; 1901 u32 page, offset; 1902 1903 log_mci = mci; 1904 1905 if (!(info->nbsh & K8_NBSH_VALID_ERROR_ADDR)) { 1906 amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n"); 1907 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); 1908 return; 1909 } 1910 1911 sys_addr = pvt->ops->get_error_address(mci, info); 1912 1913 /* 1914 * Find out which node the error address belongs to. This may be 1915 * different from the node that detected the error. 1916 */ 1917 src_mci = find_mc_by_sys_addr(mci, sys_addr); 1918 if (!src_mci) { 1919 amd64_mc_err(mci, "ERROR ADDRESS (0x%lx) NOT mapped to a MC\n", 1920 (unsigned long)sys_addr); 1921 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); 1922 return; 1923 } 1924 1925 log_mci = src_mci; 1926 1927 csrow = sys_addr_to_csrow(log_mci, sys_addr); 1928 if (csrow < 0) { 1929 amd64_mc_err(mci, "ERROR_ADDRESS (0x%lx) NOT mapped to CS\n", 1930 (unsigned long)sys_addr); 1931 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); 1932 } else { 1933 error_address_to_page_and_offset(sys_addr, &page, &offset); 1934 edac_mc_handle_ue(log_mci, page, offset, csrow, EDAC_MOD_STR); 1935 } 1936 } 1937 1938 static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci, 1939 struct err_regs *info) 1940 { 1941 u16 ec = EC(info->nbsl); 1942 u8 xec = XEC(info->nbsl, 0x1f); 1943 int ecc_type = (info->nbsh >> 13) & 0x3; 1944 1945 /* Bail early out if this was an 'observed' error */ 1946 if (PP(ec) == K8_NBSL_PP_OBS) 1947 return; 1948 1949 /* Do only ECC errors */ 1950 if (xec && xec != F10_NBSL_EXT_ERR_ECC) 1951 return; 1952 1953 if (ecc_type == 2) 1954 amd64_handle_ce(mci, info); 1955 else if (ecc_type == 1) 1956 amd64_handle_ue(mci, info); 1957 } 1958 1959 void amd64_decode_bus_error(int node_id, struct mce *m, u32 nbcfg) 1960 { 1961 struct mem_ctl_info *mci = mcis[node_id]; 1962 struct err_regs regs; 1963 1964 regs.nbsl = (u32) m->status; 1965 regs.nbsh = (u32)(m->status >> 32); 1966 regs.nbeal = (u32) m->addr; 1967 regs.nbeah = (u32)(m->addr >> 32); 1968 regs.nbcfg = nbcfg; 1969 1970 __amd64_decode_bus_error(mci, ®s); 1971 1972 /* 1973 * Check the UE bit of the NB status high register, if set generate some 1974 * logs. If NOT a GART error, then process the event as a NO-INFO event. 1975 * If it was a GART error, skip that process. 1976 * 1977 * FIXME: this should go somewhere else, if at all. 1978 */ 1979 if (regs.nbsh & K8_NBSH_UC_ERR && !report_gart_errors) 1980 edac_mc_handle_ue_no_info(mci, "UE bit is set"); 1981 1982 } 1983 1984 /* 1985 * Use pvt->F2 which contains the F2 CPU PCI device to get the related 1986 * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error. 1987 */ 1988 static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id) 1989 { 1990 /* Reserve the ADDRESS MAP Device */ 1991 pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2); 1992 if (!pvt->F1) { 1993 amd64_err("error address map device not found: " 1994 "vendor %x device 0x%x (broken BIOS?)\n", 1995 PCI_VENDOR_ID_AMD, f1_id); 1996 return -ENODEV; 1997 } 1998 1999 /* Reserve the MISC Device */ 2000 pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2); 2001 if (!pvt->F3) { 2002 pci_dev_put(pvt->F1); 2003 pvt->F1 = NULL; 2004 2005 amd64_err("error F3 device not found: " 2006 "vendor %x device 0x%x (broken BIOS?)\n", 2007 PCI_VENDOR_ID_AMD, f3_id); 2008 2009 return -ENODEV; 2010 } 2011 debugf1("F1: %s\n", pci_name(pvt->F1)); 2012 debugf1("F2: %s\n", pci_name(pvt->F2)); 2013 debugf1("F3: %s\n", pci_name(pvt->F3)); 2014 2015 return 0; 2016 } 2017 2018 static void free_mc_sibling_devs(struct amd64_pvt *pvt) 2019 { 2020 pci_dev_put(pvt->F1); 2021 pci_dev_put(pvt->F3); 2022 } 2023 2024 /* 2025 * Retrieve the hardware registers of the memory controller (this includes the 2026 * 'Address Map' and 'Misc' device regs) 2027 */ 2028 static void read_mc_regs(struct amd64_pvt *pvt) 2029 { 2030 u64 msr_val; 2031 u32 tmp; 2032 int dram; 2033 2034 /* 2035 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since 2036 * those are Read-As-Zero 2037 */ 2038 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem); 2039 debugf0(" TOP_MEM: 0x%016llx\n", pvt->top_mem); 2040 2041 /* check first whether TOP_MEM2 is enabled */ 2042 rdmsrl(MSR_K8_SYSCFG, msr_val); 2043 if (msr_val & (1U << 21)) { 2044 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2); 2045 debugf0(" TOP_MEM2: 0x%016llx\n", pvt->top_mem2); 2046 } else 2047 debugf0(" TOP_MEM2 disabled.\n"); 2048 2049 amd64_read_pci_cfg(pvt->F3, K8_NBCAP, &pvt->nbcap); 2050 2051 if (pvt->ops->read_dram_ctl_register) 2052 pvt->ops->read_dram_ctl_register(pvt); 2053 2054 for (dram = 0; dram < DRAM_REG_COUNT; dram++) { 2055 /* 2056 * Call CPU specific READ function to get the DRAM Base and 2057 * Limit values from the DCT. 2058 */ 2059 pvt->ops->read_dram_base_limit(pvt, dram); 2060 2061 /* 2062 * Only print out debug info on rows with both R and W Enabled. 2063 * Normal processing, compiler should optimize this whole 'if' 2064 * debug output block away. 2065 */ 2066 if (pvt->dram_rw_en[dram] != 0) { 2067 debugf1(" DRAM-BASE[%d]: 0x%016llx " 2068 "DRAM-LIMIT: 0x%016llx\n", 2069 dram, 2070 pvt->dram_base[dram], 2071 pvt->dram_limit[dram]); 2072 2073 debugf1(" IntlvEn=%s %s %s " 2074 "IntlvSel=%d DstNode=%d\n", 2075 pvt->dram_IntlvEn[dram] ? 2076 "Enabled" : "Disabled", 2077 (pvt->dram_rw_en[dram] & 0x2) ? "W" : "!W", 2078 (pvt->dram_rw_en[dram] & 0x1) ? "R" : "!R", 2079 pvt->dram_IntlvSel[dram], 2080 pvt->dram_DstNode[dram]); 2081 } 2082 } 2083 2084 amd64_read_dct_base_mask(pvt); 2085 2086 amd64_read_pci_cfg(pvt->F1, K8_DHAR, &pvt->dhar); 2087 amd64_read_dbam_reg(pvt); 2088 2089 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare); 2090 2091 amd64_read_pci_cfg(pvt->F2, F10_DCLR_0, &pvt->dclr0); 2092 amd64_read_pci_cfg(pvt->F2, F10_DCHR_0, &pvt->dchr0); 2093 2094 if (boot_cpu_data.x86 >= 0x10) { 2095 if (!dct_ganging_enabled(pvt)) { 2096 amd64_read_pci_cfg(pvt->F2, F10_DCLR_1, &pvt->dclr1); 2097 amd64_read_pci_cfg(pvt->F2, F10_DCHR_1, &pvt->dchr1); 2098 } 2099 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp); 2100 } 2101 2102 if (boot_cpu_data.x86 == 0x10 && 2103 boot_cpu_data.x86_model > 7 && 2104 /* F3x180[EccSymbolSize]=1 => x8 symbols */ 2105 tmp & BIT(25)) 2106 pvt->syn_type = 8; 2107 else 2108 pvt->syn_type = 4; 2109 2110 amd64_dump_misc_regs(pvt); 2111 } 2112 2113 /* 2114 * NOTE: CPU Revision Dependent code 2115 * 2116 * Input: 2117 * @csrow_nr ChipSelect Row Number (0..pvt->cs_count-1) 2118 * k8 private pointer to --> 2119 * DRAM Bank Address mapping register 2120 * node_id 2121 * DCL register where dual_channel_active is 2122 * 2123 * The DBAM register consists of 4 sets of 4 bits each definitions: 2124 * 2125 * Bits: CSROWs 2126 * 0-3 CSROWs 0 and 1 2127 * 4-7 CSROWs 2 and 3 2128 * 8-11 CSROWs 4 and 5 2129 * 12-15 CSROWs 6 and 7 2130 * 2131 * Values range from: 0 to 15 2132 * The meaning of the values depends on CPU revision and dual-channel state, 2133 * see relevant BKDG more info. 2134 * 2135 * The memory controller provides for total of only 8 CSROWs in its current 2136 * architecture. Each "pair" of CSROWs normally represents just one DIMM in 2137 * single channel or two (2) DIMMs in dual channel mode. 2138 * 2139 * The following code logic collapses the various tables for CSROW based on CPU 2140 * revision. 2141 * 2142 * Returns: 2143 * The number of PAGE_SIZE pages on the specified CSROW number it 2144 * encompasses 2145 * 2146 */ 2147 static u32 amd64_csrow_nr_pages(int csrow_nr, struct amd64_pvt *pvt) 2148 { 2149 u32 cs_mode, nr_pages; 2150 2151 /* 2152 * The math on this doesn't look right on the surface because x/2*4 can 2153 * be simplified to x*2 but this expression makes use of the fact that 2154 * it is integral math where 1/2=0. This intermediate value becomes the 2155 * number of bits to shift the DBAM register to extract the proper CSROW 2156 * field. 2157 */ 2158 cs_mode = (pvt->dbam0 >> ((csrow_nr / 2) * 4)) & 0xF; 2159 2160 nr_pages = pvt->ops->dbam_to_cs(pvt, cs_mode) << (20 - PAGE_SHIFT); 2161 2162 /* 2163 * If dual channel then double the memory size of single channel. 2164 * Channel count is 1 or 2 2165 */ 2166 nr_pages <<= (pvt->channel_count - 1); 2167 2168 debugf0(" (csrow=%d) DBAM map index= %d\n", csrow_nr, cs_mode); 2169 debugf0(" nr_pages= %u channel-count = %d\n", 2170 nr_pages, pvt->channel_count); 2171 2172 return nr_pages; 2173 } 2174 2175 /* 2176 * Initialize the array of csrow attribute instances, based on the values 2177 * from pci config hardware registers. 2178 */ 2179 static int init_csrows(struct mem_ctl_info *mci) 2180 { 2181 struct csrow_info *csrow; 2182 struct amd64_pvt *pvt = mci->pvt_info; 2183 u64 input_addr_min, input_addr_max, sys_addr; 2184 u32 val; 2185 int i, empty = 1; 2186 2187 amd64_read_pci_cfg(pvt->F3, K8_NBCFG, &val); 2188 2189 pvt->nbcfg = val; 2190 pvt->ctl_error_info.nbcfg = val; 2191 2192 debugf0("node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n", 2193 pvt->mc_node_id, val, 2194 !!(val & K8_NBCFG_CHIPKILL), !!(val & K8_NBCFG_ECC_ENABLE)); 2195 2196 for (i = 0; i < pvt->cs_count; i++) { 2197 csrow = &mci->csrows[i]; 2198 2199 if ((pvt->dcsb0[i] & K8_DCSB_CS_ENABLE) == 0) { 2200 debugf1("----CSROW %d EMPTY for node %d\n", i, 2201 pvt->mc_node_id); 2202 continue; 2203 } 2204 2205 debugf1("----CSROW %d VALID for MC node %d\n", 2206 i, pvt->mc_node_id); 2207 2208 empty = 0; 2209 csrow->nr_pages = amd64_csrow_nr_pages(i, pvt); 2210 find_csrow_limits(mci, i, &input_addr_min, &input_addr_max); 2211 sys_addr = input_addr_to_sys_addr(mci, input_addr_min); 2212 csrow->first_page = (u32) (sys_addr >> PAGE_SHIFT); 2213 sys_addr = input_addr_to_sys_addr(mci, input_addr_max); 2214 csrow->last_page = (u32) (sys_addr >> PAGE_SHIFT); 2215 csrow->page_mask = ~mask_from_dct_mask(pvt, i); 2216 /* 8 bytes of resolution */ 2217 2218 csrow->mtype = amd64_determine_memory_type(pvt, i); 2219 2220 debugf1(" for MC node %d csrow %d:\n", pvt->mc_node_id, i); 2221 debugf1(" input_addr_min: 0x%lx input_addr_max: 0x%lx\n", 2222 (unsigned long)input_addr_min, 2223 (unsigned long)input_addr_max); 2224 debugf1(" sys_addr: 0x%lx page_mask: 0x%lx\n", 2225 (unsigned long)sys_addr, csrow->page_mask); 2226 debugf1(" nr_pages: %u first_page: 0x%lx " 2227 "last_page: 0x%lx\n", 2228 (unsigned)csrow->nr_pages, 2229 csrow->first_page, csrow->last_page); 2230 2231 /* 2232 * determine whether CHIPKILL or JUST ECC or NO ECC is operating 2233 */ 2234 if (pvt->nbcfg & K8_NBCFG_ECC_ENABLE) 2235 csrow->edac_mode = 2236 (pvt->nbcfg & K8_NBCFG_CHIPKILL) ? 2237 EDAC_S4ECD4ED : EDAC_SECDED; 2238 else 2239 csrow->edac_mode = EDAC_NONE; 2240 } 2241 2242 return empty; 2243 } 2244 2245 /* get all cores on this DCT */ 2246 static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, int nid) 2247 { 2248 int cpu; 2249 2250 for_each_online_cpu(cpu) 2251 if (amd_get_nb_id(cpu) == nid) 2252 cpumask_set_cpu(cpu, mask); 2253 } 2254 2255 /* check MCG_CTL on all the cpus on this node */ 2256 static bool amd64_nb_mce_bank_enabled_on_node(int nid) 2257 { 2258 cpumask_var_t mask; 2259 int cpu, nbe; 2260 bool ret = false; 2261 2262 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { 2263 amd64_warn("%s: Error allocating mask\n", __func__); 2264 return false; 2265 } 2266 2267 get_cpus_on_this_dct_cpumask(mask, nid); 2268 2269 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs); 2270 2271 for_each_cpu(cpu, mask) { 2272 struct msr *reg = per_cpu_ptr(msrs, cpu); 2273 nbe = reg->l & K8_MSR_MCGCTL_NBE; 2274 2275 debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n", 2276 cpu, reg->q, 2277 (nbe ? "enabled" : "disabled")); 2278 2279 if (!nbe) 2280 goto out; 2281 } 2282 ret = true; 2283 2284 out: 2285 free_cpumask_var(mask); 2286 return ret; 2287 } 2288 2289 static int toggle_ecc_err_reporting(struct ecc_settings *s, u8 nid, bool on) 2290 { 2291 cpumask_var_t cmask; 2292 int cpu; 2293 2294 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) { 2295 amd64_warn("%s: error allocating mask\n", __func__); 2296 return false; 2297 } 2298 2299 get_cpus_on_this_dct_cpumask(cmask, nid); 2300 2301 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); 2302 2303 for_each_cpu(cpu, cmask) { 2304 2305 struct msr *reg = per_cpu_ptr(msrs, cpu); 2306 2307 if (on) { 2308 if (reg->l & K8_MSR_MCGCTL_NBE) 2309 s->flags.nb_mce_enable = 1; 2310 2311 reg->l |= K8_MSR_MCGCTL_NBE; 2312 } else { 2313 /* 2314 * Turn off NB MCE reporting only when it was off before 2315 */ 2316 if (!s->flags.nb_mce_enable) 2317 reg->l &= ~K8_MSR_MCGCTL_NBE; 2318 } 2319 } 2320 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); 2321 2322 free_cpumask_var(cmask); 2323 2324 return 0; 2325 } 2326 2327 static bool enable_ecc_error_reporting(struct ecc_settings *s, u8 nid, 2328 struct pci_dev *F3) 2329 { 2330 bool ret = true; 2331 u32 value, mask = K8_NBCTL_CECCEn | K8_NBCTL_UECCEn; 2332 2333 if (toggle_ecc_err_reporting(s, nid, ON)) { 2334 amd64_warn("Error enabling ECC reporting over MCGCTL!\n"); 2335 return false; 2336 } 2337 2338 amd64_read_pci_cfg(F3, K8_NBCTL, &value); 2339 2340 /* turn on UECCEn and CECCEn bits */ 2341 s->old_nbctl = value & mask; 2342 s->nbctl_valid = true; 2343 2344 value |= mask; 2345 pci_write_config_dword(F3, K8_NBCTL, value); 2346 2347 amd64_read_pci_cfg(F3, K8_NBCFG, &value); 2348 2349 debugf0("1: node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n", 2350 nid, value, 2351 !!(value & K8_NBCFG_CHIPKILL), !!(value & K8_NBCFG_ECC_ENABLE)); 2352 2353 if (!(value & K8_NBCFG_ECC_ENABLE)) { 2354 amd64_warn("DRAM ECC disabled on this node, enabling...\n"); 2355 2356 s->flags.nb_ecc_prev = 0; 2357 2358 /* Attempt to turn on DRAM ECC Enable */ 2359 value |= K8_NBCFG_ECC_ENABLE; 2360 pci_write_config_dword(F3, K8_NBCFG, value); 2361 2362 amd64_read_pci_cfg(F3, K8_NBCFG, &value); 2363 2364 if (!(value & K8_NBCFG_ECC_ENABLE)) { 2365 amd64_warn("Hardware rejected DRAM ECC enable," 2366 "check memory DIMM configuration.\n"); 2367 ret = false; 2368 } else { 2369 amd64_info("Hardware accepted DRAM ECC Enable\n"); 2370 } 2371 } else { 2372 s->flags.nb_ecc_prev = 1; 2373 } 2374 2375 debugf0("2: node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n", 2376 nid, value, 2377 !!(value & K8_NBCFG_CHIPKILL), !!(value & K8_NBCFG_ECC_ENABLE)); 2378 2379 return ret; 2380 } 2381 2382 static void restore_ecc_error_reporting(struct ecc_settings *s, u8 nid, 2383 struct pci_dev *F3) 2384 { 2385 u32 value, mask = K8_NBCTL_CECCEn | K8_NBCTL_UECCEn; 2386 2387 if (!s->nbctl_valid) 2388 return; 2389 2390 amd64_read_pci_cfg(F3, K8_NBCTL, &value); 2391 value &= ~mask; 2392 value |= s->old_nbctl; 2393 2394 pci_write_config_dword(F3, K8_NBCTL, value); 2395 2396 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */ 2397 if (!s->flags.nb_ecc_prev) { 2398 amd64_read_pci_cfg(F3, K8_NBCFG, &value); 2399 value &= ~K8_NBCFG_ECC_ENABLE; 2400 pci_write_config_dword(F3, K8_NBCFG, value); 2401 } 2402 2403 /* restore the NB Enable MCGCTL bit */ 2404 if (toggle_ecc_err_reporting(s, nid, OFF)) 2405 amd64_warn("Error restoring NB MCGCTL settings!\n"); 2406 } 2407 2408 /* 2409 * EDAC requires that the BIOS have ECC enabled before 2410 * taking over the processing of ECC errors. A command line 2411 * option allows to force-enable hardware ECC later in 2412 * enable_ecc_error_reporting(). 2413 */ 2414 static const char *ecc_msg = 2415 "ECC disabled in the BIOS or no ECC capability, module will not load.\n" 2416 " Either enable ECC checking or force module loading by setting " 2417 "'ecc_enable_override'.\n" 2418 " (Note that use of the override may cause unknown side effects.)\n"; 2419 2420 static bool ecc_enabled(struct pci_dev *F3, u8 nid) 2421 { 2422 u32 value; 2423 u8 ecc_en = 0; 2424 bool nb_mce_en = false; 2425 2426 amd64_read_pci_cfg(F3, K8_NBCFG, &value); 2427 2428 ecc_en = !!(value & K8_NBCFG_ECC_ENABLE); 2429 amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled")); 2430 2431 nb_mce_en = amd64_nb_mce_bank_enabled_on_node(nid); 2432 if (!nb_mce_en) 2433 amd64_notice("NB MCE bank disabled, set MSR " 2434 "0x%08x[4] on node %d to enable.\n", 2435 MSR_IA32_MCG_CTL, nid); 2436 2437 if (!ecc_en || !nb_mce_en) { 2438 amd64_notice("%s", ecc_msg); 2439 return false; 2440 } 2441 return true; 2442 } 2443 2444 struct mcidev_sysfs_attribute sysfs_attrs[ARRAY_SIZE(amd64_dbg_attrs) + 2445 ARRAY_SIZE(amd64_inj_attrs) + 2446 1]; 2447 2448 struct mcidev_sysfs_attribute terminator = { .attr = { .name = NULL } }; 2449 2450 static void set_mc_sysfs_attrs(struct mem_ctl_info *mci) 2451 { 2452 unsigned int i = 0, j = 0; 2453 2454 for (; i < ARRAY_SIZE(amd64_dbg_attrs); i++) 2455 sysfs_attrs[i] = amd64_dbg_attrs[i]; 2456 2457 if (boot_cpu_data.x86 >= 0x10) 2458 for (j = 0; j < ARRAY_SIZE(amd64_inj_attrs); j++, i++) 2459 sysfs_attrs[i] = amd64_inj_attrs[j]; 2460 2461 sysfs_attrs[i] = terminator; 2462 2463 mci->mc_driver_sysfs_attributes = sysfs_attrs; 2464 } 2465 2466 static void setup_mci_misc_attrs(struct mem_ctl_info *mci) 2467 { 2468 struct amd64_pvt *pvt = mci->pvt_info; 2469 2470 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2; 2471 mci->edac_ctl_cap = EDAC_FLAG_NONE; 2472 2473 if (pvt->nbcap & K8_NBCAP_SECDED) 2474 mci->edac_ctl_cap |= EDAC_FLAG_SECDED; 2475 2476 if (pvt->nbcap & K8_NBCAP_CHIPKILL) 2477 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED; 2478 2479 mci->edac_cap = amd64_determine_edac_cap(pvt); 2480 mci->mod_name = EDAC_MOD_STR; 2481 mci->mod_ver = EDAC_AMD64_VERSION; 2482 mci->ctl_name = pvt->ctl_name; 2483 mci->dev_name = pci_name(pvt->F2); 2484 mci->ctl_page_to_phys = NULL; 2485 2486 /* memory scrubber interface */ 2487 mci->set_sdram_scrub_rate = amd64_set_scrub_rate; 2488 mci->get_sdram_scrub_rate = amd64_get_scrub_rate; 2489 } 2490 2491 /* 2492 * returns a pointer to the family descriptor on success, NULL otherwise. 2493 */ 2494 static struct amd64_family_type *amd64_per_family_init(struct amd64_pvt *pvt) 2495 { 2496 u8 fam = boot_cpu_data.x86; 2497 struct amd64_family_type *fam_type = NULL; 2498 2499 switch (fam) { 2500 case 0xf: 2501 fam_type = &amd64_family_types[K8_CPUS]; 2502 pvt->ops = &amd64_family_types[K8_CPUS].ops; 2503 pvt->ctl_name = fam_type->ctl_name; 2504 pvt->min_scrubrate = K8_MIN_SCRUB_RATE_BITS; 2505 break; 2506 case 0x10: 2507 fam_type = &amd64_family_types[F10_CPUS]; 2508 pvt->ops = &amd64_family_types[F10_CPUS].ops; 2509 pvt->ctl_name = fam_type->ctl_name; 2510 pvt->min_scrubrate = F10_MIN_SCRUB_RATE_BITS; 2511 break; 2512 2513 default: 2514 amd64_err("Unsupported family!\n"); 2515 return NULL; 2516 } 2517 2518 pvt->ext_model = boot_cpu_data.x86_model >> 4; 2519 2520 amd64_info("%s %sdetected (node %d).\n", pvt->ctl_name, 2521 (fam == 0xf ? 2522 (pvt->ext_model >= K8_REV_F ? "revF or later " 2523 : "revE or earlier ") 2524 : ""), pvt->mc_node_id); 2525 return fam_type; 2526 } 2527 2528 static int amd64_init_one_instance(struct pci_dev *F2) 2529 { 2530 struct amd64_pvt *pvt = NULL; 2531 struct amd64_family_type *fam_type = NULL; 2532 struct mem_ctl_info *mci = NULL; 2533 int err = 0, ret; 2534 u8 nid = get_node_id(F2); 2535 2536 ret = -ENOMEM; 2537 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL); 2538 if (!pvt) 2539 goto err_ret; 2540 2541 pvt->mc_node_id = nid; 2542 pvt->F2 = F2; 2543 2544 ret = -EINVAL; 2545 fam_type = amd64_per_family_init(pvt); 2546 if (!fam_type) 2547 goto err_free; 2548 2549 ret = -ENODEV; 2550 err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id); 2551 if (err) 2552 goto err_free; 2553 2554 read_mc_regs(pvt); 2555 2556 /* 2557 * We need to determine how many memory channels there are. Then use 2558 * that information for calculating the size of the dynamic instance 2559 * tables in the 'mci' structure. 2560 */ 2561 ret = -EINVAL; 2562 pvt->channel_count = pvt->ops->early_channel_count(pvt); 2563 if (pvt->channel_count < 0) 2564 goto err_siblings; 2565 2566 ret = -ENOMEM; 2567 mci = edac_mc_alloc(0, pvt->cs_count, pvt->channel_count, nid); 2568 if (!mci) 2569 goto err_siblings; 2570 2571 mci->pvt_info = pvt; 2572 mci->dev = &pvt->F2->dev; 2573 2574 setup_mci_misc_attrs(mci); 2575 2576 if (init_csrows(mci)) 2577 mci->edac_cap = EDAC_FLAG_NONE; 2578 2579 set_mc_sysfs_attrs(mci); 2580 2581 ret = -ENODEV; 2582 if (edac_mc_add_mc(mci)) { 2583 debugf1("failed edac_mc_add_mc()\n"); 2584 goto err_add_mc; 2585 } 2586 2587 /* register stuff with EDAC MCE */ 2588 if (report_gart_errors) 2589 amd_report_gart_errors(true); 2590 2591 amd_register_ecc_decoder(amd64_decode_bus_error); 2592 2593 mcis[nid] = mci; 2594 2595 atomic_inc(&drv_instances); 2596 2597 return 0; 2598 2599 err_add_mc: 2600 edac_mc_free(mci); 2601 2602 err_siblings: 2603 free_mc_sibling_devs(pvt); 2604 2605 err_free: 2606 kfree(pvt); 2607 2608 err_ret: 2609 return ret; 2610 } 2611 2612 static int __devinit amd64_probe_one_instance(struct pci_dev *pdev, 2613 const struct pci_device_id *mc_type) 2614 { 2615 u8 nid = get_node_id(pdev); 2616 struct pci_dev *F3 = node_to_amd_nb(nid)->misc; 2617 struct ecc_settings *s; 2618 int ret = 0; 2619 2620 ret = pci_enable_device(pdev); 2621 if (ret < 0) { 2622 debugf0("ret=%d\n", ret); 2623 return -EIO; 2624 } 2625 2626 ret = -ENOMEM; 2627 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL); 2628 if (!s) 2629 goto err_out; 2630 2631 ecc_stngs[nid] = s; 2632 2633 if (!ecc_enabled(F3, nid)) { 2634 ret = -ENODEV; 2635 2636 if (!ecc_enable_override) 2637 goto err_enable; 2638 2639 amd64_warn("Forcing ECC on!\n"); 2640 2641 if (!enable_ecc_error_reporting(s, nid, F3)) 2642 goto err_enable; 2643 } 2644 2645 ret = amd64_init_one_instance(pdev); 2646 if (ret < 0) { 2647 amd64_err("Error probing instance: %d\n", nid); 2648 restore_ecc_error_reporting(s, nid, F3); 2649 } 2650 2651 return ret; 2652 2653 err_enable: 2654 kfree(s); 2655 ecc_stngs[nid] = NULL; 2656 2657 err_out: 2658 return ret; 2659 } 2660 2661 static void __devexit amd64_remove_one_instance(struct pci_dev *pdev) 2662 { 2663 struct mem_ctl_info *mci; 2664 struct amd64_pvt *pvt; 2665 u8 nid = get_node_id(pdev); 2666 struct pci_dev *F3 = node_to_amd_nb(nid)->misc; 2667 struct ecc_settings *s = ecc_stngs[nid]; 2668 2669 /* Remove from EDAC CORE tracking list */ 2670 mci = edac_mc_del_mc(&pdev->dev); 2671 if (!mci) 2672 return; 2673 2674 pvt = mci->pvt_info; 2675 2676 restore_ecc_error_reporting(s, nid, F3); 2677 2678 free_mc_sibling_devs(pvt); 2679 2680 /* unregister from EDAC MCE */ 2681 amd_report_gart_errors(false); 2682 amd_unregister_ecc_decoder(amd64_decode_bus_error); 2683 2684 kfree(ecc_stngs[nid]); 2685 ecc_stngs[nid] = NULL; 2686 2687 /* Free the EDAC CORE resources */ 2688 mci->pvt_info = NULL; 2689 mcis[nid] = NULL; 2690 2691 kfree(pvt); 2692 edac_mc_free(mci); 2693 } 2694 2695 /* 2696 * This table is part of the interface for loading drivers for PCI devices. The 2697 * PCI core identifies what devices are on a system during boot, and then 2698 * inquiry this table to see if this driver is for a given device found. 2699 */ 2700 static const struct pci_device_id amd64_pci_table[] __devinitdata = { 2701 { 2702 .vendor = PCI_VENDOR_ID_AMD, 2703 .device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL, 2704 .subvendor = PCI_ANY_ID, 2705 .subdevice = PCI_ANY_ID, 2706 .class = 0, 2707 .class_mask = 0, 2708 }, 2709 { 2710 .vendor = PCI_VENDOR_ID_AMD, 2711 .device = PCI_DEVICE_ID_AMD_10H_NB_DRAM, 2712 .subvendor = PCI_ANY_ID, 2713 .subdevice = PCI_ANY_ID, 2714 .class = 0, 2715 .class_mask = 0, 2716 }, 2717 {0, } 2718 }; 2719 MODULE_DEVICE_TABLE(pci, amd64_pci_table); 2720 2721 static struct pci_driver amd64_pci_driver = { 2722 .name = EDAC_MOD_STR, 2723 .probe = amd64_probe_one_instance, 2724 .remove = __devexit_p(amd64_remove_one_instance), 2725 .id_table = amd64_pci_table, 2726 }; 2727 2728 static void setup_pci_device(void) 2729 { 2730 struct mem_ctl_info *mci; 2731 struct amd64_pvt *pvt; 2732 2733 if (amd64_ctl_pci) 2734 return; 2735 2736 mci = mcis[0]; 2737 if (mci) { 2738 2739 pvt = mci->pvt_info; 2740 amd64_ctl_pci = 2741 edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR); 2742 2743 if (!amd64_ctl_pci) { 2744 pr_warning("%s(): Unable to create PCI control\n", 2745 __func__); 2746 2747 pr_warning("%s(): PCI error report via EDAC not set\n", 2748 __func__); 2749 } 2750 } 2751 } 2752 2753 static int __init amd64_edac_init(void) 2754 { 2755 int err = -ENODEV; 2756 2757 edac_printk(KERN_INFO, EDAC_MOD_STR, EDAC_AMD64_VERSION "\n"); 2758 2759 opstate_init(); 2760 2761 if (amd_cache_northbridges() < 0) 2762 goto err_ret; 2763 2764 err = -ENOMEM; 2765 mcis = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL); 2766 ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL); 2767 if (!(mcis && ecc_stngs)) 2768 goto err_ret; 2769 2770 msrs = msrs_alloc(); 2771 if (!msrs) 2772 goto err_free; 2773 2774 err = pci_register_driver(&amd64_pci_driver); 2775 if (err) 2776 goto err_pci; 2777 2778 err = -ENODEV; 2779 if (!atomic_read(&drv_instances)) 2780 goto err_no_instances; 2781 2782 setup_pci_device(); 2783 return 0; 2784 2785 err_no_instances: 2786 pci_unregister_driver(&amd64_pci_driver); 2787 2788 err_pci: 2789 msrs_free(msrs); 2790 msrs = NULL; 2791 2792 err_free: 2793 kfree(mcis); 2794 mcis = NULL; 2795 2796 kfree(ecc_stngs); 2797 ecc_stngs = NULL; 2798 2799 err_ret: 2800 return err; 2801 } 2802 2803 static void __exit amd64_edac_exit(void) 2804 { 2805 if (amd64_ctl_pci) 2806 edac_pci_release_generic_ctl(amd64_ctl_pci); 2807 2808 pci_unregister_driver(&amd64_pci_driver); 2809 2810 kfree(ecc_stngs); 2811 ecc_stngs = NULL; 2812 2813 kfree(mcis); 2814 mcis = NULL; 2815 2816 msrs_free(msrs); 2817 msrs = NULL; 2818 } 2819 2820 module_init(amd64_edac_init); 2821 module_exit(amd64_edac_exit); 2822 2823 MODULE_LICENSE("GPL"); 2824 MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, " 2825 "Dave Peterson, Thayne Harbaugh"); 2826 MODULE_DESCRIPTION("MC support for AMD64 memory controllers - " 2827 EDAC_AMD64_VERSION); 2828 2829 module_param(edac_op_state, int, 0444); 2830 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI"); 2831