1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Driver for Pondicherry2 memory controller. 4 * 5 * Copyright (c) 2016, Intel Corporation. 6 * 7 * [Derived from sb_edac.c] 8 * 9 * Translation of system physical addresses to DIMM addresses 10 * is a two stage process: 11 * 12 * First the Pondicherry 2 memory controller handles slice and channel interleaving 13 * in "sys2pmi()". This is (almost) completley common between platforms. 14 * 15 * Then a platform specific dunit (DIMM unit) completes the process to provide DIMM, 16 * rank, bank, row and column using the appropriate "dunit_ops" functions/parameters. 17 */ 18 19 #include <linux/module.h> 20 #include <linux/init.h> 21 #include <linux/pci.h> 22 #include <linux/pci_ids.h> 23 #include <linux/slab.h> 24 #include <linux/delay.h> 25 #include <linux/edac.h> 26 #include <linux/mmzone.h> 27 #include <linux/smp.h> 28 #include <linux/bitmap.h> 29 #include <linux/math64.h> 30 #include <linux/mod_devicetable.h> 31 #include <asm/cpu_device_id.h> 32 #include <asm/intel-family.h> 33 #include <asm/processor.h> 34 #include <asm/mce.h> 35 36 #include "edac_mc.h" 37 #include "edac_module.h" 38 #include "pnd2_edac.h" 39 40 #define EDAC_MOD_STR "pnd2_edac" 41 42 #define APL_NUM_CHANNELS 4 43 #define DNV_NUM_CHANNELS 2 44 #define DNV_MAX_DIMMS 2 /* Max DIMMs per channel */ 45 46 enum type { 47 APL, 48 DNV, /* All requests go to PMI CH0 on each slice (CH1 disabled) */ 49 }; 50 51 struct dram_addr { 52 int chan; 53 int dimm; 54 int rank; 55 int bank; 56 int row; 57 int col; 58 }; 59 60 struct pnd2_pvt { 61 int dimm_geom[APL_NUM_CHANNELS]; 62 u64 tolm, tohm; 63 }; 64 65 /* 66 * System address space is divided into multiple regions with 67 * different interleave rules in each. The as0/as1 regions 68 * have no interleaving at all. The as2 region is interleaved 69 * between two channels. The mot region is magic and may overlap 70 * other regions, with its interleave rules taking precedence. 71 * Addresses not in any of these regions are interleaved across 72 * all four channels. 73 */ 74 static struct region { 75 u64 base; 76 u64 limit; 77 u8 enabled; 78 } mot, as0, as1, as2; 79 80 static struct dunit_ops { 81 char *name; 82 enum type type; 83 int pmiaddr_shift; 84 int pmiidx_shift; 85 int channels; 86 int dimms_per_channel; 87 int (*rd_reg)(int port, int off, int op, void *data, size_t sz, char *name); 88 int (*get_registers)(void); 89 int (*check_ecc)(void); 90 void (*mk_region)(char *name, struct region *rp, void *asym); 91 void (*get_dimm_config)(struct mem_ctl_info *mci); 92 int (*pmi2mem)(struct mem_ctl_info *mci, u64 pmiaddr, u32 pmiidx, 93 struct dram_addr *daddr, char *msg); 94 } *ops; 95 96 static struct mem_ctl_info *pnd2_mci; 97 98 #define PND2_MSG_SIZE 256 99 100 /* Debug macros */ 101 #define pnd2_printk(level, fmt, arg...) \ 102 edac_printk(level, "pnd2", fmt, ##arg) 103 104 #define pnd2_mc_printk(mci, level, fmt, arg...) \ 105 edac_mc_chipset_printk(mci, level, "pnd2", fmt, ##arg) 106 107 #define MOT_CHAN_INTLV_BIT_1SLC_2CH 12 108 #define MOT_CHAN_INTLV_BIT_2SLC_2CH 13 109 #define SELECTOR_DISABLED (-1) 110 #define _4GB (1ul << 32) 111 112 #define PMI_ADDRESS_WIDTH 31 113 #define PND_MAX_PHYS_BIT 39 114 115 #define APL_ASYMSHIFT 28 116 #define DNV_ASYMSHIFT 31 117 #define CH_HASH_MASK_LSB 6 118 #define SLICE_HASH_MASK_LSB 6 119 #define MOT_SLC_INTLV_BIT 12 120 #define LOG2_PMI_ADDR_GRANULARITY 5 121 #define MOT_SHIFT 24 122 123 #define GET_BITFIELD(v, lo, hi) (((v) & GENMASK_ULL(hi, lo)) >> (lo)) 124 #define U64_LSHIFT(val, s) ((u64)(val) << (s)) 125 126 /* 127 * On Apollo Lake we access memory controller registers via a 128 * side-band mailbox style interface in a hidden PCI device 129 * configuration space. 130 */ 131 static struct pci_bus *p2sb_bus; 132 #define P2SB_DEVFN PCI_DEVFN(0xd, 0) 133 #define P2SB_ADDR_OFF 0xd0 134 #define P2SB_DATA_OFF 0xd4 135 #define P2SB_STAT_OFF 0xd8 136 #define P2SB_ROUT_OFF 0xda 137 #define P2SB_EADD_OFF 0xdc 138 #define P2SB_HIDE_OFF 0xe1 139 140 #define P2SB_BUSY 1 141 142 #define P2SB_READ(size, off, ptr) \ 143 pci_bus_read_config_##size(p2sb_bus, P2SB_DEVFN, off, ptr) 144 #define P2SB_WRITE(size, off, val) \ 145 pci_bus_write_config_##size(p2sb_bus, P2SB_DEVFN, off, val) 146 147 static bool p2sb_is_busy(u16 *status) 148 { 149 P2SB_READ(word, P2SB_STAT_OFF, status); 150 151 return !!(*status & P2SB_BUSY); 152 } 153 154 static int _apl_rd_reg(int port, int off, int op, u32 *data) 155 { 156 int retries = 0xff, ret; 157 u16 status; 158 u8 hidden; 159 160 /* Unhide the P2SB device, if it's hidden */ 161 P2SB_READ(byte, P2SB_HIDE_OFF, &hidden); 162 if (hidden) 163 P2SB_WRITE(byte, P2SB_HIDE_OFF, 0); 164 165 if (p2sb_is_busy(&status)) { 166 ret = -EAGAIN; 167 goto out; 168 } 169 170 P2SB_WRITE(dword, P2SB_ADDR_OFF, (port << 24) | off); 171 P2SB_WRITE(dword, P2SB_DATA_OFF, 0); 172 P2SB_WRITE(dword, P2SB_EADD_OFF, 0); 173 P2SB_WRITE(word, P2SB_ROUT_OFF, 0); 174 P2SB_WRITE(word, P2SB_STAT_OFF, (op << 8) | P2SB_BUSY); 175 176 while (p2sb_is_busy(&status)) { 177 if (retries-- == 0) { 178 ret = -EBUSY; 179 goto out; 180 } 181 } 182 183 P2SB_READ(dword, P2SB_DATA_OFF, data); 184 ret = (status >> 1) & 0x3; 185 out: 186 /* Hide the P2SB device, if it was hidden before */ 187 if (hidden) 188 P2SB_WRITE(byte, P2SB_HIDE_OFF, hidden); 189 190 return ret; 191 } 192 193 static int apl_rd_reg(int port, int off, int op, void *data, size_t sz, char *name) 194 { 195 int ret = 0; 196 197 edac_dbg(2, "Read %s port=%x off=%x op=%x\n", name, port, off, op); 198 switch (sz) { 199 case 8: 200 ret = _apl_rd_reg(port, off + 4, op, (u32 *)(data + 4)); 201 /* fall through */ 202 case 4: 203 ret |= _apl_rd_reg(port, off, op, (u32 *)data); 204 pnd2_printk(KERN_DEBUG, "%s=%x%08x ret=%d\n", name, 205 sz == 8 ? *((u32 *)(data + 4)) : 0, *((u32 *)data), ret); 206 break; 207 } 208 209 return ret; 210 } 211 212 static u64 get_mem_ctrl_hub_base_addr(void) 213 { 214 struct b_cr_mchbar_lo_pci lo; 215 struct b_cr_mchbar_hi_pci hi; 216 struct pci_dev *pdev; 217 218 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x1980, NULL); 219 if (pdev) { 220 pci_read_config_dword(pdev, 0x48, (u32 *)&lo); 221 pci_read_config_dword(pdev, 0x4c, (u32 *)&hi); 222 pci_dev_put(pdev); 223 } else { 224 return 0; 225 } 226 227 if (!lo.enable) { 228 edac_dbg(2, "MMIO via memory controller hub base address is disabled!\n"); 229 return 0; 230 } 231 232 return U64_LSHIFT(hi.base, 32) | U64_LSHIFT(lo.base, 15); 233 } 234 235 static u64 get_sideband_reg_base_addr(void) 236 { 237 struct pci_dev *pdev; 238 u32 hi, lo; 239 u8 hidden; 240 241 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x19dd, NULL); 242 if (pdev) { 243 /* Unhide the P2SB device, if it's hidden */ 244 pci_read_config_byte(pdev, 0xe1, &hidden); 245 if (hidden) 246 pci_write_config_byte(pdev, 0xe1, 0); 247 248 pci_read_config_dword(pdev, 0x10, &lo); 249 pci_read_config_dword(pdev, 0x14, &hi); 250 lo &= 0xfffffff0; 251 252 /* Hide the P2SB device, if it was hidden before */ 253 if (hidden) 254 pci_write_config_byte(pdev, 0xe1, hidden); 255 256 pci_dev_put(pdev); 257 return (U64_LSHIFT(hi, 32) | U64_LSHIFT(lo, 0)); 258 } else { 259 return 0xfd000000; 260 } 261 } 262 263 static int dnv_rd_reg(int port, int off, int op, void *data, size_t sz, char *name) 264 { 265 struct pci_dev *pdev; 266 char *base; 267 u64 addr; 268 269 if (op == 4) { 270 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x1980, NULL); 271 if (!pdev) 272 return -ENODEV; 273 274 pci_read_config_dword(pdev, off, data); 275 pci_dev_put(pdev); 276 } else { 277 /* MMIO via memory controller hub base address */ 278 if (op == 0 && port == 0x4c) { 279 addr = get_mem_ctrl_hub_base_addr(); 280 if (!addr) 281 return -ENODEV; 282 } else { 283 /* MMIO via sideband register base address */ 284 addr = get_sideband_reg_base_addr(); 285 if (!addr) 286 return -ENODEV; 287 addr += (port << 16); 288 } 289 290 base = ioremap((resource_size_t)addr, 0x10000); 291 if (!base) 292 return -ENODEV; 293 294 if (sz == 8) 295 *(u32 *)(data + 4) = *(u32 *)(base + off + 4); 296 *(u32 *)data = *(u32 *)(base + off); 297 298 iounmap(base); 299 } 300 301 edac_dbg(2, "Read %s=%.8x_%.8x\n", name, 302 (sz == 8) ? *(u32 *)(data + 4) : 0, *(u32 *)data); 303 304 return 0; 305 } 306 307 #define RD_REGP(regp, regname, port) \ 308 ops->rd_reg(port, \ 309 regname##_offset, \ 310 regname##_r_opcode, \ 311 regp, sizeof(struct regname), \ 312 #regname) 313 314 #define RD_REG(regp, regname) \ 315 ops->rd_reg(regname ## _port, \ 316 regname##_offset, \ 317 regname##_r_opcode, \ 318 regp, sizeof(struct regname), \ 319 #regname) 320 321 static u64 top_lm, top_hm; 322 static bool two_slices; 323 static bool two_channels; /* Both PMI channels in one slice enabled */ 324 325 static u8 sym_chan_mask; 326 static u8 asym_chan_mask; 327 static u8 chan_mask; 328 329 static int slice_selector = -1; 330 static int chan_selector = -1; 331 static u64 slice_hash_mask; 332 static u64 chan_hash_mask; 333 334 static void mk_region(char *name, struct region *rp, u64 base, u64 limit) 335 { 336 rp->enabled = 1; 337 rp->base = base; 338 rp->limit = limit; 339 edac_dbg(2, "Region:%s [%llx, %llx]\n", name, base, limit); 340 } 341 342 static void mk_region_mask(char *name, struct region *rp, u64 base, u64 mask) 343 { 344 if (mask == 0) { 345 pr_info(FW_BUG "MOT mask cannot be zero\n"); 346 return; 347 } 348 if (mask != GENMASK_ULL(PND_MAX_PHYS_BIT, __ffs(mask))) { 349 pr_info(FW_BUG "MOT mask not power of two\n"); 350 return; 351 } 352 if (base & ~mask) { 353 pr_info(FW_BUG "MOT region base/mask alignment error\n"); 354 return; 355 } 356 rp->base = base; 357 rp->limit = (base | ~mask) & GENMASK_ULL(PND_MAX_PHYS_BIT, 0); 358 rp->enabled = 1; 359 edac_dbg(2, "Region:%s [%llx, %llx]\n", name, base, rp->limit); 360 } 361 362 static bool in_region(struct region *rp, u64 addr) 363 { 364 if (!rp->enabled) 365 return false; 366 367 return rp->base <= addr && addr <= rp->limit; 368 } 369 370 static int gen_sym_mask(struct b_cr_slice_channel_hash *p) 371 { 372 int mask = 0; 373 374 if (!p->slice_0_mem_disabled) 375 mask |= p->sym_slice0_channel_enabled; 376 377 if (!p->slice_1_disabled) 378 mask |= p->sym_slice1_channel_enabled << 2; 379 380 if (p->ch_1_disabled || p->enable_pmi_dual_data_mode) 381 mask &= 0x5; 382 383 return mask; 384 } 385 386 static int gen_asym_mask(struct b_cr_slice_channel_hash *p, 387 struct b_cr_asym_mem_region0_mchbar *as0, 388 struct b_cr_asym_mem_region1_mchbar *as1, 389 struct b_cr_asym_2way_mem_region_mchbar *as2way) 390 { 391 const int intlv[] = { 0x5, 0xA, 0x3, 0xC }; 392 int mask = 0; 393 394 if (as2way->asym_2way_interleave_enable) 395 mask = intlv[as2way->asym_2way_intlv_mode]; 396 if (as0->slice0_asym_enable) 397 mask |= (1 << as0->slice0_asym_channel_select); 398 if (as1->slice1_asym_enable) 399 mask |= (4 << as1->slice1_asym_channel_select); 400 if (p->slice_0_mem_disabled) 401 mask &= 0xc; 402 if (p->slice_1_disabled) 403 mask &= 0x3; 404 if (p->ch_1_disabled || p->enable_pmi_dual_data_mode) 405 mask &= 0x5; 406 407 return mask; 408 } 409 410 static struct b_cr_tolud_pci tolud; 411 static struct b_cr_touud_lo_pci touud_lo; 412 static struct b_cr_touud_hi_pci touud_hi; 413 static struct b_cr_asym_mem_region0_mchbar asym0; 414 static struct b_cr_asym_mem_region1_mchbar asym1; 415 static struct b_cr_asym_2way_mem_region_mchbar asym_2way; 416 static struct b_cr_mot_out_base_mchbar mot_base; 417 static struct b_cr_mot_out_mask_mchbar mot_mask; 418 static struct b_cr_slice_channel_hash chash; 419 420 /* Apollo Lake dunit */ 421 /* 422 * Validated on board with just two DIMMs in the [0] and [2] positions 423 * in this array. Other port number matches documentation, but caution 424 * advised. 425 */ 426 static const int apl_dports[APL_NUM_CHANNELS] = { 0x18, 0x10, 0x11, 0x19 }; 427 static struct d_cr_drp0 drp0[APL_NUM_CHANNELS]; 428 429 /* Denverton dunit */ 430 static const int dnv_dports[DNV_NUM_CHANNELS] = { 0x10, 0x12 }; 431 static struct d_cr_dsch dsch; 432 static struct d_cr_ecc_ctrl ecc_ctrl[DNV_NUM_CHANNELS]; 433 static struct d_cr_drp drp[DNV_NUM_CHANNELS]; 434 static struct d_cr_dmap dmap[DNV_NUM_CHANNELS]; 435 static struct d_cr_dmap1 dmap1[DNV_NUM_CHANNELS]; 436 static struct d_cr_dmap2 dmap2[DNV_NUM_CHANNELS]; 437 static struct d_cr_dmap3 dmap3[DNV_NUM_CHANNELS]; 438 static struct d_cr_dmap4 dmap4[DNV_NUM_CHANNELS]; 439 static struct d_cr_dmap5 dmap5[DNV_NUM_CHANNELS]; 440 441 static void apl_mk_region(char *name, struct region *rp, void *asym) 442 { 443 struct b_cr_asym_mem_region0_mchbar *a = asym; 444 445 mk_region(name, rp, 446 U64_LSHIFT(a->slice0_asym_base, APL_ASYMSHIFT), 447 U64_LSHIFT(a->slice0_asym_limit, APL_ASYMSHIFT) + 448 GENMASK_ULL(APL_ASYMSHIFT - 1, 0)); 449 } 450 451 static void dnv_mk_region(char *name, struct region *rp, void *asym) 452 { 453 struct b_cr_asym_mem_region_denverton *a = asym; 454 455 mk_region(name, rp, 456 U64_LSHIFT(a->slice_asym_base, DNV_ASYMSHIFT), 457 U64_LSHIFT(a->slice_asym_limit, DNV_ASYMSHIFT) + 458 GENMASK_ULL(DNV_ASYMSHIFT - 1, 0)); 459 } 460 461 static int apl_get_registers(void) 462 { 463 int ret = -ENODEV; 464 int i; 465 466 if (RD_REG(&asym_2way, b_cr_asym_2way_mem_region_mchbar)) 467 return -ENODEV; 468 469 /* 470 * RD_REGP() will fail for unpopulated or non-existent 471 * DIMM slots. Return success if we find at least one DIMM. 472 */ 473 for (i = 0; i < APL_NUM_CHANNELS; i++) 474 if (!RD_REGP(&drp0[i], d_cr_drp0, apl_dports[i])) 475 ret = 0; 476 477 return ret; 478 } 479 480 static int dnv_get_registers(void) 481 { 482 int i; 483 484 if (RD_REG(&dsch, d_cr_dsch)) 485 return -ENODEV; 486 487 for (i = 0; i < DNV_NUM_CHANNELS; i++) 488 if (RD_REGP(&ecc_ctrl[i], d_cr_ecc_ctrl, dnv_dports[i]) || 489 RD_REGP(&drp[i], d_cr_drp, dnv_dports[i]) || 490 RD_REGP(&dmap[i], d_cr_dmap, dnv_dports[i]) || 491 RD_REGP(&dmap1[i], d_cr_dmap1, dnv_dports[i]) || 492 RD_REGP(&dmap2[i], d_cr_dmap2, dnv_dports[i]) || 493 RD_REGP(&dmap3[i], d_cr_dmap3, dnv_dports[i]) || 494 RD_REGP(&dmap4[i], d_cr_dmap4, dnv_dports[i]) || 495 RD_REGP(&dmap5[i], d_cr_dmap5, dnv_dports[i])) 496 return -ENODEV; 497 498 return 0; 499 } 500 501 /* 502 * Read all the h/w config registers once here (they don't 503 * change at run time. Figure out which address ranges have 504 * which interleave characteristics. 505 */ 506 static int get_registers(void) 507 { 508 const int intlv[] = { 10, 11, 12, 12 }; 509 510 if (RD_REG(&tolud, b_cr_tolud_pci) || 511 RD_REG(&touud_lo, b_cr_touud_lo_pci) || 512 RD_REG(&touud_hi, b_cr_touud_hi_pci) || 513 RD_REG(&asym0, b_cr_asym_mem_region0_mchbar) || 514 RD_REG(&asym1, b_cr_asym_mem_region1_mchbar) || 515 RD_REG(&mot_base, b_cr_mot_out_base_mchbar) || 516 RD_REG(&mot_mask, b_cr_mot_out_mask_mchbar) || 517 RD_REG(&chash, b_cr_slice_channel_hash)) 518 return -ENODEV; 519 520 if (ops->get_registers()) 521 return -ENODEV; 522 523 if (ops->type == DNV) { 524 /* PMI channel idx (always 0) for asymmetric region */ 525 asym0.slice0_asym_channel_select = 0; 526 asym1.slice1_asym_channel_select = 0; 527 /* PMI channel bitmap (always 1) for symmetric region */ 528 chash.sym_slice0_channel_enabled = 0x1; 529 chash.sym_slice1_channel_enabled = 0x1; 530 } 531 532 if (asym0.slice0_asym_enable) 533 ops->mk_region("as0", &as0, &asym0); 534 535 if (asym1.slice1_asym_enable) 536 ops->mk_region("as1", &as1, &asym1); 537 538 if (asym_2way.asym_2way_interleave_enable) { 539 mk_region("as2way", &as2, 540 U64_LSHIFT(asym_2way.asym_2way_base, APL_ASYMSHIFT), 541 U64_LSHIFT(asym_2way.asym_2way_limit, APL_ASYMSHIFT) + 542 GENMASK_ULL(APL_ASYMSHIFT - 1, 0)); 543 } 544 545 if (mot_base.imr_en) { 546 mk_region_mask("mot", &mot, 547 U64_LSHIFT(mot_base.mot_out_base, MOT_SHIFT), 548 U64_LSHIFT(mot_mask.mot_out_mask, MOT_SHIFT)); 549 } 550 551 top_lm = U64_LSHIFT(tolud.tolud, 20); 552 top_hm = U64_LSHIFT(touud_hi.touud, 32) | U64_LSHIFT(touud_lo.touud, 20); 553 554 two_slices = !chash.slice_1_disabled && 555 !chash.slice_0_mem_disabled && 556 (chash.sym_slice0_channel_enabled != 0) && 557 (chash.sym_slice1_channel_enabled != 0); 558 two_channels = !chash.ch_1_disabled && 559 !chash.enable_pmi_dual_data_mode && 560 ((chash.sym_slice0_channel_enabled == 3) || 561 (chash.sym_slice1_channel_enabled == 3)); 562 563 sym_chan_mask = gen_sym_mask(&chash); 564 asym_chan_mask = gen_asym_mask(&chash, &asym0, &asym1, &asym_2way); 565 chan_mask = sym_chan_mask | asym_chan_mask; 566 567 if (two_slices && !two_channels) { 568 if (chash.hvm_mode) 569 slice_selector = 29; 570 else 571 slice_selector = intlv[chash.interleave_mode]; 572 } else if (!two_slices && two_channels) { 573 if (chash.hvm_mode) 574 chan_selector = 29; 575 else 576 chan_selector = intlv[chash.interleave_mode]; 577 } else if (two_slices && two_channels) { 578 if (chash.hvm_mode) { 579 slice_selector = 29; 580 chan_selector = 30; 581 } else { 582 slice_selector = intlv[chash.interleave_mode]; 583 chan_selector = intlv[chash.interleave_mode] + 1; 584 } 585 } 586 587 if (two_slices) { 588 if (!chash.hvm_mode) 589 slice_hash_mask = chash.slice_hash_mask << SLICE_HASH_MASK_LSB; 590 if (!two_channels) 591 slice_hash_mask |= BIT_ULL(slice_selector); 592 } 593 594 if (two_channels) { 595 if (!chash.hvm_mode) 596 chan_hash_mask = chash.ch_hash_mask << CH_HASH_MASK_LSB; 597 if (!two_slices) 598 chan_hash_mask |= BIT_ULL(chan_selector); 599 } 600 601 return 0; 602 } 603 604 /* Get a contiguous memory address (remove the MMIO gap) */ 605 static u64 remove_mmio_gap(u64 sys) 606 { 607 return (sys < _4GB) ? sys : sys - (_4GB - top_lm); 608 } 609 610 /* Squeeze out one address bit, shift upper part down to fill gap */ 611 static void remove_addr_bit(u64 *addr, int bitidx) 612 { 613 u64 mask; 614 615 if (bitidx == -1) 616 return; 617 618 mask = (1ull << bitidx) - 1; 619 *addr = ((*addr >> 1) & ~mask) | (*addr & mask); 620 } 621 622 /* XOR all the bits from addr specified in mask */ 623 static int hash_by_mask(u64 addr, u64 mask) 624 { 625 u64 result = addr & mask; 626 627 result = (result >> 32) ^ result; 628 result = (result >> 16) ^ result; 629 result = (result >> 8) ^ result; 630 result = (result >> 4) ^ result; 631 result = (result >> 2) ^ result; 632 result = (result >> 1) ^ result; 633 634 return (int)result & 1; 635 } 636 637 /* 638 * First stage decode. Take the system address and figure out which 639 * second stage will deal with it based on interleave modes. 640 */ 641 static int sys2pmi(const u64 addr, u32 *pmiidx, u64 *pmiaddr, char *msg) 642 { 643 u64 contig_addr, contig_base, contig_offset, contig_base_adj; 644 int mot_intlv_bit = two_slices ? MOT_CHAN_INTLV_BIT_2SLC_2CH : 645 MOT_CHAN_INTLV_BIT_1SLC_2CH; 646 int slice_intlv_bit_rm = SELECTOR_DISABLED; 647 int chan_intlv_bit_rm = SELECTOR_DISABLED; 648 /* Determine if address is in the MOT region. */ 649 bool mot_hit = in_region(&mot, addr); 650 /* Calculate the number of symmetric regions enabled. */ 651 int sym_channels = hweight8(sym_chan_mask); 652 653 /* 654 * The amount we need to shift the asym base can be determined by the 655 * number of enabled symmetric channels. 656 * NOTE: This can only work because symmetric memory is not supposed 657 * to do a 3-way interleave. 658 */ 659 int sym_chan_shift = sym_channels >> 1; 660 661 /* Give up if address is out of range, or in MMIO gap */ 662 if (addr >= (1ul << PND_MAX_PHYS_BIT) || 663 (addr >= top_lm && addr < _4GB) || addr >= top_hm) { 664 snprintf(msg, PND2_MSG_SIZE, "Error address 0x%llx is not DRAM", addr); 665 return -EINVAL; 666 } 667 668 /* Get a contiguous memory address (remove the MMIO gap) */ 669 contig_addr = remove_mmio_gap(addr); 670 671 if (in_region(&as0, addr)) { 672 *pmiidx = asym0.slice0_asym_channel_select; 673 674 contig_base = remove_mmio_gap(as0.base); 675 contig_offset = contig_addr - contig_base; 676 contig_base_adj = (contig_base >> sym_chan_shift) * 677 ((chash.sym_slice0_channel_enabled >> (*pmiidx & 1)) & 1); 678 contig_addr = contig_offset + ((sym_channels > 0) ? contig_base_adj : 0ull); 679 } else if (in_region(&as1, addr)) { 680 *pmiidx = 2u + asym1.slice1_asym_channel_select; 681 682 contig_base = remove_mmio_gap(as1.base); 683 contig_offset = contig_addr - contig_base; 684 contig_base_adj = (contig_base >> sym_chan_shift) * 685 ((chash.sym_slice1_channel_enabled >> (*pmiidx & 1)) & 1); 686 contig_addr = contig_offset + ((sym_channels > 0) ? contig_base_adj : 0ull); 687 } else if (in_region(&as2, addr) && (asym_2way.asym_2way_intlv_mode == 0x3ul)) { 688 bool channel1; 689 690 mot_intlv_bit = MOT_CHAN_INTLV_BIT_1SLC_2CH; 691 *pmiidx = (asym_2way.asym_2way_intlv_mode & 1) << 1; 692 channel1 = mot_hit ? ((bool)((addr >> mot_intlv_bit) & 1)) : 693 hash_by_mask(contig_addr, chan_hash_mask); 694 *pmiidx |= (u32)channel1; 695 696 contig_base = remove_mmio_gap(as2.base); 697 chan_intlv_bit_rm = mot_hit ? mot_intlv_bit : chan_selector; 698 contig_offset = contig_addr - contig_base; 699 remove_addr_bit(&contig_offset, chan_intlv_bit_rm); 700 contig_addr = (contig_base >> sym_chan_shift) + contig_offset; 701 } else { 702 /* Otherwise we're in normal, boring symmetric mode. */ 703 *pmiidx = 0u; 704 705 if (two_slices) { 706 bool slice1; 707 708 if (mot_hit) { 709 slice_intlv_bit_rm = MOT_SLC_INTLV_BIT; 710 slice1 = (addr >> MOT_SLC_INTLV_BIT) & 1; 711 } else { 712 slice_intlv_bit_rm = slice_selector; 713 slice1 = hash_by_mask(addr, slice_hash_mask); 714 } 715 716 *pmiidx = (u32)slice1 << 1; 717 } 718 719 if (two_channels) { 720 bool channel1; 721 722 mot_intlv_bit = two_slices ? MOT_CHAN_INTLV_BIT_2SLC_2CH : 723 MOT_CHAN_INTLV_BIT_1SLC_2CH; 724 725 if (mot_hit) { 726 chan_intlv_bit_rm = mot_intlv_bit; 727 channel1 = (addr >> mot_intlv_bit) & 1; 728 } else { 729 chan_intlv_bit_rm = chan_selector; 730 channel1 = hash_by_mask(contig_addr, chan_hash_mask); 731 } 732 733 *pmiidx |= (u32)channel1; 734 } 735 } 736 737 /* Remove the chan_selector bit first */ 738 remove_addr_bit(&contig_addr, chan_intlv_bit_rm); 739 /* Remove the slice bit (we remove it second because it must be lower */ 740 remove_addr_bit(&contig_addr, slice_intlv_bit_rm); 741 *pmiaddr = contig_addr; 742 743 return 0; 744 } 745 746 /* Translate PMI address to memory (rank, row, bank, column) */ 747 #define C(n) (0x10 | (n)) /* column */ 748 #define B(n) (0x20 | (n)) /* bank */ 749 #define R(n) (0x40 | (n)) /* row */ 750 #define RS (0x80) /* rank */ 751 752 /* addrdec values */ 753 #define AMAP_1KB 0 754 #define AMAP_2KB 1 755 #define AMAP_4KB 2 756 #define AMAP_RSVD 3 757 758 /* dden values */ 759 #define DEN_4Gb 0 760 #define DEN_8Gb 2 761 762 /* dwid values */ 763 #define X8 0 764 #define X16 1 765 766 static struct dimm_geometry { 767 u8 addrdec; 768 u8 dden; 769 u8 dwid; 770 u8 rowbits, colbits; 771 u16 bits[PMI_ADDRESS_WIDTH]; 772 } dimms[] = { 773 { 774 .addrdec = AMAP_1KB, .dden = DEN_4Gb, .dwid = X16, 775 .rowbits = 15, .colbits = 10, 776 .bits = { 777 C(2), C(3), C(4), C(5), C(6), B(0), B(1), B(2), R(0), 778 R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), R(9), 779 R(10), C(7), C(8), C(9), R(11), RS, R(12), R(13), R(14), 780 0, 0, 0, 0 781 } 782 }, 783 { 784 .addrdec = AMAP_1KB, .dden = DEN_4Gb, .dwid = X8, 785 .rowbits = 16, .colbits = 10, 786 .bits = { 787 C(2), C(3), C(4), C(5), C(6), B(0), B(1), B(2), R(0), 788 R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), R(9), 789 R(10), C(7), C(8), C(9), R(11), RS, R(12), R(13), R(14), 790 R(15), 0, 0, 0 791 } 792 }, 793 { 794 .addrdec = AMAP_1KB, .dden = DEN_8Gb, .dwid = X16, 795 .rowbits = 16, .colbits = 10, 796 .bits = { 797 C(2), C(3), C(4), C(5), C(6), B(0), B(1), B(2), R(0), 798 R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), R(9), 799 R(10), C(7), C(8), C(9), R(11), RS, R(12), R(13), R(14), 800 R(15), 0, 0, 0 801 } 802 }, 803 { 804 .addrdec = AMAP_1KB, .dden = DEN_8Gb, .dwid = X8, 805 .rowbits = 16, .colbits = 11, 806 .bits = { 807 C(2), C(3), C(4), C(5), C(6), B(0), B(1), B(2), R(0), 808 R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), R(9), 809 R(10), C(7), C(8), C(9), R(11), RS, C(11), R(12), R(13), 810 R(14), R(15), 0, 0 811 } 812 }, 813 { 814 .addrdec = AMAP_2KB, .dden = DEN_4Gb, .dwid = X16, 815 .rowbits = 15, .colbits = 10, 816 .bits = { 817 C(2), C(3), C(4), C(5), C(6), C(7), B(0), B(1), B(2), 818 R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), 819 R(9), R(10), C(8), C(9), R(11), RS, R(12), R(13), R(14), 820 0, 0, 0, 0 821 } 822 }, 823 { 824 .addrdec = AMAP_2KB, .dden = DEN_4Gb, .dwid = X8, 825 .rowbits = 16, .colbits = 10, 826 .bits = { 827 C(2), C(3), C(4), C(5), C(6), C(7), B(0), B(1), B(2), 828 R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), 829 R(9), R(10), C(8), C(9), R(11), RS, R(12), R(13), R(14), 830 R(15), 0, 0, 0 831 } 832 }, 833 { 834 .addrdec = AMAP_2KB, .dden = DEN_8Gb, .dwid = X16, 835 .rowbits = 16, .colbits = 10, 836 .bits = { 837 C(2), C(3), C(4), C(5), C(6), C(7), B(0), B(1), B(2), 838 R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), 839 R(9), R(10), C(8), C(9), R(11), RS, R(12), R(13), R(14), 840 R(15), 0, 0, 0 841 } 842 }, 843 { 844 .addrdec = AMAP_2KB, .dden = DEN_8Gb, .dwid = X8, 845 .rowbits = 16, .colbits = 11, 846 .bits = { 847 C(2), C(3), C(4), C(5), C(6), C(7), B(0), B(1), B(2), 848 R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), 849 R(9), R(10), C(8), C(9), R(11), RS, C(11), R(12), R(13), 850 R(14), R(15), 0, 0 851 } 852 }, 853 { 854 .addrdec = AMAP_4KB, .dden = DEN_4Gb, .dwid = X16, 855 .rowbits = 15, .colbits = 10, 856 .bits = { 857 C(2), C(3), C(4), C(5), C(6), C(7), C(8), B(0), B(1), 858 B(2), R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), 859 R(8), R(9), R(10), C(9), R(11), RS, R(12), R(13), R(14), 860 0, 0, 0, 0 861 } 862 }, 863 { 864 .addrdec = AMAP_4KB, .dden = DEN_4Gb, .dwid = X8, 865 .rowbits = 16, .colbits = 10, 866 .bits = { 867 C(2), C(3), C(4), C(5), C(6), C(7), C(8), B(0), B(1), 868 B(2), R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), 869 R(8), R(9), R(10), C(9), R(11), RS, R(12), R(13), R(14), 870 R(15), 0, 0, 0 871 } 872 }, 873 { 874 .addrdec = AMAP_4KB, .dden = DEN_8Gb, .dwid = X16, 875 .rowbits = 16, .colbits = 10, 876 .bits = { 877 C(2), C(3), C(4), C(5), C(6), C(7), C(8), B(0), B(1), 878 B(2), R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), 879 R(8), R(9), R(10), C(9), R(11), RS, R(12), R(13), R(14), 880 R(15), 0, 0, 0 881 } 882 }, 883 { 884 .addrdec = AMAP_4KB, .dden = DEN_8Gb, .dwid = X8, 885 .rowbits = 16, .colbits = 11, 886 .bits = { 887 C(2), C(3), C(4), C(5), C(6), C(7), C(8), B(0), B(1), 888 B(2), R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), 889 R(8), R(9), R(10), C(9), R(11), RS, C(11), R(12), R(13), 890 R(14), R(15), 0, 0 891 } 892 } 893 }; 894 895 static int bank_hash(u64 pmiaddr, int idx, int shft) 896 { 897 int bhash = 0; 898 899 switch (idx) { 900 case 0: 901 bhash ^= ((pmiaddr >> (12 + shft)) ^ (pmiaddr >> (9 + shft))) & 1; 902 break; 903 case 1: 904 bhash ^= (((pmiaddr >> (10 + shft)) ^ (pmiaddr >> (8 + shft))) & 1) << 1; 905 bhash ^= ((pmiaddr >> 22) & 1) << 1; 906 break; 907 case 2: 908 bhash ^= (((pmiaddr >> (13 + shft)) ^ (pmiaddr >> (11 + shft))) & 1) << 2; 909 break; 910 } 911 912 return bhash; 913 } 914 915 static int rank_hash(u64 pmiaddr) 916 { 917 return ((pmiaddr >> 16) ^ (pmiaddr >> 10)) & 1; 918 } 919 920 /* Second stage decode. Compute rank, bank, row & column. */ 921 static int apl_pmi2mem(struct mem_ctl_info *mci, u64 pmiaddr, u32 pmiidx, 922 struct dram_addr *daddr, char *msg) 923 { 924 struct d_cr_drp0 *cr_drp0 = &drp0[pmiidx]; 925 struct pnd2_pvt *pvt = mci->pvt_info; 926 int g = pvt->dimm_geom[pmiidx]; 927 struct dimm_geometry *d = &dimms[g]; 928 int column = 0, bank = 0, row = 0, rank = 0; 929 int i, idx, type, skiprs = 0; 930 931 for (i = 0; i < PMI_ADDRESS_WIDTH; i++) { 932 int bit = (pmiaddr >> i) & 1; 933 934 if (i + skiprs >= PMI_ADDRESS_WIDTH) { 935 snprintf(msg, PND2_MSG_SIZE, "Bad dimm_geometry[] table\n"); 936 return -EINVAL; 937 } 938 939 type = d->bits[i + skiprs] & ~0xf; 940 idx = d->bits[i + skiprs] & 0xf; 941 942 /* 943 * On single rank DIMMs ignore the rank select bit 944 * and shift remainder of "bits[]" down one place. 945 */ 946 if (type == RS && (cr_drp0->rken0 + cr_drp0->rken1) == 1) { 947 skiprs = 1; 948 type = d->bits[i + skiprs] & ~0xf; 949 idx = d->bits[i + skiprs] & 0xf; 950 } 951 952 switch (type) { 953 case C(0): 954 column |= (bit << idx); 955 break; 956 case B(0): 957 bank |= (bit << idx); 958 if (cr_drp0->bahen) 959 bank ^= bank_hash(pmiaddr, idx, d->addrdec); 960 break; 961 case R(0): 962 row |= (bit << idx); 963 break; 964 case RS: 965 rank = bit; 966 if (cr_drp0->rsien) 967 rank ^= rank_hash(pmiaddr); 968 break; 969 default: 970 if (bit) { 971 snprintf(msg, PND2_MSG_SIZE, "Bad translation\n"); 972 return -EINVAL; 973 } 974 goto done; 975 } 976 } 977 978 done: 979 daddr->col = column; 980 daddr->bank = bank; 981 daddr->row = row; 982 daddr->rank = rank; 983 daddr->dimm = 0; 984 985 return 0; 986 } 987 988 /* Pluck bit "in" from pmiaddr and return value shifted to bit "out" */ 989 #define dnv_get_bit(pmi, in, out) ((int)(((pmi) >> (in)) & 1u) << (out)) 990 991 static int dnv_pmi2mem(struct mem_ctl_info *mci, u64 pmiaddr, u32 pmiidx, 992 struct dram_addr *daddr, char *msg) 993 { 994 /* Rank 0 or 1 */ 995 daddr->rank = dnv_get_bit(pmiaddr, dmap[pmiidx].rs0 + 13, 0); 996 /* Rank 2 or 3 */ 997 daddr->rank |= dnv_get_bit(pmiaddr, dmap[pmiidx].rs1 + 13, 1); 998 999 /* 1000 * Normally ranks 0,1 are DIMM0, and 2,3 are DIMM1, but we 1001 * flip them if DIMM1 is larger than DIMM0. 1002 */ 1003 daddr->dimm = (daddr->rank >= 2) ^ drp[pmiidx].dimmflip; 1004 1005 daddr->bank = dnv_get_bit(pmiaddr, dmap[pmiidx].ba0 + 6, 0); 1006 daddr->bank |= dnv_get_bit(pmiaddr, dmap[pmiidx].ba1 + 6, 1); 1007 daddr->bank |= dnv_get_bit(pmiaddr, dmap[pmiidx].bg0 + 6, 2); 1008 if (dsch.ddr4en) 1009 daddr->bank |= dnv_get_bit(pmiaddr, dmap[pmiidx].bg1 + 6, 3); 1010 if (dmap1[pmiidx].bxor) { 1011 if (dsch.ddr4en) { 1012 daddr->bank ^= dnv_get_bit(pmiaddr, dmap3[pmiidx].row6 + 6, 0); 1013 daddr->bank ^= dnv_get_bit(pmiaddr, dmap3[pmiidx].row7 + 6, 1); 1014 if (dsch.chan_width == 0) 1015 /* 64/72 bit dram channel width */ 1016 daddr->bank ^= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca3 + 6, 2); 1017 else 1018 /* 32/40 bit dram channel width */ 1019 daddr->bank ^= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca4 + 6, 2); 1020 daddr->bank ^= dnv_get_bit(pmiaddr, dmap2[pmiidx].row2 + 6, 3); 1021 } else { 1022 daddr->bank ^= dnv_get_bit(pmiaddr, dmap2[pmiidx].row2 + 6, 0); 1023 daddr->bank ^= dnv_get_bit(pmiaddr, dmap3[pmiidx].row6 + 6, 1); 1024 if (dsch.chan_width == 0) 1025 daddr->bank ^= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca3 + 6, 2); 1026 else 1027 daddr->bank ^= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca4 + 6, 2); 1028 } 1029 } 1030 1031 daddr->row = dnv_get_bit(pmiaddr, dmap2[pmiidx].row0 + 6, 0); 1032 daddr->row |= dnv_get_bit(pmiaddr, dmap2[pmiidx].row1 + 6, 1); 1033 daddr->row |= dnv_get_bit(pmiaddr, dmap2[pmiidx].row2 + 6, 2); 1034 daddr->row |= dnv_get_bit(pmiaddr, dmap2[pmiidx].row3 + 6, 3); 1035 daddr->row |= dnv_get_bit(pmiaddr, dmap2[pmiidx].row4 + 6, 4); 1036 daddr->row |= dnv_get_bit(pmiaddr, dmap2[pmiidx].row5 + 6, 5); 1037 daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row6 + 6, 6); 1038 daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row7 + 6, 7); 1039 daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row8 + 6, 8); 1040 daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row9 + 6, 9); 1041 daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row10 + 6, 10); 1042 daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row11 + 6, 11); 1043 daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row12 + 6, 12); 1044 daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row13 + 6, 13); 1045 if (dmap4[pmiidx].row14 != 31) 1046 daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row14 + 6, 14); 1047 if (dmap4[pmiidx].row15 != 31) 1048 daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row15 + 6, 15); 1049 if (dmap4[pmiidx].row16 != 31) 1050 daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row16 + 6, 16); 1051 if (dmap4[pmiidx].row17 != 31) 1052 daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row17 + 6, 17); 1053 1054 daddr->col = dnv_get_bit(pmiaddr, dmap5[pmiidx].ca3 + 6, 3); 1055 daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca4 + 6, 4); 1056 daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca5 + 6, 5); 1057 daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca6 + 6, 6); 1058 daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca7 + 6, 7); 1059 daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca8 + 6, 8); 1060 daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca9 + 6, 9); 1061 if (!dsch.ddr4en && dmap1[pmiidx].ca11 != 0x3f) 1062 daddr->col |= dnv_get_bit(pmiaddr, dmap1[pmiidx].ca11 + 13, 11); 1063 1064 return 0; 1065 } 1066 1067 static int check_channel(int ch) 1068 { 1069 if (drp0[ch].dramtype != 0) { 1070 pnd2_printk(KERN_INFO, "Unsupported DIMM in channel %d\n", ch); 1071 return 1; 1072 } else if (drp0[ch].eccen == 0) { 1073 pnd2_printk(KERN_INFO, "ECC disabled on channel %d\n", ch); 1074 return 1; 1075 } 1076 return 0; 1077 } 1078 1079 static int apl_check_ecc_active(void) 1080 { 1081 int i, ret = 0; 1082 1083 /* Check dramtype and ECC mode for each present DIMM */ 1084 for (i = 0; i < APL_NUM_CHANNELS; i++) 1085 if (chan_mask & BIT(i)) 1086 ret += check_channel(i); 1087 return ret ? -EINVAL : 0; 1088 } 1089 1090 #define DIMMS_PRESENT(d) ((d)->rken0 + (d)->rken1 + (d)->rken2 + (d)->rken3) 1091 1092 static int check_unit(int ch) 1093 { 1094 struct d_cr_drp *d = &drp[ch]; 1095 1096 if (DIMMS_PRESENT(d) && !ecc_ctrl[ch].eccen) { 1097 pnd2_printk(KERN_INFO, "ECC disabled on channel %d\n", ch); 1098 return 1; 1099 } 1100 return 0; 1101 } 1102 1103 static int dnv_check_ecc_active(void) 1104 { 1105 int i, ret = 0; 1106 1107 for (i = 0; i < DNV_NUM_CHANNELS; i++) 1108 ret += check_unit(i); 1109 return ret ? -EINVAL : 0; 1110 } 1111 1112 static int get_memory_error_data(struct mem_ctl_info *mci, u64 addr, 1113 struct dram_addr *daddr, char *msg) 1114 { 1115 u64 pmiaddr; 1116 u32 pmiidx; 1117 int ret; 1118 1119 ret = sys2pmi(addr, &pmiidx, &pmiaddr, msg); 1120 if (ret) 1121 return ret; 1122 1123 pmiaddr >>= ops->pmiaddr_shift; 1124 /* pmi channel idx to dimm channel idx */ 1125 pmiidx >>= ops->pmiidx_shift; 1126 daddr->chan = pmiidx; 1127 1128 ret = ops->pmi2mem(mci, pmiaddr, pmiidx, daddr, msg); 1129 if (ret) 1130 return ret; 1131 1132 edac_dbg(0, "SysAddr=%llx PmiAddr=%llx Channel=%d DIMM=%d Rank=%d Bank=%d Row=%d Column=%d\n", 1133 addr, pmiaddr, daddr->chan, daddr->dimm, daddr->rank, daddr->bank, daddr->row, daddr->col); 1134 1135 return 0; 1136 } 1137 1138 static void pnd2_mce_output_error(struct mem_ctl_info *mci, const struct mce *m, 1139 struct dram_addr *daddr) 1140 { 1141 enum hw_event_mc_err_type tp_event; 1142 char *optype, msg[PND2_MSG_SIZE]; 1143 bool ripv = m->mcgstatus & MCG_STATUS_RIPV; 1144 bool overflow = m->status & MCI_STATUS_OVER; 1145 bool uc_err = m->status & MCI_STATUS_UC; 1146 bool recov = m->status & MCI_STATUS_S; 1147 u32 core_err_cnt = GET_BITFIELD(m->status, 38, 52); 1148 u32 mscod = GET_BITFIELD(m->status, 16, 31); 1149 u32 errcode = GET_BITFIELD(m->status, 0, 15); 1150 u32 optypenum = GET_BITFIELD(m->status, 4, 6); 1151 int rc; 1152 1153 tp_event = uc_err ? (ripv ? HW_EVENT_ERR_FATAL : HW_EVENT_ERR_UNCORRECTED) : 1154 HW_EVENT_ERR_CORRECTED; 1155 1156 /* 1157 * According with Table 15-9 of the Intel Architecture spec vol 3A, 1158 * memory errors should fit in this mask: 1159 * 000f 0000 1mmm cccc (binary) 1160 * where: 1161 * f = Correction Report Filtering Bit. If 1, subsequent errors 1162 * won't be shown 1163 * mmm = error type 1164 * cccc = channel 1165 * If the mask doesn't match, report an error to the parsing logic 1166 */ 1167 if (!((errcode & 0xef80) == 0x80)) { 1168 optype = "Can't parse: it is not a mem"; 1169 } else { 1170 switch (optypenum) { 1171 case 0: 1172 optype = "generic undef request error"; 1173 break; 1174 case 1: 1175 optype = "memory read error"; 1176 break; 1177 case 2: 1178 optype = "memory write error"; 1179 break; 1180 case 3: 1181 optype = "addr/cmd error"; 1182 break; 1183 case 4: 1184 optype = "memory scrubbing error"; 1185 break; 1186 default: 1187 optype = "reserved"; 1188 break; 1189 } 1190 } 1191 1192 /* Only decode errors with an valid address (ADDRV) */ 1193 if (!(m->status & MCI_STATUS_ADDRV)) 1194 return; 1195 1196 rc = get_memory_error_data(mci, m->addr, daddr, msg); 1197 if (rc) 1198 goto address_error; 1199 1200 snprintf(msg, sizeof(msg), 1201 "%s%s err_code:%04x:%04x channel:%d DIMM:%d rank:%d row:%d bank:%d col:%d", 1202 overflow ? " OVERFLOW" : "", (uc_err && recov) ? " recoverable" : "", mscod, 1203 errcode, daddr->chan, daddr->dimm, daddr->rank, daddr->row, daddr->bank, daddr->col); 1204 1205 edac_dbg(0, "%s\n", msg); 1206 1207 /* Call the helper to output message */ 1208 edac_mc_handle_error(tp_event, mci, core_err_cnt, m->addr >> PAGE_SHIFT, 1209 m->addr & ~PAGE_MASK, 0, daddr->chan, daddr->dimm, -1, optype, msg); 1210 1211 return; 1212 1213 address_error: 1214 edac_mc_handle_error(tp_event, mci, core_err_cnt, 0, 0, 0, -1, -1, -1, msg, ""); 1215 } 1216 1217 static void apl_get_dimm_config(struct mem_ctl_info *mci) 1218 { 1219 struct pnd2_pvt *pvt = mci->pvt_info; 1220 struct dimm_info *dimm; 1221 struct d_cr_drp0 *d; 1222 u64 capacity; 1223 int i, g; 1224 1225 for (i = 0; i < APL_NUM_CHANNELS; i++) { 1226 if (!(chan_mask & BIT(i))) 1227 continue; 1228 1229 dimm = EDAC_DIMM_PTR(mci->layers, mci->dimms, mci->n_layers, i, 0, 0); 1230 if (!dimm) { 1231 edac_dbg(0, "No allocated DIMM for channel %d\n", i); 1232 continue; 1233 } 1234 1235 d = &drp0[i]; 1236 for (g = 0; g < ARRAY_SIZE(dimms); g++) 1237 if (dimms[g].addrdec == d->addrdec && 1238 dimms[g].dden == d->dden && 1239 dimms[g].dwid == d->dwid) 1240 break; 1241 1242 if (g == ARRAY_SIZE(dimms)) { 1243 edac_dbg(0, "Channel %d: unrecognized DIMM\n", i); 1244 continue; 1245 } 1246 1247 pvt->dimm_geom[i] = g; 1248 capacity = (d->rken0 + d->rken1) * 8 * (1ul << dimms[g].rowbits) * 1249 (1ul << dimms[g].colbits); 1250 edac_dbg(0, "Channel %d: %lld MByte DIMM\n", i, capacity >> (20 - 3)); 1251 dimm->nr_pages = MiB_TO_PAGES(capacity >> (20 - 3)); 1252 dimm->grain = 32; 1253 dimm->dtype = (d->dwid == 0) ? DEV_X8 : DEV_X16; 1254 dimm->mtype = MEM_DDR3; 1255 dimm->edac_mode = EDAC_SECDED; 1256 snprintf(dimm->label, sizeof(dimm->label), "Slice#%d_Chan#%d", i / 2, i % 2); 1257 } 1258 } 1259 1260 static const int dnv_dtypes[] = { 1261 DEV_X8, DEV_X4, DEV_X16, DEV_UNKNOWN 1262 }; 1263 1264 static void dnv_get_dimm_config(struct mem_ctl_info *mci) 1265 { 1266 int i, j, ranks_of_dimm[DNV_MAX_DIMMS], banks, rowbits, colbits, memtype; 1267 struct dimm_info *dimm; 1268 struct d_cr_drp *d; 1269 u64 capacity; 1270 1271 if (dsch.ddr4en) { 1272 memtype = MEM_DDR4; 1273 banks = 16; 1274 colbits = 10; 1275 } else { 1276 memtype = MEM_DDR3; 1277 banks = 8; 1278 } 1279 1280 for (i = 0; i < DNV_NUM_CHANNELS; i++) { 1281 if (dmap4[i].row14 == 31) 1282 rowbits = 14; 1283 else if (dmap4[i].row15 == 31) 1284 rowbits = 15; 1285 else if (dmap4[i].row16 == 31) 1286 rowbits = 16; 1287 else if (dmap4[i].row17 == 31) 1288 rowbits = 17; 1289 else 1290 rowbits = 18; 1291 1292 if (memtype == MEM_DDR3) { 1293 if (dmap1[i].ca11 != 0x3f) 1294 colbits = 12; 1295 else 1296 colbits = 10; 1297 } 1298 1299 d = &drp[i]; 1300 /* DIMM0 is present if rank0 and/or rank1 is enabled */ 1301 ranks_of_dimm[0] = d->rken0 + d->rken1; 1302 /* DIMM1 is present if rank2 and/or rank3 is enabled */ 1303 ranks_of_dimm[1] = d->rken2 + d->rken3; 1304 1305 for (j = 0; j < DNV_MAX_DIMMS; j++) { 1306 if (!ranks_of_dimm[j]) 1307 continue; 1308 1309 dimm = EDAC_DIMM_PTR(mci->layers, mci->dimms, mci->n_layers, i, j, 0); 1310 if (!dimm) { 1311 edac_dbg(0, "No allocated DIMM for channel %d DIMM %d\n", i, j); 1312 continue; 1313 } 1314 1315 capacity = ranks_of_dimm[j] * banks * (1ul << rowbits) * (1ul << colbits); 1316 edac_dbg(0, "Channel %d DIMM %d: %lld MByte DIMM\n", i, j, capacity >> (20 - 3)); 1317 dimm->nr_pages = MiB_TO_PAGES(capacity >> (20 - 3)); 1318 dimm->grain = 32; 1319 dimm->dtype = dnv_dtypes[j ? d->dimmdwid0 : d->dimmdwid1]; 1320 dimm->mtype = memtype; 1321 dimm->edac_mode = EDAC_SECDED; 1322 snprintf(dimm->label, sizeof(dimm->label), "Chan#%d_DIMM#%d", i, j); 1323 } 1324 } 1325 } 1326 1327 static int pnd2_register_mci(struct mem_ctl_info **ppmci) 1328 { 1329 struct edac_mc_layer layers[2]; 1330 struct mem_ctl_info *mci; 1331 struct pnd2_pvt *pvt; 1332 int rc; 1333 1334 rc = ops->check_ecc(); 1335 if (rc < 0) 1336 return rc; 1337 1338 /* Allocate a new MC control structure */ 1339 layers[0].type = EDAC_MC_LAYER_CHANNEL; 1340 layers[0].size = ops->channels; 1341 layers[0].is_virt_csrow = false; 1342 layers[1].type = EDAC_MC_LAYER_SLOT; 1343 layers[1].size = ops->dimms_per_channel; 1344 layers[1].is_virt_csrow = true; 1345 mci = edac_mc_alloc(0, ARRAY_SIZE(layers), layers, sizeof(*pvt)); 1346 if (!mci) 1347 return -ENOMEM; 1348 1349 pvt = mci->pvt_info; 1350 memset(pvt, 0, sizeof(*pvt)); 1351 1352 mci->mod_name = EDAC_MOD_STR; 1353 mci->dev_name = ops->name; 1354 mci->ctl_name = "Pondicherry2"; 1355 1356 /* Get dimm basic config and the memory layout */ 1357 ops->get_dimm_config(mci); 1358 1359 if (edac_mc_add_mc(mci)) { 1360 edac_dbg(0, "MC: failed edac_mc_add_mc()\n"); 1361 edac_mc_free(mci); 1362 return -EINVAL; 1363 } 1364 1365 *ppmci = mci; 1366 1367 return 0; 1368 } 1369 1370 static void pnd2_unregister_mci(struct mem_ctl_info *mci) 1371 { 1372 if (unlikely(!mci || !mci->pvt_info)) { 1373 pnd2_printk(KERN_ERR, "Couldn't find mci handler\n"); 1374 return; 1375 } 1376 1377 /* Remove MC sysfs nodes */ 1378 edac_mc_del_mc(NULL); 1379 edac_dbg(1, "%s: free mci struct\n", mci->ctl_name); 1380 edac_mc_free(mci); 1381 } 1382 1383 /* 1384 * Callback function registered with core kernel mce code. 1385 * Called once for each logged error. 1386 */ 1387 static int pnd2_mce_check_error(struct notifier_block *nb, unsigned long val, void *data) 1388 { 1389 struct mce *mce = (struct mce *)data; 1390 struct mem_ctl_info *mci; 1391 struct dram_addr daddr; 1392 char *type; 1393 1394 if (edac_get_report_status() == EDAC_REPORTING_DISABLED) 1395 return NOTIFY_DONE; 1396 1397 mci = pnd2_mci; 1398 if (!mci) 1399 return NOTIFY_DONE; 1400 1401 /* 1402 * Just let mcelog handle it if the error is 1403 * outside the memory controller. A memory error 1404 * is indicated by bit 7 = 1 and bits = 8-11,13-15 = 0. 1405 * bit 12 has an special meaning. 1406 */ 1407 if ((mce->status & 0xefff) >> 7 != 1) 1408 return NOTIFY_DONE; 1409 1410 if (mce->mcgstatus & MCG_STATUS_MCIP) 1411 type = "Exception"; 1412 else 1413 type = "Event"; 1414 1415 pnd2_mc_printk(mci, KERN_INFO, "HANDLING MCE MEMORY ERROR\n"); 1416 pnd2_mc_printk(mci, KERN_INFO, "CPU %u: Machine Check %s: %llx Bank %u: %llx\n", 1417 mce->extcpu, type, mce->mcgstatus, mce->bank, mce->status); 1418 pnd2_mc_printk(mci, KERN_INFO, "TSC %llx ", mce->tsc); 1419 pnd2_mc_printk(mci, KERN_INFO, "ADDR %llx ", mce->addr); 1420 pnd2_mc_printk(mci, KERN_INFO, "MISC %llx ", mce->misc); 1421 pnd2_mc_printk(mci, KERN_INFO, "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x\n", 1422 mce->cpuvendor, mce->cpuid, mce->time, mce->socketid, mce->apicid); 1423 1424 pnd2_mce_output_error(mci, mce, &daddr); 1425 1426 /* Advice mcelog that the error were handled */ 1427 return NOTIFY_STOP; 1428 } 1429 1430 static struct notifier_block pnd2_mce_dec = { 1431 .notifier_call = pnd2_mce_check_error, 1432 }; 1433 1434 #ifdef CONFIG_EDAC_DEBUG 1435 /* 1436 * Write an address to this file to exercise the address decode 1437 * logic in this driver. 1438 */ 1439 static u64 pnd2_fake_addr; 1440 #define PND2_BLOB_SIZE 1024 1441 static char pnd2_result[PND2_BLOB_SIZE]; 1442 static struct dentry *pnd2_test; 1443 static struct debugfs_blob_wrapper pnd2_blob = { 1444 .data = pnd2_result, 1445 .size = 0 1446 }; 1447 1448 static int debugfs_u64_set(void *data, u64 val) 1449 { 1450 struct dram_addr daddr; 1451 struct mce m; 1452 1453 *(u64 *)data = val; 1454 m.mcgstatus = 0; 1455 /* ADDRV + MemRd + Unknown channel */ 1456 m.status = MCI_STATUS_ADDRV + 0x9f; 1457 m.addr = val; 1458 pnd2_mce_output_error(pnd2_mci, &m, &daddr); 1459 snprintf(pnd2_blob.data, PND2_BLOB_SIZE, 1460 "SysAddr=%llx Channel=%d DIMM=%d Rank=%d Bank=%d Row=%d Column=%d\n", 1461 m.addr, daddr.chan, daddr.dimm, daddr.rank, daddr.bank, daddr.row, daddr.col); 1462 pnd2_blob.size = strlen(pnd2_blob.data); 1463 1464 return 0; 1465 } 1466 DEFINE_DEBUGFS_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n"); 1467 1468 static void setup_pnd2_debug(void) 1469 { 1470 pnd2_test = edac_debugfs_create_dir("pnd2_test"); 1471 edac_debugfs_create_file("pnd2_debug_addr", 0200, pnd2_test, 1472 &pnd2_fake_addr, &fops_u64_wo); 1473 debugfs_create_blob("pnd2_debug_results", 0400, pnd2_test, &pnd2_blob); 1474 } 1475 1476 static void teardown_pnd2_debug(void) 1477 { 1478 debugfs_remove_recursive(pnd2_test); 1479 } 1480 #else 1481 static void setup_pnd2_debug(void) {} 1482 static void teardown_pnd2_debug(void) {} 1483 #endif /* CONFIG_EDAC_DEBUG */ 1484 1485 1486 static int pnd2_probe(void) 1487 { 1488 int rc; 1489 1490 edac_dbg(2, "\n"); 1491 rc = get_registers(); 1492 if (rc) 1493 return rc; 1494 1495 return pnd2_register_mci(&pnd2_mci); 1496 } 1497 1498 static void pnd2_remove(void) 1499 { 1500 edac_dbg(0, "\n"); 1501 pnd2_unregister_mci(pnd2_mci); 1502 } 1503 1504 static struct dunit_ops apl_ops = { 1505 .name = "pnd2/apl", 1506 .type = APL, 1507 .pmiaddr_shift = LOG2_PMI_ADDR_GRANULARITY, 1508 .pmiidx_shift = 0, 1509 .channels = APL_NUM_CHANNELS, 1510 .dimms_per_channel = 1, 1511 .rd_reg = apl_rd_reg, 1512 .get_registers = apl_get_registers, 1513 .check_ecc = apl_check_ecc_active, 1514 .mk_region = apl_mk_region, 1515 .get_dimm_config = apl_get_dimm_config, 1516 .pmi2mem = apl_pmi2mem, 1517 }; 1518 1519 static struct dunit_ops dnv_ops = { 1520 .name = "pnd2/dnv", 1521 .type = DNV, 1522 .pmiaddr_shift = 0, 1523 .pmiidx_shift = 1, 1524 .channels = DNV_NUM_CHANNELS, 1525 .dimms_per_channel = 2, 1526 .rd_reg = dnv_rd_reg, 1527 .get_registers = dnv_get_registers, 1528 .check_ecc = dnv_check_ecc_active, 1529 .mk_region = dnv_mk_region, 1530 .get_dimm_config = dnv_get_dimm_config, 1531 .pmi2mem = dnv_pmi2mem, 1532 }; 1533 1534 static const struct x86_cpu_id pnd2_cpuids[] = { 1535 { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_GOLDMONT, 0, (kernel_ulong_t)&apl_ops }, 1536 { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_GOLDMONT_X, 0, (kernel_ulong_t)&dnv_ops }, 1537 { } 1538 }; 1539 MODULE_DEVICE_TABLE(x86cpu, pnd2_cpuids); 1540 1541 static int __init pnd2_init(void) 1542 { 1543 const struct x86_cpu_id *id; 1544 const char *owner; 1545 int rc; 1546 1547 edac_dbg(2, "\n"); 1548 1549 owner = edac_get_owner(); 1550 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR))) 1551 return -EBUSY; 1552 1553 id = x86_match_cpu(pnd2_cpuids); 1554 if (!id) 1555 return -ENODEV; 1556 1557 ops = (struct dunit_ops *)id->driver_data; 1558 1559 if (ops->type == APL) { 1560 p2sb_bus = pci_find_bus(0, 0); 1561 if (!p2sb_bus) 1562 return -ENODEV; 1563 } 1564 1565 /* Ensure that the OPSTATE is set correctly for POLL or NMI */ 1566 opstate_init(); 1567 1568 rc = pnd2_probe(); 1569 if (rc < 0) { 1570 pnd2_printk(KERN_ERR, "Failed to register device with error %d.\n", rc); 1571 return rc; 1572 } 1573 1574 if (!pnd2_mci) 1575 return -ENODEV; 1576 1577 mce_register_decode_chain(&pnd2_mce_dec); 1578 setup_pnd2_debug(); 1579 1580 return 0; 1581 } 1582 1583 static void __exit pnd2_exit(void) 1584 { 1585 edac_dbg(2, "\n"); 1586 teardown_pnd2_debug(); 1587 mce_unregister_decode_chain(&pnd2_mce_dec); 1588 pnd2_remove(); 1589 } 1590 1591 module_init(pnd2_init); 1592 module_exit(pnd2_exit); 1593 1594 module_param(edac_op_state, int, 0444); 1595 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI"); 1596 1597 MODULE_LICENSE("GPL v2"); 1598 MODULE_AUTHOR("Tony Luck"); 1599 MODULE_DESCRIPTION("MC Driver for Intel SoC using Pondicherry memory controller"); 1600