1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Driver for Intel(R) 10nm server memory controller. 4 * Copyright (c) 2019, Intel Corporation. 5 * 6 */ 7 8 #include <linux/kernel.h> 9 #include <linux/io.h> 10 #include <asm/cpu_device_id.h> 11 #include <asm/intel-family.h> 12 #include <asm/mce.h> 13 #include "edac_module.h" 14 #include "skx_common.h" 15 16 #define I10NM_REVISION "v0.0.5" 17 #define EDAC_MOD_STR "i10nm_edac" 18 19 /* Debug macros */ 20 #define i10nm_printk(level, fmt, arg...) \ 21 edac_printk(level, "i10nm", fmt, ##arg) 22 23 #define I10NM_GET_SCK_BAR(d, reg) \ 24 pci_read_config_dword((d)->uracu, 0xd0, &(reg)) 25 #define I10NM_GET_IMC_BAR(d, i, reg) \ 26 pci_read_config_dword((d)->uracu, 0xd8 + (i) * 4, &(reg)) 27 #define I10NM_GET_SAD(d, offset, i, reg)\ 28 pci_read_config_dword((d)->sad_all, (offset) + (i) * 8, &(reg)) 29 #define I10NM_GET_HBM_IMC_BAR(d, reg) \ 30 pci_read_config_dword((d)->uracu, 0xd4, &(reg)) 31 #define I10NM_GET_CAPID3_CFG(d, reg) \ 32 pci_read_config_dword((d)->pcu_cr3, 0x90, &(reg)) 33 #define I10NM_GET_DIMMMTR(m, i, j) \ 34 readl((m)->mbase + ((m)->hbm_mc ? 0x80c : 0x2080c) + \ 35 (i) * (m)->chan_mmio_sz + (j) * 4) 36 #define I10NM_GET_MCDDRTCFG(m, i, j) \ 37 readl((m)->mbase + ((m)->hbm_mc ? 0x970 : 0x20970) + \ 38 (i) * (m)->chan_mmio_sz + (j) * 4) 39 #define I10NM_GET_MCMTR(m, i) \ 40 readl((m)->mbase + ((m)->hbm_mc ? 0xef8 : 0x20ef8) + \ 41 (i) * (m)->chan_mmio_sz) 42 #define I10NM_GET_AMAP(m, i) \ 43 readl((m)->mbase + ((m)->hbm_mc ? 0x814 : 0x20814) + \ 44 (i) * (m)->chan_mmio_sz) 45 46 #define I10NM_GET_SCK_MMIO_BASE(reg) (GET_BITFIELD(reg, 0, 28) << 23) 47 #define I10NM_GET_IMC_MMIO_OFFSET(reg) (GET_BITFIELD(reg, 0, 10) << 12) 48 #define I10NM_GET_IMC_MMIO_SIZE(reg) ((GET_BITFIELD(reg, 13, 23) - \ 49 GET_BITFIELD(reg, 0, 10) + 1) << 12) 50 #define I10NM_GET_HBM_IMC_MMIO_OFFSET(reg) \ 51 ((GET_BITFIELD(reg, 0, 10) << 12) + 0x140000) 52 53 #define I10NM_HBM_IMC_MMIO_SIZE 0x9000 54 #define I10NM_IS_HBM_PRESENT(reg) GET_BITFIELD(reg, 27, 30) 55 #define I10NM_IS_HBM_IMC(reg) GET_BITFIELD(reg, 29, 29) 56 57 #define I10NM_MAX_SAD 16 58 #define I10NM_SAD_ENABLE(reg) GET_BITFIELD(reg, 0, 0) 59 #define I10NM_SAD_NM_CACHEABLE(reg) GET_BITFIELD(reg, 5, 5) 60 61 static struct list_head *i10nm_edac_list; 62 63 static struct pci_dev *pci_get_dev_wrapper(int dom, unsigned int bus, 64 unsigned int dev, unsigned int fun) 65 { 66 struct pci_dev *pdev; 67 68 pdev = pci_get_domain_bus_and_slot(dom, bus, PCI_DEVFN(dev, fun)); 69 if (!pdev) { 70 edac_dbg(2, "No device %02x:%02x.%x\n", 71 bus, dev, fun); 72 return NULL; 73 } 74 75 if (unlikely(pci_enable_device(pdev) < 0)) { 76 edac_dbg(2, "Failed to enable device %02x:%02x.%x\n", 77 bus, dev, fun); 78 return NULL; 79 } 80 81 pci_dev_get(pdev); 82 83 return pdev; 84 } 85 86 static bool i10nm_check_2lm(struct res_config *cfg) 87 { 88 struct skx_dev *d; 89 u32 reg; 90 int i; 91 92 list_for_each_entry(d, i10nm_edac_list, list) { 93 d->sad_all = pci_get_dev_wrapper(d->seg, d->bus[1], 94 PCI_SLOT(cfg->sad_all_devfn), 95 PCI_FUNC(cfg->sad_all_devfn)); 96 if (!d->sad_all) 97 continue; 98 99 for (i = 0; i < I10NM_MAX_SAD; i++) { 100 I10NM_GET_SAD(d, cfg->sad_all_offset, i, reg); 101 if (I10NM_SAD_ENABLE(reg) && I10NM_SAD_NM_CACHEABLE(reg)) { 102 edac_dbg(2, "2-level memory configuration.\n"); 103 return true; 104 } 105 } 106 } 107 108 return false; 109 } 110 111 static int i10nm_get_ddr_munits(void) 112 { 113 struct pci_dev *mdev; 114 void __iomem *mbase; 115 unsigned long size; 116 struct skx_dev *d; 117 int i, j = 0; 118 u32 reg, off; 119 u64 base; 120 121 list_for_each_entry(d, i10nm_edac_list, list) { 122 d->util_all = pci_get_dev_wrapper(d->seg, d->bus[1], 29, 1); 123 if (!d->util_all) 124 return -ENODEV; 125 126 d->uracu = pci_get_dev_wrapper(d->seg, d->bus[0], 0, 1); 127 if (!d->uracu) 128 return -ENODEV; 129 130 if (I10NM_GET_SCK_BAR(d, reg)) { 131 i10nm_printk(KERN_ERR, "Failed to socket bar\n"); 132 return -ENODEV; 133 } 134 135 base = I10NM_GET_SCK_MMIO_BASE(reg); 136 edac_dbg(2, "socket%d mmio base 0x%llx (reg 0x%x)\n", 137 j++, base, reg); 138 139 for (i = 0; i < I10NM_NUM_DDR_IMC; i++) { 140 mdev = pci_get_dev_wrapper(d->seg, d->bus[0], 141 12 + i, 0); 142 if (i == 0 && !mdev) { 143 i10nm_printk(KERN_ERR, "No IMC found\n"); 144 return -ENODEV; 145 } 146 if (!mdev) 147 continue; 148 149 d->imc[i].mdev = mdev; 150 151 if (I10NM_GET_IMC_BAR(d, i, reg)) { 152 i10nm_printk(KERN_ERR, "Failed to get mc bar\n"); 153 return -ENODEV; 154 } 155 156 off = I10NM_GET_IMC_MMIO_OFFSET(reg); 157 size = I10NM_GET_IMC_MMIO_SIZE(reg); 158 edac_dbg(2, "mc%d mmio base 0x%llx size 0x%lx (reg 0x%x)\n", 159 i, base + off, size, reg); 160 161 mbase = ioremap(base + off, size); 162 if (!mbase) { 163 i10nm_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", 164 base + off); 165 return -ENODEV; 166 } 167 168 d->imc[i].mbase = mbase; 169 } 170 } 171 172 return 0; 173 } 174 175 static bool i10nm_check_hbm_imc(struct skx_dev *d) 176 { 177 u32 reg; 178 179 if (I10NM_GET_CAPID3_CFG(d, reg)) { 180 i10nm_printk(KERN_ERR, "Failed to get capid3_cfg\n"); 181 return false; 182 } 183 184 return I10NM_IS_HBM_PRESENT(reg) != 0; 185 } 186 187 static int i10nm_get_hbm_munits(void) 188 { 189 struct pci_dev *mdev; 190 void __iomem *mbase; 191 u32 reg, off, mcmtr; 192 struct skx_dev *d; 193 int i, lmc; 194 u64 base; 195 196 list_for_each_entry(d, i10nm_edac_list, list) { 197 d->pcu_cr3 = pci_get_dev_wrapper(d->seg, d->bus[1], 30, 3); 198 if (!d->pcu_cr3) 199 return -ENODEV; 200 201 if (!i10nm_check_hbm_imc(d)) { 202 i10nm_printk(KERN_DEBUG, "No hbm memory\n"); 203 return -ENODEV; 204 } 205 206 if (I10NM_GET_SCK_BAR(d, reg)) { 207 i10nm_printk(KERN_ERR, "Failed to get socket bar\n"); 208 return -ENODEV; 209 } 210 base = I10NM_GET_SCK_MMIO_BASE(reg); 211 212 if (I10NM_GET_HBM_IMC_BAR(d, reg)) { 213 i10nm_printk(KERN_ERR, "Failed to get hbm mc bar\n"); 214 return -ENODEV; 215 } 216 base += I10NM_GET_HBM_IMC_MMIO_OFFSET(reg); 217 218 lmc = I10NM_NUM_DDR_IMC; 219 220 for (i = 0; i < I10NM_NUM_HBM_IMC; i++) { 221 mdev = pci_get_dev_wrapper(d->seg, d->bus[0], 222 12 + i / 4, 1 + i % 4); 223 if (i == 0 && !mdev) { 224 i10nm_printk(KERN_ERR, "No hbm mc found\n"); 225 return -ENODEV; 226 } 227 if (!mdev) 228 continue; 229 230 d->imc[lmc].mdev = mdev; 231 off = i * I10NM_HBM_IMC_MMIO_SIZE; 232 233 edac_dbg(2, "hbm mc%d mmio base 0x%llx size 0x%x\n", 234 lmc, base + off, I10NM_HBM_IMC_MMIO_SIZE); 235 236 mbase = ioremap(base + off, I10NM_HBM_IMC_MMIO_SIZE); 237 if (!mbase) { 238 i10nm_printk(KERN_ERR, "Failed to ioremap for hbm mc 0x%llx\n", 239 base + off); 240 return -ENOMEM; 241 } 242 243 d->imc[lmc].mbase = mbase; 244 d->imc[lmc].hbm_mc = true; 245 246 mcmtr = I10NM_GET_MCMTR(&d->imc[lmc], 0); 247 if (!I10NM_IS_HBM_IMC(mcmtr)) { 248 i10nm_printk(KERN_ERR, "This isn't an hbm mc!\n"); 249 return -ENODEV; 250 } 251 252 lmc++; 253 } 254 } 255 256 return 0; 257 } 258 259 static struct res_config i10nm_cfg0 = { 260 .type = I10NM, 261 .decs_did = 0x3452, 262 .busno_cfg_offset = 0xcc, 263 .ddr_chan_mmio_sz = 0x4000, 264 .sad_all_devfn = PCI_DEVFN(29, 0), 265 .sad_all_offset = 0x108, 266 }; 267 268 static struct res_config i10nm_cfg1 = { 269 .type = I10NM, 270 .decs_did = 0x3452, 271 .busno_cfg_offset = 0xd0, 272 .ddr_chan_mmio_sz = 0x4000, 273 .sad_all_devfn = PCI_DEVFN(29, 0), 274 .sad_all_offset = 0x108, 275 }; 276 277 static struct res_config spr_cfg = { 278 .type = SPR, 279 .decs_did = 0x3252, 280 .busno_cfg_offset = 0xd0, 281 .ddr_chan_mmio_sz = 0x8000, 282 .hbm_chan_mmio_sz = 0x4000, 283 .support_ddr5 = true, 284 .sad_all_devfn = PCI_DEVFN(10, 0), 285 .sad_all_offset = 0x300, 286 }; 287 288 static const struct x86_cpu_id i10nm_cpuids[] = { 289 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ATOM_TREMONT_D, X86_STEPPINGS(0x0, 0x3), &i10nm_cfg0), 290 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ATOM_TREMONT_D, X86_STEPPINGS(0x4, 0xf), &i10nm_cfg1), 291 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ICELAKE_X, X86_STEPPINGS(0x0, 0x3), &i10nm_cfg0), 292 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ICELAKE_X, X86_STEPPINGS(0x4, 0xf), &i10nm_cfg1), 293 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ICELAKE_D, X86_STEPPINGS(0x0, 0xf), &i10nm_cfg1), 294 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SAPPHIRERAPIDS_X, X86_STEPPINGS(0x0, 0xf), &spr_cfg), 295 {} 296 }; 297 MODULE_DEVICE_TABLE(x86cpu, i10nm_cpuids); 298 299 static bool i10nm_check_ecc(struct skx_imc *imc, int chan) 300 { 301 u32 mcmtr; 302 303 mcmtr = I10NM_GET_MCMTR(imc, chan); 304 edac_dbg(1, "ch%d mcmtr reg %x\n", chan, mcmtr); 305 306 return !!GET_BITFIELD(mcmtr, 2, 2); 307 } 308 309 static int i10nm_get_dimm_config(struct mem_ctl_info *mci, 310 struct res_config *cfg) 311 { 312 struct skx_pvt *pvt = mci->pvt_info; 313 struct skx_imc *imc = pvt->imc; 314 u32 mtr, amap, mcddrtcfg; 315 struct dimm_info *dimm; 316 int i, j, ndimms; 317 318 for (i = 0; i < imc->num_channels; i++) { 319 if (!imc->mbase) 320 continue; 321 322 ndimms = 0; 323 amap = I10NM_GET_AMAP(imc, i); 324 for (j = 0; j < imc->num_dimms; j++) { 325 dimm = edac_get_dimm(mci, i, j, 0); 326 mtr = I10NM_GET_DIMMMTR(imc, i, j); 327 mcddrtcfg = I10NM_GET_MCDDRTCFG(imc, i, j); 328 edac_dbg(1, "dimmmtr 0x%x mcddrtcfg 0x%x (mc%d ch%d dimm%d)\n", 329 mtr, mcddrtcfg, imc->mc, i, j); 330 331 if (IS_DIMM_PRESENT(mtr)) 332 ndimms += skx_get_dimm_info(mtr, 0, amap, dimm, 333 imc, i, j, cfg); 334 else if (IS_NVDIMM_PRESENT(mcddrtcfg, j)) 335 ndimms += skx_get_nvdimm_info(dimm, imc, i, j, 336 EDAC_MOD_STR); 337 } 338 if (ndimms && !i10nm_check_ecc(imc, i)) { 339 i10nm_printk(KERN_ERR, "ECC is disabled on imc %d channel %d\n", 340 imc->mc, i); 341 return -ENODEV; 342 } 343 } 344 345 return 0; 346 } 347 348 static struct notifier_block i10nm_mce_dec = { 349 .notifier_call = skx_mce_check_error, 350 .priority = MCE_PRIO_EDAC, 351 }; 352 353 #ifdef CONFIG_EDAC_DEBUG 354 /* 355 * Debug feature. 356 * Exercise the address decode logic by writing an address to 357 * /sys/kernel/debug/edac/i10nm_test/addr. 358 */ 359 static struct dentry *i10nm_test; 360 361 static int debugfs_u64_set(void *data, u64 val) 362 { 363 struct mce m; 364 365 pr_warn_once("Fake error to 0x%llx injected via debugfs\n", val); 366 367 memset(&m, 0, sizeof(m)); 368 /* ADDRV + MemRd + Unknown channel */ 369 m.status = MCI_STATUS_ADDRV + 0x90; 370 /* One corrected error */ 371 m.status |= BIT_ULL(MCI_STATUS_CEC_SHIFT); 372 m.addr = val; 373 skx_mce_check_error(NULL, 0, &m); 374 375 return 0; 376 } 377 DEFINE_SIMPLE_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n"); 378 379 static void setup_i10nm_debug(void) 380 { 381 i10nm_test = edac_debugfs_create_dir("i10nm_test"); 382 if (!i10nm_test) 383 return; 384 385 if (!edac_debugfs_create_file("addr", 0200, i10nm_test, 386 NULL, &fops_u64_wo)) { 387 debugfs_remove(i10nm_test); 388 i10nm_test = NULL; 389 } 390 } 391 392 static void teardown_i10nm_debug(void) 393 { 394 debugfs_remove_recursive(i10nm_test); 395 } 396 #else 397 static inline void setup_i10nm_debug(void) {} 398 static inline void teardown_i10nm_debug(void) {} 399 #endif /*CONFIG_EDAC_DEBUG*/ 400 401 static int __init i10nm_init(void) 402 { 403 u8 mc = 0, src_id = 0, node_id = 0; 404 const struct x86_cpu_id *id; 405 struct res_config *cfg; 406 const char *owner; 407 struct skx_dev *d; 408 int rc, i, off[3] = {0xd0, 0xc8, 0xcc}; 409 u64 tolm, tohm; 410 411 edac_dbg(2, "\n"); 412 413 owner = edac_get_owner(); 414 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR))) 415 return -EBUSY; 416 417 if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) 418 return -ENODEV; 419 420 id = x86_match_cpu(i10nm_cpuids); 421 if (!id) 422 return -ENODEV; 423 424 cfg = (struct res_config *)id->driver_data; 425 426 rc = skx_get_hi_lo(0x09a2, off, &tolm, &tohm); 427 if (rc) 428 return rc; 429 430 rc = skx_get_all_bus_mappings(cfg, &i10nm_edac_list); 431 if (rc < 0) 432 goto fail; 433 if (rc == 0) { 434 i10nm_printk(KERN_ERR, "No memory controllers found\n"); 435 return -ENODEV; 436 } 437 438 skx_set_mem_cfg(i10nm_check_2lm(cfg)); 439 440 rc = i10nm_get_ddr_munits(); 441 442 if (i10nm_get_hbm_munits() && rc) 443 goto fail; 444 445 list_for_each_entry(d, i10nm_edac_list, list) { 446 rc = skx_get_src_id(d, 0xf8, &src_id); 447 if (rc < 0) 448 goto fail; 449 450 rc = skx_get_node_id(d, &node_id); 451 if (rc < 0) 452 goto fail; 453 454 edac_dbg(2, "src_id = %d node_id = %d\n", src_id, node_id); 455 for (i = 0; i < I10NM_NUM_IMC; i++) { 456 if (!d->imc[i].mdev) 457 continue; 458 459 d->imc[i].mc = mc++; 460 d->imc[i].lmc = i; 461 d->imc[i].src_id = src_id; 462 d->imc[i].node_id = node_id; 463 if (d->imc[i].hbm_mc) { 464 d->imc[i].chan_mmio_sz = cfg->hbm_chan_mmio_sz; 465 d->imc[i].num_channels = I10NM_NUM_HBM_CHANNELS; 466 d->imc[i].num_dimms = I10NM_NUM_HBM_DIMMS; 467 } else { 468 d->imc[i].chan_mmio_sz = cfg->ddr_chan_mmio_sz; 469 d->imc[i].num_channels = I10NM_NUM_DDR_CHANNELS; 470 d->imc[i].num_dimms = I10NM_NUM_DDR_DIMMS; 471 } 472 473 rc = skx_register_mci(&d->imc[i], d->imc[i].mdev, 474 "Intel_10nm Socket", EDAC_MOD_STR, 475 i10nm_get_dimm_config, cfg); 476 if (rc < 0) 477 goto fail; 478 } 479 } 480 481 rc = skx_adxl_get(); 482 if (rc) 483 goto fail; 484 485 opstate_init(); 486 mce_register_decode_chain(&i10nm_mce_dec); 487 setup_i10nm_debug(); 488 489 i10nm_printk(KERN_INFO, "%s\n", I10NM_REVISION); 490 491 return 0; 492 fail: 493 skx_remove(); 494 return rc; 495 } 496 497 static void __exit i10nm_exit(void) 498 { 499 edac_dbg(2, "\n"); 500 teardown_i10nm_debug(); 501 mce_unregister_decode_chain(&i10nm_mce_dec); 502 skx_adxl_put(); 503 skx_remove(); 504 } 505 506 module_init(i10nm_init); 507 module_exit(i10nm_exit); 508 509 MODULE_LICENSE("GPL v2"); 510 MODULE_DESCRIPTION("MC Driver for Intel 10nm server processors"); 511