1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Freescale Memory Controller kernel module 4 * 5 * Support Power-based SoCs including MPC85xx, MPC86xx, MPC83xx and 6 * ARM-based Layerscape SoCs including LS2xxx and LS1021A. Originally 7 * split out from mpc85xx_edac EDAC driver. 8 * 9 * Parts Copyrighted (c) 2013 by Freescale Semiconductor, Inc. 10 * 11 * Author: Dave Jiang <djiang@mvista.com> 12 * 13 * 2006-2007 (c) MontaVista Software, Inc. 14 */ 15 #include <linux/module.h> 16 #include <linux/init.h> 17 #include <linux/interrupt.h> 18 #include <linux/ctype.h> 19 #include <linux/io.h> 20 #include <linux/mod_devicetable.h> 21 #include <linux/edac.h> 22 #include <linux/smp.h> 23 #include <linux/gfp.h> 24 25 #include <linux/of.h> 26 #include <linux/of_address.h> 27 #include "edac_module.h" 28 #include "fsl_ddr_edac.h" 29 30 #define EDAC_MOD_STR "fsl_ddr_edac" 31 32 static int edac_mc_idx; 33 34 static u32 orig_ddr_err_disable; 35 static u32 orig_ddr_err_sbe; 36 static bool little_endian; 37 38 static inline u32 ddr_in32(void __iomem *addr) 39 { 40 return little_endian ? ioread32(addr) : ioread32be(addr); 41 } 42 43 static inline void ddr_out32(void __iomem *addr, u32 value) 44 { 45 if (little_endian) 46 iowrite32(value, addr); 47 else 48 iowrite32be(value, addr); 49 } 50 51 #ifdef CONFIG_EDAC_DEBUG 52 /************************ MC SYSFS parts ***********************************/ 53 54 #define to_mci(k) container_of(k, struct mem_ctl_info, dev) 55 56 static ssize_t fsl_mc_inject_data_hi_show(struct device *dev, 57 struct device_attribute *mattr, 58 char *data) 59 { 60 struct mem_ctl_info *mci = to_mci(dev); 61 struct fsl_mc_pdata *pdata = mci->pvt_info; 62 return sprintf(data, "0x%08x", 63 ddr_in32(pdata->mc_vbase + FSL_MC_DATA_ERR_INJECT_HI)); 64 } 65 66 static ssize_t fsl_mc_inject_data_lo_show(struct device *dev, 67 struct device_attribute *mattr, 68 char *data) 69 { 70 struct mem_ctl_info *mci = to_mci(dev); 71 struct fsl_mc_pdata *pdata = mci->pvt_info; 72 return sprintf(data, "0x%08x", 73 ddr_in32(pdata->mc_vbase + FSL_MC_DATA_ERR_INJECT_LO)); 74 } 75 76 static ssize_t fsl_mc_inject_ctrl_show(struct device *dev, 77 struct device_attribute *mattr, 78 char *data) 79 { 80 struct mem_ctl_info *mci = to_mci(dev); 81 struct fsl_mc_pdata *pdata = mci->pvt_info; 82 return sprintf(data, "0x%08x", 83 ddr_in32(pdata->mc_vbase + FSL_MC_ECC_ERR_INJECT)); 84 } 85 86 static ssize_t fsl_mc_inject_data_hi_store(struct device *dev, 87 struct device_attribute *mattr, 88 const char *data, size_t count) 89 { 90 struct mem_ctl_info *mci = to_mci(dev); 91 struct fsl_mc_pdata *pdata = mci->pvt_info; 92 unsigned long val; 93 int rc; 94 95 if (isdigit(*data)) { 96 rc = kstrtoul(data, 0, &val); 97 if (rc) 98 return rc; 99 100 ddr_out32(pdata->mc_vbase + FSL_MC_DATA_ERR_INJECT_HI, val); 101 return count; 102 } 103 return 0; 104 } 105 106 static ssize_t fsl_mc_inject_data_lo_store(struct device *dev, 107 struct device_attribute *mattr, 108 const char *data, size_t count) 109 { 110 struct mem_ctl_info *mci = to_mci(dev); 111 struct fsl_mc_pdata *pdata = mci->pvt_info; 112 unsigned long val; 113 int rc; 114 115 if (isdigit(*data)) { 116 rc = kstrtoul(data, 0, &val); 117 if (rc) 118 return rc; 119 120 ddr_out32(pdata->mc_vbase + FSL_MC_DATA_ERR_INJECT_LO, val); 121 return count; 122 } 123 return 0; 124 } 125 126 static ssize_t fsl_mc_inject_ctrl_store(struct device *dev, 127 struct device_attribute *mattr, 128 const char *data, size_t count) 129 { 130 struct mem_ctl_info *mci = to_mci(dev); 131 struct fsl_mc_pdata *pdata = mci->pvt_info; 132 unsigned long val; 133 int rc; 134 135 if (isdigit(*data)) { 136 rc = kstrtoul(data, 0, &val); 137 if (rc) 138 return rc; 139 140 ddr_out32(pdata->mc_vbase + FSL_MC_ECC_ERR_INJECT, val); 141 return count; 142 } 143 return 0; 144 } 145 146 static DEVICE_ATTR(inject_data_hi, S_IRUGO | S_IWUSR, 147 fsl_mc_inject_data_hi_show, fsl_mc_inject_data_hi_store); 148 static DEVICE_ATTR(inject_data_lo, S_IRUGO | S_IWUSR, 149 fsl_mc_inject_data_lo_show, fsl_mc_inject_data_lo_store); 150 static DEVICE_ATTR(inject_ctrl, S_IRUGO | S_IWUSR, 151 fsl_mc_inject_ctrl_show, fsl_mc_inject_ctrl_store); 152 #endif /* CONFIG_EDAC_DEBUG */ 153 154 static struct attribute *fsl_ddr_dev_attrs[] = { 155 #ifdef CONFIG_EDAC_DEBUG 156 &dev_attr_inject_data_hi.attr, 157 &dev_attr_inject_data_lo.attr, 158 &dev_attr_inject_ctrl.attr, 159 #endif 160 NULL 161 }; 162 163 ATTRIBUTE_GROUPS(fsl_ddr_dev); 164 165 /**************************** MC Err device ***************************/ 166 167 /* 168 * Taken from table 8-55 in the MPC8641 User's Manual and/or 9-61 in the 169 * MPC8572 User's Manual. Each line represents a syndrome bit column as a 170 * 64-bit value, but split into an upper and lower 32-bit chunk. The labels 171 * below correspond to Freescale's manuals. 172 */ 173 static unsigned int ecc_table[16] = { 174 /* MSB LSB */ 175 /* [0:31] [32:63] */ 176 0xf00fe11e, 0xc33c0ff7, /* Syndrome bit 7 */ 177 0x00ff00ff, 0x00fff0ff, 178 0x0f0f0f0f, 0x0f0fff00, 179 0x11113333, 0x7777000f, 180 0x22224444, 0x8888222f, 181 0x44448888, 0xffff4441, 182 0x8888ffff, 0x11118882, 183 0xffff1111, 0x22221114, /* Syndrome bit 0 */ 184 }; 185 186 /* 187 * Calculate the correct ECC value for a 64-bit value specified by high:low 188 */ 189 static u8 calculate_ecc(u32 high, u32 low) 190 { 191 u32 mask_low; 192 u32 mask_high; 193 int bit_cnt; 194 u8 ecc = 0; 195 int i; 196 int j; 197 198 for (i = 0; i < 8; i++) { 199 mask_high = ecc_table[i * 2]; 200 mask_low = ecc_table[i * 2 + 1]; 201 bit_cnt = 0; 202 203 for (j = 0; j < 32; j++) { 204 if ((mask_high >> j) & 1) 205 bit_cnt ^= (high >> j) & 1; 206 if ((mask_low >> j) & 1) 207 bit_cnt ^= (low >> j) & 1; 208 } 209 210 ecc |= bit_cnt << i; 211 } 212 213 return ecc; 214 } 215 216 /* 217 * Create the syndrome code which is generated if the data line specified by 218 * 'bit' failed. Eg generate an 8-bit codes seen in Table 8-55 in the MPC8641 219 * User's Manual and 9-61 in the MPC8572 User's Manual. 220 */ 221 static u8 syndrome_from_bit(unsigned int bit) { 222 int i; 223 u8 syndrome = 0; 224 225 /* 226 * Cycle through the upper or lower 32-bit portion of each value in 227 * ecc_table depending on if 'bit' is in the upper or lower half of 228 * 64-bit data. 229 */ 230 for (i = bit < 32; i < 16; i += 2) 231 syndrome |= ((ecc_table[i] >> (bit % 32)) & 1) << (i / 2); 232 233 return syndrome; 234 } 235 236 /* 237 * Decode data and ecc syndrome to determine what went wrong 238 * Note: This can only decode single-bit errors 239 */ 240 static void sbe_ecc_decode(u32 cap_high, u32 cap_low, u32 cap_ecc, 241 int *bad_data_bit, int *bad_ecc_bit) 242 { 243 int i; 244 u8 syndrome; 245 246 *bad_data_bit = -1; 247 *bad_ecc_bit = -1; 248 249 /* 250 * Calculate the ECC of the captured data and XOR it with the captured 251 * ECC to find an ECC syndrome value we can search for 252 */ 253 syndrome = calculate_ecc(cap_high, cap_low) ^ cap_ecc; 254 255 /* Check if a data line is stuck... */ 256 for (i = 0; i < 64; i++) { 257 if (syndrome == syndrome_from_bit(i)) { 258 *bad_data_bit = i; 259 return; 260 } 261 } 262 263 /* If data is correct, check ECC bits for errors... */ 264 for (i = 0; i < 8; i++) { 265 if ((syndrome >> i) & 0x1) { 266 *bad_ecc_bit = i; 267 return; 268 } 269 } 270 } 271 272 #define make64(high, low) (((u64)(high) << 32) | (low)) 273 274 static void fsl_mc_check(struct mem_ctl_info *mci) 275 { 276 struct fsl_mc_pdata *pdata = mci->pvt_info; 277 struct csrow_info *csrow; 278 u32 bus_width; 279 u32 err_detect; 280 u32 syndrome; 281 u64 err_addr; 282 u32 pfn; 283 int row_index; 284 u32 cap_high; 285 u32 cap_low; 286 int bad_data_bit; 287 int bad_ecc_bit; 288 289 err_detect = ddr_in32(pdata->mc_vbase + FSL_MC_ERR_DETECT); 290 if (!err_detect) 291 return; 292 293 fsl_mc_printk(mci, KERN_ERR, "Err Detect Register: %#8.8x\n", 294 err_detect); 295 296 /* no more processing if not ECC bit errors */ 297 if (!(err_detect & (DDR_EDE_SBE | DDR_EDE_MBE))) { 298 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_DETECT, err_detect); 299 return; 300 } 301 302 syndrome = ddr_in32(pdata->mc_vbase + FSL_MC_CAPTURE_ECC); 303 304 /* Mask off appropriate bits of syndrome based on bus width */ 305 bus_width = (ddr_in32(pdata->mc_vbase + FSL_MC_DDR_SDRAM_CFG) & 306 DSC_DBW_MASK) ? 32 : 64; 307 if (bus_width == 64) 308 syndrome &= 0xff; 309 else 310 syndrome &= 0xffff; 311 312 err_addr = make64( 313 ddr_in32(pdata->mc_vbase + FSL_MC_CAPTURE_EXT_ADDRESS), 314 ddr_in32(pdata->mc_vbase + FSL_MC_CAPTURE_ADDRESS)); 315 pfn = err_addr >> PAGE_SHIFT; 316 317 for (row_index = 0; row_index < mci->nr_csrows; row_index++) { 318 csrow = mci->csrows[row_index]; 319 if ((pfn >= csrow->first_page) && (pfn <= csrow->last_page)) 320 break; 321 } 322 323 cap_high = ddr_in32(pdata->mc_vbase + FSL_MC_CAPTURE_DATA_HI); 324 cap_low = ddr_in32(pdata->mc_vbase + FSL_MC_CAPTURE_DATA_LO); 325 326 /* 327 * Analyze single-bit errors on 64-bit wide buses 328 * TODO: Add support for 32-bit wide buses 329 */ 330 if ((err_detect & DDR_EDE_SBE) && (bus_width == 64)) { 331 u64 cap = (u64)cap_high << 32 | cap_low; 332 u32 s = syndrome; 333 334 sbe_ecc_decode(cap_high, cap_low, syndrome, 335 &bad_data_bit, &bad_ecc_bit); 336 337 if (bad_data_bit >= 0) { 338 fsl_mc_printk(mci, KERN_ERR, "Faulty Data bit: %d\n", bad_data_bit); 339 cap ^= 1ULL << bad_data_bit; 340 } 341 342 if (bad_ecc_bit >= 0) { 343 fsl_mc_printk(mci, KERN_ERR, "Faulty ECC bit: %d\n", bad_ecc_bit); 344 s ^= 1 << bad_ecc_bit; 345 } 346 347 fsl_mc_printk(mci, KERN_ERR, 348 "Expected Data / ECC:\t%#8.8x_%08x / %#2.2x\n", 349 upper_32_bits(cap), lower_32_bits(cap), s); 350 } 351 352 fsl_mc_printk(mci, KERN_ERR, 353 "Captured Data / ECC:\t%#8.8x_%08x / %#2.2x\n", 354 cap_high, cap_low, syndrome); 355 fsl_mc_printk(mci, KERN_ERR, "Err addr: %#8.8llx\n", err_addr); 356 fsl_mc_printk(mci, KERN_ERR, "PFN: %#8.8x\n", pfn); 357 358 /* we are out of range */ 359 if (row_index == mci->nr_csrows) 360 fsl_mc_printk(mci, KERN_ERR, "PFN out of range!\n"); 361 362 if (err_detect & DDR_EDE_SBE) 363 edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 1, 364 pfn, err_addr & ~PAGE_MASK, syndrome, 365 row_index, 0, -1, 366 mci->ctl_name, ""); 367 368 if (err_detect & DDR_EDE_MBE) 369 edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1, 370 pfn, err_addr & ~PAGE_MASK, syndrome, 371 row_index, 0, -1, 372 mci->ctl_name, ""); 373 374 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_DETECT, err_detect); 375 } 376 377 static irqreturn_t fsl_mc_isr(int irq, void *dev_id) 378 { 379 struct mem_ctl_info *mci = dev_id; 380 struct fsl_mc_pdata *pdata = mci->pvt_info; 381 u32 err_detect; 382 383 err_detect = ddr_in32(pdata->mc_vbase + FSL_MC_ERR_DETECT); 384 if (!err_detect) 385 return IRQ_NONE; 386 387 fsl_mc_check(mci); 388 389 return IRQ_HANDLED; 390 } 391 392 static void fsl_ddr_init_csrows(struct mem_ctl_info *mci) 393 { 394 struct fsl_mc_pdata *pdata = mci->pvt_info; 395 struct csrow_info *csrow; 396 struct dimm_info *dimm; 397 u32 sdram_ctl; 398 u32 sdtype; 399 enum mem_type mtype; 400 u32 cs_bnds; 401 int index; 402 403 sdram_ctl = ddr_in32(pdata->mc_vbase + FSL_MC_DDR_SDRAM_CFG); 404 405 sdtype = sdram_ctl & DSC_SDTYPE_MASK; 406 if (sdram_ctl & DSC_RD_EN) { 407 switch (sdtype) { 408 case 0x02000000: 409 mtype = MEM_RDDR; 410 break; 411 case 0x03000000: 412 mtype = MEM_RDDR2; 413 break; 414 case 0x07000000: 415 mtype = MEM_RDDR3; 416 break; 417 case 0x05000000: 418 mtype = MEM_RDDR4; 419 break; 420 default: 421 mtype = MEM_UNKNOWN; 422 break; 423 } 424 } else { 425 switch (sdtype) { 426 case 0x02000000: 427 mtype = MEM_DDR; 428 break; 429 case 0x03000000: 430 mtype = MEM_DDR2; 431 break; 432 case 0x07000000: 433 mtype = MEM_DDR3; 434 break; 435 case 0x05000000: 436 mtype = MEM_DDR4; 437 break; 438 default: 439 mtype = MEM_UNKNOWN; 440 break; 441 } 442 } 443 444 for (index = 0; index < mci->nr_csrows; index++) { 445 u32 start; 446 u32 end; 447 448 csrow = mci->csrows[index]; 449 dimm = csrow->channels[0]->dimm; 450 451 cs_bnds = ddr_in32(pdata->mc_vbase + FSL_MC_CS_BNDS_0 + 452 (index * FSL_MC_CS_BNDS_OFS)); 453 454 start = (cs_bnds & 0xffff0000) >> 16; 455 end = (cs_bnds & 0x0000ffff); 456 457 if (start == end) 458 continue; /* not populated */ 459 460 start <<= (24 - PAGE_SHIFT); 461 end <<= (24 - PAGE_SHIFT); 462 end |= (1 << (24 - PAGE_SHIFT)) - 1; 463 464 csrow->first_page = start; 465 csrow->last_page = end; 466 467 dimm->nr_pages = end + 1 - start; 468 dimm->grain = 8; 469 dimm->mtype = mtype; 470 dimm->dtype = DEV_UNKNOWN; 471 if (sdram_ctl & DSC_X32_EN) 472 dimm->dtype = DEV_X32; 473 dimm->edac_mode = EDAC_SECDED; 474 } 475 } 476 477 int fsl_mc_err_probe(struct platform_device *op) 478 { 479 struct mem_ctl_info *mci; 480 struct edac_mc_layer layers[2]; 481 struct fsl_mc_pdata *pdata; 482 struct resource r; 483 u32 sdram_ctl; 484 int res; 485 486 if (!devres_open_group(&op->dev, fsl_mc_err_probe, GFP_KERNEL)) 487 return -ENOMEM; 488 489 layers[0].type = EDAC_MC_LAYER_CHIP_SELECT; 490 layers[0].size = 4; 491 layers[0].is_virt_csrow = true; 492 layers[1].type = EDAC_MC_LAYER_CHANNEL; 493 layers[1].size = 1; 494 layers[1].is_virt_csrow = false; 495 mci = edac_mc_alloc(edac_mc_idx, ARRAY_SIZE(layers), layers, 496 sizeof(*pdata)); 497 if (!mci) { 498 devres_release_group(&op->dev, fsl_mc_err_probe); 499 return -ENOMEM; 500 } 501 502 pdata = mci->pvt_info; 503 pdata->name = "fsl_mc_err"; 504 mci->pdev = &op->dev; 505 pdata->edac_idx = edac_mc_idx++; 506 dev_set_drvdata(mci->pdev, mci); 507 mci->ctl_name = pdata->name; 508 mci->dev_name = pdata->name; 509 510 /* 511 * Get the endianness of DDR controller registers. 512 * Default is big endian. 513 */ 514 little_endian = of_property_read_bool(op->dev.of_node, "little-endian"); 515 516 res = of_address_to_resource(op->dev.of_node, 0, &r); 517 if (res) { 518 pr_err("%s: Unable to get resource for MC err regs\n", 519 __func__); 520 goto err; 521 } 522 523 if (!devm_request_mem_region(&op->dev, r.start, resource_size(&r), 524 pdata->name)) { 525 pr_err("%s: Error while requesting mem region\n", 526 __func__); 527 res = -EBUSY; 528 goto err; 529 } 530 531 pdata->mc_vbase = devm_ioremap(&op->dev, r.start, resource_size(&r)); 532 if (!pdata->mc_vbase) { 533 pr_err("%s: Unable to setup MC err regs\n", __func__); 534 res = -ENOMEM; 535 goto err; 536 } 537 538 sdram_ctl = ddr_in32(pdata->mc_vbase + FSL_MC_DDR_SDRAM_CFG); 539 if (!(sdram_ctl & DSC_ECC_EN)) { 540 /* no ECC */ 541 pr_warn("%s: No ECC DIMMs discovered\n", __func__); 542 res = -ENODEV; 543 goto err; 544 } 545 546 edac_dbg(3, "init mci\n"); 547 mci->mtype_cap = MEM_FLAG_DDR | MEM_FLAG_RDDR | 548 MEM_FLAG_DDR2 | MEM_FLAG_RDDR2 | 549 MEM_FLAG_DDR3 | MEM_FLAG_RDDR3 | 550 MEM_FLAG_DDR4 | MEM_FLAG_RDDR4; 551 mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED; 552 mci->edac_cap = EDAC_FLAG_SECDED; 553 mci->mod_name = EDAC_MOD_STR; 554 555 if (edac_op_state == EDAC_OPSTATE_POLL) 556 mci->edac_check = fsl_mc_check; 557 558 mci->ctl_page_to_phys = NULL; 559 560 mci->scrub_mode = SCRUB_SW_SRC; 561 562 fsl_ddr_init_csrows(mci); 563 564 /* store the original error disable bits */ 565 orig_ddr_err_disable = ddr_in32(pdata->mc_vbase + FSL_MC_ERR_DISABLE); 566 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_DISABLE, 0); 567 568 /* clear all error bits */ 569 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_DETECT, ~0); 570 571 res = edac_mc_add_mc_with_groups(mci, fsl_ddr_dev_groups); 572 if (res) { 573 edac_dbg(3, "failed edac_mc_add_mc()\n"); 574 goto err; 575 } 576 577 if (edac_op_state == EDAC_OPSTATE_INT) { 578 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_INT_EN, 579 DDR_EIE_MBEE | DDR_EIE_SBEE); 580 581 /* store the original error management threshold */ 582 orig_ddr_err_sbe = ddr_in32(pdata->mc_vbase + 583 FSL_MC_ERR_SBE) & 0xff0000; 584 585 /* set threshold to 1 error per interrupt */ 586 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_SBE, 0x10000); 587 588 /* register interrupts */ 589 pdata->irq = platform_get_irq(op, 0); 590 res = devm_request_irq(&op->dev, pdata->irq, 591 fsl_mc_isr, 592 IRQF_SHARED, 593 "[EDAC] MC err", mci); 594 if (res < 0) { 595 pr_err("%s: Unable to request irq %d for FSL DDR DRAM ERR\n", 596 __func__, pdata->irq); 597 res = -ENODEV; 598 goto err2; 599 } 600 601 pr_info(EDAC_MOD_STR " acquired irq %d for MC\n", 602 pdata->irq); 603 } 604 605 devres_remove_group(&op->dev, fsl_mc_err_probe); 606 edac_dbg(3, "success\n"); 607 pr_info(EDAC_MOD_STR " MC err registered\n"); 608 609 return 0; 610 611 err2: 612 edac_mc_del_mc(&op->dev); 613 err: 614 devres_release_group(&op->dev, fsl_mc_err_probe); 615 edac_mc_free(mci); 616 return res; 617 } 618 619 int fsl_mc_err_remove(struct platform_device *op) 620 { 621 struct mem_ctl_info *mci = dev_get_drvdata(&op->dev); 622 struct fsl_mc_pdata *pdata = mci->pvt_info; 623 624 edac_dbg(0, "\n"); 625 626 if (edac_op_state == EDAC_OPSTATE_INT) { 627 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_INT_EN, 0); 628 } 629 630 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_DISABLE, 631 orig_ddr_err_disable); 632 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_SBE, orig_ddr_err_sbe); 633 634 edac_mc_del_mc(&op->dev); 635 edac_mc_free(mci); 636 return 0; 637 } 638