1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Freescale Memory Controller kernel module 4 * 5 * Support Power-based SoCs including MPC85xx, MPC86xx, MPC83xx and 6 * ARM-based Layerscape SoCs including LS2xxx and LS1021A. Originally 7 * split out from mpc85xx_edac EDAC driver. 8 * 9 * Parts Copyrighted (c) 2013 by Freescale Semiconductor, Inc. 10 * 11 * Author: Dave Jiang <djiang@mvista.com> 12 * 13 * 2006-2007 (c) MontaVista Software, Inc. 14 */ 15 #include <linux/module.h> 16 #include <linux/init.h> 17 #include <linux/interrupt.h> 18 #include <linux/ctype.h> 19 #include <linux/io.h> 20 #include <linux/mod_devicetable.h> 21 #include <linux/edac.h> 22 #include <linux/smp.h> 23 #include <linux/gfp.h> 24 25 #include <linux/of_platform.h> 26 #include <linux/of_device.h> 27 #include <linux/of_address.h> 28 #include "edac_module.h" 29 #include "fsl_ddr_edac.h" 30 31 #define EDAC_MOD_STR "fsl_ddr_edac" 32 33 static int edac_mc_idx; 34 35 static u32 orig_ddr_err_disable; 36 static u32 orig_ddr_err_sbe; 37 static bool little_endian; 38 39 static inline u32 ddr_in32(void __iomem *addr) 40 { 41 return little_endian ? ioread32(addr) : ioread32be(addr); 42 } 43 44 static inline void ddr_out32(void __iomem *addr, u32 value) 45 { 46 if (little_endian) 47 iowrite32(value, addr); 48 else 49 iowrite32be(value, addr); 50 } 51 52 #ifdef CONFIG_EDAC_DEBUG 53 /************************ MC SYSFS parts ***********************************/ 54 55 #define to_mci(k) container_of(k, struct mem_ctl_info, dev) 56 57 static ssize_t fsl_mc_inject_data_hi_show(struct device *dev, 58 struct device_attribute *mattr, 59 char *data) 60 { 61 struct mem_ctl_info *mci = to_mci(dev); 62 struct fsl_mc_pdata *pdata = mci->pvt_info; 63 return sprintf(data, "0x%08x", 64 ddr_in32(pdata->mc_vbase + FSL_MC_DATA_ERR_INJECT_HI)); 65 } 66 67 static ssize_t fsl_mc_inject_data_lo_show(struct device *dev, 68 struct device_attribute *mattr, 69 char *data) 70 { 71 struct mem_ctl_info *mci = to_mci(dev); 72 struct fsl_mc_pdata *pdata = mci->pvt_info; 73 return sprintf(data, "0x%08x", 74 ddr_in32(pdata->mc_vbase + FSL_MC_DATA_ERR_INJECT_LO)); 75 } 76 77 static ssize_t fsl_mc_inject_ctrl_show(struct device *dev, 78 struct device_attribute *mattr, 79 char *data) 80 { 81 struct mem_ctl_info *mci = to_mci(dev); 82 struct fsl_mc_pdata *pdata = mci->pvt_info; 83 return sprintf(data, "0x%08x", 84 ddr_in32(pdata->mc_vbase + FSL_MC_ECC_ERR_INJECT)); 85 } 86 87 static ssize_t fsl_mc_inject_data_hi_store(struct device *dev, 88 struct device_attribute *mattr, 89 const char *data, size_t count) 90 { 91 struct mem_ctl_info *mci = to_mci(dev); 92 struct fsl_mc_pdata *pdata = mci->pvt_info; 93 unsigned long val; 94 int rc; 95 96 if (isdigit(*data)) { 97 rc = kstrtoul(data, 0, &val); 98 if (rc) 99 return rc; 100 101 ddr_out32(pdata->mc_vbase + FSL_MC_DATA_ERR_INJECT_HI, val); 102 return count; 103 } 104 return 0; 105 } 106 107 static ssize_t fsl_mc_inject_data_lo_store(struct device *dev, 108 struct device_attribute *mattr, 109 const char *data, size_t count) 110 { 111 struct mem_ctl_info *mci = to_mci(dev); 112 struct fsl_mc_pdata *pdata = mci->pvt_info; 113 unsigned long val; 114 int rc; 115 116 if (isdigit(*data)) { 117 rc = kstrtoul(data, 0, &val); 118 if (rc) 119 return rc; 120 121 ddr_out32(pdata->mc_vbase + FSL_MC_DATA_ERR_INJECT_LO, val); 122 return count; 123 } 124 return 0; 125 } 126 127 static ssize_t fsl_mc_inject_ctrl_store(struct device *dev, 128 struct device_attribute *mattr, 129 const char *data, size_t count) 130 { 131 struct mem_ctl_info *mci = to_mci(dev); 132 struct fsl_mc_pdata *pdata = mci->pvt_info; 133 unsigned long val; 134 int rc; 135 136 if (isdigit(*data)) { 137 rc = kstrtoul(data, 0, &val); 138 if (rc) 139 return rc; 140 141 ddr_out32(pdata->mc_vbase + FSL_MC_ECC_ERR_INJECT, val); 142 return count; 143 } 144 return 0; 145 } 146 147 static DEVICE_ATTR(inject_data_hi, S_IRUGO | S_IWUSR, 148 fsl_mc_inject_data_hi_show, fsl_mc_inject_data_hi_store); 149 static DEVICE_ATTR(inject_data_lo, S_IRUGO | S_IWUSR, 150 fsl_mc_inject_data_lo_show, fsl_mc_inject_data_lo_store); 151 static DEVICE_ATTR(inject_ctrl, S_IRUGO | S_IWUSR, 152 fsl_mc_inject_ctrl_show, fsl_mc_inject_ctrl_store); 153 #endif /* CONFIG_EDAC_DEBUG */ 154 155 static struct attribute *fsl_ddr_dev_attrs[] = { 156 #ifdef CONFIG_EDAC_DEBUG 157 &dev_attr_inject_data_hi.attr, 158 &dev_attr_inject_data_lo.attr, 159 &dev_attr_inject_ctrl.attr, 160 #endif 161 NULL 162 }; 163 164 ATTRIBUTE_GROUPS(fsl_ddr_dev); 165 166 /**************************** MC Err device ***************************/ 167 168 /* 169 * Taken from table 8-55 in the MPC8641 User's Manual and/or 9-61 in the 170 * MPC8572 User's Manual. Each line represents a syndrome bit column as a 171 * 64-bit value, but split into an upper and lower 32-bit chunk. The labels 172 * below correspond to Freescale's manuals. 173 */ 174 static unsigned int ecc_table[16] = { 175 /* MSB LSB */ 176 /* [0:31] [32:63] */ 177 0xf00fe11e, 0xc33c0ff7, /* Syndrome bit 7 */ 178 0x00ff00ff, 0x00fff0ff, 179 0x0f0f0f0f, 0x0f0fff00, 180 0x11113333, 0x7777000f, 181 0x22224444, 0x8888222f, 182 0x44448888, 0xffff4441, 183 0x8888ffff, 0x11118882, 184 0xffff1111, 0x22221114, /* Syndrome bit 0 */ 185 }; 186 187 /* 188 * Calculate the correct ECC value for a 64-bit value specified by high:low 189 */ 190 static u8 calculate_ecc(u32 high, u32 low) 191 { 192 u32 mask_low; 193 u32 mask_high; 194 int bit_cnt; 195 u8 ecc = 0; 196 int i; 197 int j; 198 199 for (i = 0; i < 8; i++) { 200 mask_high = ecc_table[i * 2]; 201 mask_low = ecc_table[i * 2 + 1]; 202 bit_cnt = 0; 203 204 for (j = 0; j < 32; j++) { 205 if ((mask_high >> j) & 1) 206 bit_cnt ^= (high >> j) & 1; 207 if ((mask_low >> j) & 1) 208 bit_cnt ^= (low >> j) & 1; 209 } 210 211 ecc |= bit_cnt << i; 212 } 213 214 return ecc; 215 } 216 217 /* 218 * Create the syndrome code which is generated if the data line specified by 219 * 'bit' failed. Eg generate an 8-bit codes seen in Table 8-55 in the MPC8641 220 * User's Manual and 9-61 in the MPC8572 User's Manual. 221 */ 222 static u8 syndrome_from_bit(unsigned int bit) { 223 int i; 224 u8 syndrome = 0; 225 226 /* 227 * Cycle through the upper or lower 32-bit portion of each value in 228 * ecc_table depending on if 'bit' is in the upper or lower half of 229 * 64-bit data. 230 */ 231 for (i = bit < 32; i < 16; i += 2) 232 syndrome |= ((ecc_table[i] >> (bit % 32)) & 1) << (i / 2); 233 234 return syndrome; 235 } 236 237 /* 238 * Decode data and ecc syndrome to determine what went wrong 239 * Note: This can only decode single-bit errors 240 */ 241 static void sbe_ecc_decode(u32 cap_high, u32 cap_low, u32 cap_ecc, 242 int *bad_data_bit, int *bad_ecc_bit) 243 { 244 int i; 245 u8 syndrome; 246 247 *bad_data_bit = -1; 248 *bad_ecc_bit = -1; 249 250 /* 251 * Calculate the ECC of the captured data and XOR it with the captured 252 * ECC to find an ECC syndrome value we can search for 253 */ 254 syndrome = calculate_ecc(cap_high, cap_low) ^ cap_ecc; 255 256 /* Check if a data line is stuck... */ 257 for (i = 0; i < 64; i++) { 258 if (syndrome == syndrome_from_bit(i)) { 259 *bad_data_bit = i; 260 return; 261 } 262 } 263 264 /* If data is correct, check ECC bits for errors... */ 265 for (i = 0; i < 8; i++) { 266 if ((syndrome >> i) & 0x1) { 267 *bad_ecc_bit = i; 268 return; 269 } 270 } 271 } 272 273 #define make64(high, low) (((u64)(high) << 32) | (low)) 274 275 static void fsl_mc_check(struct mem_ctl_info *mci) 276 { 277 struct fsl_mc_pdata *pdata = mci->pvt_info; 278 struct csrow_info *csrow; 279 u32 bus_width; 280 u32 err_detect; 281 u32 syndrome; 282 u64 err_addr; 283 u32 pfn; 284 int row_index; 285 u32 cap_high; 286 u32 cap_low; 287 int bad_data_bit; 288 int bad_ecc_bit; 289 290 err_detect = ddr_in32(pdata->mc_vbase + FSL_MC_ERR_DETECT); 291 if (!err_detect) 292 return; 293 294 fsl_mc_printk(mci, KERN_ERR, "Err Detect Register: %#8.8x\n", 295 err_detect); 296 297 /* no more processing if not ECC bit errors */ 298 if (!(err_detect & (DDR_EDE_SBE | DDR_EDE_MBE))) { 299 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_DETECT, err_detect); 300 return; 301 } 302 303 syndrome = ddr_in32(pdata->mc_vbase + FSL_MC_CAPTURE_ECC); 304 305 /* Mask off appropriate bits of syndrome based on bus width */ 306 bus_width = (ddr_in32(pdata->mc_vbase + FSL_MC_DDR_SDRAM_CFG) & 307 DSC_DBW_MASK) ? 32 : 64; 308 if (bus_width == 64) 309 syndrome &= 0xff; 310 else 311 syndrome &= 0xffff; 312 313 err_addr = make64( 314 ddr_in32(pdata->mc_vbase + FSL_MC_CAPTURE_EXT_ADDRESS), 315 ddr_in32(pdata->mc_vbase + FSL_MC_CAPTURE_ADDRESS)); 316 pfn = err_addr >> PAGE_SHIFT; 317 318 for (row_index = 0; row_index < mci->nr_csrows; row_index++) { 319 csrow = mci->csrows[row_index]; 320 if ((pfn >= csrow->first_page) && (pfn <= csrow->last_page)) 321 break; 322 } 323 324 cap_high = ddr_in32(pdata->mc_vbase + FSL_MC_CAPTURE_DATA_HI); 325 cap_low = ddr_in32(pdata->mc_vbase + FSL_MC_CAPTURE_DATA_LO); 326 327 /* 328 * Analyze single-bit errors on 64-bit wide buses 329 * TODO: Add support for 32-bit wide buses 330 */ 331 if ((err_detect & DDR_EDE_SBE) && (bus_width == 64)) { 332 sbe_ecc_decode(cap_high, cap_low, syndrome, 333 &bad_data_bit, &bad_ecc_bit); 334 335 if (bad_data_bit != -1) 336 fsl_mc_printk(mci, KERN_ERR, 337 "Faulty Data bit: %d\n", bad_data_bit); 338 if (bad_ecc_bit != -1) 339 fsl_mc_printk(mci, KERN_ERR, 340 "Faulty ECC bit: %d\n", bad_ecc_bit); 341 342 fsl_mc_printk(mci, KERN_ERR, 343 "Expected Data / ECC:\t%#8.8x_%08x / %#2.2x\n", 344 cap_high ^ (1 << (bad_data_bit - 32)), 345 cap_low ^ (1 << bad_data_bit), 346 syndrome ^ (1 << bad_ecc_bit)); 347 } 348 349 fsl_mc_printk(mci, KERN_ERR, 350 "Captured Data / ECC:\t%#8.8x_%08x / %#2.2x\n", 351 cap_high, cap_low, syndrome); 352 fsl_mc_printk(mci, KERN_ERR, "Err addr: %#8.8llx\n", err_addr); 353 fsl_mc_printk(mci, KERN_ERR, "PFN: %#8.8x\n", pfn); 354 355 /* we are out of range */ 356 if (row_index == mci->nr_csrows) 357 fsl_mc_printk(mci, KERN_ERR, "PFN out of range!\n"); 358 359 if (err_detect & DDR_EDE_SBE) 360 edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 1, 361 pfn, err_addr & ~PAGE_MASK, syndrome, 362 row_index, 0, -1, 363 mci->ctl_name, ""); 364 365 if (err_detect & DDR_EDE_MBE) 366 edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1, 367 pfn, err_addr & ~PAGE_MASK, syndrome, 368 row_index, 0, -1, 369 mci->ctl_name, ""); 370 371 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_DETECT, err_detect); 372 } 373 374 static irqreturn_t fsl_mc_isr(int irq, void *dev_id) 375 { 376 struct mem_ctl_info *mci = dev_id; 377 struct fsl_mc_pdata *pdata = mci->pvt_info; 378 u32 err_detect; 379 380 err_detect = ddr_in32(pdata->mc_vbase + FSL_MC_ERR_DETECT); 381 if (!err_detect) 382 return IRQ_NONE; 383 384 fsl_mc_check(mci); 385 386 return IRQ_HANDLED; 387 } 388 389 static void fsl_ddr_init_csrows(struct mem_ctl_info *mci) 390 { 391 struct fsl_mc_pdata *pdata = mci->pvt_info; 392 struct csrow_info *csrow; 393 struct dimm_info *dimm; 394 u32 sdram_ctl; 395 u32 sdtype; 396 enum mem_type mtype; 397 u32 cs_bnds; 398 int index; 399 400 sdram_ctl = ddr_in32(pdata->mc_vbase + FSL_MC_DDR_SDRAM_CFG); 401 402 sdtype = sdram_ctl & DSC_SDTYPE_MASK; 403 if (sdram_ctl & DSC_RD_EN) { 404 switch (sdtype) { 405 case 0x02000000: 406 mtype = MEM_RDDR; 407 break; 408 case 0x03000000: 409 mtype = MEM_RDDR2; 410 break; 411 case 0x07000000: 412 mtype = MEM_RDDR3; 413 break; 414 case 0x05000000: 415 mtype = MEM_RDDR4; 416 break; 417 default: 418 mtype = MEM_UNKNOWN; 419 break; 420 } 421 } else { 422 switch (sdtype) { 423 case 0x02000000: 424 mtype = MEM_DDR; 425 break; 426 case 0x03000000: 427 mtype = MEM_DDR2; 428 break; 429 case 0x07000000: 430 mtype = MEM_DDR3; 431 break; 432 case 0x05000000: 433 mtype = MEM_DDR4; 434 break; 435 default: 436 mtype = MEM_UNKNOWN; 437 break; 438 } 439 } 440 441 for (index = 0; index < mci->nr_csrows; index++) { 442 u32 start; 443 u32 end; 444 445 csrow = mci->csrows[index]; 446 dimm = csrow->channels[0]->dimm; 447 448 cs_bnds = ddr_in32(pdata->mc_vbase + FSL_MC_CS_BNDS_0 + 449 (index * FSL_MC_CS_BNDS_OFS)); 450 451 start = (cs_bnds & 0xffff0000) >> 16; 452 end = (cs_bnds & 0x0000ffff); 453 454 if (start == end) 455 continue; /* not populated */ 456 457 start <<= (24 - PAGE_SHIFT); 458 end <<= (24 - PAGE_SHIFT); 459 end |= (1 << (24 - PAGE_SHIFT)) - 1; 460 461 csrow->first_page = start; 462 csrow->last_page = end; 463 464 dimm->nr_pages = end + 1 - start; 465 dimm->grain = 8; 466 dimm->mtype = mtype; 467 dimm->dtype = DEV_UNKNOWN; 468 if (sdram_ctl & DSC_X32_EN) 469 dimm->dtype = DEV_X32; 470 dimm->edac_mode = EDAC_SECDED; 471 } 472 } 473 474 int fsl_mc_err_probe(struct platform_device *op) 475 { 476 struct mem_ctl_info *mci; 477 struct edac_mc_layer layers[2]; 478 struct fsl_mc_pdata *pdata; 479 struct resource r; 480 u32 sdram_ctl; 481 int res; 482 483 if (!devres_open_group(&op->dev, fsl_mc_err_probe, GFP_KERNEL)) 484 return -ENOMEM; 485 486 layers[0].type = EDAC_MC_LAYER_CHIP_SELECT; 487 layers[0].size = 4; 488 layers[0].is_virt_csrow = true; 489 layers[1].type = EDAC_MC_LAYER_CHANNEL; 490 layers[1].size = 1; 491 layers[1].is_virt_csrow = false; 492 mci = edac_mc_alloc(edac_mc_idx, ARRAY_SIZE(layers), layers, 493 sizeof(*pdata)); 494 if (!mci) { 495 devres_release_group(&op->dev, fsl_mc_err_probe); 496 return -ENOMEM; 497 } 498 499 pdata = mci->pvt_info; 500 pdata->name = "fsl_mc_err"; 501 mci->pdev = &op->dev; 502 pdata->edac_idx = edac_mc_idx++; 503 dev_set_drvdata(mci->pdev, mci); 504 mci->ctl_name = pdata->name; 505 mci->dev_name = pdata->name; 506 507 /* 508 * Get the endianness of DDR controller registers. 509 * Default is big endian. 510 */ 511 little_endian = of_property_read_bool(op->dev.of_node, "little-endian"); 512 513 res = of_address_to_resource(op->dev.of_node, 0, &r); 514 if (res) { 515 pr_err("%s: Unable to get resource for MC err regs\n", 516 __func__); 517 goto err; 518 } 519 520 if (!devm_request_mem_region(&op->dev, r.start, resource_size(&r), 521 pdata->name)) { 522 pr_err("%s: Error while requesting mem region\n", 523 __func__); 524 res = -EBUSY; 525 goto err; 526 } 527 528 pdata->mc_vbase = devm_ioremap(&op->dev, r.start, resource_size(&r)); 529 if (!pdata->mc_vbase) { 530 pr_err("%s: Unable to setup MC err regs\n", __func__); 531 res = -ENOMEM; 532 goto err; 533 } 534 535 sdram_ctl = ddr_in32(pdata->mc_vbase + FSL_MC_DDR_SDRAM_CFG); 536 if (!(sdram_ctl & DSC_ECC_EN)) { 537 /* no ECC */ 538 pr_warn("%s: No ECC DIMMs discovered\n", __func__); 539 res = -ENODEV; 540 goto err; 541 } 542 543 edac_dbg(3, "init mci\n"); 544 mci->mtype_cap = MEM_FLAG_DDR | MEM_FLAG_RDDR | 545 MEM_FLAG_DDR2 | MEM_FLAG_RDDR2 | 546 MEM_FLAG_DDR3 | MEM_FLAG_RDDR3 | 547 MEM_FLAG_DDR4 | MEM_FLAG_RDDR4; 548 mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED; 549 mci->edac_cap = EDAC_FLAG_SECDED; 550 mci->mod_name = EDAC_MOD_STR; 551 552 if (edac_op_state == EDAC_OPSTATE_POLL) 553 mci->edac_check = fsl_mc_check; 554 555 mci->ctl_page_to_phys = NULL; 556 557 mci->scrub_mode = SCRUB_SW_SRC; 558 559 fsl_ddr_init_csrows(mci); 560 561 /* store the original error disable bits */ 562 orig_ddr_err_disable = ddr_in32(pdata->mc_vbase + FSL_MC_ERR_DISABLE); 563 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_DISABLE, 0); 564 565 /* clear all error bits */ 566 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_DETECT, ~0); 567 568 res = edac_mc_add_mc_with_groups(mci, fsl_ddr_dev_groups); 569 if (res) { 570 edac_dbg(3, "failed edac_mc_add_mc()\n"); 571 goto err; 572 } 573 574 if (edac_op_state == EDAC_OPSTATE_INT) { 575 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_INT_EN, 576 DDR_EIE_MBEE | DDR_EIE_SBEE); 577 578 /* store the original error management threshold */ 579 orig_ddr_err_sbe = ddr_in32(pdata->mc_vbase + 580 FSL_MC_ERR_SBE) & 0xff0000; 581 582 /* set threshold to 1 error per interrupt */ 583 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_SBE, 0x10000); 584 585 /* register interrupts */ 586 pdata->irq = platform_get_irq(op, 0); 587 res = devm_request_irq(&op->dev, pdata->irq, 588 fsl_mc_isr, 589 IRQF_SHARED, 590 "[EDAC] MC err", mci); 591 if (res < 0) { 592 pr_err("%s: Unable to request irq %d for FSL DDR DRAM ERR\n", 593 __func__, pdata->irq); 594 res = -ENODEV; 595 goto err2; 596 } 597 598 pr_info(EDAC_MOD_STR " acquired irq %d for MC\n", 599 pdata->irq); 600 } 601 602 devres_remove_group(&op->dev, fsl_mc_err_probe); 603 edac_dbg(3, "success\n"); 604 pr_info(EDAC_MOD_STR " MC err registered\n"); 605 606 return 0; 607 608 err2: 609 edac_mc_del_mc(&op->dev); 610 err: 611 devres_release_group(&op->dev, fsl_mc_err_probe); 612 edac_mc_free(mci); 613 return res; 614 } 615 616 int fsl_mc_err_remove(struct platform_device *op) 617 { 618 struct mem_ctl_info *mci = dev_get_drvdata(&op->dev); 619 struct fsl_mc_pdata *pdata = mci->pvt_info; 620 621 edac_dbg(0, "\n"); 622 623 if (edac_op_state == EDAC_OPSTATE_INT) { 624 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_INT_EN, 0); 625 } 626 627 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_DISABLE, 628 orig_ddr_err_disable); 629 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_SBE, orig_ddr_err_sbe); 630 631 edac_mc_del_mc(&op->dev); 632 edac_mc_free(mci); 633 return 0; 634 } 635