1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * MTK ECC controller driver. 4 * Copyright (C) 2016 MediaTek Inc. 5 * Authors: Xiaolei Li <xiaolei.li@mediatek.com> 6 * Jorge Ramirez-Ortiz <jorge.ramirez-ortiz@linaro.org> 7 */ 8 9 #include <linux/platform_device.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/interrupt.h> 12 #include <linux/clk.h> 13 #include <linux/module.h> 14 #include <linux/iopoll.h> 15 #include <linux/of.h> 16 #include <linux/of_platform.h> 17 #include <linux/mutex.h> 18 #include <linux/mtd/nand-ecc-mtk.h> 19 20 #define ECC_IDLE_MASK BIT(0) 21 #define ECC_IRQ_EN BIT(0) 22 #define ECC_PG_IRQ_SEL BIT(1) 23 #define ECC_OP_ENABLE (1) 24 #define ECC_OP_DISABLE (0) 25 26 #define ECC_ENCCON (0x00) 27 #define ECC_ENCCNFG (0x04) 28 #define ECC_MS_SHIFT (16) 29 #define ECC_ENCDIADDR (0x08) 30 #define ECC_ENCIDLE (0x0C) 31 #define ECC_DECCON (0x100) 32 #define ECC_DECCNFG (0x104) 33 #define DEC_EMPTY_EN BIT(31) 34 #define DEC_CNFG_CORRECT (0x3 << 12) 35 #define ECC_DECIDLE (0x10C) 36 #define ECC_DECENUM0 (0x114) 37 38 #define ECC_TIMEOUT (500000) 39 40 #define ECC_IDLE_REG(op) ((op) == ECC_ENCODE ? ECC_ENCIDLE : ECC_DECIDLE) 41 #define ECC_CTL_REG(op) ((op) == ECC_ENCODE ? ECC_ENCCON : ECC_DECCON) 42 43 struct mtk_ecc_caps { 44 u32 err_mask; 45 u32 err_shift; 46 const u8 *ecc_strength; 47 const u32 *ecc_regs; 48 u8 num_ecc_strength; 49 u8 ecc_mode_shift; 50 u32 parity_bits; 51 int pg_irq_sel; 52 }; 53 54 struct mtk_ecc { 55 struct device *dev; 56 const struct mtk_ecc_caps *caps; 57 void __iomem *regs; 58 struct clk *clk; 59 60 struct completion done; 61 struct mutex lock; 62 u32 sectors; 63 64 u8 *eccdata; 65 }; 66 67 /* ecc strength that each IP supports */ 68 static const u8 ecc_strength_mt2701[] = { 69 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 28, 32, 36, 70 40, 44, 48, 52, 56, 60 71 }; 72 73 static const u8 ecc_strength_mt2712[] = { 74 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 28, 32, 36, 75 40, 44, 48, 52, 56, 60, 68, 72, 80 76 }; 77 78 static const u8 ecc_strength_mt7622[] = { 79 4, 6, 8, 10, 12 80 }; 81 82 enum mtk_ecc_regs { 83 ECC_ENCPAR00, 84 ECC_ENCIRQ_EN, 85 ECC_ENCIRQ_STA, 86 ECC_DECDONE, 87 ECC_DECIRQ_EN, 88 ECC_DECIRQ_STA, 89 }; 90 91 static int mt2701_ecc_regs[] = { 92 [ECC_ENCPAR00] = 0x10, 93 [ECC_ENCIRQ_EN] = 0x80, 94 [ECC_ENCIRQ_STA] = 0x84, 95 [ECC_DECDONE] = 0x124, 96 [ECC_DECIRQ_EN] = 0x200, 97 [ECC_DECIRQ_STA] = 0x204, 98 }; 99 100 static int mt2712_ecc_regs[] = { 101 [ECC_ENCPAR00] = 0x300, 102 [ECC_ENCIRQ_EN] = 0x80, 103 [ECC_ENCIRQ_STA] = 0x84, 104 [ECC_DECDONE] = 0x124, 105 [ECC_DECIRQ_EN] = 0x200, 106 [ECC_DECIRQ_STA] = 0x204, 107 }; 108 109 static int mt7622_ecc_regs[] = { 110 [ECC_ENCPAR00] = 0x10, 111 [ECC_ENCIRQ_EN] = 0x30, 112 [ECC_ENCIRQ_STA] = 0x34, 113 [ECC_DECDONE] = 0x11c, 114 [ECC_DECIRQ_EN] = 0x140, 115 [ECC_DECIRQ_STA] = 0x144, 116 }; 117 118 static inline void mtk_ecc_wait_idle(struct mtk_ecc *ecc, 119 enum mtk_ecc_operation op) 120 { 121 struct device *dev = ecc->dev; 122 u32 val; 123 int ret; 124 125 ret = readl_poll_timeout_atomic(ecc->regs + ECC_IDLE_REG(op), val, 126 val & ECC_IDLE_MASK, 127 10, ECC_TIMEOUT); 128 if (ret) 129 dev_warn(dev, "%s NOT idle\n", 130 op == ECC_ENCODE ? "encoder" : "decoder"); 131 } 132 133 static irqreturn_t mtk_ecc_irq(int irq, void *id) 134 { 135 struct mtk_ecc *ecc = id; 136 u32 dec, enc; 137 138 dec = readw(ecc->regs + ecc->caps->ecc_regs[ECC_DECIRQ_STA]) 139 & ECC_IRQ_EN; 140 if (dec) { 141 dec = readw(ecc->regs + ecc->caps->ecc_regs[ECC_DECDONE]); 142 if (dec & ecc->sectors) { 143 /* 144 * Clear decode IRQ status once again to ensure that 145 * there will be no extra IRQ. 146 */ 147 readw(ecc->regs + ecc->caps->ecc_regs[ECC_DECIRQ_STA]); 148 ecc->sectors = 0; 149 complete(&ecc->done); 150 } else { 151 return IRQ_HANDLED; 152 } 153 } else { 154 enc = readl(ecc->regs + ecc->caps->ecc_regs[ECC_ENCIRQ_STA]) 155 & ECC_IRQ_EN; 156 if (enc) 157 complete(&ecc->done); 158 else 159 return IRQ_NONE; 160 } 161 162 return IRQ_HANDLED; 163 } 164 165 static int mtk_ecc_config(struct mtk_ecc *ecc, struct mtk_ecc_config *config) 166 { 167 u32 ecc_bit, dec_sz, enc_sz; 168 u32 reg, i; 169 170 for (i = 0; i < ecc->caps->num_ecc_strength; i++) { 171 if (ecc->caps->ecc_strength[i] == config->strength) 172 break; 173 } 174 175 if (i == ecc->caps->num_ecc_strength) { 176 dev_err(ecc->dev, "invalid ecc strength %d\n", 177 config->strength); 178 return -EINVAL; 179 } 180 181 ecc_bit = i; 182 183 if (config->op == ECC_ENCODE) { 184 /* configure ECC encoder (in bits) */ 185 enc_sz = config->len << 3; 186 187 reg = ecc_bit | (config->mode << ecc->caps->ecc_mode_shift); 188 reg |= (enc_sz << ECC_MS_SHIFT); 189 writel(reg, ecc->regs + ECC_ENCCNFG); 190 191 if (config->mode != ECC_NFI_MODE) 192 writel(lower_32_bits(config->addr), 193 ecc->regs + ECC_ENCDIADDR); 194 195 } else { 196 /* configure ECC decoder (in bits) */ 197 dec_sz = (config->len << 3) + 198 config->strength * ecc->caps->parity_bits; 199 200 reg = ecc_bit | (config->mode << ecc->caps->ecc_mode_shift); 201 reg |= (dec_sz << ECC_MS_SHIFT) | DEC_CNFG_CORRECT; 202 reg |= DEC_EMPTY_EN; 203 writel(reg, ecc->regs + ECC_DECCNFG); 204 205 if (config->sectors) 206 ecc->sectors = 1 << (config->sectors - 1); 207 } 208 209 return 0; 210 } 211 212 void mtk_ecc_get_stats(struct mtk_ecc *ecc, struct mtk_ecc_stats *stats, 213 int sectors) 214 { 215 u32 offset, i, err; 216 u32 bitflips = 0; 217 218 stats->corrected = 0; 219 stats->failed = 0; 220 221 for (i = 0; i < sectors; i++) { 222 offset = (i >> 2) << 2; 223 err = readl(ecc->regs + ECC_DECENUM0 + offset); 224 err = err >> ((i % 4) * ecc->caps->err_shift); 225 err &= ecc->caps->err_mask; 226 if (err == ecc->caps->err_mask) { 227 /* uncorrectable errors */ 228 stats->failed++; 229 continue; 230 } 231 232 stats->corrected += err; 233 bitflips = max_t(u32, bitflips, err); 234 } 235 236 stats->bitflips = bitflips; 237 } 238 EXPORT_SYMBOL(mtk_ecc_get_stats); 239 240 void mtk_ecc_release(struct mtk_ecc *ecc) 241 { 242 clk_disable_unprepare(ecc->clk); 243 put_device(ecc->dev); 244 } 245 EXPORT_SYMBOL(mtk_ecc_release); 246 247 static void mtk_ecc_hw_init(struct mtk_ecc *ecc) 248 { 249 mtk_ecc_wait_idle(ecc, ECC_ENCODE); 250 writew(ECC_OP_DISABLE, ecc->regs + ECC_ENCCON); 251 252 mtk_ecc_wait_idle(ecc, ECC_DECODE); 253 writel(ECC_OP_DISABLE, ecc->regs + ECC_DECCON); 254 } 255 256 static struct mtk_ecc *mtk_ecc_get(struct device_node *np) 257 { 258 struct platform_device *pdev; 259 struct mtk_ecc *ecc; 260 261 pdev = of_find_device_by_node(np); 262 if (!pdev) 263 return ERR_PTR(-EPROBE_DEFER); 264 265 ecc = platform_get_drvdata(pdev); 266 if (!ecc) { 267 put_device(&pdev->dev); 268 return ERR_PTR(-EPROBE_DEFER); 269 } 270 271 clk_prepare_enable(ecc->clk); 272 mtk_ecc_hw_init(ecc); 273 274 return ecc; 275 } 276 277 struct mtk_ecc *of_mtk_ecc_get(struct device_node *of_node) 278 { 279 struct mtk_ecc *ecc = NULL; 280 struct device_node *np; 281 282 np = of_parse_phandle(of_node, "nand-ecc-engine", 0); 283 /* for backward compatibility */ 284 if (!np) 285 np = of_parse_phandle(of_node, "ecc-engine", 0); 286 if (np) { 287 ecc = mtk_ecc_get(np); 288 of_node_put(np); 289 } 290 291 return ecc; 292 } 293 EXPORT_SYMBOL(of_mtk_ecc_get); 294 295 int mtk_ecc_enable(struct mtk_ecc *ecc, struct mtk_ecc_config *config) 296 { 297 enum mtk_ecc_operation op = config->op; 298 u16 reg_val; 299 int ret; 300 301 ret = mutex_lock_interruptible(&ecc->lock); 302 if (ret) { 303 dev_err(ecc->dev, "interrupted when attempting to lock\n"); 304 return ret; 305 } 306 307 mtk_ecc_wait_idle(ecc, op); 308 309 ret = mtk_ecc_config(ecc, config); 310 if (ret) { 311 mutex_unlock(&ecc->lock); 312 return ret; 313 } 314 315 if (config->mode != ECC_NFI_MODE || op != ECC_ENCODE) { 316 init_completion(&ecc->done); 317 reg_val = ECC_IRQ_EN; 318 /* 319 * For ECC_NFI_MODE, if ecc->caps->pg_irq_sel is 1, then it 320 * means this chip can only generate one ecc irq during page 321 * read / write. If is 0, generate one ecc irq each ecc step. 322 */ 323 if (ecc->caps->pg_irq_sel && config->mode == ECC_NFI_MODE) 324 reg_val |= ECC_PG_IRQ_SEL; 325 if (op == ECC_ENCODE) 326 writew(reg_val, ecc->regs + 327 ecc->caps->ecc_regs[ECC_ENCIRQ_EN]); 328 else 329 writew(reg_val, ecc->regs + 330 ecc->caps->ecc_regs[ECC_DECIRQ_EN]); 331 } 332 333 writew(ECC_OP_ENABLE, ecc->regs + ECC_CTL_REG(op)); 334 335 return 0; 336 } 337 EXPORT_SYMBOL(mtk_ecc_enable); 338 339 void mtk_ecc_disable(struct mtk_ecc *ecc) 340 { 341 enum mtk_ecc_operation op = ECC_ENCODE; 342 343 /* find out the running operation */ 344 if (readw(ecc->regs + ECC_CTL_REG(op)) != ECC_OP_ENABLE) 345 op = ECC_DECODE; 346 347 /* disable it */ 348 mtk_ecc_wait_idle(ecc, op); 349 if (op == ECC_DECODE) { 350 /* 351 * Clear decode IRQ status in case there is a timeout to wait 352 * decode IRQ. 353 */ 354 readw(ecc->regs + ecc->caps->ecc_regs[ECC_DECDONE]); 355 writew(0, ecc->regs + ecc->caps->ecc_regs[ECC_DECIRQ_EN]); 356 } else { 357 writew(0, ecc->regs + ecc->caps->ecc_regs[ECC_ENCIRQ_EN]); 358 } 359 360 writew(ECC_OP_DISABLE, ecc->regs + ECC_CTL_REG(op)); 361 362 mutex_unlock(&ecc->lock); 363 } 364 EXPORT_SYMBOL(mtk_ecc_disable); 365 366 int mtk_ecc_wait_done(struct mtk_ecc *ecc, enum mtk_ecc_operation op) 367 { 368 int ret; 369 370 ret = wait_for_completion_timeout(&ecc->done, msecs_to_jiffies(500)); 371 if (!ret) { 372 dev_err(ecc->dev, "%s timeout - interrupt did not arrive)\n", 373 (op == ECC_ENCODE) ? "encoder" : "decoder"); 374 return -ETIMEDOUT; 375 } 376 377 return 0; 378 } 379 EXPORT_SYMBOL(mtk_ecc_wait_done); 380 381 int mtk_ecc_encode(struct mtk_ecc *ecc, struct mtk_ecc_config *config, 382 u8 *data, u32 bytes) 383 { 384 dma_addr_t addr; 385 u32 len; 386 int ret; 387 388 addr = dma_map_single(ecc->dev, data, bytes, DMA_TO_DEVICE); 389 ret = dma_mapping_error(ecc->dev, addr); 390 if (ret) { 391 dev_err(ecc->dev, "dma mapping error\n"); 392 return -EINVAL; 393 } 394 395 config->op = ECC_ENCODE; 396 config->addr = addr; 397 ret = mtk_ecc_enable(ecc, config); 398 if (ret) { 399 dma_unmap_single(ecc->dev, addr, bytes, DMA_TO_DEVICE); 400 return ret; 401 } 402 403 ret = mtk_ecc_wait_done(ecc, ECC_ENCODE); 404 if (ret) 405 goto timeout; 406 407 mtk_ecc_wait_idle(ecc, ECC_ENCODE); 408 409 /* Program ECC bytes to OOB: per sector oob = FDM + ECC + SPARE */ 410 len = (config->strength * ecc->caps->parity_bits + 7) >> 3; 411 412 /* write the parity bytes generated by the ECC back to temp buffer */ 413 __ioread32_copy(ecc->eccdata, 414 ecc->regs + ecc->caps->ecc_regs[ECC_ENCPAR00], 415 round_up(len, 4)); 416 417 /* copy into possibly unaligned OOB region with actual length */ 418 memcpy(data + bytes, ecc->eccdata, len); 419 timeout: 420 421 dma_unmap_single(ecc->dev, addr, bytes, DMA_TO_DEVICE); 422 mtk_ecc_disable(ecc); 423 424 return ret; 425 } 426 EXPORT_SYMBOL(mtk_ecc_encode); 427 428 void mtk_ecc_adjust_strength(struct mtk_ecc *ecc, u32 *p) 429 { 430 const u8 *ecc_strength = ecc->caps->ecc_strength; 431 int i; 432 433 for (i = 0; i < ecc->caps->num_ecc_strength; i++) { 434 if (*p <= ecc_strength[i]) { 435 if (!i) 436 *p = ecc_strength[i]; 437 else if (*p != ecc_strength[i]) 438 *p = ecc_strength[i - 1]; 439 return; 440 } 441 } 442 443 *p = ecc_strength[ecc->caps->num_ecc_strength - 1]; 444 } 445 EXPORT_SYMBOL(mtk_ecc_adjust_strength); 446 447 unsigned int mtk_ecc_get_parity_bits(struct mtk_ecc *ecc) 448 { 449 return ecc->caps->parity_bits; 450 } 451 EXPORT_SYMBOL(mtk_ecc_get_parity_bits); 452 453 static const struct mtk_ecc_caps mtk_ecc_caps_mt2701 = { 454 .err_mask = 0x3f, 455 .err_shift = 8, 456 .ecc_strength = ecc_strength_mt2701, 457 .ecc_regs = mt2701_ecc_regs, 458 .num_ecc_strength = 20, 459 .ecc_mode_shift = 5, 460 .parity_bits = 14, 461 .pg_irq_sel = 0, 462 }; 463 464 static const struct mtk_ecc_caps mtk_ecc_caps_mt2712 = { 465 .err_mask = 0x7f, 466 .err_shift = 8, 467 .ecc_strength = ecc_strength_mt2712, 468 .ecc_regs = mt2712_ecc_regs, 469 .num_ecc_strength = 23, 470 .ecc_mode_shift = 5, 471 .parity_bits = 14, 472 .pg_irq_sel = 1, 473 }; 474 475 static const struct mtk_ecc_caps mtk_ecc_caps_mt7622 = { 476 .err_mask = 0x1f, 477 .err_shift = 5, 478 .ecc_strength = ecc_strength_mt7622, 479 .ecc_regs = mt7622_ecc_regs, 480 .num_ecc_strength = 5, 481 .ecc_mode_shift = 4, 482 .parity_bits = 13, 483 .pg_irq_sel = 0, 484 }; 485 486 static const struct of_device_id mtk_ecc_dt_match[] = { 487 { 488 .compatible = "mediatek,mt2701-ecc", 489 .data = &mtk_ecc_caps_mt2701, 490 }, { 491 .compatible = "mediatek,mt2712-ecc", 492 .data = &mtk_ecc_caps_mt2712, 493 }, { 494 .compatible = "mediatek,mt7622-ecc", 495 .data = &mtk_ecc_caps_mt7622, 496 }, 497 {}, 498 }; 499 500 static int mtk_ecc_probe(struct platform_device *pdev) 501 { 502 struct device *dev = &pdev->dev; 503 struct mtk_ecc *ecc; 504 u32 max_eccdata_size; 505 int irq, ret; 506 507 ecc = devm_kzalloc(dev, sizeof(*ecc), GFP_KERNEL); 508 if (!ecc) 509 return -ENOMEM; 510 511 ecc->caps = of_device_get_match_data(dev); 512 513 max_eccdata_size = ecc->caps->num_ecc_strength - 1; 514 max_eccdata_size = ecc->caps->ecc_strength[max_eccdata_size]; 515 max_eccdata_size = (max_eccdata_size * ecc->caps->parity_bits + 7) >> 3; 516 max_eccdata_size = round_up(max_eccdata_size, 4); 517 ecc->eccdata = devm_kzalloc(dev, max_eccdata_size, GFP_KERNEL); 518 if (!ecc->eccdata) 519 return -ENOMEM; 520 521 ecc->regs = devm_platform_ioremap_resource(pdev, 0); 522 if (IS_ERR(ecc->regs)) 523 return PTR_ERR(ecc->regs); 524 525 ecc->clk = devm_clk_get(dev, NULL); 526 if (IS_ERR(ecc->clk)) { 527 dev_err(dev, "failed to get clock: %ld\n", PTR_ERR(ecc->clk)); 528 return PTR_ERR(ecc->clk); 529 } 530 531 irq = platform_get_irq(pdev, 0); 532 if (irq < 0) 533 return irq; 534 535 ret = dma_set_mask(dev, DMA_BIT_MASK(32)); 536 if (ret) { 537 dev_err(dev, "failed to set DMA mask\n"); 538 return ret; 539 } 540 541 ret = devm_request_irq(dev, irq, mtk_ecc_irq, 0x0, "mtk-ecc", ecc); 542 if (ret) { 543 dev_err(dev, "failed to request irq\n"); 544 return -EINVAL; 545 } 546 547 ecc->dev = dev; 548 mutex_init(&ecc->lock); 549 platform_set_drvdata(pdev, ecc); 550 dev_info(dev, "probed\n"); 551 552 return 0; 553 } 554 555 #ifdef CONFIG_PM_SLEEP 556 static int mtk_ecc_suspend(struct device *dev) 557 { 558 struct mtk_ecc *ecc = dev_get_drvdata(dev); 559 560 clk_disable_unprepare(ecc->clk); 561 562 return 0; 563 } 564 565 static int mtk_ecc_resume(struct device *dev) 566 { 567 struct mtk_ecc *ecc = dev_get_drvdata(dev); 568 int ret; 569 570 ret = clk_prepare_enable(ecc->clk); 571 if (ret) { 572 dev_err(dev, "failed to enable clk\n"); 573 return ret; 574 } 575 576 return 0; 577 } 578 579 static SIMPLE_DEV_PM_OPS(mtk_ecc_pm_ops, mtk_ecc_suspend, mtk_ecc_resume); 580 #endif 581 582 MODULE_DEVICE_TABLE(of, mtk_ecc_dt_match); 583 584 static struct platform_driver mtk_ecc_driver = { 585 .probe = mtk_ecc_probe, 586 .driver = { 587 .name = "mtk-ecc", 588 .of_match_table = mtk_ecc_dt_match, 589 #ifdef CONFIG_PM_SLEEP 590 .pm = &mtk_ecc_pm_ops, 591 #endif 592 }, 593 }; 594 595 module_platform_driver(mtk_ecc_driver); 596 597 MODULE_AUTHOR("Xiaolei Li <xiaolei.li@mediatek.com>"); 598 MODULE_DESCRIPTION("MTK Nand ECC Driver"); 599 MODULE_LICENSE("Dual MIT/GPL"); 600