1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Support for Macronix external hardware ECC engine for NAND devices, also 4 * called DPE for Data Processing Engine. 5 * 6 * Copyright © 2019 Macronix 7 * Author: Miquel Raynal <miquel.raynal@bootlin.com> 8 */ 9 10 #include <linux/dma-mapping.h> 11 #include <linux/init.h> 12 #include <linux/interrupt.h> 13 #include <linux/io.h> 14 #include <linux/iopoll.h> 15 #include <linux/kernel.h> 16 #include <linux/module.h> 17 #include <linux/mtd/mtd.h> 18 #include <linux/mtd/nand.h> 19 #include <linux/mtd/nand-ecc-mxic.h> 20 #include <linux/mutex.h> 21 #include <linux/of_device.h> 22 #include <linux/of_platform.h> 23 #include <linux/platform_device.h> 24 #include <linux/slab.h> 25 26 /* DPE Configuration */ 27 #define DP_CONFIG 0x00 28 #define ECC_EN BIT(0) 29 #define ECC_TYP(idx) (((idx) << 3) & GENMASK(6, 3)) 30 /* DPE Interrupt Status */ 31 #define INTRPT_STS 0x04 32 #define TRANS_CMPLT BIT(0) 33 #define SDMA_MAIN BIT(1) 34 #define SDMA_SPARE BIT(2) 35 #define ECC_ERR BIT(3) 36 #define TO_SPARE BIT(4) 37 #define TO_MAIN BIT(5) 38 /* DPE Interrupt Status Enable */ 39 #define INTRPT_STS_EN 0x08 40 /* DPE Interrupt Signal Enable */ 41 #define INTRPT_SIG_EN 0x0C 42 /* Host Controller Configuration */ 43 #define HC_CONFIG 0x10 44 #define DEV2MEM 0 /* TRANS_TYP_DMA in the spec */ 45 #define MEM2MEM BIT(4) /* TRANS_TYP_IO in the spec */ 46 #define MAPPING BIT(5) /* TRANS_TYP_MAPPING in the spec */ 47 #define ECC_PACKED 0 /* LAYOUT_TYP_INTEGRATED in the spec */ 48 #define ECC_INTERLEAVED BIT(2) /* LAYOUT_TYP_DISTRIBUTED in the spec */ 49 #define BURST_TYP_FIXED 0 50 #define BURST_TYP_INCREASING BIT(0) 51 /* Host Controller Slave Address */ 52 #define HC_SLV_ADDR 0x14 53 /* ECC Chunk Size */ 54 #define CHUNK_SIZE 0x20 55 /* Main Data Size */ 56 #define MAIN_SIZE 0x24 57 /* Spare Data Size */ 58 #define SPARE_SIZE 0x28 59 #define META_SZ(reg) ((reg) & GENMASK(7, 0)) 60 #define PARITY_SZ(reg) (((reg) & GENMASK(15, 8)) >> 8) 61 #define RSV_SZ(reg) (((reg) & GENMASK(23, 16)) >> 16) 62 #define SPARE_SZ(reg) ((reg) >> 24) 63 /* ECC Chunk Count */ 64 #define CHUNK_CNT 0x30 65 /* SDMA Control */ 66 #define SDMA_CTRL 0x40 67 #define WRITE_NAND 0 68 #define READ_NAND BIT(1) 69 #define CONT_NAND BIT(29) 70 #define CONT_SYSM BIT(30) /* Continue System Memory? */ 71 #define SDMA_STRT BIT(31) 72 /* SDMA Address of Main Data */ 73 #define SDMA_MAIN_ADDR 0x44 74 /* SDMA Address of Spare Data */ 75 #define SDMA_SPARE_ADDR 0x48 76 /* DPE Version Number */ 77 #define DP_VER 0xD0 78 #define DP_VER_OFFSET 16 79 80 /* Status bytes between each chunk of spare data */ 81 #define STAT_BYTES 4 82 #define NO_ERR 0x00 83 #define MAX_CORR_ERR 0x28 84 #define UNCORR_ERR 0xFE 85 #define ERASED_CHUNK 0xFF 86 87 struct mxic_ecc_engine { 88 struct device *dev; 89 void __iomem *regs; 90 int irq; 91 struct completion complete; 92 struct nand_ecc_engine external_engine; 93 struct nand_ecc_engine pipelined_engine; 94 struct mutex lock; 95 }; 96 97 struct mxic_ecc_ctx { 98 /* ECC machinery */ 99 unsigned int data_step_sz; 100 unsigned int oob_step_sz; 101 unsigned int parity_sz; 102 unsigned int meta_sz; 103 u8 *status; 104 int steps; 105 106 /* DMA boilerplate */ 107 struct nand_ecc_req_tweak_ctx req_ctx; 108 u8 *oobwithstat; 109 struct scatterlist sg[2]; 110 struct nand_page_io_req *req; 111 unsigned int pageoffs; 112 }; 113 114 static struct mxic_ecc_engine *ext_ecc_eng_to_mxic(struct nand_ecc_engine *eng) 115 { 116 return container_of(eng, struct mxic_ecc_engine, external_engine); 117 } 118 119 static struct mxic_ecc_engine *pip_ecc_eng_to_mxic(struct nand_ecc_engine *eng) 120 { 121 return container_of(eng, struct mxic_ecc_engine, pipelined_engine); 122 } 123 124 static struct mxic_ecc_engine *nand_to_mxic(struct nand_device *nand) 125 { 126 struct nand_ecc_engine *eng = nand->ecc.engine; 127 128 if (eng->integration == NAND_ECC_ENGINE_INTEGRATION_EXTERNAL) 129 return ext_ecc_eng_to_mxic(eng); 130 else 131 return pip_ecc_eng_to_mxic(eng); 132 } 133 134 static int mxic_ecc_ooblayout_ecc(struct mtd_info *mtd, int section, 135 struct mtd_oob_region *oobregion) 136 { 137 struct nand_device *nand = mtd_to_nanddev(mtd); 138 struct mxic_ecc_ctx *ctx = nand_to_ecc_ctx(nand); 139 140 if (section < 0 || section >= ctx->steps) 141 return -ERANGE; 142 143 oobregion->offset = (section * ctx->oob_step_sz) + ctx->meta_sz; 144 oobregion->length = ctx->parity_sz; 145 146 return 0; 147 } 148 149 static int mxic_ecc_ooblayout_free(struct mtd_info *mtd, int section, 150 struct mtd_oob_region *oobregion) 151 { 152 struct nand_device *nand = mtd_to_nanddev(mtd); 153 struct mxic_ecc_ctx *ctx = nand_to_ecc_ctx(nand); 154 155 if (section < 0 || section >= ctx->steps) 156 return -ERANGE; 157 158 if (!section) { 159 oobregion->offset = 2; 160 oobregion->length = ctx->meta_sz - 2; 161 } else { 162 oobregion->offset = section * ctx->oob_step_sz; 163 oobregion->length = ctx->meta_sz; 164 } 165 166 return 0; 167 } 168 169 static const struct mtd_ooblayout_ops mxic_ecc_ooblayout_ops = { 170 .ecc = mxic_ecc_ooblayout_ecc, 171 .free = mxic_ecc_ooblayout_free, 172 }; 173 174 static void mxic_ecc_disable_engine(struct mxic_ecc_engine *mxic) 175 { 176 u32 reg; 177 178 reg = readl(mxic->regs + DP_CONFIG); 179 reg &= ~ECC_EN; 180 writel(reg, mxic->regs + DP_CONFIG); 181 } 182 183 static void mxic_ecc_enable_engine(struct mxic_ecc_engine *mxic) 184 { 185 u32 reg; 186 187 reg = readl(mxic->regs + DP_CONFIG); 188 reg |= ECC_EN; 189 writel(reg, mxic->regs + DP_CONFIG); 190 } 191 192 static void mxic_ecc_disable_int(struct mxic_ecc_engine *mxic) 193 { 194 writel(0, mxic->regs + INTRPT_SIG_EN); 195 } 196 197 static void mxic_ecc_enable_int(struct mxic_ecc_engine *mxic) 198 { 199 writel(TRANS_CMPLT, mxic->regs + INTRPT_SIG_EN); 200 } 201 202 static irqreturn_t mxic_ecc_isr(int irq, void *dev_id) 203 { 204 struct mxic_ecc_engine *mxic = dev_id; 205 u32 sts; 206 207 sts = readl(mxic->regs + INTRPT_STS); 208 if (!sts) 209 return IRQ_NONE; 210 211 if (sts & TRANS_CMPLT) 212 complete(&mxic->complete); 213 214 writel(sts, mxic->regs + INTRPT_STS); 215 216 return IRQ_HANDLED; 217 } 218 219 static int mxic_ecc_init_ctx(struct nand_device *nand, struct device *dev) 220 { 221 struct mxic_ecc_engine *mxic = nand_to_mxic(nand); 222 struct nand_ecc_props *conf = &nand->ecc.ctx.conf; 223 struct nand_ecc_props *reqs = &nand->ecc.requirements; 224 struct nand_ecc_props *user = &nand->ecc.user_conf; 225 struct mtd_info *mtd = nanddev_to_mtd(nand); 226 int step_size = 0, strength = 0, desired_correction = 0, steps, idx; 227 static const int possible_strength[] = {4, 8, 40, 48}; 228 static const int spare_size[] = {32, 32, 96, 96}; 229 struct mxic_ecc_ctx *ctx; 230 u32 spare_reg; 231 int ret; 232 233 ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL); 234 if (!ctx) 235 return -ENOMEM; 236 237 nand->ecc.ctx.priv = ctx; 238 239 /* Only large page NAND chips may use BCH */ 240 if (mtd->oobsize < 64) { 241 pr_err("BCH cannot be used with small page NAND chips\n"); 242 return -EINVAL; 243 } 244 245 mtd_set_ooblayout(mtd, &mxic_ecc_ooblayout_ops); 246 247 /* Enable all status bits */ 248 writel(TRANS_CMPLT | SDMA_MAIN | SDMA_SPARE | ECC_ERR | 249 TO_SPARE | TO_MAIN, mxic->regs + INTRPT_STS_EN); 250 251 /* Configure the correction depending on the NAND device topology */ 252 if (user->step_size && user->strength) { 253 step_size = user->step_size; 254 strength = user->strength; 255 } else if (reqs->step_size && reqs->strength) { 256 step_size = reqs->step_size; 257 strength = reqs->strength; 258 } 259 260 if (step_size && strength) { 261 steps = mtd->writesize / step_size; 262 desired_correction = steps * strength; 263 } 264 265 /* Step size is fixed to 1kiB, strength may vary (4 possible values) */ 266 conf->step_size = SZ_1K; 267 steps = mtd->writesize / conf->step_size; 268 269 ctx->status = devm_kzalloc(dev, steps * sizeof(u8), GFP_KERNEL); 270 if (!ctx->status) 271 return -ENOMEM; 272 273 if (desired_correction) { 274 strength = desired_correction / steps; 275 276 for (idx = 0; idx < ARRAY_SIZE(possible_strength); idx++) 277 if (possible_strength[idx] >= strength) 278 break; 279 280 idx = min_t(unsigned int, idx, 281 ARRAY_SIZE(possible_strength) - 1); 282 } else { 283 /* Missing data, maximize the correction */ 284 idx = ARRAY_SIZE(possible_strength) - 1; 285 } 286 287 /* Tune the selected strength until it fits in the OOB area */ 288 for (; idx >= 0; idx--) { 289 if (spare_size[idx] * steps <= mtd->oobsize) 290 break; 291 } 292 293 /* This engine cannot be used with this NAND device */ 294 if (idx < 0) 295 return -EINVAL; 296 297 /* Configure the engine for the desired strength */ 298 writel(ECC_TYP(idx), mxic->regs + DP_CONFIG); 299 conf->strength = possible_strength[idx]; 300 spare_reg = readl(mxic->regs + SPARE_SIZE); 301 302 ctx->steps = steps; 303 ctx->data_step_sz = mtd->writesize / steps; 304 ctx->oob_step_sz = mtd->oobsize / steps; 305 ctx->parity_sz = PARITY_SZ(spare_reg); 306 ctx->meta_sz = META_SZ(spare_reg); 307 308 /* Ensure buffers will contain enough bytes to store the STAT_BYTES */ 309 ctx->req_ctx.oob_buffer_size = nanddev_per_page_oobsize(nand) + 310 (ctx->steps * STAT_BYTES); 311 ret = nand_ecc_init_req_tweaking(&ctx->req_ctx, nand); 312 if (ret) 313 return ret; 314 315 ctx->oobwithstat = kmalloc(mtd->oobsize + (ctx->steps * STAT_BYTES), 316 GFP_KERNEL); 317 if (!ctx->oobwithstat) { 318 ret = -ENOMEM; 319 goto cleanup_req_tweak; 320 } 321 322 sg_init_table(ctx->sg, 2); 323 324 /* Configuration dump and sanity checks */ 325 dev_err(dev, "DPE version number: %d\n", 326 readl(mxic->regs + DP_VER) >> DP_VER_OFFSET); 327 dev_err(dev, "Chunk size: %d\n", readl(mxic->regs + CHUNK_SIZE)); 328 dev_err(dev, "Main size: %d\n", readl(mxic->regs + MAIN_SIZE)); 329 dev_err(dev, "Spare size: %d\n", SPARE_SZ(spare_reg)); 330 dev_err(dev, "Rsv size: %ld\n", RSV_SZ(spare_reg)); 331 dev_err(dev, "Parity size: %d\n", ctx->parity_sz); 332 dev_err(dev, "Meta size: %d\n", ctx->meta_sz); 333 334 if ((ctx->meta_sz + ctx->parity_sz + RSV_SZ(spare_reg)) != 335 SPARE_SZ(spare_reg)) { 336 dev_err(dev, "Wrong OOB configuration: %d + %d + %ld != %d\n", 337 ctx->meta_sz, ctx->parity_sz, RSV_SZ(spare_reg), 338 SPARE_SZ(spare_reg)); 339 ret = -EINVAL; 340 goto free_oobwithstat; 341 } 342 343 if (ctx->oob_step_sz != SPARE_SZ(spare_reg)) { 344 dev_err(dev, "Wrong OOB configuration: %d != %d\n", 345 ctx->oob_step_sz, SPARE_SZ(spare_reg)); 346 ret = -EINVAL; 347 goto free_oobwithstat; 348 } 349 350 return 0; 351 352 free_oobwithstat: 353 kfree(ctx->oobwithstat); 354 cleanup_req_tweak: 355 nand_ecc_cleanup_req_tweaking(&ctx->req_ctx); 356 357 return ret; 358 } 359 360 static int mxic_ecc_init_ctx_external(struct nand_device *nand) 361 { 362 struct mxic_ecc_engine *mxic = nand_to_mxic(nand); 363 struct device *dev = nand->ecc.engine->dev; 364 int ret; 365 366 dev_info(dev, "Macronix ECC engine in external mode\n"); 367 368 ret = mxic_ecc_init_ctx(nand, dev); 369 if (ret) 370 return ret; 371 372 /* Trigger each step manually */ 373 writel(1, mxic->regs + CHUNK_CNT); 374 writel(BURST_TYP_INCREASING | ECC_PACKED | MEM2MEM, 375 mxic->regs + HC_CONFIG); 376 377 return 0; 378 } 379 380 static int mxic_ecc_init_ctx_pipelined(struct nand_device *nand) 381 { 382 struct mxic_ecc_engine *mxic = nand_to_mxic(nand); 383 struct mxic_ecc_ctx *ctx; 384 struct device *dev; 385 int ret; 386 387 dev = nand_ecc_get_engine_dev(nand->ecc.engine->dev); 388 if (!dev) 389 return -EINVAL; 390 391 dev_info(dev, "Macronix ECC engine in pipelined/mapping mode\n"); 392 393 ret = mxic_ecc_init_ctx(nand, dev); 394 if (ret) 395 return ret; 396 397 ctx = nand_to_ecc_ctx(nand); 398 399 /* All steps should be handled in one go directly by the internal DMA */ 400 writel(ctx->steps, mxic->regs + CHUNK_CNT); 401 402 /* 403 * Interleaved ECC scheme cannot be used otherwise factory bad block 404 * markers would be lost. A packed layout is mandatory. 405 */ 406 writel(BURST_TYP_INCREASING | ECC_PACKED | MAPPING, 407 mxic->regs + HC_CONFIG); 408 409 return 0; 410 } 411 412 static void mxic_ecc_cleanup_ctx(struct nand_device *nand) 413 { 414 struct mxic_ecc_ctx *ctx = nand_to_ecc_ctx(nand); 415 416 if (ctx) { 417 nand_ecc_cleanup_req_tweaking(&ctx->req_ctx); 418 kfree(ctx->oobwithstat); 419 } 420 } 421 422 static int mxic_ecc_data_xfer_wait_for_completion(struct mxic_ecc_engine *mxic) 423 { 424 u32 val; 425 int ret; 426 427 if (mxic->irq) { 428 reinit_completion(&mxic->complete); 429 mxic_ecc_enable_int(mxic); 430 ret = wait_for_completion_timeout(&mxic->complete, 431 msecs_to_jiffies(1000)); 432 mxic_ecc_disable_int(mxic); 433 } else { 434 ret = readl_poll_timeout(mxic->regs + INTRPT_STS, val, 435 val & TRANS_CMPLT, 10, USEC_PER_SEC); 436 writel(val, mxic->regs + INTRPT_STS); 437 } 438 439 if (ret) { 440 dev_err(mxic->dev, "Timeout on data xfer completion\n"); 441 return -ETIMEDOUT; 442 } 443 444 return 0; 445 } 446 447 static int mxic_ecc_process_data(struct mxic_ecc_engine *mxic, 448 unsigned int direction) 449 { 450 unsigned int dir = (direction == NAND_PAGE_READ) ? 451 READ_NAND : WRITE_NAND; 452 int ret; 453 454 mxic_ecc_enable_engine(mxic); 455 456 /* Trigger processing */ 457 writel(SDMA_STRT | dir, mxic->regs + SDMA_CTRL); 458 459 /* Wait for completion */ 460 ret = mxic_ecc_data_xfer_wait_for_completion(mxic); 461 462 mxic_ecc_disable_engine(mxic); 463 464 return ret; 465 } 466 467 int mxic_ecc_process_data_pipelined(struct nand_ecc_engine *eng, 468 unsigned int direction, dma_addr_t dirmap) 469 { 470 struct mxic_ecc_engine *mxic = pip_ecc_eng_to_mxic(eng); 471 472 if (dirmap) 473 writel(dirmap, mxic->regs + HC_SLV_ADDR); 474 475 return mxic_ecc_process_data(mxic, direction); 476 } 477 EXPORT_SYMBOL_GPL(mxic_ecc_process_data_pipelined); 478 479 static void mxic_ecc_extract_status_bytes(struct mxic_ecc_ctx *ctx) 480 { 481 u8 *buf = ctx->oobwithstat; 482 int next_stat_pos; 483 int step; 484 485 /* Extract the ECC status */ 486 for (step = 0; step < ctx->steps; step++) { 487 next_stat_pos = ctx->oob_step_sz + 488 ((STAT_BYTES + ctx->oob_step_sz) * step); 489 490 ctx->status[step] = buf[next_stat_pos]; 491 } 492 } 493 494 static void mxic_ecc_reconstruct_oobbuf(struct mxic_ecc_ctx *ctx, 495 u8 *dst, const u8 *src) 496 { 497 int step; 498 499 /* Reconstruct the OOB buffer linearly (without the ECC status bytes) */ 500 for (step = 0; step < ctx->steps; step++) 501 memcpy(dst + (step * ctx->oob_step_sz), 502 src + (step * (ctx->oob_step_sz + STAT_BYTES)), 503 ctx->oob_step_sz); 504 } 505 506 static void mxic_ecc_add_room_in_oobbuf(struct mxic_ecc_ctx *ctx, 507 u8 *dst, const u8 *src) 508 { 509 int step; 510 511 /* Add some space in the OOB buffer for the status bytes */ 512 for (step = 0; step < ctx->steps; step++) 513 memcpy(dst + (step * (ctx->oob_step_sz + STAT_BYTES)), 514 src + (step * ctx->oob_step_sz), 515 ctx->oob_step_sz); 516 } 517 518 static int mxic_ecc_count_biterrs(struct mxic_ecc_engine *mxic, 519 struct nand_device *nand) 520 { 521 struct mxic_ecc_ctx *ctx = nand_to_ecc_ctx(nand); 522 struct mtd_info *mtd = nanddev_to_mtd(nand); 523 struct device *dev = mxic->dev; 524 unsigned int max_bf = 0; 525 bool failure = false; 526 int step; 527 528 for (step = 0; step < ctx->steps; step++) { 529 u8 stat = ctx->status[step]; 530 531 if (stat == NO_ERR) { 532 dev_dbg(dev, "ECC step %d: no error\n", step); 533 } else if (stat == ERASED_CHUNK) { 534 dev_dbg(dev, "ECC step %d: erased\n", step); 535 } else if (stat == UNCORR_ERR || stat > MAX_CORR_ERR) { 536 dev_dbg(dev, "ECC step %d: uncorrectable\n", step); 537 mtd->ecc_stats.failed++; 538 failure = true; 539 } else { 540 dev_dbg(dev, "ECC step %d: %d bits corrected\n", 541 step, stat); 542 max_bf = max_t(unsigned int, max_bf, stat); 543 mtd->ecc_stats.corrected += stat; 544 } 545 } 546 547 return failure ? -EBADMSG : max_bf; 548 } 549 550 /* External ECC engine helpers */ 551 static int mxic_ecc_prepare_io_req_external(struct nand_device *nand, 552 struct nand_page_io_req *req) 553 { 554 struct mxic_ecc_engine *mxic = nand_to_mxic(nand); 555 struct mxic_ecc_ctx *ctx = nand_to_ecc_ctx(nand); 556 struct mtd_info *mtd = nanddev_to_mtd(nand); 557 int offset, nents, step, ret; 558 559 if (req->mode == MTD_OPS_RAW) 560 return 0; 561 562 nand_ecc_tweak_req(&ctx->req_ctx, req); 563 ctx->req = req; 564 565 if (req->type == NAND_PAGE_READ) 566 return 0; 567 568 mxic_ecc_add_room_in_oobbuf(ctx, ctx->oobwithstat, 569 ctx->req->oobbuf.out); 570 571 sg_set_buf(&ctx->sg[0], req->databuf.out, req->datalen); 572 sg_set_buf(&ctx->sg[1], ctx->oobwithstat, 573 req->ooblen + (ctx->steps * STAT_BYTES)); 574 575 nents = dma_map_sg(mxic->dev, ctx->sg, 2, DMA_BIDIRECTIONAL); 576 if (!nents) 577 return -EINVAL; 578 579 mutex_lock(&mxic->lock); 580 581 for (step = 0; step < ctx->steps; step++) { 582 writel(sg_dma_address(&ctx->sg[0]) + (step * ctx->data_step_sz), 583 mxic->regs + SDMA_MAIN_ADDR); 584 writel(sg_dma_address(&ctx->sg[1]) + (step * (ctx->oob_step_sz + STAT_BYTES)), 585 mxic->regs + SDMA_SPARE_ADDR); 586 ret = mxic_ecc_process_data(mxic, ctx->req->type); 587 if (ret) 588 break; 589 } 590 591 mutex_unlock(&mxic->lock); 592 593 dma_unmap_sg(mxic->dev, ctx->sg, 2, DMA_BIDIRECTIONAL); 594 595 if (ret) 596 return ret; 597 598 /* Retrieve the calculated ECC bytes */ 599 for (step = 0; step < ctx->steps; step++) { 600 offset = ctx->meta_sz + (step * ctx->oob_step_sz); 601 mtd_ooblayout_get_eccbytes(mtd, 602 (u8 *)ctx->req->oobbuf.out + offset, 603 ctx->oobwithstat + (step * STAT_BYTES), 604 step * ctx->parity_sz, 605 ctx->parity_sz); 606 } 607 608 return 0; 609 } 610 611 static int mxic_ecc_finish_io_req_external(struct nand_device *nand, 612 struct nand_page_io_req *req) 613 { 614 struct mxic_ecc_engine *mxic = nand_to_mxic(nand); 615 struct mxic_ecc_ctx *ctx = nand_to_ecc_ctx(nand); 616 int nents, step, ret; 617 618 if (req->mode == MTD_OPS_RAW) 619 return 0; 620 621 if (req->type == NAND_PAGE_WRITE) { 622 nand_ecc_restore_req(&ctx->req_ctx, req); 623 return 0; 624 } 625 626 /* Copy the OOB buffer and add room for the ECC engine status bytes */ 627 mxic_ecc_add_room_in_oobbuf(ctx, ctx->oobwithstat, ctx->req->oobbuf.in); 628 629 sg_set_buf(&ctx->sg[0], req->databuf.in, req->datalen); 630 sg_set_buf(&ctx->sg[1], ctx->oobwithstat, 631 req->ooblen + (ctx->steps * STAT_BYTES)); 632 nents = dma_map_sg(mxic->dev, ctx->sg, 2, DMA_BIDIRECTIONAL); 633 if (!nents) 634 return -EINVAL; 635 636 mutex_lock(&mxic->lock); 637 638 for (step = 0; step < ctx->steps; step++) { 639 writel(sg_dma_address(&ctx->sg[0]) + (step * ctx->data_step_sz), 640 mxic->regs + SDMA_MAIN_ADDR); 641 writel(sg_dma_address(&ctx->sg[1]) + (step * (ctx->oob_step_sz + STAT_BYTES)), 642 mxic->regs + SDMA_SPARE_ADDR); 643 ret = mxic_ecc_process_data(mxic, ctx->req->type); 644 if (ret) 645 break; 646 } 647 648 mutex_unlock(&mxic->lock); 649 650 dma_unmap_sg(mxic->dev, ctx->sg, 2, DMA_BIDIRECTIONAL); 651 652 if (ret) { 653 nand_ecc_restore_req(&ctx->req_ctx, req); 654 return ret; 655 } 656 657 /* Extract the status bytes and reconstruct the buffer */ 658 mxic_ecc_extract_status_bytes(ctx); 659 mxic_ecc_reconstruct_oobbuf(ctx, ctx->req->oobbuf.in, ctx->oobwithstat); 660 661 nand_ecc_restore_req(&ctx->req_ctx, req); 662 663 return mxic_ecc_count_biterrs(mxic, nand); 664 } 665 666 /* Pipelined ECC engine helpers */ 667 static int mxic_ecc_prepare_io_req_pipelined(struct nand_device *nand, 668 struct nand_page_io_req *req) 669 { 670 struct mxic_ecc_engine *mxic = nand_to_mxic(nand); 671 struct mxic_ecc_ctx *ctx = nand_to_ecc_ctx(nand); 672 int nents; 673 674 if (req->mode == MTD_OPS_RAW) 675 return 0; 676 677 nand_ecc_tweak_req(&ctx->req_ctx, req); 678 ctx->req = req; 679 680 /* Copy the OOB buffer and add room for the ECC engine status bytes */ 681 mxic_ecc_add_room_in_oobbuf(ctx, ctx->oobwithstat, ctx->req->oobbuf.in); 682 683 sg_set_buf(&ctx->sg[0], req->databuf.in, req->datalen); 684 sg_set_buf(&ctx->sg[1], ctx->oobwithstat, 685 req->ooblen + (ctx->steps * STAT_BYTES)); 686 687 nents = dma_map_sg(mxic->dev, ctx->sg, 2, DMA_BIDIRECTIONAL); 688 if (!nents) 689 return -EINVAL; 690 691 mutex_lock(&mxic->lock); 692 693 writel(sg_dma_address(&ctx->sg[0]), mxic->regs + SDMA_MAIN_ADDR); 694 writel(sg_dma_address(&ctx->sg[1]), mxic->regs + SDMA_SPARE_ADDR); 695 696 return 0; 697 } 698 699 static int mxic_ecc_finish_io_req_pipelined(struct nand_device *nand, 700 struct nand_page_io_req *req) 701 { 702 struct mxic_ecc_engine *mxic = nand_to_mxic(nand); 703 struct mxic_ecc_ctx *ctx = nand_to_ecc_ctx(nand); 704 int ret = 0; 705 706 if (req->mode == MTD_OPS_RAW) 707 return 0; 708 709 mutex_unlock(&mxic->lock); 710 711 dma_unmap_sg(mxic->dev, ctx->sg, 2, DMA_BIDIRECTIONAL); 712 713 if (req->type == NAND_PAGE_READ) { 714 mxic_ecc_extract_status_bytes(ctx); 715 mxic_ecc_reconstruct_oobbuf(ctx, ctx->req->oobbuf.in, 716 ctx->oobwithstat); 717 ret = mxic_ecc_count_biterrs(mxic, nand); 718 } 719 720 nand_ecc_restore_req(&ctx->req_ctx, req); 721 722 return ret; 723 } 724 725 static struct nand_ecc_engine_ops mxic_ecc_engine_external_ops = { 726 .init_ctx = mxic_ecc_init_ctx_external, 727 .cleanup_ctx = mxic_ecc_cleanup_ctx, 728 .prepare_io_req = mxic_ecc_prepare_io_req_external, 729 .finish_io_req = mxic_ecc_finish_io_req_external, 730 }; 731 732 static struct nand_ecc_engine_ops mxic_ecc_engine_pipelined_ops = { 733 .init_ctx = mxic_ecc_init_ctx_pipelined, 734 .cleanup_ctx = mxic_ecc_cleanup_ctx, 735 .prepare_io_req = mxic_ecc_prepare_io_req_pipelined, 736 .finish_io_req = mxic_ecc_finish_io_req_pipelined, 737 }; 738 739 struct nand_ecc_engine_ops *mxic_ecc_get_pipelined_ops(void) 740 { 741 return &mxic_ecc_engine_pipelined_ops; 742 } 743 EXPORT_SYMBOL_GPL(mxic_ecc_get_pipelined_ops); 744 745 static struct platform_device * 746 mxic_ecc_get_pdev(struct platform_device *spi_pdev) 747 { 748 struct platform_device *eng_pdev; 749 struct device_node *np; 750 751 /* Retrieve the nand-ecc-engine phandle */ 752 np = of_parse_phandle(spi_pdev->dev.of_node, "nand-ecc-engine", 0); 753 if (!np) 754 return NULL; 755 756 /* Jump to the engine's device node */ 757 eng_pdev = of_find_device_by_node(np); 758 of_node_put(np); 759 760 return eng_pdev; 761 } 762 763 void mxic_ecc_put_pipelined_engine(struct nand_ecc_engine *eng) 764 { 765 struct mxic_ecc_engine *mxic = pip_ecc_eng_to_mxic(eng); 766 767 platform_device_put(to_platform_device(mxic->dev)); 768 } 769 EXPORT_SYMBOL_GPL(mxic_ecc_put_pipelined_engine); 770 771 struct nand_ecc_engine * 772 mxic_ecc_get_pipelined_engine(struct platform_device *spi_pdev) 773 { 774 struct platform_device *eng_pdev; 775 struct mxic_ecc_engine *mxic; 776 777 eng_pdev = mxic_ecc_get_pdev(spi_pdev); 778 if (!eng_pdev) 779 return ERR_PTR(-ENODEV); 780 781 mxic = platform_get_drvdata(eng_pdev); 782 if (!mxic) { 783 platform_device_put(eng_pdev); 784 return ERR_PTR(-EPROBE_DEFER); 785 } 786 787 return &mxic->pipelined_engine; 788 } 789 EXPORT_SYMBOL_GPL(mxic_ecc_get_pipelined_engine); 790 791 /* 792 * Only the external ECC engine is exported as the pipelined is SoC specific, so 793 * it is registered directly by the drivers that wrap it. 794 */ 795 static int mxic_ecc_probe(struct platform_device *pdev) 796 { 797 struct device *dev = &pdev->dev; 798 struct mxic_ecc_engine *mxic; 799 int ret; 800 801 mxic = devm_kzalloc(&pdev->dev, sizeof(*mxic), GFP_KERNEL); 802 if (!mxic) 803 return -ENOMEM; 804 805 mxic->dev = &pdev->dev; 806 807 /* 808 * Both memory regions for the ECC engine itself and the AXI slave 809 * address are mandatory. 810 */ 811 mxic->regs = devm_platform_ioremap_resource(pdev, 0); 812 if (IS_ERR(mxic->regs)) { 813 dev_err(&pdev->dev, "Missing memory region\n"); 814 return PTR_ERR(mxic->regs); 815 } 816 817 mxic_ecc_disable_engine(mxic); 818 mxic_ecc_disable_int(mxic); 819 820 /* IRQ is optional yet much more efficient */ 821 mxic->irq = platform_get_irq_byname_optional(pdev, "ecc-engine"); 822 if (mxic->irq > 0) { 823 ret = devm_request_irq(&pdev->dev, mxic->irq, mxic_ecc_isr, 0, 824 "mxic-ecc", mxic); 825 if (ret) 826 return ret; 827 } else { 828 dev_info(dev, "Invalid or missing IRQ, fallback to polling\n"); 829 mxic->irq = 0; 830 } 831 832 mutex_init(&mxic->lock); 833 834 /* 835 * In external mode, the device is the ECC engine. In pipelined mode, 836 * the device is the host controller. The device is used to match the 837 * right ECC engine based on the DT properties. 838 */ 839 mxic->external_engine.dev = &pdev->dev; 840 mxic->external_engine.integration = NAND_ECC_ENGINE_INTEGRATION_EXTERNAL; 841 mxic->external_engine.ops = &mxic_ecc_engine_external_ops; 842 843 nand_ecc_register_on_host_hw_engine(&mxic->external_engine); 844 845 platform_set_drvdata(pdev, mxic); 846 847 return 0; 848 } 849 850 static int mxic_ecc_remove(struct platform_device *pdev) 851 { 852 struct mxic_ecc_engine *mxic = platform_get_drvdata(pdev); 853 854 nand_ecc_unregister_on_host_hw_engine(&mxic->external_engine); 855 856 return 0; 857 } 858 859 static const struct of_device_id mxic_ecc_of_ids[] = { 860 { 861 .compatible = "mxicy,nand-ecc-engine-rev3", 862 }, 863 { /* sentinel */ }, 864 }; 865 MODULE_DEVICE_TABLE(of, mxic_ecc_of_ids); 866 867 static struct platform_driver mxic_ecc_driver = { 868 .driver = { 869 .name = "mxic-nand-ecc-engine", 870 .of_match_table = mxic_ecc_of_ids, 871 }, 872 .probe = mxic_ecc_probe, 873 .remove = mxic_ecc_remove, 874 }; 875 module_platform_driver(mxic_ecc_driver); 876 877 MODULE_LICENSE("GPL"); 878 MODULE_AUTHOR("Miquel Raynal <miquel.raynal@bootlin.com>"); 879 MODULE_DESCRIPTION("Macronix NAND hardware ECC controller"); 880