1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Freescale GPMI NAND Flash Driver 4 * 5 * Copyright (C) 2010-2015 Freescale Semiconductor, Inc. 6 * Copyright (C) 2008 Embedded Alley Solutions, Inc. 7 */ 8 #include <linux/clk.h> 9 #include <linux/delay.h> 10 #include <linux/slab.h> 11 #include <linux/sched/task_stack.h> 12 #include <linux/interrupt.h> 13 #include <linux/module.h> 14 #include <linux/mtd/partitions.h> 15 #include <linux/of.h> 16 #include <linux/of_device.h> 17 #include <linux/pm_runtime.h> 18 #include <linux/dma/mxs-dma.h> 19 #include "gpmi-nand.h" 20 #include "gpmi-regs.h" 21 #include "bch-regs.h" 22 23 /* Resource names for the GPMI NAND driver. */ 24 #define GPMI_NAND_GPMI_REGS_ADDR_RES_NAME "gpmi-nand" 25 #define GPMI_NAND_BCH_REGS_ADDR_RES_NAME "bch" 26 #define GPMI_NAND_BCH_INTERRUPT_RES_NAME "bch" 27 28 /* Converts time to clock cycles */ 29 #define TO_CYCLES(duration, period) DIV_ROUND_UP_ULL(duration, period) 30 31 #define MXS_SET_ADDR 0x4 32 #define MXS_CLR_ADDR 0x8 33 /* 34 * Clear the bit and poll it cleared. This is usually called with 35 * a reset address and mask being either SFTRST(bit 31) or CLKGATE 36 * (bit 30). 37 */ 38 static int clear_poll_bit(void __iomem *addr, u32 mask) 39 { 40 int timeout = 0x400; 41 42 /* clear the bit */ 43 writel(mask, addr + MXS_CLR_ADDR); 44 45 /* 46 * SFTRST needs 3 GPMI clocks to settle, the reference manual 47 * recommends to wait 1us. 48 */ 49 udelay(1); 50 51 /* poll the bit becoming clear */ 52 while ((readl(addr) & mask) && --timeout) 53 /* nothing */; 54 55 return !timeout; 56 } 57 58 #define MODULE_CLKGATE (1 << 30) 59 #define MODULE_SFTRST (1 << 31) 60 /* 61 * The current mxs_reset_block() will do two things: 62 * [1] enable the module. 63 * [2] reset the module. 64 * 65 * In most of the cases, it's ok. 66 * But in MX23, there is a hardware bug in the BCH block (see erratum #2847). 67 * If you try to soft reset the BCH block, it becomes unusable until 68 * the next hard reset. This case occurs in the NAND boot mode. When the board 69 * boots by NAND, the ROM of the chip will initialize the BCH blocks itself. 70 * So If the driver tries to reset the BCH again, the BCH will not work anymore. 71 * You will see a DMA timeout in this case. The bug has been fixed 72 * in the following chips, such as MX28. 73 * 74 * To avoid this bug, just add a new parameter `just_enable` for 75 * the mxs_reset_block(), and rewrite it here. 76 */ 77 static int gpmi_reset_block(void __iomem *reset_addr, bool just_enable) 78 { 79 int ret; 80 int timeout = 0x400; 81 82 /* clear and poll SFTRST */ 83 ret = clear_poll_bit(reset_addr, MODULE_SFTRST); 84 if (unlikely(ret)) 85 goto error; 86 87 /* clear CLKGATE */ 88 writel(MODULE_CLKGATE, reset_addr + MXS_CLR_ADDR); 89 90 if (!just_enable) { 91 /* set SFTRST to reset the block */ 92 writel(MODULE_SFTRST, reset_addr + MXS_SET_ADDR); 93 udelay(1); 94 95 /* poll CLKGATE becoming set */ 96 while ((!(readl(reset_addr) & MODULE_CLKGATE)) && --timeout) 97 /* nothing */; 98 if (unlikely(!timeout)) 99 goto error; 100 } 101 102 /* clear and poll SFTRST */ 103 ret = clear_poll_bit(reset_addr, MODULE_SFTRST); 104 if (unlikely(ret)) 105 goto error; 106 107 /* clear and poll CLKGATE */ 108 ret = clear_poll_bit(reset_addr, MODULE_CLKGATE); 109 if (unlikely(ret)) 110 goto error; 111 112 return 0; 113 114 error: 115 pr_err("%s(%p): module reset timeout\n", __func__, reset_addr); 116 return -ETIMEDOUT; 117 } 118 119 static int __gpmi_enable_clk(struct gpmi_nand_data *this, bool v) 120 { 121 struct clk *clk; 122 int ret; 123 int i; 124 125 for (i = 0; i < GPMI_CLK_MAX; i++) { 126 clk = this->resources.clock[i]; 127 if (!clk) 128 break; 129 130 if (v) { 131 ret = clk_prepare_enable(clk); 132 if (ret) 133 goto err_clk; 134 } else { 135 clk_disable_unprepare(clk); 136 } 137 } 138 return 0; 139 140 err_clk: 141 for (; i > 0; i--) 142 clk_disable_unprepare(this->resources.clock[i - 1]); 143 return ret; 144 } 145 146 static int gpmi_init(struct gpmi_nand_data *this) 147 { 148 struct resources *r = &this->resources; 149 int ret; 150 151 ret = pm_runtime_get_sync(this->dev); 152 if (ret < 0) { 153 pm_runtime_put_noidle(this->dev); 154 return ret; 155 } 156 157 ret = gpmi_reset_block(r->gpmi_regs, false); 158 if (ret) 159 goto err_out; 160 161 /* 162 * Reset BCH here, too. We got failures otherwise :( 163 * See later BCH reset for explanation of MX23 and MX28 handling 164 */ 165 ret = gpmi_reset_block(r->bch_regs, GPMI_IS_MXS(this)); 166 if (ret) 167 goto err_out; 168 169 /* Choose NAND mode. */ 170 writel(BM_GPMI_CTRL1_GPMI_MODE, r->gpmi_regs + HW_GPMI_CTRL1_CLR); 171 172 /* Set the IRQ polarity. */ 173 writel(BM_GPMI_CTRL1_ATA_IRQRDY_POLARITY, 174 r->gpmi_regs + HW_GPMI_CTRL1_SET); 175 176 /* Disable Write-Protection. */ 177 writel(BM_GPMI_CTRL1_DEV_RESET, r->gpmi_regs + HW_GPMI_CTRL1_SET); 178 179 /* Select BCH ECC. */ 180 writel(BM_GPMI_CTRL1_BCH_MODE, r->gpmi_regs + HW_GPMI_CTRL1_SET); 181 182 /* 183 * Decouple the chip select from dma channel. We use dma0 for all 184 * the chips, force all NAND RDY_BUSY inputs to be sourced from 185 * RDY_BUSY0. 186 */ 187 writel(BM_GPMI_CTRL1_DECOUPLE_CS | BM_GPMI_CTRL1_GANGED_RDYBUSY, 188 r->gpmi_regs + HW_GPMI_CTRL1_SET); 189 190 err_out: 191 pm_runtime_mark_last_busy(this->dev); 192 pm_runtime_put_autosuspend(this->dev); 193 return ret; 194 } 195 196 /* This function is very useful. It is called only when the bug occur. */ 197 static void gpmi_dump_info(struct gpmi_nand_data *this) 198 { 199 struct resources *r = &this->resources; 200 struct bch_geometry *geo = &this->bch_geometry; 201 u32 reg; 202 int i; 203 204 dev_err(this->dev, "Show GPMI registers :\n"); 205 for (i = 0; i <= HW_GPMI_DEBUG / 0x10 + 1; i++) { 206 reg = readl(r->gpmi_regs + i * 0x10); 207 dev_err(this->dev, "offset 0x%.3x : 0x%.8x\n", i * 0x10, reg); 208 } 209 210 /* start to print out the BCH info */ 211 dev_err(this->dev, "Show BCH registers :\n"); 212 for (i = 0; i <= HW_BCH_VERSION / 0x10 + 1; i++) { 213 reg = readl(r->bch_regs + i * 0x10); 214 dev_err(this->dev, "offset 0x%.3x : 0x%.8x\n", i * 0x10, reg); 215 } 216 dev_err(this->dev, "BCH Geometry :\n" 217 "GF length : %u\n" 218 "ECC Strength : %u\n" 219 "Page Size in Bytes : %u\n" 220 "Metadata Size in Bytes : %u\n" 221 "ECC0 Chunk Size in Bytes: %u\n" 222 "ECCn Chunk Size in Bytes: %u\n" 223 "ECC Chunk Count : %u\n" 224 "Payload Size in Bytes : %u\n" 225 "Auxiliary Size in Bytes: %u\n" 226 "Auxiliary Status Offset: %u\n" 227 "Block Mark Byte Offset : %u\n" 228 "Block Mark Bit Offset : %u\n", 229 geo->gf_len, 230 geo->ecc_strength, 231 geo->page_size, 232 geo->metadata_size, 233 geo->ecc0_chunk_size, 234 geo->eccn_chunk_size, 235 geo->ecc_chunk_count, 236 geo->payload_size, 237 geo->auxiliary_size, 238 geo->auxiliary_status_offset, 239 geo->block_mark_byte_offset, 240 geo->block_mark_bit_offset); 241 } 242 243 static bool gpmi_check_ecc(struct gpmi_nand_data *this) 244 { 245 struct nand_chip *chip = &this->nand; 246 struct bch_geometry *geo = &this->bch_geometry; 247 struct nand_device *nand = &chip->base; 248 struct nand_ecc_props *conf = &nand->ecc.ctx.conf; 249 250 conf->step_size = geo->eccn_chunk_size; 251 conf->strength = geo->ecc_strength; 252 253 /* Do the sanity check. */ 254 if (GPMI_IS_MXS(this)) { 255 /* The mx23/mx28 only support the GF13. */ 256 if (geo->gf_len == 14) 257 return false; 258 } 259 260 if (geo->ecc_strength > this->devdata->bch_max_ecc_strength) 261 return false; 262 263 if (!nand_ecc_is_strong_enough(nand)) 264 return false; 265 266 return true; 267 } 268 269 /* check if bbm locates in data chunk rather than ecc chunk */ 270 static bool bbm_in_data_chunk(struct gpmi_nand_data *this, 271 unsigned int *chunk_num) 272 { 273 struct bch_geometry *geo = &this->bch_geometry; 274 struct nand_chip *chip = &this->nand; 275 struct mtd_info *mtd = nand_to_mtd(chip); 276 unsigned int i, j; 277 278 if (geo->ecc0_chunk_size != geo->eccn_chunk_size) { 279 dev_err(this->dev, 280 "The size of ecc0_chunk must equal to eccn_chunk\n"); 281 return false; 282 } 283 284 i = (mtd->writesize * 8 - geo->metadata_size * 8) / 285 (geo->gf_len * geo->ecc_strength + 286 geo->eccn_chunk_size * 8); 287 288 j = (mtd->writesize * 8 - geo->metadata_size * 8) - 289 (geo->gf_len * geo->ecc_strength + 290 geo->eccn_chunk_size * 8) * i; 291 292 if (j < geo->eccn_chunk_size * 8) { 293 *chunk_num = i+1; 294 dev_dbg(this->dev, "Set ecc to %d and bbm in chunk %d\n", 295 geo->ecc_strength, *chunk_num); 296 return true; 297 } 298 299 return false; 300 } 301 302 /* 303 * If we can get the ECC information from the nand chip, we do not 304 * need to calculate them ourselves. 305 * 306 * We may have available oob space in this case. 307 */ 308 static int set_geometry_by_ecc_info(struct gpmi_nand_data *this, 309 unsigned int ecc_strength, 310 unsigned int ecc_step) 311 { 312 struct bch_geometry *geo = &this->bch_geometry; 313 struct nand_chip *chip = &this->nand; 314 struct mtd_info *mtd = nand_to_mtd(chip); 315 unsigned int block_mark_bit_offset; 316 317 switch (ecc_step) { 318 case SZ_512: 319 geo->gf_len = 13; 320 break; 321 case SZ_1K: 322 geo->gf_len = 14; 323 break; 324 default: 325 dev_err(this->dev, 326 "unsupported nand chip. ecc bits : %d, ecc size : %d\n", 327 nanddev_get_ecc_requirements(&chip->base)->strength, 328 nanddev_get_ecc_requirements(&chip->base)->step_size); 329 return -EINVAL; 330 } 331 geo->ecc0_chunk_size = ecc_step; 332 geo->eccn_chunk_size = ecc_step; 333 geo->ecc_strength = round_up(ecc_strength, 2); 334 if (!gpmi_check_ecc(this)) 335 return -EINVAL; 336 337 /* Keep the C >= O */ 338 if (geo->eccn_chunk_size < mtd->oobsize) { 339 dev_err(this->dev, 340 "unsupported nand chip. ecc size: %d, oob size : %d\n", 341 ecc_step, mtd->oobsize); 342 return -EINVAL; 343 } 344 345 /* The default value, see comment in the legacy_set_geometry(). */ 346 geo->metadata_size = 10; 347 348 geo->ecc_chunk_count = mtd->writesize / geo->eccn_chunk_size; 349 350 /* 351 * Now, the NAND chip with 2K page(data chunk is 512byte) shows below: 352 * 353 * | P | 354 * |<----------------------------------------------------->| 355 * | | 356 * | (Block Mark) | 357 * | P' | | | | 358 * |<-------------------------------------------->| D | | O' | 359 * | |<---->| |<--->| 360 * V V V V V 361 * +---+----------+-+----------+-+----------+-+----------+-+-----+ 362 * | M | data |E| data |E| data |E| data |E| | 363 * +---+----------+-+----------+-+----------+-+----------+-+-----+ 364 * ^ ^ 365 * | O | 366 * |<------------>| 367 * | | 368 * 369 * P : the page size for BCH module. 370 * E : The ECC strength. 371 * G : the length of Galois Field. 372 * N : The chunk count of per page. 373 * M : the metasize of per page. 374 * C : the ecc chunk size, aka the "data" above. 375 * P': the nand chip's page size. 376 * O : the nand chip's oob size. 377 * O': the free oob. 378 * 379 * The formula for P is : 380 * 381 * E * G * N 382 * P = ------------ + P' + M 383 * 8 384 * 385 * The position of block mark moves forward in the ECC-based view 386 * of page, and the delta is: 387 * 388 * E * G * (N - 1) 389 * D = (---------------- + M) 390 * 8 391 * 392 * Please see the comment in legacy_set_geometry(). 393 * With the condition C >= O , we still can get same result. 394 * So the bit position of the physical block mark within the ECC-based 395 * view of the page is : 396 * (P' - D) * 8 397 */ 398 geo->page_size = mtd->writesize + geo->metadata_size + 399 (geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8; 400 401 geo->payload_size = mtd->writesize; 402 403 geo->auxiliary_status_offset = ALIGN(geo->metadata_size, 4); 404 geo->auxiliary_size = ALIGN(geo->metadata_size, 4) 405 + ALIGN(geo->ecc_chunk_count, 4); 406 407 if (!this->swap_block_mark) 408 return 0; 409 410 /* For bit swap. */ 411 block_mark_bit_offset = mtd->writesize * 8 - 412 (geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - 1) 413 + geo->metadata_size * 8); 414 415 geo->block_mark_byte_offset = block_mark_bit_offset / 8; 416 geo->block_mark_bit_offset = block_mark_bit_offset % 8; 417 return 0; 418 } 419 420 /* 421 * Calculate the ECC strength by hand: 422 * E : The ECC strength. 423 * G : the length of Galois Field. 424 * N : The chunk count of per page. 425 * O : the oobsize of the NAND chip. 426 * M : the metasize of per page. 427 * 428 * The formula is : 429 * E * G * N 430 * ------------ <= (O - M) 431 * 8 432 * 433 * So, we get E by: 434 * (O - M) * 8 435 * E <= ------------- 436 * G * N 437 */ 438 static inline int get_ecc_strength(struct gpmi_nand_data *this) 439 { 440 struct bch_geometry *geo = &this->bch_geometry; 441 struct mtd_info *mtd = nand_to_mtd(&this->nand); 442 int ecc_strength; 443 444 ecc_strength = ((mtd->oobsize - geo->metadata_size) * 8) 445 / (geo->gf_len * geo->ecc_chunk_count); 446 447 /* We need the minor even number. */ 448 return round_down(ecc_strength, 2); 449 } 450 451 static int set_geometry_for_large_oob(struct gpmi_nand_data *this) 452 { 453 struct bch_geometry *geo = &this->bch_geometry; 454 struct nand_chip *chip = &this->nand; 455 struct mtd_info *mtd = nand_to_mtd(chip); 456 const struct nand_ecc_props *requirements = 457 nanddev_get_ecc_requirements(&chip->base); 458 unsigned int block_mark_bit_offset; 459 unsigned int max_ecc; 460 unsigned int bbm_chunk; 461 unsigned int i; 462 463 /* sanity check for the minimum ecc nand required */ 464 if (!(requirements->strength > 0 && 465 requirements->step_size > 0)) 466 return -EINVAL; 467 geo->ecc_strength = requirements->strength; 468 469 /* check if platform can support this nand */ 470 if (!gpmi_check_ecc(this)) { 471 dev_err(this->dev, 472 "unsupported NAND chip, minimum ecc required %d\n", 473 geo->ecc_strength); 474 return -EINVAL; 475 } 476 477 /* calculate the maximum ecc platform can support*/ 478 geo->metadata_size = 10; 479 geo->gf_len = 14; 480 geo->ecc0_chunk_size = 1024; 481 geo->eccn_chunk_size = 1024; 482 geo->ecc_chunk_count = mtd->writesize / geo->eccn_chunk_size; 483 max_ecc = min(get_ecc_strength(this), 484 this->devdata->bch_max_ecc_strength); 485 486 /* 487 * search a supported ecc strength that makes bbm 488 * located in data chunk 489 */ 490 geo->ecc_strength = max_ecc; 491 while (!(geo->ecc_strength < requirements->strength)) { 492 if (bbm_in_data_chunk(this, &bbm_chunk)) 493 goto geo_setting; 494 geo->ecc_strength -= 2; 495 } 496 497 /* if none of them works, keep using the minimum ecc */ 498 /* nand required but changing ecc page layout */ 499 geo->ecc_strength = requirements->strength; 500 /* add extra ecc for meta data */ 501 geo->ecc0_chunk_size = 0; 502 geo->ecc_chunk_count = (mtd->writesize / geo->eccn_chunk_size) + 1; 503 geo->ecc_for_meta = 1; 504 /* check if oob can afford this extra ecc chunk */ 505 if (mtd->oobsize * 8 < geo->metadata_size * 8 + 506 geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) { 507 dev_err(this->dev, "unsupported NAND chip with new layout\n"); 508 return -EINVAL; 509 } 510 511 /* calculate in which chunk bbm located */ 512 bbm_chunk = (mtd->writesize * 8 - geo->metadata_size * 8 - 513 geo->gf_len * geo->ecc_strength) / 514 (geo->gf_len * geo->ecc_strength + 515 geo->eccn_chunk_size * 8) + 1; 516 517 geo_setting: 518 519 geo->page_size = mtd->writesize + geo->metadata_size + 520 (geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8; 521 geo->payload_size = mtd->writesize; 522 523 /* 524 * The auxiliary buffer contains the metadata and the ECC status. The 525 * metadata is padded to the nearest 32-bit boundary. The ECC status 526 * contains one byte for every ECC chunk, and is also padded to the 527 * nearest 32-bit boundary. 528 */ 529 geo->auxiliary_status_offset = ALIGN(geo->metadata_size, 4); 530 geo->auxiliary_size = ALIGN(geo->metadata_size, 4) 531 + ALIGN(geo->ecc_chunk_count, 4); 532 533 if (!this->swap_block_mark) 534 return 0; 535 536 /* calculate the number of ecc chunk behind the bbm */ 537 i = (mtd->writesize / geo->eccn_chunk_size) - bbm_chunk + 1; 538 539 block_mark_bit_offset = mtd->writesize * 8 - 540 (geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - i) 541 + geo->metadata_size * 8); 542 543 geo->block_mark_byte_offset = block_mark_bit_offset / 8; 544 geo->block_mark_bit_offset = block_mark_bit_offset % 8; 545 546 dev_dbg(this->dev, "BCH Geometry :\n" 547 "GF length : %u\n" 548 "ECC Strength : %u\n" 549 "Page Size in Bytes : %u\n" 550 "Metadata Size in Bytes : %u\n" 551 "ECC0 Chunk Size in Bytes: %u\n" 552 "ECCn Chunk Size in Bytes: %u\n" 553 "ECC Chunk Count : %u\n" 554 "Payload Size in Bytes : %u\n" 555 "Auxiliary Size in Bytes: %u\n" 556 "Auxiliary Status Offset: %u\n" 557 "Block Mark Byte Offset : %u\n" 558 "Block Mark Bit Offset : %u\n" 559 "Block Mark in chunk : %u\n" 560 "Ecc for Meta data : %u\n", 561 geo->gf_len, 562 geo->ecc_strength, 563 geo->page_size, 564 geo->metadata_size, 565 geo->ecc0_chunk_size, 566 geo->eccn_chunk_size, 567 geo->ecc_chunk_count, 568 geo->payload_size, 569 geo->auxiliary_size, 570 geo->auxiliary_status_offset, 571 geo->block_mark_byte_offset, 572 geo->block_mark_bit_offset, 573 bbm_chunk, 574 geo->ecc_for_meta); 575 576 return 0; 577 } 578 579 static int legacy_set_geometry(struct gpmi_nand_data *this) 580 { 581 struct bch_geometry *geo = &this->bch_geometry; 582 struct mtd_info *mtd = nand_to_mtd(&this->nand); 583 unsigned int metadata_size; 584 unsigned int status_size; 585 unsigned int block_mark_bit_offset; 586 587 /* 588 * The size of the metadata can be changed, though we set it to 10 589 * bytes now. But it can't be too large, because we have to save 590 * enough space for BCH. 591 */ 592 geo->metadata_size = 10; 593 594 /* The default for the length of Galois Field. */ 595 geo->gf_len = 13; 596 597 /* The default for chunk size. */ 598 geo->ecc0_chunk_size = 512; 599 geo->eccn_chunk_size = 512; 600 while (geo->eccn_chunk_size < mtd->oobsize) { 601 geo->ecc0_chunk_size *= 2; /* keep C >= O */ 602 geo->eccn_chunk_size *= 2; /* keep C >= O */ 603 geo->gf_len = 14; 604 } 605 606 geo->ecc_chunk_count = mtd->writesize / geo->eccn_chunk_size; 607 608 /* We use the same ECC strength for all chunks. */ 609 geo->ecc_strength = get_ecc_strength(this); 610 if (!gpmi_check_ecc(this)) { 611 dev_err(this->dev, 612 "ecc strength: %d cannot be supported by the controller (%d)\n" 613 "try to use minimum ecc strength that NAND chip required\n", 614 geo->ecc_strength, 615 this->devdata->bch_max_ecc_strength); 616 return -EINVAL; 617 } 618 619 geo->page_size = mtd->writesize + geo->metadata_size + 620 (geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8; 621 geo->payload_size = mtd->writesize; 622 623 /* 624 * The auxiliary buffer contains the metadata and the ECC status. The 625 * metadata is padded to the nearest 32-bit boundary. The ECC status 626 * contains one byte for every ECC chunk, and is also padded to the 627 * nearest 32-bit boundary. 628 */ 629 metadata_size = ALIGN(geo->metadata_size, 4); 630 status_size = ALIGN(geo->ecc_chunk_count, 4); 631 632 geo->auxiliary_size = metadata_size + status_size; 633 geo->auxiliary_status_offset = metadata_size; 634 635 if (!this->swap_block_mark) 636 return 0; 637 638 /* 639 * We need to compute the byte and bit offsets of 640 * the physical block mark within the ECC-based view of the page. 641 * 642 * NAND chip with 2K page shows below: 643 * (Block Mark) 644 * | | 645 * | D | 646 * |<---->| 647 * V V 648 * +---+----------+-+----------+-+----------+-+----------+-+ 649 * | M | data |E| data |E| data |E| data |E| 650 * +---+----------+-+----------+-+----------+-+----------+-+ 651 * 652 * The position of block mark moves forward in the ECC-based view 653 * of page, and the delta is: 654 * 655 * E * G * (N - 1) 656 * D = (---------------- + M) 657 * 8 658 * 659 * With the formula to compute the ECC strength, and the condition 660 * : C >= O (C is the ecc chunk size) 661 * 662 * It's easy to deduce to the following result: 663 * 664 * E * G (O - M) C - M C - M 665 * ----------- <= ------- <= -------- < --------- 666 * 8 N N (N - 1) 667 * 668 * So, we get: 669 * 670 * E * G * (N - 1) 671 * D = (---------------- + M) < C 672 * 8 673 * 674 * The above inequality means the position of block mark 675 * within the ECC-based view of the page is still in the data chunk, 676 * and it's NOT in the ECC bits of the chunk. 677 * 678 * Use the following to compute the bit position of the 679 * physical block mark within the ECC-based view of the page: 680 * (page_size - D) * 8 681 * 682 * --Huang Shijie 683 */ 684 block_mark_bit_offset = mtd->writesize * 8 - 685 (geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - 1) 686 + geo->metadata_size * 8); 687 688 geo->block_mark_byte_offset = block_mark_bit_offset / 8; 689 geo->block_mark_bit_offset = block_mark_bit_offset % 8; 690 return 0; 691 } 692 693 static int common_nfc_set_geometry(struct gpmi_nand_data *this) 694 { 695 struct nand_chip *chip = &this->nand; 696 struct mtd_info *mtd = nand_to_mtd(&this->nand); 697 const struct nand_ecc_props *requirements = 698 nanddev_get_ecc_requirements(&chip->base); 699 bool use_minimun_ecc; 700 int err; 701 702 use_minimun_ecc = of_property_read_bool(this->dev->of_node, 703 "fsl,use-minimum-ecc"); 704 705 /* use legacy bch geometry settings by default*/ 706 if ((!use_minimun_ecc && mtd->oobsize < 1024) || 707 !(requirements->strength > 0 && requirements->step_size > 0)) { 708 dev_dbg(this->dev, "use legacy bch geometry\n"); 709 err = legacy_set_geometry(this); 710 if (!err) 711 return 0; 712 } 713 714 /* for large oob nand */ 715 if (mtd->oobsize > 1024) { 716 dev_dbg(this->dev, "use large oob bch geometry\n"); 717 err = set_geometry_for_large_oob(this); 718 if (!err) 719 return 0; 720 } 721 722 /* otherwise use the minimum ecc nand chip required */ 723 dev_dbg(this->dev, "use minimum ecc bch geometry\n"); 724 err = set_geometry_by_ecc_info(this, requirements->strength, 725 requirements->step_size); 726 if (err) 727 dev_err(this->dev, "none of the bch geometry setting works\n"); 728 729 return err; 730 } 731 732 /* Configures the geometry for BCH. */ 733 static int bch_set_geometry(struct gpmi_nand_data *this) 734 { 735 struct resources *r = &this->resources; 736 int ret; 737 738 ret = common_nfc_set_geometry(this); 739 if (ret) 740 return ret; 741 742 ret = pm_runtime_get_sync(this->dev); 743 if (ret < 0) { 744 pm_runtime_put_autosuspend(this->dev); 745 return ret; 746 } 747 748 /* 749 * Due to erratum #2847 of the MX23, the BCH cannot be soft reset on this 750 * chip, otherwise it will lock up. So we skip resetting BCH on the MX23. 751 * and MX28. 752 */ 753 ret = gpmi_reset_block(r->bch_regs, GPMI_IS_MXS(this)); 754 if (ret) 755 goto err_out; 756 757 /* Set *all* chip selects to use layout 0. */ 758 writel(0, r->bch_regs + HW_BCH_LAYOUTSELECT); 759 760 ret = 0; 761 err_out: 762 pm_runtime_mark_last_busy(this->dev); 763 pm_runtime_put_autosuspend(this->dev); 764 765 return ret; 766 } 767 768 /* 769 * <1> Firstly, we should know what's the GPMI-clock means. 770 * The GPMI-clock is the internal clock in the gpmi nand controller. 771 * If you set 100MHz to gpmi nand controller, the GPMI-clock's period 772 * is 10ns. Mark the GPMI-clock's period as GPMI-clock-period. 773 * 774 * <2> Secondly, we should know what's the frequency on the nand chip pins. 775 * The frequency on the nand chip pins is derived from the GPMI-clock. 776 * We can get it from the following equation: 777 * 778 * F = G / (DS + DH) 779 * 780 * F : the frequency on the nand chip pins. 781 * G : the GPMI clock, such as 100MHz. 782 * DS : GPMI_HW_GPMI_TIMING0:DATA_SETUP 783 * DH : GPMI_HW_GPMI_TIMING0:DATA_HOLD 784 * 785 * <3> Thirdly, when the frequency on the nand chip pins is above 33MHz, 786 * the nand EDO(extended Data Out) timing could be applied. 787 * The GPMI implements a feedback read strobe to sample the read data. 788 * The feedback read strobe can be delayed to support the nand EDO timing 789 * where the read strobe may deasserts before the read data is valid, and 790 * read data is valid for some time after read strobe. 791 * 792 * The following figure illustrates some aspects of a NAND Flash read: 793 * 794 * |<---tREA---->| 795 * | | 796 * | | | 797 * |<--tRP-->| | 798 * | | | 799 * __ ___|__________________________________ 800 * RDN \________/ | 801 * | 802 * /---------\ 803 * Read Data --------------< >--------- 804 * \---------/ 805 * | | 806 * |<-D->| 807 * FeedbackRDN ________ ____________ 808 * \___________/ 809 * 810 * D stands for delay, set in the HW_GPMI_CTRL1:RDN_DELAY. 811 * 812 * 813 * <4> Now, we begin to describe how to compute the right RDN_DELAY. 814 * 815 * 4.1) From the aspect of the nand chip pins: 816 * Delay = (tREA + C - tRP) {1} 817 * 818 * tREA : the maximum read access time. 819 * C : a constant to adjust the delay. default is 4000ps. 820 * tRP : the read pulse width, which is exactly: 821 * tRP = (GPMI-clock-period) * DATA_SETUP 822 * 823 * 4.2) From the aspect of the GPMI nand controller: 824 * Delay = RDN_DELAY * 0.125 * RP {2} 825 * 826 * RP : the DLL reference period. 827 * if (GPMI-clock-period > DLL_THRETHOLD) 828 * RP = GPMI-clock-period / 2; 829 * else 830 * RP = GPMI-clock-period; 831 * 832 * Set the HW_GPMI_CTRL1:HALF_PERIOD if GPMI-clock-period 833 * is greater DLL_THRETHOLD. In other SOCs, the DLL_THRETHOLD 834 * is 16000ps, but in mx6q, we use 12000ps. 835 * 836 * 4.3) since {1} equals {2}, we get: 837 * 838 * (tREA + 4000 - tRP) * 8 839 * RDN_DELAY = ----------------------- {3} 840 * RP 841 */ 842 static int gpmi_nfc_compute_timings(struct gpmi_nand_data *this, 843 const struct nand_sdr_timings *sdr) 844 { 845 struct gpmi_nfc_hardware_timing *hw = &this->hw; 846 struct resources *r = &this->resources; 847 unsigned int dll_threshold_ps = this->devdata->max_chain_delay; 848 unsigned int period_ps, reference_period_ps; 849 unsigned int data_setup_cycles, data_hold_cycles, addr_setup_cycles; 850 unsigned int tRP_ps; 851 bool use_half_period; 852 int sample_delay_ps, sample_delay_factor; 853 unsigned int busy_timeout_cycles; 854 u8 wrn_dly_sel; 855 unsigned long clk_rate, min_rate; 856 u64 busy_timeout_ps; 857 858 if (sdr->tRC_min >= 30000) { 859 /* ONFI non-EDO modes [0-3] */ 860 hw->clk_rate = 22000000; 861 min_rate = 0; 862 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_4_TO_8NS; 863 } else if (sdr->tRC_min >= 25000) { 864 /* ONFI EDO mode 4 */ 865 hw->clk_rate = 80000000; 866 min_rate = 22000000; 867 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_NO_DELAY; 868 } else { 869 /* ONFI EDO mode 5 */ 870 hw->clk_rate = 100000000; 871 min_rate = 80000000; 872 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_NO_DELAY; 873 } 874 875 clk_rate = clk_round_rate(r->clock[0], hw->clk_rate); 876 if (clk_rate <= min_rate) { 877 dev_err(this->dev, "clock setting: expected %ld, got %ld\n", 878 hw->clk_rate, clk_rate); 879 return -ENOTSUPP; 880 } 881 882 hw->clk_rate = clk_rate; 883 /* SDR core timings are given in picoseconds */ 884 period_ps = div_u64((u64)NSEC_PER_SEC * 1000, hw->clk_rate); 885 886 addr_setup_cycles = TO_CYCLES(sdr->tALS_min, period_ps); 887 data_setup_cycles = TO_CYCLES(sdr->tDS_min, period_ps); 888 data_hold_cycles = TO_CYCLES(sdr->tDH_min, period_ps); 889 busy_timeout_ps = max(sdr->tBERS_max, sdr->tPROG_max); 890 busy_timeout_cycles = TO_CYCLES(busy_timeout_ps, period_ps); 891 892 hw->timing0 = BF_GPMI_TIMING0_ADDRESS_SETUP(addr_setup_cycles) | 893 BF_GPMI_TIMING0_DATA_HOLD(data_hold_cycles) | 894 BF_GPMI_TIMING0_DATA_SETUP(data_setup_cycles); 895 hw->timing1 = BF_GPMI_TIMING1_BUSY_TIMEOUT(DIV_ROUND_UP(busy_timeout_cycles, 4096)); 896 897 /* 898 * Derive NFC ideal delay from {3}: 899 * 900 * (tREA + 4000 - tRP) * 8 901 * RDN_DELAY = ----------------------- 902 * RP 903 */ 904 if (period_ps > dll_threshold_ps) { 905 use_half_period = true; 906 reference_period_ps = period_ps / 2; 907 } else { 908 use_half_period = false; 909 reference_period_ps = period_ps; 910 } 911 912 tRP_ps = data_setup_cycles * period_ps; 913 sample_delay_ps = (sdr->tREA_max + 4000 - tRP_ps) * 8; 914 if (sample_delay_ps > 0) 915 sample_delay_factor = sample_delay_ps / reference_period_ps; 916 else 917 sample_delay_factor = 0; 918 919 hw->ctrl1n = BF_GPMI_CTRL1_WRN_DLY_SEL(wrn_dly_sel); 920 if (sample_delay_factor) 921 hw->ctrl1n |= BF_GPMI_CTRL1_RDN_DELAY(sample_delay_factor) | 922 BM_GPMI_CTRL1_DLL_ENABLE | 923 (use_half_period ? BM_GPMI_CTRL1_HALF_PERIOD : 0); 924 return 0; 925 } 926 927 static int gpmi_nfc_apply_timings(struct gpmi_nand_data *this) 928 { 929 struct gpmi_nfc_hardware_timing *hw = &this->hw; 930 struct resources *r = &this->resources; 931 void __iomem *gpmi_regs = r->gpmi_regs; 932 unsigned int dll_wait_time_us; 933 int ret; 934 935 /* Clock dividers do NOT guarantee a clean clock signal on its output 936 * during the change of the divide factor on i.MX6Q/UL/SX. On i.MX7/8, 937 * all clock dividers provide these guarantee. 938 */ 939 if (GPMI_IS_MX6Q(this) || GPMI_IS_MX6SX(this)) 940 clk_disable_unprepare(r->clock[0]); 941 942 ret = clk_set_rate(r->clock[0], hw->clk_rate); 943 if (ret) { 944 dev_err(this->dev, "cannot set clock rate to %lu Hz: %d\n", hw->clk_rate, ret); 945 return ret; 946 } 947 948 if (GPMI_IS_MX6Q(this) || GPMI_IS_MX6SX(this)) { 949 ret = clk_prepare_enable(r->clock[0]); 950 if (ret) 951 return ret; 952 } 953 954 writel(hw->timing0, gpmi_regs + HW_GPMI_TIMING0); 955 writel(hw->timing1, gpmi_regs + HW_GPMI_TIMING1); 956 957 /* 958 * Clear several CTRL1 fields, DLL must be disabled when setting 959 * RDN_DELAY or HALF_PERIOD. 960 */ 961 writel(BM_GPMI_CTRL1_CLEAR_MASK, gpmi_regs + HW_GPMI_CTRL1_CLR); 962 writel(hw->ctrl1n, gpmi_regs + HW_GPMI_CTRL1_SET); 963 964 /* Wait 64 clock cycles before using the GPMI after enabling the DLL */ 965 dll_wait_time_us = USEC_PER_SEC / hw->clk_rate * 64; 966 if (!dll_wait_time_us) 967 dll_wait_time_us = 1; 968 969 /* Wait for the DLL to settle. */ 970 udelay(dll_wait_time_us); 971 972 return 0; 973 } 974 975 static int gpmi_setup_interface(struct nand_chip *chip, int chipnr, 976 const struct nand_interface_config *conf) 977 { 978 struct gpmi_nand_data *this = nand_get_controller_data(chip); 979 const struct nand_sdr_timings *sdr; 980 int ret; 981 982 /* Retrieve required NAND timings */ 983 sdr = nand_get_sdr_timings(conf); 984 if (IS_ERR(sdr)) 985 return PTR_ERR(sdr); 986 987 /* Only MX28/MX6 GPMI controller can reach EDO timings */ 988 if (sdr->tRC_min <= 25000 && !GPMI_IS_MX28(this) && !GPMI_IS_MX6(this)) 989 return -ENOTSUPP; 990 991 /* Stop here if this call was just a check */ 992 if (chipnr < 0) 993 return 0; 994 995 /* Do the actual derivation of the controller timings */ 996 ret = gpmi_nfc_compute_timings(this, sdr); 997 if (ret) 998 return ret; 999 1000 this->hw.must_apply_timings = true; 1001 1002 return 0; 1003 } 1004 1005 /* Clears a BCH interrupt. */ 1006 static void gpmi_clear_bch(struct gpmi_nand_data *this) 1007 { 1008 struct resources *r = &this->resources; 1009 writel(BM_BCH_CTRL_COMPLETE_IRQ, r->bch_regs + HW_BCH_CTRL_CLR); 1010 } 1011 1012 static struct dma_chan *get_dma_chan(struct gpmi_nand_data *this) 1013 { 1014 /* We use the DMA channel 0 to access all the nand chips. */ 1015 return this->dma_chans[0]; 1016 } 1017 1018 /* This will be called after the DMA operation is finished. */ 1019 static void dma_irq_callback(void *param) 1020 { 1021 struct gpmi_nand_data *this = param; 1022 struct completion *dma_c = &this->dma_done; 1023 1024 complete(dma_c); 1025 } 1026 1027 static irqreturn_t bch_irq(int irq, void *cookie) 1028 { 1029 struct gpmi_nand_data *this = cookie; 1030 1031 gpmi_clear_bch(this); 1032 complete(&this->bch_done); 1033 return IRQ_HANDLED; 1034 } 1035 1036 static int gpmi_raw_len_to_len(struct gpmi_nand_data *this, int raw_len) 1037 { 1038 /* 1039 * raw_len is the length to read/write including bch data which 1040 * we are passed in exec_op. Calculate the data length from it. 1041 */ 1042 if (this->bch) 1043 return ALIGN_DOWN(raw_len, this->bch_geometry.eccn_chunk_size); 1044 else 1045 return raw_len; 1046 } 1047 1048 /* Can we use the upper's buffer directly for DMA? */ 1049 static bool prepare_data_dma(struct gpmi_nand_data *this, const void *buf, 1050 int raw_len, struct scatterlist *sgl, 1051 enum dma_data_direction dr) 1052 { 1053 int ret; 1054 int len = gpmi_raw_len_to_len(this, raw_len); 1055 1056 /* first try to map the upper buffer directly */ 1057 if (virt_addr_valid(buf) && !object_is_on_stack(buf)) { 1058 sg_init_one(sgl, buf, len); 1059 ret = dma_map_sg(this->dev, sgl, 1, dr); 1060 if (ret == 0) 1061 goto map_fail; 1062 1063 return true; 1064 } 1065 1066 map_fail: 1067 /* We have to use our own DMA buffer. */ 1068 sg_init_one(sgl, this->data_buffer_dma, len); 1069 1070 if (dr == DMA_TO_DEVICE && buf != this->data_buffer_dma) 1071 memcpy(this->data_buffer_dma, buf, len); 1072 1073 dma_map_sg(this->dev, sgl, 1, dr); 1074 1075 return false; 1076 } 1077 1078 /* add our owner bbt descriptor */ 1079 static uint8_t scan_ff_pattern[] = { 0xff }; 1080 static struct nand_bbt_descr gpmi_bbt_descr = { 1081 .options = 0, 1082 .offs = 0, 1083 .len = 1, 1084 .pattern = scan_ff_pattern 1085 }; 1086 1087 /* 1088 * We may change the layout if we can get the ECC info from the datasheet, 1089 * else we will use all the (page + OOB). 1090 */ 1091 static int gpmi_ooblayout_ecc(struct mtd_info *mtd, int section, 1092 struct mtd_oob_region *oobregion) 1093 { 1094 struct nand_chip *chip = mtd_to_nand(mtd); 1095 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1096 struct bch_geometry *geo = &this->bch_geometry; 1097 1098 if (section) 1099 return -ERANGE; 1100 1101 oobregion->offset = 0; 1102 oobregion->length = geo->page_size - mtd->writesize; 1103 1104 return 0; 1105 } 1106 1107 static int gpmi_ooblayout_free(struct mtd_info *mtd, int section, 1108 struct mtd_oob_region *oobregion) 1109 { 1110 struct nand_chip *chip = mtd_to_nand(mtd); 1111 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1112 struct bch_geometry *geo = &this->bch_geometry; 1113 1114 if (section) 1115 return -ERANGE; 1116 1117 /* The available oob size we have. */ 1118 if (geo->page_size < mtd->writesize + mtd->oobsize) { 1119 oobregion->offset = geo->page_size - mtd->writesize; 1120 oobregion->length = mtd->oobsize - oobregion->offset; 1121 } 1122 1123 return 0; 1124 } 1125 1126 static const char * const gpmi_clks_for_mx2x[] = { 1127 "gpmi_io", 1128 }; 1129 1130 static const struct mtd_ooblayout_ops gpmi_ooblayout_ops = { 1131 .ecc = gpmi_ooblayout_ecc, 1132 .free = gpmi_ooblayout_free, 1133 }; 1134 1135 static const struct gpmi_devdata gpmi_devdata_imx23 = { 1136 .type = IS_MX23, 1137 .bch_max_ecc_strength = 20, 1138 .max_chain_delay = 16000, 1139 .clks = gpmi_clks_for_mx2x, 1140 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx2x), 1141 }; 1142 1143 static const struct gpmi_devdata gpmi_devdata_imx28 = { 1144 .type = IS_MX28, 1145 .bch_max_ecc_strength = 20, 1146 .max_chain_delay = 16000, 1147 .clks = gpmi_clks_for_mx2x, 1148 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx2x), 1149 }; 1150 1151 static const char * const gpmi_clks_for_mx6[] = { 1152 "gpmi_io", "gpmi_apb", "gpmi_bch", "gpmi_bch_apb", "per1_bch", 1153 }; 1154 1155 static const struct gpmi_devdata gpmi_devdata_imx6q = { 1156 .type = IS_MX6Q, 1157 .bch_max_ecc_strength = 40, 1158 .max_chain_delay = 12000, 1159 .clks = gpmi_clks_for_mx6, 1160 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx6), 1161 }; 1162 1163 static const struct gpmi_devdata gpmi_devdata_imx6sx = { 1164 .type = IS_MX6SX, 1165 .bch_max_ecc_strength = 62, 1166 .max_chain_delay = 12000, 1167 .clks = gpmi_clks_for_mx6, 1168 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx6), 1169 }; 1170 1171 static const char * const gpmi_clks_for_mx7d[] = { 1172 "gpmi_io", "gpmi_bch_apb", 1173 }; 1174 1175 static const struct gpmi_devdata gpmi_devdata_imx7d = { 1176 .type = IS_MX7D, 1177 .bch_max_ecc_strength = 62, 1178 .max_chain_delay = 12000, 1179 .clks = gpmi_clks_for_mx7d, 1180 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx7d), 1181 }; 1182 1183 static int acquire_register_block(struct gpmi_nand_data *this, 1184 const char *res_name) 1185 { 1186 struct platform_device *pdev = this->pdev; 1187 struct resources *res = &this->resources; 1188 void __iomem *p; 1189 1190 p = devm_platform_ioremap_resource_byname(pdev, res_name); 1191 if (IS_ERR(p)) 1192 return PTR_ERR(p); 1193 1194 if (!strcmp(res_name, GPMI_NAND_GPMI_REGS_ADDR_RES_NAME)) 1195 res->gpmi_regs = p; 1196 else if (!strcmp(res_name, GPMI_NAND_BCH_REGS_ADDR_RES_NAME)) 1197 res->bch_regs = p; 1198 else 1199 dev_err(this->dev, "unknown resource name : %s\n", res_name); 1200 1201 return 0; 1202 } 1203 1204 static int acquire_bch_irq(struct gpmi_nand_data *this, irq_handler_t irq_h) 1205 { 1206 struct platform_device *pdev = this->pdev; 1207 const char *res_name = GPMI_NAND_BCH_INTERRUPT_RES_NAME; 1208 int err; 1209 1210 err = platform_get_irq_byname(pdev, res_name); 1211 if (err < 0) 1212 return err; 1213 1214 err = devm_request_irq(this->dev, err, irq_h, 0, res_name, this); 1215 if (err) 1216 dev_err(this->dev, "error requesting BCH IRQ\n"); 1217 1218 return err; 1219 } 1220 1221 static void release_dma_channels(struct gpmi_nand_data *this) 1222 { 1223 unsigned int i; 1224 for (i = 0; i < DMA_CHANS; i++) 1225 if (this->dma_chans[i]) { 1226 dma_release_channel(this->dma_chans[i]); 1227 this->dma_chans[i] = NULL; 1228 } 1229 } 1230 1231 static int acquire_dma_channels(struct gpmi_nand_data *this) 1232 { 1233 struct platform_device *pdev = this->pdev; 1234 struct dma_chan *dma_chan; 1235 int ret = 0; 1236 1237 /* request dma channel */ 1238 dma_chan = dma_request_chan(&pdev->dev, "rx-tx"); 1239 if (IS_ERR(dma_chan)) { 1240 ret = dev_err_probe(this->dev, PTR_ERR(dma_chan), 1241 "DMA channel request failed\n"); 1242 release_dma_channels(this); 1243 } else { 1244 this->dma_chans[0] = dma_chan; 1245 } 1246 1247 return ret; 1248 } 1249 1250 static int gpmi_get_clks(struct gpmi_nand_data *this) 1251 { 1252 struct resources *r = &this->resources; 1253 struct clk *clk; 1254 int err, i; 1255 1256 for (i = 0; i < this->devdata->clks_count; i++) { 1257 clk = devm_clk_get(this->dev, this->devdata->clks[i]); 1258 if (IS_ERR(clk)) { 1259 err = PTR_ERR(clk); 1260 goto err_clock; 1261 } 1262 1263 r->clock[i] = clk; 1264 } 1265 1266 return 0; 1267 1268 err_clock: 1269 dev_dbg(this->dev, "failed in finding the clocks.\n"); 1270 return err; 1271 } 1272 1273 static int acquire_resources(struct gpmi_nand_data *this) 1274 { 1275 int ret; 1276 1277 ret = acquire_register_block(this, GPMI_NAND_GPMI_REGS_ADDR_RES_NAME); 1278 if (ret) 1279 goto exit_regs; 1280 1281 ret = acquire_register_block(this, GPMI_NAND_BCH_REGS_ADDR_RES_NAME); 1282 if (ret) 1283 goto exit_regs; 1284 1285 ret = acquire_bch_irq(this, bch_irq); 1286 if (ret) 1287 goto exit_regs; 1288 1289 ret = acquire_dma_channels(this); 1290 if (ret) 1291 goto exit_regs; 1292 1293 ret = gpmi_get_clks(this); 1294 if (ret) 1295 goto exit_clock; 1296 return 0; 1297 1298 exit_clock: 1299 release_dma_channels(this); 1300 exit_regs: 1301 return ret; 1302 } 1303 1304 static void release_resources(struct gpmi_nand_data *this) 1305 { 1306 release_dma_channels(this); 1307 } 1308 1309 static void gpmi_free_dma_buffer(struct gpmi_nand_data *this) 1310 { 1311 struct device *dev = this->dev; 1312 struct bch_geometry *geo = &this->bch_geometry; 1313 1314 if (this->auxiliary_virt && virt_addr_valid(this->auxiliary_virt)) 1315 dma_free_coherent(dev, geo->auxiliary_size, 1316 this->auxiliary_virt, 1317 this->auxiliary_phys); 1318 kfree(this->data_buffer_dma); 1319 kfree(this->raw_buffer); 1320 1321 this->data_buffer_dma = NULL; 1322 this->raw_buffer = NULL; 1323 } 1324 1325 /* Allocate the DMA buffers */ 1326 static int gpmi_alloc_dma_buffer(struct gpmi_nand_data *this) 1327 { 1328 struct bch_geometry *geo = &this->bch_geometry; 1329 struct device *dev = this->dev; 1330 struct mtd_info *mtd = nand_to_mtd(&this->nand); 1331 1332 /* 1333 * [2] Allocate a read/write data buffer. 1334 * The gpmi_alloc_dma_buffer can be called twice. 1335 * We allocate a PAGE_SIZE length buffer if gpmi_alloc_dma_buffer 1336 * is called before the NAND identification; and we allocate a 1337 * buffer of the real NAND page size when the gpmi_alloc_dma_buffer 1338 * is called after. 1339 */ 1340 this->data_buffer_dma = kzalloc(mtd->writesize ?: PAGE_SIZE, 1341 GFP_DMA | GFP_KERNEL); 1342 if (this->data_buffer_dma == NULL) 1343 goto error_alloc; 1344 1345 this->auxiliary_virt = dma_alloc_coherent(dev, geo->auxiliary_size, 1346 &this->auxiliary_phys, GFP_DMA); 1347 if (!this->auxiliary_virt) 1348 goto error_alloc; 1349 1350 this->raw_buffer = kzalloc((mtd->writesize ?: PAGE_SIZE) + mtd->oobsize, GFP_KERNEL); 1351 if (!this->raw_buffer) 1352 goto error_alloc; 1353 1354 return 0; 1355 1356 error_alloc: 1357 gpmi_free_dma_buffer(this); 1358 return -ENOMEM; 1359 } 1360 1361 /* 1362 * Handles block mark swapping. 1363 * It can be called in swapping the block mark, or swapping it back, 1364 * because the operations are the same. 1365 */ 1366 static void block_mark_swapping(struct gpmi_nand_data *this, 1367 void *payload, void *auxiliary) 1368 { 1369 struct bch_geometry *nfc_geo = &this->bch_geometry; 1370 unsigned char *p; 1371 unsigned char *a; 1372 unsigned int bit; 1373 unsigned char mask; 1374 unsigned char from_data; 1375 unsigned char from_oob; 1376 1377 if (!this->swap_block_mark) 1378 return; 1379 1380 /* 1381 * If control arrives here, we're swapping. Make some convenience 1382 * variables. 1383 */ 1384 bit = nfc_geo->block_mark_bit_offset; 1385 p = payload + nfc_geo->block_mark_byte_offset; 1386 a = auxiliary; 1387 1388 /* 1389 * Get the byte from the data area that overlays the block mark. Since 1390 * the ECC engine applies its own view to the bits in the page, the 1391 * physical block mark won't (in general) appear on a byte boundary in 1392 * the data. 1393 */ 1394 from_data = (p[0] >> bit) | (p[1] << (8 - bit)); 1395 1396 /* Get the byte from the OOB. */ 1397 from_oob = a[0]; 1398 1399 /* Swap them. */ 1400 a[0] = from_data; 1401 1402 mask = (0x1 << bit) - 1; 1403 p[0] = (p[0] & mask) | (from_oob << bit); 1404 1405 mask = ~0 << bit; 1406 p[1] = (p[1] & mask) | (from_oob >> (8 - bit)); 1407 } 1408 1409 static int gpmi_count_bitflips(struct nand_chip *chip, void *buf, int first, 1410 int last, int meta) 1411 { 1412 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1413 struct bch_geometry *nfc_geo = &this->bch_geometry; 1414 struct mtd_info *mtd = nand_to_mtd(chip); 1415 int i; 1416 unsigned char *status; 1417 unsigned int max_bitflips = 0; 1418 1419 /* Loop over status bytes, accumulating ECC status. */ 1420 status = this->auxiliary_virt + ALIGN(meta, 4); 1421 1422 for (i = first; i < last; i++, status++) { 1423 if ((*status == STATUS_GOOD) || (*status == STATUS_ERASED)) 1424 continue; 1425 1426 if (*status == STATUS_UNCORRECTABLE) { 1427 int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len; 1428 u8 *eccbuf = this->raw_buffer; 1429 int offset, bitoffset; 1430 int eccbytes; 1431 int flips; 1432 1433 /* Read ECC bytes into our internal raw_buffer */ 1434 offset = nfc_geo->metadata_size * 8; 1435 offset += ((8 * nfc_geo->eccn_chunk_size) + eccbits) * (i + 1); 1436 offset -= eccbits; 1437 bitoffset = offset % 8; 1438 eccbytes = DIV_ROUND_UP(offset + eccbits, 8); 1439 offset /= 8; 1440 eccbytes -= offset; 1441 nand_change_read_column_op(chip, offset, eccbuf, 1442 eccbytes, false); 1443 1444 /* 1445 * ECC data are not byte aligned and we may have 1446 * in-band data in the first and last byte of 1447 * eccbuf. Set non-eccbits to one so that 1448 * nand_check_erased_ecc_chunk() does not count them 1449 * as bitflips. 1450 */ 1451 if (bitoffset) 1452 eccbuf[0] |= GENMASK(bitoffset - 1, 0); 1453 1454 bitoffset = (bitoffset + eccbits) % 8; 1455 if (bitoffset) 1456 eccbuf[eccbytes - 1] |= GENMASK(7, bitoffset); 1457 1458 /* 1459 * The ECC hardware has an uncorrectable ECC status 1460 * code in case we have bitflips in an erased page. As 1461 * nothing was written into this subpage the ECC is 1462 * obviously wrong and we can not trust it. We assume 1463 * at this point that we are reading an erased page and 1464 * try to correct the bitflips in buffer up to 1465 * ecc_strength bitflips. If this is a page with random 1466 * data, we exceed this number of bitflips and have a 1467 * ECC failure. Otherwise we use the corrected buffer. 1468 */ 1469 if (i == 0) { 1470 /* The first block includes metadata */ 1471 flips = nand_check_erased_ecc_chunk( 1472 buf + i * nfc_geo->eccn_chunk_size, 1473 nfc_geo->eccn_chunk_size, 1474 eccbuf, eccbytes, 1475 this->auxiliary_virt, 1476 nfc_geo->metadata_size, 1477 nfc_geo->ecc_strength); 1478 } else { 1479 flips = nand_check_erased_ecc_chunk( 1480 buf + i * nfc_geo->eccn_chunk_size, 1481 nfc_geo->eccn_chunk_size, 1482 eccbuf, eccbytes, 1483 NULL, 0, 1484 nfc_geo->ecc_strength); 1485 } 1486 1487 if (flips > 0) { 1488 max_bitflips = max_t(unsigned int, max_bitflips, 1489 flips); 1490 mtd->ecc_stats.corrected += flips; 1491 continue; 1492 } 1493 1494 mtd->ecc_stats.failed++; 1495 continue; 1496 } 1497 1498 mtd->ecc_stats.corrected += *status; 1499 max_bitflips = max_t(unsigned int, max_bitflips, *status); 1500 } 1501 1502 return max_bitflips; 1503 } 1504 1505 static void gpmi_bch_layout_std(struct gpmi_nand_data *this) 1506 { 1507 struct bch_geometry *geo = &this->bch_geometry; 1508 unsigned int ecc_strength = geo->ecc_strength >> 1; 1509 unsigned int gf_len = geo->gf_len; 1510 unsigned int block0_size = geo->ecc0_chunk_size; 1511 unsigned int blockn_size = geo->eccn_chunk_size; 1512 1513 this->bch_flashlayout0 = 1514 BF_BCH_FLASH0LAYOUT0_NBLOCKS(geo->ecc_chunk_count - 1) | 1515 BF_BCH_FLASH0LAYOUT0_META_SIZE(geo->metadata_size) | 1516 BF_BCH_FLASH0LAYOUT0_ECC0(ecc_strength, this) | 1517 BF_BCH_FLASH0LAYOUT0_GF(gf_len, this) | 1518 BF_BCH_FLASH0LAYOUT0_DATA0_SIZE(block0_size, this); 1519 1520 this->bch_flashlayout1 = 1521 BF_BCH_FLASH0LAYOUT1_PAGE_SIZE(geo->page_size) | 1522 BF_BCH_FLASH0LAYOUT1_ECCN(ecc_strength, this) | 1523 BF_BCH_FLASH0LAYOUT1_GF(gf_len, this) | 1524 BF_BCH_FLASH0LAYOUT1_DATAN_SIZE(blockn_size, this); 1525 } 1526 1527 static int gpmi_ecc_read_page(struct nand_chip *chip, uint8_t *buf, 1528 int oob_required, int page) 1529 { 1530 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1531 struct mtd_info *mtd = nand_to_mtd(chip); 1532 struct bch_geometry *geo = &this->bch_geometry; 1533 unsigned int max_bitflips; 1534 int ret; 1535 1536 gpmi_bch_layout_std(this); 1537 this->bch = true; 1538 1539 ret = nand_read_page_op(chip, page, 0, buf, geo->page_size); 1540 if (ret) 1541 return ret; 1542 1543 max_bitflips = gpmi_count_bitflips(chip, buf, 0, 1544 geo->ecc_chunk_count, 1545 geo->auxiliary_status_offset); 1546 1547 /* handle the block mark swapping */ 1548 block_mark_swapping(this, buf, this->auxiliary_virt); 1549 1550 if (oob_required) { 1551 /* 1552 * It's time to deliver the OOB bytes. See gpmi_ecc_read_oob() 1553 * for details about our policy for delivering the OOB. 1554 * 1555 * We fill the caller's buffer with set bits, and then copy the 1556 * block mark to th caller's buffer. Note that, if block mark 1557 * swapping was necessary, it has already been done, so we can 1558 * rely on the first byte of the auxiliary buffer to contain 1559 * the block mark. 1560 */ 1561 memset(chip->oob_poi, ~0, mtd->oobsize); 1562 chip->oob_poi[0] = ((uint8_t *)this->auxiliary_virt)[0]; 1563 } 1564 1565 return max_bitflips; 1566 } 1567 1568 /* Fake a virtual small page for the subpage read */ 1569 static int gpmi_ecc_read_subpage(struct nand_chip *chip, uint32_t offs, 1570 uint32_t len, uint8_t *buf, int page) 1571 { 1572 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1573 struct bch_geometry *geo = &this->bch_geometry; 1574 int size = chip->ecc.size; /* ECC chunk size */ 1575 int meta, n, page_size; 1576 unsigned int max_bitflips; 1577 unsigned int ecc_strength; 1578 int first, last, marker_pos; 1579 int ecc_parity_size; 1580 int col = 0; 1581 int ret; 1582 1583 /* The size of ECC parity */ 1584 ecc_parity_size = geo->gf_len * geo->ecc_strength / 8; 1585 1586 /* Align it with the chunk size */ 1587 first = offs / size; 1588 last = (offs + len - 1) / size; 1589 1590 if (this->swap_block_mark) { 1591 /* 1592 * Find the chunk which contains the Block Marker. 1593 * If this chunk is in the range of [first, last], 1594 * we have to read out the whole page. 1595 * Why? since we had swapped the data at the position of Block 1596 * Marker to the metadata which is bound with the chunk 0. 1597 */ 1598 marker_pos = geo->block_mark_byte_offset / size; 1599 if (last >= marker_pos && first <= marker_pos) { 1600 dev_dbg(this->dev, 1601 "page:%d, first:%d, last:%d, marker at:%d\n", 1602 page, first, last, marker_pos); 1603 return gpmi_ecc_read_page(chip, buf, 0, page); 1604 } 1605 } 1606 1607 /* 1608 * if there is an ECC dedicate for meta: 1609 * - need to add an extra ECC size when calculating col and page_size, 1610 * if the meta size is NOT zero. 1611 * - ecc0_chunk size need to set to the same size as other chunks, 1612 * if the meta size is zero. 1613 */ 1614 1615 meta = geo->metadata_size; 1616 if (first) { 1617 if (geo->ecc_for_meta) 1618 col = meta + ecc_parity_size 1619 + (size + ecc_parity_size) * first; 1620 else 1621 col = meta + (size + ecc_parity_size) * first; 1622 1623 meta = 0; 1624 buf = buf + first * size; 1625 } 1626 1627 ecc_parity_size = geo->gf_len * geo->ecc_strength / 8; 1628 n = last - first + 1; 1629 1630 if (geo->ecc_for_meta && meta) 1631 page_size = meta + ecc_parity_size 1632 + (size + ecc_parity_size) * n; 1633 else 1634 page_size = meta + (size + ecc_parity_size) * n; 1635 1636 ecc_strength = geo->ecc_strength >> 1; 1637 1638 this->bch_flashlayout0 = BF_BCH_FLASH0LAYOUT0_NBLOCKS( 1639 (geo->ecc_for_meta ? n : n - 1)) | 1640 BF_BCH_FLASH0LAYOUT0_META_SIZE(meta) | 1641 BF_BCH_FLASH0LAYOUT0_ECC0(ecc_strength, this) | 1642 BF_BCH_FLASH0LAYOUT0_GF(geo->gf_len, this) | 1643 BF_BCH_FLASH0LAYOUT0_DATA0_SIZE((geo->ecc_for_meta ? 1644 0 : geo->ecc0_chunk_size), this); 1645 1646 this->bch_flashlayout1 = BF_BCH_FLASH0LAYOUT1_PAGE_SIZE(page_size) | 1647 BF_BCH_FLASH0LAYOUT1_ECCN(ecc_strength, this) | 1648 BF_BCH_FLASH0LAYOUT1_GF(geo->gf_len, this) | 1649 BF_BCH_FLASH0LAYOUT1_DATAN_SIZE(geo->eccn_chunk_size, this); 1650 1651 this->bch = true; 1652 1653 ret = nand_read_page_op(chip, page, col, buf, page_size); 1654 if (ret) 1655 return ret; 1656 1657 dev_dbg(this->dev, "page:%d(%d:%d)%d, chunk:(%d:%d), BCH PG size:%d\n", 1658 page, offs, len, col, first, n, page_size); 1659 1660 max_bitflips = gpmi_count_bitflips(chip, buf, first, last, meta); 1661 1662 return max_bitflips; 1663 } 1664 1665 static int gpmi_ecc_write_page(struct nand_chip *chip, const uint8_t *buf, 1666 int oob_required, int page) 1667 { 1668 struct mtd_info *mtd = nand_to_mtd(chip); 1669 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1670 struct bch_geometry *nfc_geo = &this->bch_geometry; 1671 1672 dev_dbg(this->dev, "ecc write page.\n"); 1673 1674 gpmi_bch_layout_std(this); 1675 this->bch = true; 1676 1677 memcpy(this->auxiliary_virt, chip->oob_poi, nfc_geo->auxiliary_size); 1678 1679 if (this->swap_block_mark) { 1680 /* 1681 * When doing bad block marker swapping we must always copy the 1682 * input buffer as we can't modify the const buffer. 1683 */ 1684 memcpy(this->data_buffer_dma, buf, mtd->writesize); 1685 buf = this->data_buffer_dma; 1686 block_mark_swapping(this, this->data_buffer_dma, 1687 this->auxiliary_virt); 1688 } 1689 1690 return nand_prog_page_op(chip, page, 0, buf, nfc_geo->page_size); 1691 } 1692 1693 /* 1694 * There are several places in this driver where we have to handle the OOB and 1695 * block marks. This is the function where things are the most complicated, so 1696 * this is where we try to explain it all. All the other places refer back to 1697 * here. 1698 * 1699 * These are the rules, in order of decreasing importance: 1700 * 1701 * 1) Nothing the caller does can be allowed to imperil the block mark. 1702 * 1703 * 2) In read operations, the first byte of the OOB we return must reflect the 1704 * true state of the block mark, no matter where that block mark appears in 1705 * the physical page. 1706 * 1707 * 3) ECC-based read operations return an OOB full of set bits (since we never 1708 * allow ECC-based writes to the OOB, it doesn't matter what ECC-based reads 1709 * return). 1710 * 1711 * 4) "Raw" read operations return a direct view of the physical bytes in the 1712 * page, using the conventional definition of which bytes are data and which 1713 * are OOB. This gives the caller a way to see the actual, physical bytes 1714 * in the page, without the distortions applied by our ECC engine. 1715 * 1716 * 1717 * What we do for this specific read operation depends on two questions: 1718 * 1719 * 1) Are we doing a "raw" read, or an ECC-based read? 1720 * 1721 * 2) Are we using block mark swapping or transcription? 1722 * 1723 * There are four cases, illustrated by the following Karnaugh map: 1724 * 1725 * | Raw | ECC-based | 1726 * -------------+-------------------------+-------------------------+ 1727 * | Read the conventional | | 1728 * | OOB at the end of the | | 1729 * Swapping | page and return it. It | | 1730 * | contains exactly what | | 1731 * | we want. | Read the block mark and | 1732 * -------------+-------------------------+ return it in a buffer | 1733 * | Read the conventional | full of set bits. | 1734 * | OOB at the end of the | | 1735 * | page and also the block | | 1736 * Transcribing | mark in the metadata. | | 1737 * | Copy the block mark | | 1738 * | into the first byte of | | 1739 * | the OOB. | | 1740 * -------------+-------------------------+-------------------------+ 1741 * 1742 * Note that we break rule #4 in the Transcribing/Raw case because we're not 1743 * giving an accurate view of the actual, physical bytes in the page (we're 1744 * overwriting the block mark). That's OK because it's more important to follow 1745 * rule #2. 1746 * 1747 * It turns out that knowing whether we want an "ECC-based" or "raw" read is not 1748 * easy. When reading a page, for example, the NAND Flash MTD code calls our 1749 * ecc.read_page or ecc.read_page_raw function. Thus, the fact that MTD wants an 1750 * ECC-based or raw view of the page is implicit in which function it calls 1751 * (there is a similar pair of ECC-based/raw functions for writing). 1752 */ 1753 static int gpmi_ecc_read_oob(struct nand_chip *chip, int page) 1754 { 1755 struct mtd_info *mtd = nand_to_mtd(chip); 1756 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1757 int ret; 1758 1759 /* clear the OOB buffer */ 1760 memset(chip->oob_poi, ~0, mtd->oobsize); 1761 1762 /* Read out the conventional OOB. */ 1763 ret = nand_read_page_op(chip, page, mtd->writesize, chip->oob_poi, 1764 mtd->oobsize); 1765 if (ret) 1766 return ret; 1767 1768 /* 1769 * Now, we want to make sure the block mark is correct. In the 1770 * non-transcribing case (!GPMI_IS_MX23()), we already have it. 1771 * Otherwise, we need to explicitly read it. 1772 */ 1773 if (GPMI_IS_MX23(this)) { 1774 /* Read the block mark into the first byte of the OOB buffer. */ 1775 ret = nand_read_page_op(chip, page, 0, chip->oob_poi, 1); 1776 if (ret) 1777 return ret; 1778 } 1779 1780 return 0; 1781 } 1782 1783 static int gpmi_ecc_write_oob(struct nand_chip *chip, int page) 1784 { 1785 struct mtd_info *mtd = nand_to_mtd(chip); 1786 struct mtd_oob_region of = { }; 1787 1788 /* Do we have available oob area? */ 1789 mtd_ooblayout_free(mtd, 0, &of); 1790 if (!of.length) 1791 return -EPERM; 1792 1793 if (!nand_is_slc(chip)) 1794 return -EPERM; 1795 1796 return nand_prog_page_op(chip, page, mtd->writesize + of.offset, 1797 chip->oob_poi + of.offset, of.length); 1798 } 1799 1800 /* 1801 * This function reads a NAND page without involving the ECC engine (no HW 1802 * ECC correction). 1803 * The tricky part in the GPMI/BCH controller is that it stores ECC bits 1804 * inline (interleaved with payload DATA), and do not align data chunk on 1805 * byte boundaries. 1806 * We thus need to take care moving the payload data and ECC bits stored in the 1807 * page into the provided buffers, which is why we're using nand_extract_bits(). 1808 * 1809 * See set_geometry_by_ecc_info inline comments to have a full description 1810 * of the layout used by the GPMI controller. 1811 */ 1812 static int gpmi_ecc_read_page_raw(struct nand_chip *chip, uint8_t *buf, 1813 int oob_required, int page) 1814 { 1815 struct mtd_info *mtd = nand_to_mtd(chip); 1816 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1817 struct bch_geometry *nfc_geo = &this->bch_geometry; 1818 int eccsize = nfc_geo->eccn_chunk_size; 1819 int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len; 1820 u8 *tmp_buf = this->raw_buffer; 1821 size_t src_bit_off; 1822 size_t oob_bit_off; 1823 size_t oob_byte_off; 1824 uint8_t *oob = chip->oob_poi; 1825 int step; 1826 int ret; 1827 1828 ret = nand_read_page_op(chip, page, 0, tmp_buf, 1829 mtd->writesize + mtd->oobsize); 1830 if (ret) 1831 return ret; 1832 1833 /* 1834 * If required, swap the bad block marker and the data stored in the 1835 * metadata section, so that we don't wrongly consider a block as bad. 1836 * 1837 * See the layout description for a detailed explanation on why this 1838 * is needed. 1839 */ 1840 if (this->swap_block_mark) 1841 swap(tmp_buf[0], tmp_buf[mtd->writesize]); 1842 1843 /* 1844 * Copy the metadata section into the oob buffer (this section is 1845 * guaranteed to be aligned on a byte boundary). 1846 */ 1847 if (oob_required) 1848 memcpy(oob, tmp_buf, nfc_geo->metadata_size); 1849 1850 oob_bit_off = nfc_geo->metadata_size * 8; 1851 src_bit_off = oob_bit_off; 1852 1853 /* Extract interleaved payload data and ECC bits */ 1854 for (step = 0; step < nfc_geo->ecc_chunk_count; step++) { 1855 if (buf) 1856 nand_extract_bits(buf, step * eccsize * 8, tmp_buf, 1857 src_bit_off, eccsize * 8); 1858 src_bit_off += eccsize * 8; 1859 1860 /* Align last ECC block to align a byte boundary */ 1861 if (step == nfc_geo->ecc_chunk_count - 1 && 1862 (oob_bit_off + eccbits) % 8) 1863 eccbits += 8 - ((oob_bit_off + eccbits) % 8); 1864 1865 if (oob_required) 1866 nand_extract_bits(oob, oob_bit_off, tmp_buf, 1867 src_bit_off, eccbits); 1868 1869 src_bit_off += eccbits; 1870 oob_bit_off += eccbits; 1871 } 1872 1873 if (oob_required) { 1874 oob_byte_off = oob_bit_off / 8; 1875 1876 if (oob_byte_off < mtd->oobsize) 1877 memcpy(oob + oob_byte_off, 1878 tmp_buf + mtd->writesize + oob_byte_off, 1879 mtd->oobsize - oob_byte_off); 1880 } 1881 1882 return 0; 1883 } 1884 1885 /* 1886 * This function writes a NAND page without involving the ECC engine (no HW 1887 * ECC generation). 1888 * The tricky part in the GPMI/BCH controller is that it stores ECC bits 1889 * inline (interleaved with payload DATA), and do not align data chunk on 1890 * byte boundaries. 1891 * We thus need to take care moving the OOB area at the right place in the 1892 * final page, which is why we're using nand_extract_bits(). 1893 * 1894 * See set_geometry_by_ecc_info inline comments to have a full description 1895 * of the layout used by the GPMI controller. 1896 */ 1897 static int gpmi_ecc_write_page_raw(struct nand_chip *chip, const uint8_t *buf, 1898 int oob_required, int page) 1899 { 1900 struct mtd_info *mtd = nand_to_mtd(chip); 1901 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1902 struct bch_geometry *nfc_geo = &this->bch_geometry; 1903 int eccsize = nfc_geo->eccn_chunk_size; 1904 int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len; 1905 u8 *tmp_buf = this->raw_buffer; 1906 uint8_t *oob = chip->oob_poi; 1907 size_t dst_bit_off; 1908 size_t oob_bit_off; 1909 size_t oob_byte_off; 1910 int step; 1911 1912 /* 1913 * Initialize all bits to 1 in case we don't have a buffer for the 1914 * payload or oob data in order to leave unspecified bits of data 1915 * to their initial state. 1916 */ 1917 if (!buf || !oob_required) 1918 memset(tmp_buf, 0xff, mtd->writesize + mtd->oobsize); 1919 1920 /* 1921 * First copy the metadata section (stored in oob buffer) at the 1922 * beginning of the page, as imposed by the GPMI layout. 1923 */ 1924 memcpy(tmp_buf, oob, nfc_geo->metadata_size); 1925 oob_bit_off = nfc_geo->metadata_size * 8; 1926 dst_bit_off = oob_bit_off; 1927 1928 /* Interleave payload data and ECC bits */ 1929 for (step = 0; step < nfc_geo->ecc_chunk_count; step++) { 1930 if (buf) 1931 nand_extract_bits(tmp_buf, dst_bit_off, buf, 1932 step * eccsize * 8, eccsize * 8); 1933 dst_bit_off += eccsize * 8; 1934 1935 /* Align last ECC block to align a byte boundary */ 1936 if (step == nfc_geo->ecc_chunk_count - 1 && 1937 (oob_bit_off + eccbits) % 8) 1938 eccbits += 8 - ((oob_bit_off + eccbits) % 8); 1939 1940 if (oob_required) 1941 nand_extract_bits(tmp_buf, dst_bit_off, oob, 1942 oob_bit_off, eccbits); 1943 1944 dst_bit_off += eccbits; 1945 oob_bit_off += eccbits; 1946 } 1947 1948 oob_byte_off = oob_bit_off / 8; 1949 1950 if (oob_required && oob_byte_off < mtd->oobsize) 1951 memcpy(tmp_buf + mtd->writesize + oob_byte_off, 1952 oob + oob_byte_off, mtd->oobsize - oob_byte_off); 1953 1954 /* 1955 * If required, swap the bad block marker and the first byte of the 1956 * metadata section, so that we don't modify the bad block marker. 1957 * 1958 * See the layout description for a detailed explanation on why this 1959 * is needed. 1960 */ 1961 if (this->swap_block_mark) 1962 swap(tmp_buf[0], tmp_buf[mtd->writesize]); 1963 1964 return nand_prog_page_op(chip, page, 0, tmp_buf, 1965 mtd->writesize + mtd->oobsize); 1966 } 1967 1968 static int gpmi_ecc_read_oob_raw(struct nand_chip *chip, int page) 1969 { 1970 return gpmi_ecc_read_page_raw(chip, NULL, 1, page); 1971 } 1972 1973 static int gpmi_ecc_write_oob_raw(struct nand_chip *chip, int page) 1974 { 1975 return gpmi_ecc_write_page_raw(chip, NULL, 1, page); 1976 } 1977 1978 static int gpmi_block_markbad(struct nand_chip *chip, loff_t ofs) 1979 { 1980 struct mtd_info *mtd = nand_to_mtd(chip); 1981 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1982 int ret = 0; 1983 uint8_t *block_mark; 1984 int column, page, chipnr; 1985 1986 chipnr = (int)(ofs >> chip->chip_shift); 1987 nand_select_target(chip, chipnr); 1988 1989 column = !GPMI_IS_MX23(this) ? mtd->writesize : 0; 1990 1991 /* Write the block mark. */ 1992 block_mark = this->data_buffer_dma; 1993 block_mark[0] = 0; /* bad block marker */ 1994 1995 /* Shift to get page */ 1996 page = (int)(ofs >> chip->page_shift); 1997 1998 ret = nand_prog_page_op(chip, page, column, block_mark, 1); 1999 2000 nand_deselect_target(chip); 2001 2002 return ret; 2003 } 2004 2005 static int nand_boot_set_geometry(struct gpmi_nand_data *this) 2006 { 2007 struct boot_rom_geometry *geometry = &this->rom_geometry; 2008 2009 /* 2010 * Set the boot block stride size. 2011 * 2012 * In principle, we should be reading this from the OTP bits, since 2013 * that's where the ROM is going to get it. In fact, we don't have any 2014 * way to read the OTP bits, so we go with the default and hope for the 2015 * best. 2016 */ 2017 geometry->stride_size_in_pages = 64; 2018 2019 /* 2020 * Set the search area stride exponent. 2021 * 2022 * In principle, we should be reading this from the OTP bits, since 2023 * that's where the ROM is going to get it. In fact, we don't have any 2024 * way to read the OTP bits, so we go with the default and hope for the 2025 * best. 2026 */ 2027 geometry->search_area_stride_exponent = 2; 2028 return 0; 2029 } 2030 2031 static const char *fingerprint = "STMP"; 2032 static int mx23_check_transcription_stamp(struct gpmi_nand_data *this) 2033 { 2034 struct boot_rom_geometry *rom_geo = &this->rom_geometry; 2035 struct device *dev = this->dev; 2036 struct nand_chip *chip = &this->nand; 2037 unsigned int search_area_size_in_strides; 2038 unsigned int stride; 2039 unsigned int page; 2040 u8 *buffer = nand_get_data_buf(chip); 2041 int found_an_ncb_fingerprint = false; 2042 int ret; 2043 2044 /* Compute the number of strides in a search area. */ 2045 search_area_size_in_strides = 1 << rom_geo->search_area_stride_exponent; 2046 2047 nand_select_target(chip, 0); 2048 2049 /* 2050 * Loop through the first search area, looking for the NCB fingerprint. 2051 */ 2052 dev_dbg(dev, "Scanning for an NCB fingerprint...\n"); 2053 2054 for (stride = 0; stride < search_area_size_in_strides; stride++) { 2055 /* Compute the page addresses. */ 2056 page = stride * rom_geo->stride_size_in_pages; 2057 2058 dev_dbg(dev, "Looking for a fingerprint in page 0x%x\n", page); 2059 2060 /* 2061 * Read the NCB fingerprint. The fingerprint is four bytes long 2062 * and starts in the 12th byte of the page. 2063 */ 2064 ret = nand_read_page_op(chip, page, 12, buffer, 2065 strlen(fingerprint)); 2066 if (ret) 2067 continue; 2068 2069 /* Look for the fingerprint. */ 2070 if (!memcmp(buffer, fingerprint, strlen(fingerprint))) { 2071 found_an_ncb_fingerprint = true; 2072 break; 2073 } 2074 2075 } 2076 2077 nand_deselect_target(chip); 2078 2079 if (found_an_ncb_fingerprint) 2080 dev_dbg(dev, "\tFound a fingerprint\n"); 2081 else 2082 dev_dbg(dev, "\tNo fingerprint found\n"); 2083 return found_an_ncb_fingerprint; 2084 } 2085 2086 /* Writes a transcription stamp. */ 2087 static int mx23_write_transcription_stamp(struct gpmi_nand_data *this) 2088 { 2089 struct device *dev = this->dev; 2090 struct boot_rom_geometry *rom_geo = &this->rom_geometry; 2091 struct nand_chip *chip = &this->nand; 2092 struct mtd_info *mtd = nand_to_mtd(chip); 2093 unsigned int block_size_in_pages; 2094 unsigned int search_area_size_in_strides; 2095 unsigned int search_area_size_in_pages; 2096 unsigned int search_area_size_in_blocks; 2097 unsigned int block; 2098 unsigned int stride; 2099 unsigned int page; 2100 u8 *buffer = nand_get_data_buf(chip); 2101 int status; 2102 2103 /* Compute the search area geometry. */ 2104 block_size_in_pages = mtd->erasesize / mtd->writesize; 2105 search_area_size_in_strides = 1 << rom_geo->search_area_stride_exponent; 2106 search_area_size_in_pages = search_area_size_in_strides * 2107 rom_geo->stride_size_in_pages; 2108 search_area_size_in_blocks = 2109 (search_area_size_in_pages + (block_size_in_pages - 1)) / 2110 block_size_in_pages; 2111 2112 dev_dbg(dev, "Search Area Geometry :\n"); 2113 dev_dbg(dev, "\tin Blocks : %u\n", search_area_size_in_blocks); 2114 dev_dbg(dev, "\tin Strides: %u\n", search_area_size_in_strides); 2115 dev_dbg(dev, "\tin Pages : %u\n", search_area_size_in_pages); 2116 2117 nand_select_target(chip, 0); 2118 2119 /* Loop over blocks in the first search area, erasing them. */ 2120 dev_dbg(dev, "Erasing the search area...\n"); 2121 2122 for (block = 0; block < search_area_size_in_blocks; block++) { 2123 /* Erase this block. */ 2124 dev_dbg(dev, "\tErasing block 0x%x\n", block); 2125 status = nand_erase_op(chip, block); 2126 if (status) 2127 dev_err(dev, "[%s] Erase failed.\n", __func__); 2128 } 2129 2130 /* Write the NCB fingerprint into the page buffer. */ 2131 memset(buffer, ~0, mtd->writesize); 2132 memcpy(buffer + 12, fingerprint, strlen(fingerprint)); 2133 2134 /* Loop through the first search area, writing NCB fingerprints. */ 2135 dev_dbg(dev, "Writing NCB fingerprints...\n"); 2136 for (stride = 0; stride < search_area_size_in_strides; stride++) { 2137 /* Compute the page addresses. */ 2138 page = stride * rom_geo->stride_size_in_pages; 2139 2140 /* Write the first page of the current stride. */ 2141 dev_dbg(dev, "Writing an NCB fingerprint in page 0x%x\n", page); 2142 2143 status = chip->ecc.write_page_raw(chip, buffer, 0, page); 2144 if (status) 2145 dev_err(dev, "[%s] Write failed.\n", __func__); 2146 } 2147 2148 nand_deselect_target(chip); 2149 2150 return 0; 2151 } 2152 2153 static int mx23_boot_init(struct gpmi_nand_data *this) 2154 { 2155 struct device *dev = this->dev; 2156 struct nand_chip *chip = &this->nand; 2157 struct mtd_info *mtd = nand_to_mtd(chip); 2158 unsigned int block_count; 2159 unsigned int block; 2160 int chipnr; 2161 int page; 2162 loff_t byte; 2163 uint8_t block_mark; 2164 int ret = 0; 2165 2166 /* 2167 * If control arrives here, we can't use block mark swapping, which 2168 * means we're forced to use transcription. First, scan for the 2169 * transcription stamp. If we find it, then we don't have to do 2170 * anything -- the block marks are already transcribed. 2171 */ 2172 if (mx23_check_transcription_stamp(this)) 2173 return 0; 2174 2175 /* 2176 * If control arrives here, we couldn't find a transcription stamp, so 2177 * so we presume the block marks are in the conventional location. 2178 */ 2179 dev_dbg(dev, "Transcribing bad block marks...\n"); 2180 2181 /* Compute the number of blocks in the entire medium. */ 2182 block_count = nanddev_eraseblocks_per_target(&chip->base); 2183 2184 /* 2185 * Loop over all the blocks in the medium, transcribing block marks as 2186 * we go. 2187 */ 2188 for (block = 0; block < block_count; block++) { 2189 /* 2190 * Compute the chip, page and byte addresses for this block's 2191 * conventional mark. 2192 */ 2193 chipnr = block >> (chip->chip_shift - chip->phys_erase_shift); 2194 page = block << (chip->phys_erase_shift - chip->page_shift); 2195 byte = block << chip->phys_erase_shift; 2196 2197 /* Send the command to read the conventional block mark. */ 2198 nand_select_target(chip, chipnr); 2199 ret = nand_read_page_op(chip, page, mtd->writesize, &block_mark, 2200 1); 2201 nand_deselect_target(chip); 2202 2203 if (ret) 2204 continue; 2205 2206 /* 2207 * Check if the block is marked bad. If so, we need to mark it 2208 * again, but this time the result will be a mark in the 2209 * location where we transcribe block marks. 2210 */ 2211 if (block_mark != 0xff) { 2212 dev_dbg(dev, "Transcribing mark in block %u\n", block); 2213 ret = chip->legacy.block_markbad(chip, byte); 2214 if (ret) 2215 dev_err(dev, 2216 "Failed to mark block bad with ret %d\n", 2217 ret); 2218 } 2219 } 2220 2221 /* Write the stamp that indicates we've transcribed the block marks. */ 2222 mx23_write_transcription_stamp(this); 2223 return 0; 2224 } 2225 2226 static int nand_boot_init(struct gpmi_nand_data *this) 2227 { 2228 nand_boot_set_geometry(this); 2229 2230 /* This is ROM arch-specific initilization before the BBT scanning. */ 2231 if (GPMI_IS_MX23(this)) 2232 return mx23_boot_init(this); 2233 return 0; 2234 } 2235 2236 static int gpmi_set_geometry(struct gpmi_nand_data *this) 2237 { 2238 int ret; 2239 2240 /* Free the temporary DMA memory for reading ID. */ 2241 gpmi_free_dma_buffer(this); 2242 2243 /* Set up the NFC geometry which is used by BCH. */ 2244 ret = bch_set_geometry(this); 2245 if (ret) { 2246 dev_err(this->dev, "Error setting BCH geometry : %d\n", ret); 2247 return ret; 2248 } 2249 2250 /* Alloc the new DMA buffers according to the pagesize and oobsize */ 2251 return gpmi_alloc_dma_buffer(this); 2252 } 2253 2254 static int gpmi_init_last(struct gpmi_nand_data *this) 2255 { 2256 struct nand_chip *chip = &this->nand; 2257 struct mtd_info *mtd = nand_to_mtd(chip); 2258 struct nand_ecc_ctrl *ecc = &chip->ecc; 2259 struct bch_geometry *bch_geo = &this->bch_geometry; 2260 int ret; 2261 2262 /* Set up the medium geometry */ 2263 ret = gpmi_set_geometry(this); 2264 if (ret) 2265 return ret; 2266 2267 /* Init the nand_ecc_ctrl{} */ 2268 ecc->read_page = gpmi_ecc_read_page; 2269 ecc->write_page = gpmi_ecc_write_page; 2270 ecc->read_oob = gpmi_ecc_read_oob; 2271 ecc->write_oob = gpmi_ecc_write_oob; 2272 ecc->read_page_raw = gpmi_ecc_read_page_raw; 2273 ecc->write_page_raw = gpmi_ecc_write_page_raw; 2274 ecc->read_oob_raw = gpmi_ecc_read_oob_raw; 2275 ecc->write_oob_raw = gpmi_ecc_write_oob_raw; 2276 ecc->engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; 2277 ecc->size = bch_geo->eccn_chunk_size; 2278 ecc->strength = bch_geo->ecc_strength; 2279 mtd_set_ooblayout(mtd, &gpmi_ooblayout_ops); 2280 2281 /* 2282 * We only enable the subpage read when: 2283 * (1) the chip is imx6, and 2284 * (2) the size of the ECC parity is byte aligned. 2285 */ 2286 if (GPMI_IS_MX6(this) && 2287 ((bch_geo->gf_len * bch_geo->ecc_strength) % 8) == 0) { 2288 ecc->read_subpage = gpmi_ecc_read_subpage; 2289 chip->options |= NAND_SUBPAGE_READ; 2290 } 2291 2292 return 0; 2293 } 2294 2295 static int gpmi_nand_attach_chip(struct nand_chip *chip) 2296 { 2297 struct gpmi_nand_data *this = nand_get_controller_data(chip); 2298 int ret; 2299 2300 if (chip->bbt_options & NAND_BBT_USE_FLASH) { 2301 chip->bbt_options |= NAND_BBT_NO_OOB; 2302 2303 if (of_property_read_bool(this->dev->of_node, 2304 "fsl,no-blockmark-swap")) 2305 this->swap_block_mark = false; 2306 } 2307 dev_dbg(this->dev, "Blockmark swapping %sabled\n", 2308 this->swap_block_mark ? "en" : "dis"); 2309 2310 ret = gpmi_init_last(this); 2311 if (ret) 2312 return ret; 2313 2314 chip->options |= NAND_SKIP_BBTSCAN; 2315 2316 return 0; 2317 } 2318 2319 static struct gpmi_transfer *get_next_transfer(struct gpmi_nand_data *this) 2320 { 2321 struct gpmi_transfer *transfer = &this->transfers[this->ntransfers]; 2322 2323 this->ntransfers++; 2324 2325 if (this->ntransfers == GPMI_MAX_TRANSFERS) 2326 return NULL; 2327 2328 return transfer; 2329 } 2330 2331 static struct dma_async_tx_descriptor *gpmi_chain_command( 2332 struct gpmi_nand_data *this, u8 cmd, const u8 *addr, int naddr) 2333 { 2334 struct dma_chan *channel = get_dma_chan(this); 2335 struct dma_async_tx_descriptor *desc; 2336 struct gpmi_transfer *transfer; 2337 int chip = this->nand.cur_cs; 2338 u32 pio[3]; 2339 2340 /* [1] send out the PIO words */ 2341 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WRITE) 2342 | BM_GPMI_CTRL0_WORD_LENGTH 2343 | BF_GPMI_CTRL0_CS(chip, this) 2344 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this) 2345 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_CLE) 2346 | BM_GPMI_CTRL0_ADDRESS_INCREMENT 2347 | BF_GPMI_CTRL0_XFER_COUNT(naddr + 1); 2348 pio[1] = 0; 2349 pio[2] = 0; 2350 desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio), 2351 DMA_TRANS_NONE, 0); 2352 if (!desc) 2353 return NULL; 2354 2355 transfer = get_next_transfer(this); 2356 if (!transfer) 2357 return NULL; 2358 2359 transfer->cmdbuf[0] = cmd; 2360 if (naddr) 2361 memcpy(&transfer->cmdbuf[1], addr, naddr); 2362 2363 sg_init_one(&transfer->sgl, transfer->cmdbuf, naddr + 1); 2364 dma_map_sg(this->dev, &transfer->sgl, 1, DMA_TO_DEVICE); 2365 2366 transfer->direction = DMA_TO_DEVICE; 2367 2368 desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1, DMA_MEM_TO_DEV, 2369 MXS_DMA_CTRL_WAIT4END); 2370 return desc; 2371 } 2372 2373 static struct dma_async_tx_descriptor *gpmi_chain_wait_ready( 2374 struct gpmi_nand_data *this) 2375 { 2376 struct dma_chan *channel = get_dma_chan(this); 2377 u32 pio[2]; 2378 2379 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WAIT_FOR_READY) 2380 | BM_GPMI_CTRL0_WORD_LENGTH 2381 | BF_GPMI_CTRL0_CS(this->nand.cur_cs, this) 2382 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this) 2383 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA) 2384 | BF_GPMI_CTRL0_XFER_COUNT(0); 2385 pio[1] = 0; 2386 2387 return mxs_dmaengine_prep_pio(channel, pio, 2, DMA_TRANS_NONE, 2388 MXS_DMA_CTRL_WAIT4END | MXS_DMA_CTRL_WAIT4RDY); 2389 } 2390 2391 static struct dma_async_tx_descriptor *gpmi_chain_data_read( 2392 struct gpmi_nand_data *this, void *buf, int raw_len, bool *direct) 2393 { 2394 struct dma_async_tx_descriptor *desc; 2395 struct dma_chan *channel = get_dma_chan(this); 2396 struct gpmi_transfer *transfer; 2397 u32 pio[6] = {}; 2398 2399 transfer = get_next_transfer(this); 2400 if (!transfer) 2401 return NULL; 2402 2403 transfer->direction = DMA_FROM_DEVICE; 2404 2405 *direct = prepare_data_dma(this, buf, raw_len, &transfer->sgl, 2406 DMA_FROM_DEVICE); 2407 2408 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__READ) 2409 | BM_GPMI_CTRL0_WORD_LENGTH 2410 | BF_GPMI_CTRL0_CS(this->nand.cur_cs, this) 2411 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this) 2412 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA) 2413 | BF_GPMI_CTRL0_XFER_COUNT(raw_len); 2414 2415 if (this->bch) { 2416 pio[2] = BM_GPMI_ECCCTRL_ENABLE_ECC 2417 | BF_GPMI_ECCCTRL_ECC_CMD(BV_GPMI_ECCCTRL_ECC_CMD__BCH_DECODE) 2418 | BF_GPMI_ECCCTRL_BUFFER_MASK(BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_PAGE 2419 | BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_AUXONLY); 2420 pio[3] = raw_len; 2421 pio[4] = transfer->sgl.dma_address; 2422 pio[5] = this->auxiliary_phys; 2423 } 2424 2425 desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio), 2426 DMA_TRANS_NONE, 0); 2427 if (!desc) 2428 return NULL; 2429 2430 if (!this->bch) 2431 desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1, 2432 DMA_DEV_TO_MEM, 2433 MXS_DMA_CTRL_WAIT4END); 2434 2435 return desc; 2436 } 2437 2438 static struct dma_async_tx_descriptor *gpmi_chain_data_write( 2439 struct gpmi_nand_data *this, const void *buf, int raw_len) 2440 { 2441 struct dma_chan *channel = get_dma_chan(this); 2442 struct dma_async_tx_descriptor *desc; 2443 struct gpmi_transfer *transfer; 2444 u32 pio[6] = {}; 2445 2446 transfer = get_next_transfer(this); 2447 if (!transfer) 2448 return NULL; 2449 2450 transfer->direction = DMA_TO_DEVICE; 2451 2452 prepare_data_dma(this, buf, raw_len, &transfer->sgl, DMA_TO_DEVICE); 2453 2454 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WRITE) 2455 | BM_GPMI_CTRL0_WORD_LENGTH 2456 | BF_GPMI_CTRL0_CS(this->nand.cur_cs, this) 2457 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this) 2458 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA) 2459 | BF_GPMI_CTRL0_XFER_COUNT(raw_len); 2460 2461 if (this->bch) { 2462 pio[2] = BM_GPMI_ECCCTRL_ENABLE_ECC 2463 | BF_GPMI_ECCCTRL_ECC_CMD(BV_GPMI_ECCCTRL_ECC_CMD__BCH_ENCODE) 2464 | BF_GPMI_ECCCTRL_BUFFER_MASK(BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_PAGE | 2465 BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_AUXONLY); 2466 pio[3] = raw_len; 2467 pio[4] = transfer->sgl.dma_address; 2468 pio[5] = this->auxiliary_phys; 2469 } 2470 2471 desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio), 2472 DMA_TRANS_NONE, 2473 (this->bch ? MXS_DMA_CTRL_WAIT4END : 0)); 2474 if (!desc) 2475 return NULL; 2476 2477 if (!this->bch) 2478 desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1, 2479 DMA_MEM_TO_DEV, 2480 MXS_DMA_CTRL_WAIT4END); 2481 2482 return desc; 2483 } 2484 2485 static int gpmi_nfc_exec_op(struct nand_chip *chip, 2486 const struct nand_operation *op, 2487 bool check_only) 2488 { 2489 const struct nand_op_instr *instr; 2490 struct gpmi_nand_data *this = nand_get_controller_data(chip); 2491 struct dma_async_tx_descriptor *desc = NULL; 2492 int i, ret, buf_len = 0, nbufs = 0; 2493 u8 cmd = 0; 2494 void *buf_read = NULL; 2495 const void *buf_write = NULL; 2496 bool direct = false; 2497 struct completion *dma_completion, *bch_completion; 2498 unsigned long to; 2499 2500 if (check_only) 2501 return 0; 2502 2503 this->ntransfers = 0; 2504 for (i = 0; i < GPMI_MAX_TRANSFERS; i++) 2505 this->transfers[i].direction = DMA_NONE; 2506 2507 ret = pm_runtime_get_sync(this->dev); 2508 if (ret < 0) { 2509 pm_runtime_put_noidle(this->dev); 2510 return ret; 2511 } 2512 2513 /* 2514 * This driver currently supports only one NAND chip. Plus, dies share 2515 * the same configuration. So once timings have been applied on the 2516 * controller side, they will not change anymore. When the time will 2517 * come, the check on must_apply_timings will have to be dropped. 2518 */ 2519 if (this->hw.must_apply_timings) { 2520 this->hw.must_apply_timings = false; 2521 ret = gpmi_nfc_apply_timings(this); 2522 if (ret) 2523 goto out_pm; 2524 } 2525 2526 dev_dbg(this->dev, "%s: %d instructions\n", __func__, op->ninstrs); 2527 2528 for (i = 0; i < op->ninstrs; i++) { 2529 instr = &op->instrs[i]; 2530 2531 nand_op_trace(" ", instr); 2532 2533 switch (instr->type) { 2534 case NAND_OP_WAITRDY_INSTR: 2535 desc = gpmi_chain_wait_ready(this); 2536 break; 2537 case NAND_OP_CMD_INSTR: 2538 cmd = instr->ctx.cmd.opcode; 2539 2540 /* 2541 * When this command has an address cycle chain it 2542 * together with the address cycle 2543 */ 2544 if (i + 1 != op->ninstrs && 2545 op->instrs[i + 1].type == NAND_OP_ADDR_INSTR) 2546 continue; 2547 2548 desc = gpmi_chain_command(this, cmd, NULL, 0); 2549 2550 break; 2551 case NAND_OP_ADDR_INSTR: 2552 desc = gpmi_chain_command(this, cmd, instr->ctx.addr.addrs, 2553 instr->ctx.addr.naddrs); 2554 break; 2555 case NAND_OP_DATA_OUT_INSTR: 2556 buf_write = instr->ctx.data.buf.out; 2557 buf_len = instr->ctx.data.len; 2558 nbufs++; 2559 2560 desc = gpmi_chain_data_write(this, buf_write, buf_len); 2561 2562 break; 2563 case NAND_OP_DATA_IN_INSTR: 2564 if (!instr->ctx.data.len) 2565 break; 2566 buf_read = instr->ctx.data.buf.in; 2567 buf_len = instr->ctx.data.len; 2568 nbufs++; 2569 2570 desc = gpmi_chain_data_read(this, buf_read, buf_len, 2571 &direct); 2572 break; 2573 } 2574 2575 if (!desc) { 2576 ret = -ENXIO; 2577 goto unmap; 2578 } 2579 } 2580 2581 dev_dbg(this->dev, "%s setup done\n", __func__); 2582 2583 if (nbufs > 1) { 2584 dev_err(this->dev, "Multiple data instructions not supported\n"); 2585 ret = -EINVAL; 2586 goto unmap; 2587 } 2588 2589 if (this->bch) { 2590 writel(this->bch_flashlayout0, 2591 this->resources.bch_regs + HW_BCH_FLASH0LAYOUT0); 2592 writel(this->bch_flashlayout1, 2593 this->resources.bch_regs + HW_BCH_FLASH0LAYOUT1); 2594 } 2595 2596 desc->callback = dma_irq_callback; 2597 desc->callback_param = this; 2598 dma_completion = &this->dma_done; 2599 bch_completion = NULL; 2600 2601 init_completion(dma_completion); 2602 2603 if (this->bch && buf_read) { 2604 writel(BM_BCH_CTRL_COMPLETE_IRQ_EN, 2605 this->resources.bch_regs + HW_BCH_CTRL_SET); 2606 bch_completion = &this->bch_done; 2607 init_completion(bch_completion); 2608 } 2609 2610 dmaengine_submit(desc); 2611 dma_async_issue_pending(get_dma_chan(this)); 2612 2613 to = wait_for_completion_timeout(dma_completion, msecs_to_jiffies(1000)); 2614 if (!to) { 2615 dev_err(this->dev, "DMA timeout, last DMA\n"); 2616 gpmi_dump_info(this); 2617 ret = -ETIMEDOUT; 2618 goto unmap; 2619 } 2620 2621 if (this->bch && buf_read) { 2622 to = wait_for_completion_timeout(bch_completion, msecs_to_jiffies(1000)); 2623 if (!to) { 2624 dev_err(this->dev, "BCH timeout, last DMA\n"); 2625 gpmi_dump_info(this); 2626 ret = -ETIMEDOUT; 2627 goto unmap; 2628 } 2629 } 2630 2631 writel(BM_BCH_CTRL_COMPLETE_IRQ_EN, 2632 this->resources.bch_regs + HW_BCH_CTRL_CLR); 2633 gpmi_clear_bch(this); 2634 2635 ret = 0; 2636 2637 unmap: 2638 for (i = 0; i < this->ntransfers; i++) { 2639 struct gpmi_transfer *transfer = &this->transfers[i]; 2640 2641 if (transfer->direction != DMA_NONE) 2642 dma_unmap_sg(this->dev, &transfer->sgl, 1, 2643 transfer->direction); 2644 } 2645 2646 if (!ret && buf_read && !direct) 2647 memcpy(buf_read, this->data_buffer_dma, 2648 gpmi_raw_len_to_len(this, buf_len)); 2649 2650 this->bch = false; 2651 2652 out_pm: 2653 pm_runtime_mark_last_busy(this->dev); 2654 pm_runtime_put_autosuspend(this->dev); 2655 2656 return ret; 2657 } 2658 2659 static const struct nand_controller_ops gpmi_nand_controller_ops = { 2660 .attach_chip = gpmi_nand_attach_chip, 2661 .setup_interface = gpmi_setup_interface, 2662 .exec_op = gpmi_nfc_exec_op, 2663 }; 2664 2665 static int gpmi_nand_init(struct gpmi_nand_data *this) 2666 { 2667 struct nand_chip *chip = &this->nand; 2668 struct mtd_info *mtd = nand_to_mtd(chip); 2669 int ret; 2670 2671 /* init the MTD data structures */ 2672 mtd->name = "gpmi-nand"; 2673 mtd->dev.parent = this->dev; 2674 2675 /* init the nand_chip{}, we don't support a 16-bit NAND Flash bus. */ 2676 nand_set_controller_data(chip, this); 2677 nand_set_flash_node(chip, this->pdev->dev.of_node); 2678 chip->legacy.block_markbad = gpmi_block_markbad; 2679 chip->badblock_pattern = &gpmi_bbt_descr; 2680 chip->options |= NAND_NO_SUBPAGE_WRITE; 2681 2682 /* Set up swap_block_mark, must be set before the gpmi_set_geometry() */ 2683 this->swap_block_mark = !GPMI_IS_MX23(this); 2684 2685 /* 2686 * Allocate a temporary DMA buffer for reading ID in the 2687 * nand_scan_ident(). 2688 */ 2689 this->bch_geometry.payload_size = 1024; 2690 this->bch_geometry.auxiliary_size = 128; 2691 ret = gpmi_alloc_dma_buffer(this); 2692 if (ret) 2693 return ret; 2694 2695 nand_controller_init(&this->base); 2696 this->base.ops = &gpmi_nand_controller_ops; 2697 chip->controller = &this->base; 2698 2699 ret = nand_scan(chip, GPMI_IS_MX6(this) ? 2 : 1); 2700 if (ret) 2701 goto err_out; 2702 2703 ret = nand_boot_init(this); 2704 if (ret) 2705 goto err_nand_cleanup; 2706 ret = nand_create_bbt(chip); 2707 if (ret) 2708 goto err_nand_cleanup; 2709 2710 ret = mtd_device_register(mtd, NULL, 0); 2711 if (ret) 2712 goto err_nand_cleanup; 2713 return 0; 2714 2715 err_nand_cleanup: 2716 nand_cleanup(chip); 2717 err_out: 2718 gpmi_free_dma_buffer(this); 2719 return ret; 2720 } 2721 2722 static const struct of_device_id gpmi_nand_id_table[] = { 2723 { .compatible = "fsl,imx23-gpmi-nand", .data = &gpmi_devdata_imx23, }, 2724 { .compatible = "fsl,imx28-gpmi-nand", .data = &gpmi_devdata_imx28, }, 2725 { .compatible = "fsl,imx6q-gpmi-nand", .data = &gpmi_devdata_imx6q, }, 2726 { .compatible = "fsl,imx6sx-gpmi-nand", .data = &gpmi_devdata_imx6sx, }, 2727 { .compatible = "fsl,imx7d-gpmi-nand", .data = &gpmi_devdata_imx7d,}, 2728 {} 2729 }; 2730 MODULE_DEVICE_TABLE(of, gpmi_nand_id_table); 2731 2732 static int gpmi_nand_probe(struct platform_device *pdev) 2733 { 2734 struct gpmi_nand_data *this; 2735 int ret; 2736 2737 this = devm_kzalloc(&pdev->dev, sizeof(*this), GFP_KERNEL); 2738 if (!this) 2739 return -ENOMEM; 2740 2741 this->devdata = of_device_get_match_data(&pdev->dev); 2742 platform_set_drvdata(pdev, this); 2743 this->pdev = pdev; 2744 this->dev = &pdev->dev; 2745 2746 ret = acquire_resources(this); 2747 if (ret) 2748 goto exit_acquire_resources; 2749 2750 ret = __gpmi_enable_clk(this, true); 2751 if (ret) 2752 goto exit_acquire_resources; 2753 2754 pm_runtime_set_autosuspend_delay(&pdev->dev, 500); 2755 pm_runtime_use_autosuspend(&pdev->dev); 2756 pm_runtime_set_active(&pdev->dev); 2757 pm_runtime_enable(&pdev->dev); 2758 pm_runtime_get_sync(&pdev->dev); 2759 2760 ret = gpmi_init(this); 2761 if (ret) 2762 goto exit_nfc_init; 2763 2764 ret = gpmi_nand_init(this); 2765 if (ret) 2766 goto exit_nfc_init; 2767 2768 pm_runtime_mark_last_busy(&pdev->dev); 2769 pm_runtime_put_autosuspend(&pdev->dev); 2770 2771 dev_info(this->dev, "driver registered.\n"); 2772 2773 return 0; 2774 2775 exit_nfc_init: 2776 pm_runtime_put(&pdev->dev); 2777 pm_runtime_disable(&pdev->dev); 2778 release_resources(this); 2779 exit_acquire_resources: 2780 2781 return ret; 2782 } 2783 2784 static int gpmi_nand_remove(struct platform_device *pdev) 2785 { 2786 struct gpmi_nand_data *this = platform_get_drvdata(pdev); 2787 struct nand_chip *chip = &this->nand; 2788 int ret; 2789 2790 pm_runtime_put_sync(&pdev->dev); 2791 pm_runtime_disable(&pdev->dev); 2792 2793 ret = mtd_device_unregister(nand_to_mtd(chip)); 2794 WARN_ON(ret); 2795 nand_cleanup(chip); 2796 gpmi_free_dma_buffer(this); 2797 release_resources(this); 2798 return 0; 2799 } 2800 2801 #ifdef CONFIG_PM_SLEEP 2802 static int gpmi_pm_suspend(struct device *dev) 2803 { 2804 struct gpmi_nand_data *this = dev_get_drvdata(dev); 2805 2806 release_dma_channels(this); 2807 return 0; 2808 } 2809 2810 static int gpmi_pm_resume(struct device *dev) 2811 { 2812 struct gpmi_nand_data *this = dev_get_drvdata(dev); 2813 int ret; 2814 2815 ret = acquire_dma_channels(this); 2816 if (ret < 0) 2817 return ret; 2818 2819 /* re-init the GPMI registers */ 2820 ret = gpmi_init(this); 2821 if (ret) { 2822 dev_err(this->dev, "Error setting GPMI : %d\n", ret); 2823 return ret; 2824 } 2825 2826 /* Set flag to get timing setup restored for next exec_op */ 2827 if (this->hw.clk_rate) 2828 this->hw.must_apply_timings = true; 2829 2830 /* re-init the BCH registers */ 2831 ret = bch_set_geometry(this); 2832 if (ret) { 2833 dev_err(this->dev, "Error setting BCH : %d\n", ret); 2834 return ret; 2835 } 2836 2837 return 0; 2838 } 2839 #endif /* CONFIG_PM_SLEEP */ 2840 2841 static int __maybe_unused gpmi_runtime_suspend(struct device *dev) 2842 { 2843 struct gpmi_nand_data *this = dev_get_drvdata(dev); 2844 2845 return __gpmi_enable_clk(this, false); 2846 } 2847 2848 static int __maybe_unused gpmi_runtime_resume(struct device *dev) 2849 { 2850 struct gpmi_nand_data *this = dev_get_drvdata(dev); 2851 2852 return __gpmi_enable_clk(this, true); 2853 } 2854 2855 static const struct dev_pm_ops gpmi_pm_ops = { 2856 SET_SYSTEM_SLEEP_PM_OPS(gpmi_pm_suspend, gpmi_pm_resume) 2857 SET_RUNTIME_PM_OPS(gpmi_runtime_suspend, gpmi_runtime_resume, NULL) 2858 }; 2859 2860 static struct platform_driver gpmi_nand_driver = { 2861 .driver = { 2862 .name = "gpmi-nand", 2863 .pm = &gpmi_pm_ops, 2864 .of_match_table = gpmi_nand_id_table, 2865 }, 2866 .probe = gpmi_nand_probe, 2867 .remove = gpmi_nand_remove, 2868 }; 2869 module_platform_driver(gpmi_nand_driver); 2870 2871 MODULE_AUTHOR("Freescale Semiconductor, Inc."); 2872 MODULE_DESCRIPTION("i.MX GPMI NAND Flash Controller Driver"); 2873 MODULE_LICENSE("GPL"); 2874