1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2017 Free Electrons 4 * Copyright (C) 2017 NextThing Co 5 * 6 * Author: Boris Brezillon <boris.brezillon@free-electrons.com> 7 */ 8 9 #include <linux/slab.h> 10 11 #include "internals.h" 12 13 /* 14 * Special Micron status bit 3 indicates that the block has been 15 * corrected by on-die ECC and should be rewritten. 16 */ 17 #define NAND_ECC_STATUS_WRITE_RECOMMENDED BIT(3) 18 19 /* 20 * On chips with 8-bit ECC and additional bit can be used to distinguish 21 * cases where a errors were corrected without needing a rewrite 22 * 23 * Bit 4 Bit 3 Bit 0 Description 24 * ----- ----- ----- ----------- 25 * 0 0 0 No Errors 26 * 0 0 1 Multiple uncorrected errors 27 * 0 1 0 4 - 6 errors corrected, recommend rewrite 28 * 0 1 1 Reserved 29 * 1 0 0 1 - 3 errors corrected 30 * 1 0 1 Reserved 31 * 1 1 0 7 - 8 errors corrected, recommend rewrite 32 */ 33 #define NAND_ECC_STATUS_MASK (BIT(4) | BIT(3) | BIT(0)) 34 #define NAND_ECC_STATUS_UNCORRECTABLE BIT(0) 35 #define NAND_ECC_STATUS_4_6_CORRECTED BIT(3) 36 #define NAND_ECC_STATUS_1_3_CORRECTED BIT(4) 37 #define NAND_ECC_STATUS_7_8_CORRECTED (BIT(4) | BIT(3)) 38 39 struct nand_onfi_vendor_micron { 40 u8 two_plane_read; 41 u8 read_cache; 42 u8 read_unique_id; 43 u8 dq_imped; 44 u8 dq_imped_num_settings; 45 u8 dq_imped_feat_addr; 46 u8 rb_pulldown_strength; 47 u8 rb_pulldown_strength_feat_addr; 48 u8 rb_pulldown_strength_num_settings; 49 u8 otp_mode; 50 u8 otp_page_start; 51 u8 otp_data_prot_addr; 52 u8 otp_num_pages; 53 u8 otp_feat_addr; 54 u8 read_retry_options; 55 u8 reserved[72]; 56 u8 param_revision; 57 } __packed; 58 59 struct micron_on_die_ecc { 60 bool forced; 61 bool enabled; 62 void *rawbuf; 63 }; 64 65 struct micron_nand { 66 struct micron_on_die_ecc ecc; 67 }; 68 69 static int micron_nand_setup_read_retry(struct nand_chip *chip, int retry_mode) 70 { 71 u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = {retry_mode}; 72 73 return nand_set_features(chip, ONFI_FEATURE_ADDR_READ_RETRY, feature); 74 } 75 76 /* 77 * Configure chip properties from Micron vendor-specific ONFI table 78 */ 79 static int micron_nand_onfi_init(struct nand_chip *chip) 80 { 81 struct nand_parameters *p = &chip->parameters; 82 83 if (p->onfi) { 84 struct nand_onfi_vendor_micron *micron = (void *)p->onfi->vendor; 85 86 chip->read_retries = micron->read_retry_options; 87 chip->ops.setup_read_retry = micron_nand_setup_read_retry; 88 } 89 90 if (p->supports_set_get_features) { 91 set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->set_feature_list); 92 set_bit(ONFI_FEATURE_ON_DIE_ECC, p->set_feature_list); 93 set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->get_feature_list); 94 set_bit(ONFI_FEATURE_ON_DIE_ECC, p->get_feature_list); 95 } 96 97 return 0; 98 } 99 100 static int micron_nand_on_die_4_ooblayout_ecc(struct mtd_info *mtd, 101 int section, 102 struct mtd_oob_region *oobregion) 103 { 104 if (section >= 4) 105 return -ERANGE; 106 107 oobregion->offset = (section * 16) + 8; 108 oobregion->length = 8; 109 110 return 0; 111 } 112 113 static int micron_nand_on_die_4_ooblayout_free(struct mtd_info *mtd, 114 int section, 115 struct mtd_oob_region *oobregion) 116 { 117 if (section >= 4) 118 return -ERANGE; 119 120 oobregion->offset = (section * 16) + 2; 121 oobregion->length = 6; 122 123 return 0; 124 } 125 126 static const struct mtd_ooblayout_ops micron_nand_on_die_4_ooblayout_ops = { 127 .ecc = micron_nand_on_die_4_ooblayout_ecc, 128 .free = micron_nand_on_die_4_ooblayout_free, 129 }; 130 131 static int micron_nand_on_die_8_ooblayout_ecc(struct mtd_info *mtd, 132 int section, 133 struct mtd_oob_region *oobregion) 134 { 135 struct nand_chip *chip = mtd_to_nand(mtd); 136 137 if (section) 138 return -ERANGE; 139 140 oobregion->offset = mtd->oobsize - chip->ecc.total; 141 oobregion->length = chip->ecc.total; 142 143 return 0; 144 } 145 146 static int micron_nand_on_die_8_ooblayout_free(struct mtd_info *mtd, 147 int section, 148 struct mtd_oob_region *oobregion) 149 { 150 struct nand_chip *chip = mtd_to_nand(mtd); 151 152 if (section) 153 return -ERANGE; 154 155 oobregion->offset = 2; 156 oobregion->length = mtd->oobsize - chip->ecc.total - 2; 157 158 return 0; 159 } 160 161 static const struct mtd_ooblayout_ops micron_nand_on_die_8_ooblayout_ops = { 162 .ecc = micron_nand_on_die_8_ooblayout_ecc, 163 .free = micron_nand_on_die_8_ooblayout_free, 164 }; 165 166 static int micron_nand_on_die_ecc_setup(struct nand_chip *chip, bool enable) 167 { 168 struct micron_nand *micron = nand_get_manufacturer_data(chip); 169 u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = { 0, }; 170 int ret; 171 172 if (micron->ecc.forced) 173 return 0; 174 175 if (micron->ecc.enabled == enable) 176 return 0; 177 178 if (enable) 179 feature[0] |= ONFI_FEATURE_ON_DIE_ECC_EN; 180 181 ret = nand_set_features(chip, ONFI_FEATURE_ON_DIE_ECC, feature); 182 if (!ret) 183 micron->ecc.enabled = enable; 184 185 return ret; 186 } 187 188 static int micron_nand_on_die_ecc_status_4(struct nand_chip *chip, u8 status, 189 void *buf, int page, 190 int oob_required) 191 { 192 struct micron_nand *micron = nand_get_manufacturer_data(chip); 193 struct mtd_info *mtd = nand_to_mtd(chip); 194 unsigned int step, max_bitflips = 0; 195 bool use_datain = false; 196 int ret; 197 198 if (!(status & NAND_ECC_STATUS_WRITE_RECOMMENDED)) { 199 if (status & NAND_STATUS_FAIL) 200 mtd->ecc_stats.failed++; 201 202 return 0; 203 } 204 205 /* 206 * The internal ECC doesn't tell us the number of bitflips that have 207 * been corrected, but tells us if it recommends to rewrite the block. 208 * If it's the case, we need to read the page in raw mode and compare 209 * its content to the corrected version to extract the actual number of 210 * bitflips. 211 * But before we do that, we must make sure we have all OOB bytes read 212 * in non-raw mode, even if the user did not request those bytes. 213 */ 214 if (!oob_required) { 215 /* 216 * We first check which operation is supported by the controller 217 * before running it. This trick makes it possible to support 218 * all controllers, even the most constraints, without almost 219 * any performance hit. 220 * 221 * TODO: could be enhanced to avoid repeating the same check 222 * over and over in the fast path. 223 */ 224 if (!nand_has_exec_op(chip) || 225 !nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, false, 226 true)) 227 use_datain = true; 228 229 if (use_datain) 230 ret = nand_read_data_op(chip, chip->oob_poi, 231 mtd->oobsize, false, false); 232 else 233 ret = nand_change_read_column_op(chip, mtd->writesize, 234 chip->oob_poi, 235 mtd->oobsize, false); 236 if (ret) 237 return ret; 238 } 239 240 micron_nand_on_die_ecc_setup(chip, false); 241 242 ret = nand_read_page_op(chip, page, 0, micron->ecc.rawbuf, 243 mtd->writesize + mtd->oobsize); 244 if (ret) 245 return ret; 246 247 for (step = 0; step < chip->ecc.steps; step++) { 248 unsigned int offs, i, nbitflips = 0; 249 u8 *rawbuf, *corrbuf; 250 251 offs = step * chip->ecc.size; 252 rawbuf = micron->ecc.rawbuf + offs; 253 corrbuf = buf + offs; 254 255 for (i = 0; i < chip->ecc.size; i++) 256 nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]); 257 258 offs = (step * 16) + 4; 259 rawbuf = micron->ecc.rawbuf + mtd->writesize + offs; 260 corrbuf = chip->oob_poi + offs; 261 262 for (i = 0; i < chip->ecc.bytes + 4; i++) 263 nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]); 264 265 if (WARN_ON(nbitflips > chip->ecc.strength)) 266 return -EINVAL; 267 268 max_bitflips = max(nbitflips, max_bitflips); 269 mtd->ecc_stats.corrected += nbitflips; 270 } 271 272 return max_bitflips; 273 } 274 275 static int micron_nand_on_die_ecc_status_8(struct nand_chip *chip, u8 status) 276 { 277 struct mtd_info *mtd = nand_to_mtd(chip); 278 279 /* 280 * With 8/512 we have more information but still don't know precisely 281 * how many bit-flips were seen. 282 */ 283 switch (status & NAND_ECC_STATUS_MASK) { 284 case NAND_ECC_STATUS_UNCORRECTABLE: 285 mtd->ecc_stats.failed++; 286 return 0; 287 case NAND_ECC_STATUS_1_3_CORRECTED: 288 mtd->ecc_stats.corrected += 3; 289 return 3; 290 case NAND_ECC_STATUS_4_6_CORRECTED: 291 mtd->ecc_stats.corrected += 6; 292 /* rewrite recommended */ 293 return 6; 294 case NAND_ECC_STATUS_7_8_CORRECTED: 295 mtd->ecc_stats.corrected += 8; 296 /* rewrite recommended */ 297 return 8; 298 default: 299 return 0; 300 } 301 } 302 303 static int 304 micron_nand_read_page_on_die_ecc(struct nand_chip *chip, uint8_t *buf, 305 int oob_required, int page) 306 { 307 struct mtd_info *mtd = nand_to_mtd(chip); 308 bool use_datain = false; 309 u8 status; 310 int ret, max_bitflips = 0; 311 312 ret = micron_nand_on_die_ecc_setup(chip, true); 313 if (ret) 314 return ret; 315 316 ret = nand_read_page_op(chip, page, 0, NULL, 0); 317 if (ret) 318 goto out; 319 320 ret = nand_status_op(chip, &status); 321 if (ret) 322 goto out; 323 324 /* 325 * We first check which operation is supported by the controller before 326 * running it. This trick makes it possible to support all controllers, 327 * even the most constraints, without almost any performance hit. 328 * 329 * TODO: could be enhanced to avoid repeating the same check over and 330 * over in the fast path. 331 */ 332 if (!nand_has_exec_op(chip) || 333 !nand_read_data_op(chip, buf, mtd->writesize, false, true)) 334 use_datain = true; 335 336 if (use_datain) { 337 ret = nand_exit_status_op(chip); 338 if (ret) 339 goto out; 340 341 ret = nand_read_data_op(chip, buf, mtd->writesize, false, 342 false); 343 if (!ret && oob_required) 344 ret = nand_read_data_op(chip, chip->oob_poi, 345 mtd->oobsize, false, false); 346 } else { 347 ret = nand_change_read_column_op(chip, 0, buf, mtd->writesize, 348 false); 349 if (!ret && oob_required) 350 ret = nand_change_read_column_op(chip, mtd->writesize, 351 chip->oob_poi, 352 mtd->oobsize, false); 353 } 354 355 if (chip->ecc.strength == 4) 356 max_bitflips = micron_nand_on_die_ecc_status_4(chip, status, 357 buf, page, 358 oob_required); 359 else 360 max_bitflips = micron_nand_on_die_ecc_status_8(chip, status); 361 362 out: 363 micron_nand_on_die_ecc_setup(chip, false); 364 365 return ret ? ret : max_bitflips; 366 } 367 368 static int 369 micron_nand_write_page_on_die_ecc(struct nand_chip *chip, const uint8_t *buf, 370 int oob_required, int page) 371 { 372 int ret; 373 374 ret = micron_nand_on_die_ecc_setup(chip, true); 375 if (ret) 376 return ret; 377 378 ret = nand_write_page_raw(chip, buf, oob_required, page); 379 micron_nand_on_die_ecc_setup(chip, false); 380 381 return ret; 382 } 383 384 enum { 385 /* The NAND flash doesn't support on-die ECC */ 386 MICRON_ON_DIE_UNSUPPORTED, 387 388 /* 389 * The NAND flash supports on-die ECC and it can be 390 * enabled/disabled by a set features command. 391 */ 392 MICRON_ON_DIE_SUPPORTED, 393 394 /* 395 * The NAND flash supports on-die ECC, and it cannot be 396 * disabled. 397 */ 398 MICRON_ON_DIE_MANDATORY, 399 }; 400 401 #define MICRON_ID_INTERNAL_ECC_MASK GENMASK(1, 0) 402 #define MICRON_ID_ECC_ENABLED BIT(7) 403 404 /* 405 * Try to detect if the NAND support on-die ECC. To do this, we enable 406 * the feature, and read back if it has been enabled as expected. We 407 * also check if it can be disabled, because some Micron NANDs do not 408 * allow disabling the on-die ECC and we don't support such NANDs for 409 * now. 410 * 411 * This function also has the side effect of disabling on-die ECC if 412 * it had been left enabled by the firmware/bootloader. 413 */ 414 static int micron_supports_on_die_ecc(struct nand_chip *chip) 415 { 416 const struct nand_ecc_props *requirements = 417 nanddev_get_ecc_requirements(&chip->base); 418 u8 id[5]; 419 int ret; 420 421 if (!chip->parameters.onfi) 422 return MICRON_ON_DIE_UNSUPPORTED; 423 424 if (nanddev_bits_per_cell(&chip->base) != 1) 425 return MICRON_ON_DIE_UNSUPPORTED; 426 427 /* 428 * We only support on-die ECC of 4/512 or 8/512 429 */ 430 if (requirements->strength != 4 && requirements->strength != 8) 431 return MICRON_ON_DIE_UNSUPPORTED; 432 433 /* 0x2 means on-die ECC is available. */ 434 if (chip->id.len != 5 || 435 (chip->id.data[4] & MICRON_ID_INTERNAL_ECC_MASK) != 0x2) 436 return MICRON_ON_DIE_UNSUPPORTED; 437 438 /* 439 * It seems that there are devices which do not support ECC officially. 440 * At least the MT29F2G08ABAGA / MT29F2G08ABBGA devices supports 441 * enabling the ECC feature but don't reflect that to the READ_ID table. 442 * So we have to guarantee that we disable the ECC feature directly 443 * after we did the READ_ID table command. Later we can evaluate the 444 * ECC_ENABLE support. 445 */ 446 ret = micron_nand_on_die_ecc_setup(chip, true); 447 if (ret) 448 return MICRON_ON_DIE_UNSUPPORTED; 449 450 ret = nand_readid_op(chip, 0, id, sizeof(id)); 451 if (ret) 452 return MICRON_ON_DIE_UNSUPPORTED; 453 454 ret = micron_nand_on_die_ecc_setup(chip, false); 455 if (ret) 456 return MICRON_ON_DIE_UNSUPPORTED; 457 458 if (!(id[4] & MICRON_ID_ECC_ENABLED)) 459 return MICRON_ON_DIE_UNSUPPORTED; 460 461 ret = nand_readid_op(chip, 0, id, sizeof(id)); 462 if (ret) 463 return MICRON_ON_DIE_UNSUPPORTED; 464 465 if (id[4] & MICRON_ID_ECC_ENABLED) 466 return MICRON_ON_DIE_MANDATORY; 467 468 /* 469 * We only support on-die ECC of 4/512 or 8/512 470 */ 471 if (requirements->strength != 4 && requirements->strength != 8) 472 return MICRON_ON_DIE_UNSUPPORTED; 473 474 return MICRON_ON_DIE_SUPPORTED; 475 } 476 477 static int micron_nand_init(struct nand_chip *chip) 478 { 479 struct nand_device *base = &chip->base; 480 const struct nand_ecc_props *requirements = 481 nanddev_get_ecc_requirements(base); 482 struct mtd_info *mtd = nand_to_mtd(chip); 483 struct micron_nand *micron; 484 int ondie; 485 int ret; 486 487 micron = kzalloc(sizeof(*micron), GFP_KERNEL); 488 if (!micron) 489 return -ENOMEM; 490 491 nand_set_manufacturer_data(chip, micron); 492 493 ret = micron_nand_onfi_init(chip); 494 if (ret) 495 goto err_free_manuf_data; 496 497 chip->options |= NAND_BBM_FIRSTPAGE; 498 499 if (mtd->writesize == 2048) 500 chip->options |= NAND_BBM_SECONDPAGE; 501 502 ondie = micron_supports_on_die_ecc(chip); 503 504 if (ondie == MICRON_ON_DIE_MANDATORY && 505 chip->ecc.engine_type != NAND_ECC_ENGINE_TYPE_ON_DIE) { 506 pr_err("On-die ECC forcefully enabled, not supported\n"); 507 ret = -EINVAL; 508 goto err_free_manuf_data; 509 } 510 511 if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE) { 512 if (ondie == MICRON_ON_DIE_UNSUPPORTED) { 513 pr_err("On-die ECC selected but not supported\n"); 514 ret = -EINVAL; 515 goto err_free_manuf_data; 516 } 517 518 if (ondie == MICRON_ON_DIE_MANDATORY) { 519 micron->ecc.forced = true; 520 micron->ecc.enabled = true; 521 } 522 523 /* 524 * In case of 4bit on-die ECC, we need a buffer to store a 525 * page dumped in raw mode so that we can compare its content 526 * to the same page after ECC correction happened and extract 527 * the real number of bitflips from this comparison. 528 * That's not needed for 8-bit ECC, because the status expose 529 * a better approximation of the number of bitflips in a page. 530 */ 531 if (requirements->strength == 4) { 532 micron->ecc.rawbuf = kmalloc(mtd->writesize + 533 mtd->oobsize, 534 GFP_KERNEL); 535 if (!micron->ecc.rawbuf) { 536 ret = -ENOMEM; 537 goto err_free_manuf_data; 538 } 539 } 540 541 if (requirements->strength == 4) 542 mtd_set_ooblayout(mtd, 543 µn_nand_on_die_4_ooblayout_ops); 544 else 545 mtd_set_ooblayout(mtd, 546 µn_nand_on_die_8_ooblayout_ops); 547 548 chip->ecc.bytes = requirements->strength * 2; 549 chip->ecc.size = 512; 550 chip->ecc.strength = requirements->strength; 551 chip->ecc.algo = NAND_ECC_ALGO_BCH; 552 chip->ecc.read_page = micron_nand_read_page_on_die_ecc; 553 chip->ecc.write_page = micron_nand_write_page_on_die_ecc; 554 555 if (ondie == MICRON_ON_DIE_MANDATORY) { 556 chip->ecc.read_page_raw = nand_read_page_raw_notsupp; 557 chip->ecc.write_page_raw = nand_write_page_raw_notsupp; 558 } else { 559 if (!chip->ecc.read_page_raw) 560 chip->ecc.read_page_raw = nand_read_page_raw; 561 if (!chip->ecc.write_page_raw) 562 chip->ecc.write_page_raw = nand_write_page_raw; 563 } 564 } 565 566 return 0; 567 568 err_free_manuf_data: 569 kfree(micron->ecc.rawbuf); 570 kfree(micron); 571 572 return ret; 573 } 574 575 static void micron_nand_cleanup(struct nand_chip *chip) 576 { 577 struct micron_nand *micron = nand_get_manufacturer_data(chip); 578 579 kfree(micron->ecc.rawbuf); 580 kfree(micron); 581 } 582 583 static void micron_fixup_onfi_param_page(struct nand_chip *chip, 584 struct nand_onfi_params *p) 585 { 586 /* 587 * MT29F1G08ABAFAWP-ITE:F and possibly others report 00 00 for the 588 * revision number field of the ONFI parameter page. Assume ONFI 589 * version 1.0 if the revision number is 00 00. 590 */ 591 if (le16_to_cpu(p->revision) == 0) 592 p->revision = cpu_to_le16(ONFI_VERSION_1_0); 593 } 594 595 const struct nand_manufacturer_ops micron_nand_manuf_ops = { 596 .init = micron_nand_init, 597 .cleanup = micron_nand_cleanup, 598 .fixup_onfi_param_page = micron_fixup_onfi_param_page, 599 }; 600