1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2017 Free Electrons 4 * Copyright (C) 2017 NextThing Co 5 * 6 * Author: Boris Brezillon <boris.brezillon@free-electrons.com> 7 */ 8 9 #include <linux/slab.h> 10 11 #include "internals.h" 12 13 /* 14 * Special Micron status bit 3 indicates that the block has been 15 * corrected by on-die ECC and should be rewritten. 16 */ 17 #define NAND_ECC_STATUS_WRITE_RECOMMENDED BIT(3) 18 19 /* 20 * On chips with 8-bit ECC and additional bit can be used to distinguish 21 * cases where a errors were corrected without needing a rewrite 22 * 23 * Bit 4 Bit 3 Bit 0 Description 24 * ----- ----- ----- ----------- 25 * 0 0 0 No Errors 26 * 0 0 1 Multiple uncorrected errors 27 * 0 1 0 4 - 6 errors corrected, recommend rewrite 28 * 0 1 1 Reserved 29 * 1 0 0 1 - 3 errors corrected 30 * 1 0 1 Reserved 31 * 1 1 0 7 - 8 errors corrected, recommend rewrite 32 */ 33 #define NAND_ECC_STATUS_MASK (BIT(4) | BIT(3) | BIT(0)) 34 #define NAND_ECC_STATUS_UNCORRECTABLE BIT(0) 35 #define NAND_ECC_STATUS_4_6_CORRECTED BIT(3) 36 #define NAND_ECC_STATUS_1_3_CORRECTED BIT(4) 37 #define NAND_ECC_STATUS_7_8_CORRECTED (BIT(4) | BIT(3)) 38 39 struct nand_onfi_vendor_micron { 40 u8 two_plane_read; 41 u8 read_cache; 42 u8 read_unique_id; 43 u8 dq_imped; 44 u8 dq_imped_num_settings; 45 u8 dq_imped_feat_addr; 46 u8 rb_pulldown_strength; 47 u8 rb_pulldown_strength_feat_addr; 48 u8 rb_pulldown_strength_num_settings; 49 u8 otp_mode; 50 u8 otp_page_start; 51 u8 otp_data_prot_addr; 52 u8 otp_num_pages; 53 u8 otp_feat_addr; 54 u8 read_retry_options; 55 u8 reserved[72]; 56 u8 param_revision; 57 } __packed; 58 59 struct micron_on_die_ecc { 60 bool forced; 61 bool enabled; 62 void *rawbuf; 63 }; 64 65 struct micron_nand { 66 struct micron_on_die_ecc ecc; 67 }; 68 69 static int micron_nand_setup_read_retry(struct nand_chip *chip, int retry_mode) 70 { 71 u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = {retry_mode}; 72 73 return nand_set_features(chip, ONFI_FEATURE_ADDR_READ_RETRY, feature); 74 } 75 76 /* 77 * Configure chip properties from Micron vendor-specific ONFI table 78 */ 79 static int micron_nand_onfi_init(struct nand_chip *chip) 80 { 81 struct nand_parameters *p = &chip->parameters; 82 83 if (p->onfi) { 84 struct nand_onfi_vendor_micron *micron = (void *)p->onfi->vendor; 85 86 chip->read_retries = micron->read_retry_options; 87 chip->setup_read_retry = micron_nand_setup_read_retry; 88 } 89 90 if (p->supports_set_get_features) { 91 set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->set_feature_list); 92 set_bit(ONFI_FEATURE_ON_DIE_ECC, p->set_feature_list); 93 set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->get_feature_list); 94 set_bit(ONFI_FEATURE_ON_DIE_ECC, p->get_feature_list); 95 } 96 97 return 0; 98 } 99 100 static int micron_nand_on_die_4_ooblayout_ecc(struct mtd_info *mtd, 101 int section, 102 struct mtd_oob_region *oobregion) 103 { 104 if (section >= 4) 105 return -ERANGE; 106 107 oobregion->offset = (section * 16) + 8; 108 oobregion->length = 8; 109 110 return 0; 111 } 112 113 static int micron_nand_on_die_4_ooblayout_free(struct mtd_info *mtd, 114 int section, 115 struct mtd_oob_region *oobregion) 116 { 117 if (section >= 4) 118 return -ERANGE; 119 120 oobregion->offset = (section * 16) + 2; 121 oobregion->length = 6; 122 123 return 0; 124 } 125 126 static const struct mtd_ooblayout_ops micron_nand_on_die_4_ooblayout_ops = { 127 .ecc = micron_nand_on_die_4_ooblayout_ecc, 128 .free = micron_nand_on_die_4_ooblayout_free, 129 }; 130 131 static int micron_nand_on_die_8_ooblayout_ecc(struct mtd_info *mtd, 132 int section, 133 struct mtd_oob_region *oobregion) 134 { 135 struct nand_chip *chip = mtd_to_nand(mtd); 136 137 if (section) 138 return -ERANGE; 139 140 oobregion->offset = mtd->oobsize - chip->ecc.total; 141 oobregion->length = chip->ecc.total; 142 143 return 0; 144 } 145 146 static int micron_nand_on_die_8_ooblayout_free(struct mtd_info *mtd, 147 int section, 148 struct mtd_oob_region *oobregion) 149 { 150 struct nand_chip *chip = mtd_to_nand(mtd); 151 152 if (section) 153 return -ERANGE; 154 155 oobregion->offset = 2; 156 oobregion->length = mtd->oobsize - chip->ecc.total - 2; 157 158 return 0; 159 } 160 161 static const struct mtd_ooblayout_ops micron_nand_on_die_8_ooblayout_ops = { 162 .ecc = micron_nand_on_die_8_ooblayout_ecc, 163 .free = micron_nand_on_die_8_ooblayout_free, 164 }; 165 166 static int micron_nand_on_die_ecc_setup(struct nand_chip *chip, bool enable) 167 { 168 struct micron_nand *micron = nand_get_manufacturer_data(chip); 169 u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = { 0, }; 170 int ret; 171 172 if (micron->ecc.forced) 173 return 0; 174 175 if (micron->ecc.enabled == enable) 176 return 0; 177 178 if (enable) 179 feature[0] |= ONFI_FEATURE_ON_DIE_ECC_EN; 180 181 ret = nand_set_features(chip, ONFI_FEATURE_ON_DIE_ECC, feature); 182 if (!ret) 183 micron->ecc.enabled = enable; 184 185 return ret; 186 } 187 188 static int micron_nand_on_die_ecc_status_4(struct nand_chip *chip, u8 status, 189 void *buf, int page, 190 int oob_required) 191 { 192 struct micron_nand *micron = nand_get_manufacturer_data(chip); 193 struct mtd_info *mtd = nand_to_mtd(chip); 194 unsigned int step, max_bitflips = 0; 195 bool use_datain = false; 196 int ret; 197 198 if (!(status & NAND_ECC_STATUS_WRITE_RECOMMENDED)) { 199 if (status & NAND_STATUS_FAIL) 200 mtd->ecc_stats.failed++; 201 202 return 0; 203 } 204 205 /* 206 * The internal ECC doesn't tell us the number of bitflips that have 207 * been corrected, but tells us if it recommends to rewrite the block. 208 * If it's the case, we need to read the page in raw mode and compare 209 * its content to the corrected version to extract the actual number of 210 * bitflips. 211 * But before we do that, we must make sure we have all OOB bytes read 212 * in non-raw mode, even if the user did not request those bytes. 213 */ 214 if (!oob_required) { 215 /* 216 * We first check which operation is supported by the controller 217 * before running it. This trick makes it possible to support 218 * all controllers, even the most constraints, without almost 219 * any performance hit. 220 * 221 * TODO: could be enhanced to avoid repeating the same check 222 * over and over in the fast path. 223 */ 224 if (!nand_has_exec_op(chip) || 225 !nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, false, 226 true)) 227 use_datain = true; 228 229 if (use_datain) 230 ret = nand_read_data_op(chip, chip->oob_poi, 231 mtd->oobsize, false, false); 232 else 233 ret = nand_change_read_column_op(chip, mtd->writesize, 234 chip->oob_poi, 235 mtd->oobsize, false); 236 if (ret) 237 return ret; 238 } 239 240 micron_nand_on_die_ecc_setup(chip, false); 241 242 ret = nand_read_page_op(chip, page, 0, micron->ecc.rawbuf, 243 mtd->writesize + mtd->oobsize); 244 if (ret) 245 return ret; 246 247 for (step = 0; step < chip->ecc.steps; step++) { 248 unsigned int offs, i, nbitflips = 0; 249 u8 *rawbuf, *corrbuf; 250 251 offs = step * chip->ecc.size; 252 rawbuf = micron->ecc.rawbuf + offs; 253 corrbuf = buf + offs; 254 255 for (i = 0; i < chip->ecc.size; i++) 256 nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]); 257 258 offs = (step * 16) + 4; 259 rawbuf = micron->ecc.rawbuf + mtd->writesize + offs; 260 corrbuf = chip->oob_poi + offs; 261 262 for (i = 0; i < chip->ecc.bytes + 4; i++) 263 nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]); 264 265 if (WARN_ON(nbitflips > chip->ecc.strength)) 266 return -EINVAL; 267 268 max_bitflips = max(nbitflips, max_bitflips); 269 mtd->ecc_stats.corrected += nbitflips; 270 } 271 272 return max_bitflips; 273 } 274 275 static int micron_nand_on_die_ecc_status_8(struct nand_chip *chip, u8 status) 276 { 277 struct mtd_info *mtd = nand_to_mtd(chip); 278 279 /* 280 * With 8/512 we have more information but still don't know precisely 281 * how many bit-flips were seen. 282 */ 283 switch (status & NAND_ECC_STATUS_MASK) { 284 case NAND_ECC_STATUS_UNCORRECTABLE: 285 mtd->ecc_stats.failed++; 286 return 0; 287 case NAND_ECC_STATUS_1_3_CORRECTED: 288 mtd->ecc_stats.corrected += 3; 289 return 3; 290 case NAND_ECC_STATUS_4_6_CORRECTED: 291 mtd->ecc_stats.corrected += 6; 292 /* rewrite recommended */ 293 return 6; 294 case NAND_ECC_STATUS_7_8_CORRECTED: 295 mtd->ecc_stats.corrected += 8; 296 /* rewrite recommended */ 297 return 8; 298 default: 299 return 0; 300 } 301 } 302 303 static int 304 micron_nand_read_page_on_die_ecc(struct nand_chip *chip, uint8_t *buf, 305 int oob_required, int page) 306 { 307 struct mtd_info *mtd = nand_to_mtd(chip); 308 bool use_datain = false; 309 u8 status; 310 int ret, max_bitflips = 0; 311 312 ret = micron_nand_on_die_ecc_setup(chip, true); 313 if (ret) 314 return ret; 315 316 ret = nand_read_page_op(chip, page, 0, NULL, 0); 317 if (ret) 318 goto out; 319 320 ret = nand_status_op(chip, &status); 321 if (ret) 322 goto out; 323 324 /* 325 * We first check which operation is supported by the controller before 326 * running it. This trick makes it possible to support all controllers, 327 * even the most constraints, without almost any performance hit. 328 * 329 * TODO: could be enhanced to avoid repeating the same check over and 330 * over in the fast path. 331 */ 332 if (!nand_has_exec_op(chip) || 333 !nand_read_data_op(chip, buf, mtd->writesize, false, true)) 334 use_datain = true; 335 336 if (use_datain) { 337 ret = nand_exit_status_op(chip); 338 if (ret) 339 goto out; 340 341 ret = nand_read_data_op(chip, buf, mtd->writesize, false, 342 false); 343 if (!ret && oob_required) 344 ret = nand_read_data_op(chip, chip->oob_poi, 345 mtd->oobsize, false, false); 346 } else { 347 ret = nand_change_read_column_op(chip, 0, buf, mtd->writesize, 348 false); 349 if (!ret && oob_required) 350 ret = nand_change_read_column_op(chip, mtd->writesize, 351 chip->oob_poi, 352 mtd->oobsize, false); 353 } 354 355 if (chip->ecc.strength == 4) 356 max_bitflips = micron_nand_on_die_ecc_status_4(chip, status, 357 buf, page, 358 oob_required); 359 else 360 max_bitflips = micron_nand_on_die_ecc_status_8(chip, status); 361 362 out: 363 micron_nand_on_die_ecc_setup(chip, false); 364 365 return ret ? ret : max_bitflips; 366 } 367 368 static int 369 micron_nand_write_page_on_die_ecc(struct nand_chip *chip, const uint8_t *buf, 370 int oob_required, int page) 371 { 372 int ret; 373 374 ret = micron_nand_on_die_ecc_setup(chip, true); 375 if (ret) 376 return ret; 377 378 ret = nand_write_page_raw(chip, buf, oob_required, page); 379 micron_nand_on_die_ecc_setup(chip, false); 380 381 return ret; 382 } 383 384 enum { 385 /* The NAND flash doesn't support on-die ECC */ 386 MICRON_ON_DIE_UNSUPPORTED, 387 388 /* 389 * The NAND flash supports on-die ECC and it can be 390 * enabled/disabled by a set features command. 391 */ 392 MICRON_ON_DIE_SUPPORTED, 393 394 /* 395 * The NAND flash supports on-die ECC, and it cannot be 396 * disabled. 397 */ 398 MICRON_ON_DIE_MANDATORY, 399 }; 400 401 #define MICRON_ID_INTERNAL_ECC_MASK GENMASK(1, 0) 402 #define MICRON_ID_ECC_ENABLED BIT(7) 403 404 /* 405 * Try to detect if the NAND support on-die ECC. To do this, we enable 406 * the feature, and read back if it has been enabled as expected. We 407 * also check if it can be disabled, because some Micron NANDs do not 408 * allow disabling the on-die ECC and we don't support such NANDs for 409 * now. 410 * 411 * This function also has the side effect of disabling on-die ECC if 412 * it had been left enabled by the firmware/bootloader. 413 */ 414 static int micron_supports_on_die_ecc(struct nand_chip *chip) 415 { 416 u8 id[5]; 417 int ret; 418 419 if (!chip->parameters.onfi) 420 return MICRON_ON_DIE_UNSUPPORTED; 421 422 if (nanddev_bits_per_cell(&chip->base) != 1) 423 return MICRON_ON_DIE_UNSUPPORTED; 424 425 /* 426 * We only support on-die ECC of 4/512 or 8/512 427 */ 428 if (chip->base.eccreq.strength != 4 && chip->base.eccreq.strength != 8) 429 return MICRON_ON_DIE_UNSUPPORTED; 430 431 /* 0x2 means on-die ECC is available. */ 432 if (chip->id.len != 5 || 433 (chip->id.data[4] & MICRON_ID_INTERNAL_ECC_MASK) != 0x2) 434 return MICRON_ON_DIE_UNSUPPORTED; 435 436 /* 437 * It seems that there are devices which do not support ECC officially. 438 * At least the MT29F2G08ABAGA / MT29F2G08ABBGA devices supports 439 * enabling the ECC feature but don't reflect that to the READ_ID table. 440 * So we have to guarantee that we disable the ECC feature directly 441 * after we did the READ_ID table command. Later we can evaluate the 442 * ECC_ENABLE support. 443 */ 444 ret = micron_nand_on_die_ecc_setup(chip, true); 445 if (ret) 446 return MICRON_ON_DIE_UNSUPPORTED; 447 448 ret = nand_readid_op(chip, 0, id, sizeof(id)); 449 if (ret) 450 return MICRON_ON_DIE_UNSUPPORTED; 451 452 ret = micron_nand_on_die_ecc_setup(chip, false); 453 if (ret) 454 return MICRON_ON_DIE_UNSUPPORTED; 455 456 if (!(id[4] & MICRON_ID_ECC_ENABLED)) 457 return MICRON_ON_DIE_UNSUPPORTED; 458 459 ret = nand_readid_op(chip, 0, id, sizeof(id)); 460 if (ret) 461 return MICRON_ON_DIE_UNSUPPORTED; 462 463 if (id[4] & MICRON_ID_ECC_ENABLED) 464 return MICRON_ON_DIE_MANDATORY; 465 466 /* 467 * We only support on-die ECC of 4/512 or 8/512 468 */ 469 if (chip->base.eccreq.strength != 4 && chip->base.eccreq.strength != 8) 470 return MICRON_ON_DIE_UNSUPPORTED; 471 472 return MICRON_ON_DIE_SUPPORTED; 473 } 474 475 static int micron_nand_init(struct nand_chip *chip) 476 { 477 struct mtd_info *mtd = nand_to_mtd(chip); 478 struct micron_nand *micron; 479 int ondie; 480 int ret; 481 482 micron = kzalloc(sizeof(*micron), GFP_KERNEL); 483 if (!micron) 484 return -ENOMEM; 485 486 nand_set_manufacturer_data(chip, micron); 487 488 ret = micron_nand_onfi_init(chip); 489 if (ret) 490 goto err_free_manuf_data; 491 492 chip->options |= NAND_BBM_FIRSTPAGE; 493 494 if (mtd->writesize == 2048) 495 chip->options |= NAND_BBM_SECONDPAGE; 496 497 ondie = micron_supports_on_die_ecc(chip); 498 499 if (ondie == MICRON_ON_DIE_MANDATORY && 500 chip->ecc.mode != NAND_ECC_ON_DIE) { 501 pr_err("On-die ECC forcefully enabled, not supported\n"); 502 ret = -EINVAL; 503 goto err_free_manuf_data; 504 } 505 506 if (chip->ecc.mode == NAND_ECC_ON_DIE) { 507 if (ondie == MICRON_ON_DIE_UNSUPPORTED) { 508 pr_err("On-die ECC selected but not supported\n"); 509 ret = -EINVAL; 510 goto err_free_manuf_data; 511 } 512 513 if (ondie == MICRON_ON_DIE_MANDATORY) { 514 micron->ecc.forced = true; 515 micron->ecc.enabled = true; 516 } 517 518 /* 519 * In case of 4bit on-die ECC, we need a buffer to store a 520 * page dumped in raw mode so that we can compare its content 521 * to the same page after ECC correction happened and extract 522 * the real number of bitflips from this comparison. 523 * That's not needed for 8-bit ECC, because the status expose 524 * a better approximation of the number of bitflips in a page. 525 */ 526 if (chip->base.eccreq.strength == 4) { 527 micron->ecc.rawbuf = kmalloc(mtd->writesize + 528 mtd->oobsize, 529 GFP_KERNEL); 530 if (!micron->ecc.rawbuf) { 531 ret = -ENOMEM; 532 goto err_free_manuf_data; 533 } 534 } 535 536 if (chip->base.eccreq.strength == 4) 537 mtd_set_ooblayout(mtd, 538 µn_nand_on_die_4_ooblayout_ops); 539 else 540 mtd_set_ooblayout(mtd, 541 µn_nand_on_die_8_ooblayout_ops); 542 543 chip->ecc.bytes = chip->base.eccreq.strength * 2; 544 chip->ecc.size = 512; 545 chip->ecc.strength = chip->base.eccreq.strength; 546 chip->ecc.algo = NAND_ECC_BCH; 547 chip->ecc.read_page = micron_nand_read_page_on_die_ecc; 548 chip->ecc.write_page = micron_nand_write_page_on_die_ecc; 549 550 if (ondie == MICRON_ON_DIE_MANDATORY) { 551 chip->ecc.read_page_raw = nand_read_page_raw_notsupp; 552 chip->ecc.write_page_raw = nand_write_page_raw_notsupp; 553 } else { 554 if (!chip->ecc.read_page_raw) 555 chip->ecc.read_page_raw = nand_read_page_raw; 556 if (!chip->ecc.write_page_raw) 557 chip->ecc.write_page_raw = nand_write_page_raw; 558 } 559 } 560 561 return 0; 562 563 err_free_manuf_data: 564 kfree(micron->ecc.rawbuf); 565 kfree(micron); 566 567 return ret; 568 } 569 570 static void micron_nand_cleanup(struct nand_chip *chip) 571 { 572 struct micron_nand *micron = nand_get_manufacturer_data(chip); 573 574 kfree(micron->ecc.rawbuf); 575 kfree(micron); 576 } 577 578 static void micron_fixup_onfi_param_page(struct nand_chip *chip, 579 struct nand_onfi_params *p) 580 { 581 /* 582 * MT29F1G08ABAFAWP-ITE:F and possibly others report 00 00 for the 583 * revision number field of the ONFI parameter page. Assume ONFI 584 * version 1.0 if the revision number is 00 00. 585 */ 586 if (le16_to_cpu(p->revision) == 0) 587 p->revision = cpu_to_le16(ONFI_VERSION_1_0); 588 } 589 590 const struct nand_manufacturer_ops micron_nand_manuf_ops = { 591 .init = micron_nand_init, 592 .cleanup = micron_nand_cleanup, 593 .fixup_onfi_param_page = micron_fixup_onfi_param_page, 594 }; 595