1 /* 2 * Copyright (C) 2017 Free Electrons 3 * Copyright (C) 2017 NextThing Co 4 * 5 * Author: Boris Brezillon <boris.brezillon@free-electrons.com> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License as published by 9 * the Free Software Foundation; either version 2 of the License, or 10 * (at your option) any later version. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 */ 17 18 #include <linux/slab.h> 19 20 #include "internals.h" 21 22 /* 23 * Special Micron status bit 3 indicates that the block has been 24 * corrected by on-die ECC and should be rewritten. 25 */ 26 #define NAND_ECC_STATUS_WRITE_RECOMMENDED BIT(3) 27 28 /* 29 * On chips with 8-bit ECC and additional bit can be used to distinguish 30 * cases where a errors were corrected without needing a rewrite 31 * 32 * Bit 4 Bit 3 Bit 0 Description 33 * ----- ----- ----- ----------- 34 * 0 0 0 No Errors 35 * 0 0 1 Multiple uncorrected errors 36 * 0 1 0 4 - 6 errors corrected, recommend rewrite 37 * 0 1 1 Reserved 38 * 1 0 0 1 - 3 errors corrected 39 * 1 0 1 Reserved 40 * 1 1 0 7 - 8 errors corrected, recommend rewrite 41 */ 42 #define NAND_ECC_STATUS_MASK (BIT(4) | BIT(3) | BIT(0)) 43 #define NAND_ECC_STATUS_UNCORRECTABLE BIT(0) 44 #define NAND_ECC_STATUS_4_6_CORRECTED BIT(3) 45 #define NAND_ECC_STATUS_1_3_CORRECTED BIT(4) 46 #define NAND_ECC_STATUS_7_8_CORRECTED (BIT(4) | BIT(3)) 47 48 struct nand_onfi_vendor_micron { 49 u8 two_plane_read; 50 u8 read_cache; 51 u8 read_unique_id; 52 u8 dq_imped; 53 u8 dq_imped_num_settings; 54 u8 dq_imped_feat_addr; 55 u8 rb_pulldown_strength; 56 u8 rb_pulldown_strength_feat_addr; 57 u8 rb_pulldown_strength_num_settings; 58 u8 otp_mode; 59 u8 otp_page_start; 60 u8 otp_data_prot_addr; 61 u8 otp_num_pages; 62 u8 otp_feat_addr; 63 u8 read_retry_options; 64 u8 reserved[72]; 65 u8 param_revision; 66 } __packed; 67 68 struct micron_on_die_ecc { 69 bool forced; 70 bool enabled; 71 void *rawbuf; 72 }; 73 74 struct micron_nand { 75 struct micron_on_die_ecc ecc; 76 }; 77 78 static int micron_nand_setup_read_retry(struct nand_chip *chip, int retry_mode) 79 { 80 u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = {retry_mode}; 81 82 return nand_set_features(chip, ONFI_FEATURE_ADDR_READ_RETRY, feature); 83 } 84 85 /* 86 * Configure chip properties from Micron vendor-specific ONFI table 87 */ 88 static int micron_nand_onfi_init(struct nand_chip *chip) 89 { 90 struct nand_parameters *p = &chip->parameters; 91 92 if (p->onfi) { 93 struct nand_onfi_vendor_micron *micron = (void *)p->onfi->vendor; 94 95 chip->read_retries = micron->read_retry_options; 96 chip->setup_read_retry = micron_nand_setup_read_retry; 97 } 98 99 if (p->supports_set_get_features) { 100 set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->set_feature_list); 101 set_bit(ONFI_FEATURE_ON_DIE_ECC, p->set_feature_list); 102 set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->get_feature_list); 103 set_bit(ONFI_FEATURE_ON_DIE_ECC, p->get_feature_list); 104 } 105 106 return 0; 107 } 108 109 static int micron_nand_on_die_4_ooblayout_ecc(struct mtd_info *mtd, 110 int section, 111 struct mtd_oob_region *oobregion) 112 { 113 if (section >= 4) 114 return -ERANGE; 115 116 oobregion->offset = (section * 16) + 8; 117 oobregion->length = 8; 118 119 return 0; 120 } 121 122 static int micron_nand_on_die_4_ooblayout_free(struct mtd_info *mtd, 123 int section, 124 struct mtd_oob_region *oobregion) 125 { 126 if (section >= 4) 127 return -ERANGE; 128 129 oobregion->offset = (section * 16) + 2; 130 oobregion->length = 6; 131 132 return 0; 133 } 134 135 static const struct mtd_ooblayout_ops micron_nand_on_die_4_ooblayout_ops = { 136 .ecc = micron_nand_on_die_4_ooblayout_ecc, 137 .free = micron_nand_on_die_4_ooblayout_free, 138 }; 139 140 static int micron_nand_on_die_8_ooblayout_ecc(struct mtd_info *mtd, 141 int section, 142 struct mtd_oob_region *oobregion) 143 { 144 struct nand_chip *chip = mtd_to_nand(mtd); 145 146 if (section) 147 return -ERANGE; 148 149 oobregion->offset = mtd->oobsize - chip->ecc.total; 150 oobregion->length = chip->ecc.total; 151 152 return 0; 153 } 154 155 static int micron_nand_on_die_8_ooblayout_free(struct mtd_info *mtd, 156 int section, 157 struct mtd_oob_region *oobregion) 158 { 159 struct nand_chip *chip = mtd_to_nand(mtd); 160 161 if (section) 162 return -ERANGE; 163 164 oobregion->offset = 2; 165 oobregion->length = mtd->oobsize - chip->ecc.total - 2; 166 167 return 0; 168 } 169 170 static const struct mtd_ooblayout_ops micron_nand_on_die_8_ooblayout_ops = { 171 .ecc = micron_nand_on_die_8_ooblayout_ecc, 172 .free = micron_nand_on_die_8_ooblayout_free, 173 }; 174 175 static int micron_nand_on_die_ecc_setup(struct nand_chip *chip, bool enable) 176 { 177 struct micron_nand *micron = nand_get_manufacturer_data(chip); 178 u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = { 0, }; 179 int ret; 180 181 if (micron->ecc.forced) 182 return 0; 183 184 if (micron->ecc.enabled == enable) 185 return 0; 186 187 if (enable) 188 feature[0] |= ONFI_FEATURE_ON_DIE_ECC_EN; 189 190 ret = nand_set_features(chip, ONFI_FEATURE_ON_DIE_ECC, feature); 191 if (!ret) 192 micron->ecc.enabled = enable; 193 194 return ret; 195 } 196 197 static int micron_nand_on_die_ecc_status_4(struct nand_chip *chip, u8 status, 198 void *buf, int page, 199 int oob_required) 200 { 201 struct micron_nand *micron = nand_get_manufacturer_data(chip); 202 struct mtd_info *mtd = nand_to_mtd(chip); 203 unsigned int step, max_bitflips = 0; 204 int ret; 205 206 if (!(status & NAND_ECC_STATUS_WRITE_RECOMMENDED)) { 207 if (status & NAND_STATUS_FAIL) 208 mtd->ecc_stats.failed++; 209 210 return 0; 211 } 212 213 /* 214 * The internal ECC doesn't tell us the number of bitflips that have 215 * been corrected, but tells us if it recommends to rewrite the block. 216 * If it's the case, we need to read the page in raw mode and compare 217 * its content to the corrected version to extract the actual number of 218 * bitflips. 219 * But before we do that, we must make sure we have all OOB bytes read 220 * in non-raw mode, even if the user did not request those bytes. 221 */ 222 if (!oob_required) { 223 ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, 224 false); 225 if (ret) 226 return ret; 227 } 228 229 micron_nand_on_die_ecc_setup(chip, false); 230 231 ret = nand_read_page_op(chip, page, 0, micron->ecc.rawbuf, 232 mtd->writesize + mtd->oobsize); 233 if (ret) 234 return ret; 235 236 for (step = 0; step < chip->ecc.steps; step++) { 237 unsigned int offs, i, nbitflips = 0; 238 u8 *rawbuf, *corrbuf; 239 240 offs = step * chip->ecc.size; 241 rawbuf = micron->ecc.rawbuf + offs; 242 corrbuf = buf + offs; 243 244 for (i = 0; i < chip->ecc.size; i++) 245 nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]); 246 247 offs = (step * 16) + 4; 248 rawbuf = micron->ecc.rawbuf + mtd->writesize + offs; 249 corrbuf = chip->oob_poi + offs; 250 251 for (i = 0; i < chip->ecc.bytes + 4; i++) 252 nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]); 253 254 if (WARN_ON(nbitflips > chip->ecc.strength)) 255 return -EINVAL; 256 257 max_bitflips = max(nbitflips, max_bitflips); 258 mtd->ecc_stats.corrected += nbitflips; 259 } 260 261 return max_bitflips; 262 } 263 264 static int micron_nand_on_die_ecc_status_8(struct nand_chip *chip, u8 status) 265 { 266 struct mtd_info *mtd = nand_to_mtd(chip); 267 268 /* 269 * With 8/512 we have more information but still don't know precisely 270 * how many bit-flips were seen. 271 */ 272 switch (status & NAND_ECC_STATUS_MASK) { 273 case NAND_ECC_STATUS_UNCORRECTABLE: 274 mtd->ecc_stats.failed++; 275 return 0; 276 case NAND_ECC_STATUS_1_3_CORRECTED: 277 mtd->ecc_stats.corrected += 3; 278 return 3; 279 case NAND_ECC_STATUS_4_6_CORRECTED: 280 mtd->ecc_stats.corrected += 6; 281 /* rewrite recommended */ 282 return 6; 283 case NAND_ECC_STATUS_7_8_CORRECTED: 284 mtd->ecc_stats.corrected += 8; 285 /* rewrite recommended */ 286 return 8; 287 default: 288 return 0; 289 } 290 } 291 292 static int 293 micron_nand_read_page_on_die_ecc(struct nand_chip *chip, uint8_t *buf, 294 int oob_required, int page) 295 { 296 struct mtd_info *mtd = nand_to_mtd(chip); 297 u8 status; 298 int ret, max_bitflips = 0; 299 300 ret = micron_nand_on_die_ecc_setup(chip, true); 301 if (ret) 302 return ret; 303 304 ret = nand_read_page_op(chip, page, 0, NULL, 0); 305 if (ret) 306 goto out; 307 308 ret = nand_status_op(chip, &status); 309 if (ret) 310 goto out; 311 312 ret = nand_exit_status_op(chip); 313 if (ret) 314 goto out; 315 316 ret = nand_read_data_op(chip, buf, mtd->writesize, false); 317 if (!ret && oob_required) 318 ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, 319 false); 320 321 if (chip->ecc.strength == 4) 322 max_bitflips = micron_nand_on_die_ecc_status_4(chip, status, 323 buf, page, 324 oob_required); 325 else 326 max_bitflips = micron_nand_on_die_ecc_status_8(chip, status); 327 328 out: 329 micron_nand_on_die_ecc_setup(chip, false); 330 331 return ret ? ret : max_bitflips; 332 } 333 334 static int 335 micron_nand_write_page_on_die_ecc(struct nand_chip *chip, const uint8_t *buf, 336 int oob_required, int page) 337 { 338 int ret; 339 340 ret = micron_nand_on_die_ecc_setup(chip, true); 341 if (ret) 342 return ret; 343 344 ret = nand_write_page_raw(chip, buf, oob_required, page); 345 micron_nand_on_die_ecc_setup(chip, false); 346 347 return ret; 348 } 349 350 enum { 351 /* The NAND flash doesn't support on-die ECC */ 352 MICRON_ON_DIE_UNSUPPORTED, 353 354 /* 355 * The NAND flash supports on-die ECC and it can be 356 * enabled/disabled by a set features command. 357 */ 358 MICRON_ON_DIE_SUPPORTED, 359 360 /* 361 * The NAND flash supports on-die ECC, and it cannot be 362 * disabled. 363 */ 364 MICRON_ON_DIE_MANDATORY, 365 }; 366 367 #define MICRON_ID_INTERNAL_ECC_MASK GENMASK(1, 0) 368 #define MICRON_ID_ECC_ENABLED BIT(7) 369 370 /* 371 * Try to detect if the NAND support on-die ECC. To do this, we enable 372 * the feature, and read back if it has been enabled as expected. We 373 * also check if it can be disabled, because some Micron NANDs do not 374 * allow disabling the on-die ECC and we don't support such NANDs for 375 * now. 376 * 377 * This function also has the side effect of disabling on-die ECC if 378 * it had been left enabled by the firmware/bootloader. 379 */ 380 static int micron_supports_on_die_ecc(struct nand_chip *chip) 381 { 382 u8 id[5]; 383 int ret; 384 385 if (!chip->parameters.onfi) 386 return MICRON_ON_DIE_UNSUPPORTED; 387 388 if (chip->bits_per_cell != 1) 389 return MICRON_ON_DIE_UNSUPPORTED; 390 391 /* 392 * We only support on-die ECC of 4/512 or 8/512 393 */ 394 if (chip->ecc_strength_ds != 4 && chip->ecc_strength_ds != 8) 395 return MICRON_ON_DIE_UNSUPPORTED; 396 397 /* 0x2 means on-die ECC is available. */ 398 if (chip->id.len != 5 || 399 (chip->id.data[4] & MICRON_ID_INTERNAL_ECC_MASK) != 0x2) 400 return MICRON_ON_DIE_UNSUPPORTED; 401 402 ret = micron_nand_on_die_ecc_setup(chip, true); 403 if (ret) 404 return MICRON_ON_DIE_UNSUPPORTED; 405 406 ret = nand_readid_op(chip, 0, id, sizeof(id)); 407 if (ret) 408 return MICRON_ON_DIE_UNSUPPORTED; 409 410 if (!(id[4] & MICRON_ID_ECC_ENABLED)) 411 return MICRON_ON_DIE_UNSUPPORTED; 412 413 ret = micron_nand_on_die_ecc_setup(chip, false); 414 if (ret) 415 return MICRON_ON_DIE_UNSUPPORTED; 416 417 ret = nand_readid_op(chip, 0, id, sizeof(id)); 418 if (ret) 419 return MICRON_ON_DIE_UNSUPPORTED; 420 421 if (id[4] & MICRON_ID_ECC_ENABLED) 422 return MICRON_ON_DIE_MANDATORY; 423 424 /* 425 * We only support on-die ECC of 4/512 or 8/512 426 */ 427 if (chip->ecc_strength_ds != 4 && chip->ecc_strength_ds != 8) 428 return MICRON_ON_DIE_UNSUPPORTED; 429 430 return MICRON_ON_DIE_SUPPORTED; 431 } 432 433 static int micron_nand_init(struct nand_chip *chip) 434 { 435 struct mtd_info *mtd = nand_to_mtd(chip); 436 struct micron_nand *micron; 437 int ondie; 438 int ret; 439 440 micron = kzalloc(sizeof(*micron), GFP_KERNEL); 441 if (!micron) 442 return -ENOMEM; 443 444 nand_set_manufacturer_data(chip, micron); 445 446 ret = micron_nand_onfi_init(chip); 447 if (ret) 448 goto err_free_manuf_data; 449 450 if (mtd->writesize == 2048) 451 chip->bbt_options |= NAND_BBT_SCAN2NDPAGE; 452 453 ondie = micron_supports_on_die_ecc(chip); 454 455 if (ondie == MICRON_ON_DIE_MANDATORY && 456 chip->ecc.mode != NAND_ECC_ON_DIE) { 457 pr_err("On-die ECC forcefully enabled, not supported\n"); 458 ret = -EINVAL; 459 goto err_free_manuf_data; 460 } 461 462 if (chip->ecc.mode == NAND_ECC_ON_DIE) { 463 if (ondie == MICRON_ON_DIE_UNSUPPORTED) { 464 pr_err("On-die ECC selected but not supported\n"); 465 ret = -EINVAL; 466 goto err_free_manuf_data; 467 } 468 469 if (ondie == MICRON_ON_DIE_MANDATORY) { 470 micron->ecc.forced = true; 471 micron->ecc.enabled = true; 472 } 473 474 /* 475 * In case of 4bit on-die ECC, we need a buffer to store a 476 * page dumped in raw mode so that we can compare its content 477 * to the same page after ECC correction happened and extract 478 * the real number of bitflips from this comparison. 479 * That's not needed for 8-bit ECC, because the status expose 480 * a better approximation of the number of bitflips in a page. 481 */ 482 if (chip->ecc_strength_ds == 4) { 483 micron->ecc.rawbuf = kmalloc(mtd->writesize + 484 mtd->oobsize, 485 GFP_KERNEL); 486 if (!micron->ecc.rawbuf) { 487 ret = -ENOMEM; 488 goto err_free_manuf_data; 489 } 490 } 491 492 if (chip->ecc_strength_ds == 4) 493 mtd_set_ooblayout(mtd, 494 µn_nand_on_die_4_ooblayout_ops); 495 else 496 mtd_set_ooblayout(mtd, 497 µn_nand_on_die_8_ooblayout_ops); 498 499 chip->ecc.bytes = chip->ecc_strength_ds * 2; 500 chip->ecc.size = 512; 501 chip->ecc.strength = chip->ecc_strength_ds; 502 chip->ecc.algo = NAND_ECC_BCH; 503 chip->ecc.read_page = micron_nand_read_page_on_die_ecc; 504 chip->ecc.write_page = micron_nand_write_page_on_die_ecc; 505 506 if (ondie == MICRON_ON_DIE_MANDATORY) { 507 chip->ecc.read_page_raw = nand_read_page_raw_notsupp; 508 chip->ecc.write_page_raw = nand_write_page_raw_notsupp; 509 } else { 510 chip->ecc.read_page_raw = nand_read_page_raw; 511 chip->ecc.write_page_raw = nand_write_page_raw; 512 } 513 } 514 515 return 0; 516 517 err_free_manuf_data: 518 kfree(micron->ecc.rawbuf); 519 kfree(micron); 520 521 return ret; 522 } 523 524 static void micron_nand_cleanup(struct nand_chip *chip) 525 { 526 struct micron_nand *micron = nand_get_manufacturer_data(chip); 527 528 kfree(micron->ecc.rawbuf); 529 kfree(micron); 530 } 531 532 static void micron_fixup_onfi_param_page(struct nand_chip *chip, 533 struct nand_onfi_params *p) 534 { 535 /* 536 * MT29F1G08ABAFAWP-ITE:F and possibly others report 00 00 for the 537 * revision number field of the ONFI parameter page. Assume ONFI 538 * version 1.0 if the revision number is 00 00. 539 */ 540 if (le16_to_cpu(p->revision) == 0) 541 p->revision = cpu_to_le16(ONFI_VERSION_1_0); 542 } 543 544 const struct nand_manufacturer_ops micron_nand_manuf_ops = { 545 .init = micron_nand_init, 546 .cleanup = micron_nand_cleanup, 547 .fixup_onfi_param_page = micron_fixup_onfi_param_page, 548 }; 549