1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Overview:
4 * This is the generic MTD driver for NAND flash devices. It should be
5 * capable of working with almost all NAND chips currently available.
6 *
7 * Additional technical information is available on
8 * http://www.linux-mtd.infradead.org/doc/nand.html
9 *
10 * Copyright (C) 2000 Steven J. Hill (sjhill@realitydiluted.com)
11 * 2002-2006 Thomas Gleixner (tglx@linutronix.de)
12 *
13 * Credits:
14 * David Woodhouse for adding multichip support
15 *
16 * Aleph One Ltd. and Toby Churchill Ltd. for supporting the
17 * rework for 2K page size chips
18 *
19 * TODO:
20 * Enable cached programming for 2k page size chips
21 * Check, if mtd->ecctype should be set to MTD_ECC_HW
22 * if we have HW ECC support.
23 * BBT table is not serialized, has to be fixed
24 */
25
26 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
27
28 #include <linux/module.h>
29 #include <linux/delay.h>
30 #include <linux/errno.h>
31 #include <linux/err.h>
32 #include <linux/sched.h>
33 #include <linux/slab.h>
34 #include <linux/mm.h>
35 #include <linux/types.h>
36 #include <linux/mtd/mtd.h>
37 #include <linux/mtd/nand.h>
38 #include <linux/mtd/nand-ecc-sw-hamming.h>
39 #include <linux/mtd/nand-ecc-sw-bch.h>
40 #include <linux/interrupt.h>
41 #include <linux/bitops.h>
42 #include <linux/io.h>
43 #include <linux/mtd/partitions.h>
44 #include <linux/of.h>
45 #include <linux/of_gpio.h>
46 #include <linux/gpio/consumer.h>
47
48 #include "internals.h"
49
nand_pairing_dist3_get_info(struct mtd_info * mtd,int page,struct mtd_pairing_info * info)50 static int nand_pairing_dist3_get_info(struct mtd_info *mtd, int page,
51 struct mtd_pairing_info *info)
52 {
53 int lastpage = (mtd->erasesize / mtd->writesize) - 1;
54 int dist = 3;
55
56 if (page == lastpage)
57 dist = 2;
58
59 if (!page || (page & 1)) {
60 info->group = 0;
61 info->pair = (page + 1) / 2;
62 } else {
63 info->group = 1;
64 info->pair = (page + 1 - dist) / 2;
65 }
66
67 return 0;
68 }
69
nand_pairing_dist3_get_wunit(struct mtd_info * mtd,const struct mtd_pairing_info * info)70 static int nand_pairing_dist3_get_wunit(struct mtd_info *mtd,
71 const struct mtd_pairing_info *info)
72 {
73 int lastpair = ((mtd->erasesize / mtd->writesize) - 1) / 2;
74 int page = info->pair * 2;
75 int dist = 3;
76
77 if (!info->group && !info->pair)
78 return 0;
79
80 if (info->pair == lastpair && info->group)
81 dist = 2;
82
83 if (!info->group)
84 page--;
85 else if (info->pair)
86 page += dist - 1;
87
88 if (page >= mtd->erasesize / mtd->writesize)
89 return -EINVAL;
90
91 return page;
92 }
93
94 const struct mtd_pairing_scheme dist3_pairing_scheme = {
95 .ngroups = 2,
96 .get_info = nand_pairing_dist3_get_info,
97 .get_wunit = nand_pairing_dist3_get_wunit,
98 };
99
check_offs_len(struct nand_chip * chip,loff_t ofs,uint64_t len)100 static int check_offs_len(struct nand_chip *chip, loff_t ofs, uint64_t len)
101 {
102 int ret = 0;
103
104 /* Start address must align on block boundary */
105 if (ofs & ((1ULL << chip->phys_erase_shift) - 1)) {
106 pr_debug("%s: unaligned address\n", __func__);
107 ret = -EINVAL;
108 }
109
110 /* Length must align on block boundary */
111 if (len & ((1ULL << chip->phys_erase_shift) - 1)) {
112 pr_debug("%s: length not block aligned\n", __func__);
113 ret = -EINVAL;
114 }
115
116 return ret;
117 }
118
119 /**
120 * nand_extract_bits - Copy unaligned bits from one buffer to another one
121 * @dst: destination buffer
122 * @dst_off: bit offset at which the writing starts
123 * @src: source buffer
124 * @src_off: bit offset at which the reading starts
125 * @nbits: number of bits to copy from @src to @dst
126 *
127 * Copy bits from one memory region to another (overlap authorized).
128 */
nand_extract_bits(u8 * dst,unsigned int dst_off,const u8 * src,unsigned int src_off,unsigned int nbits)129 void nand_extract_bits(u8 *dst, unsigned int dst_off, const u8 *src,
130 unsigned int src_off, unsigned int nbits)
131 {
132 unsigned int tmp, n;
133
134 dst += dst_off / 8;
135 dst_off %= 8;
136 src += src_off / 8;
137 src_off %= 8;
138
139 while (nbits) {
140 n = min3(8 - dst_off, 8 - src_off, nbits);
141
142 tmp = (*src >> src_off) & GENMASK(n - 1, 0);
143 *dst &= ~GENMASK(n - 1 + dst_off, dst_off);
144 *dst |= tmp << dst_off;
145
146 dst_off += n;
147 if (dst_off >= 8) {
148 dst++;
149 dst_off -= 8;
150 }
151
152 src_off += n;
153 if (src_off >= 8) {
154 src++;
155 src_off -= 8;
156 }
157
158 nbits -= n;
159 }
160 }
161 EXPORT_SYMBOL_GPL(nand_extract_bits);
162
163 /**
164 * nand_select_target() - Select a NAND target (A.K.A. die)
165 * @chip: NAND chip object
166 * @cs: the CS line to select. Note that this CS id is always from the chip
167 * PoV, not the controller one
168 *
169 * Select a NAND target so that further operations executed on @chip go to the
170 * selected NAND target.
171 */
nand_select_target(struct nand_chip * chip,unsigned int cs)172 void nand_select_target(struct nand_chip *chip, unsigned int cs)
173 {
174 /*
175 * cs should always lie between 0 and nanddev_ntargets(), when that's
176 * not the case it's a bug and the caller should be fixed.
177 */
178 if (WARN_ON(cs > nanddev_ntargets(&chip->base)))
179 return;
180
181 chip->cur_cs = cs;
182
183 if (chip->legacy.select_chip)
184 chip->legacy.select_chip(chip, cs);
185 }
186 EXPORT_SYMBOL_GPL(nand_select_target);
187
188 /**
189 * nand_deselect_target() - Deselect the currently selected target
190 * @chip: NAND chip object
191 *
192 * Deselect the currently selected NAND target. The result of operations
193 * executed on @chip after the target has been deselected is undefined.
194 */
nand_deselect_target(struct nand_chip * chip)195 void nand_deselect_target(struct nand_chip *chip)
196 {
197 if (chip->legacy.select_chip)
198 chip->legacy.select_chip(chip, -1);
199
200 chip->cur_cs = -1;
201 }
202 EXPORT_SYMBOL_GPL(nand_deselect_target);
203
204 /**
205 * nand_release_device - [GENERIC] release chip
206 * @chip: NAND chip object
207 *
208 * Release chip lock and wake up anyone waiting on the device.
209 */
nand_release_device(struct nand_chip * chip)210 static void nand_release_device(struct nand_chip *chip)
211 {
212 /* Release the controller and the chip */
213 mutex_unlock(&chip->controller->lock);
214 mutex_unlock(&chip->lock);
215 }
216
217 /**
218 * nand_bbm_get_next_page - Get the next page for bad block markers
219 * @chip: NAND chip object
220 * @page: First page to start checking for bad block marker usage
221 *
222 * Returns an integer that corresponds to the page offset within a block, for
223 * a page that is used to store bad block markers. If no more pages are
224 * available, -EINVAL is returned.
225 */
nand_bbm_get_next_page(struct nand_chip * chip,int page)226 int nand_bbm_get_next_page(struct nand_chip *chip, int page)
227 {
228 struct mtd_info *mtd = nand_to_mtd(chip);
229 int last_page = ((mtd->erasesize - mtd->writesize) >>
230 chip->page_shift) & chip->pagemask;
231 unsigned int bbm_flags = NAND_BBM_FIRSTPAGE | NAND_BBM_SECONDPAGE
232 | NAND_BBM_LASTPAGE;
233
234 if (page == 0 && !(chip->options & bbm_flags))
235 return 0;
236 if (page == 0 && chip->options & NAND_BBM_FIRSTPAGE)
237 return 0;
238 if (page <= 1 && chip->options & NAND_BBM_SECONDPAGE)
239 return 1;
240 if (page <= last_page && chip->options & NAND_BBM_LASTPAGE)
241 return last_page;
242
243 return -EINVAL;
244 }
245
246 /**
247 * nand_block_bad - [DEFAULT] Read bad block marker from the chip
248 * @chip: NAND chip object
249 * @ofs: offset from device start
250 *
251 * Check, if the block is bad.
252 */
nand_block_bad(struct nand_chip * chip,loff_t ofs)253 static int nand_block_bad(struct nand_chip *chip, loff_t ofs)
254 {
255 int first_page, page_offset;
256 int res;
257 u8 bad;
258
259 first_page = (int)(ofs >> chip->page_shift) & chip->pagemask;
260 page_offset = nand_bbm_get_next_page(chip, 0);
261
262 while (page_offset >= 0) {
263 res = chip->ecc.read_oob(chip, first_page + page_offset);
264 if (res < 0)
265 return res;
266
267 bad = chip->oob_poi[chip->badblockpos];
268
269 if (likely(chip->badblockbits == 8))
270 res = bad != 0xFF;
271 else
272 res = hweight8(bad) < chip->badblockbits;
273 if (res)
274 return res;
275
276 page_offset = nand_bbm_get_next_page(chip, page_offset + 1);
277 }
278
279 return 0;
280 }
281
282 /**
283 * nand_region_is_secured() - Check if the region is secured
284 * @chip: NAND chip object
285 * @offset: Offset of the region to check
286 * @size: Size of the region to check
287 *
288 * Checks if the region is secured by comparing the offset and size with the
289 * list of secure regions obtained from DT. Returns true if the region is
290 * secured else false.
291 */
nand_region_is_secured(struct nand_chip * chip,loff_t offset,u64 size)292 static bool nand_region_is_secured(struct nand_chip *chip, loff_t offset, u64 size)
293 {
294 int i;
295
296 /* Skip touching the secure regions if present */
297 for (i = 0; i < chip->nr_secure_regions; i++) {
298 const struct nand_secure_region *region = &chip->secure_regions[i];
299
300 if (offset + size <= region->offset ||
301 offset >= region->offset + region->size)
302 continue;
303
304 pr_debug("%s: Region 0x%llx - 0x%llx is secured!",
305 __func__, offset, offset + size);
306
307 return true;
308 }
309
310 return false;
311 }
312
nand_isbad_bbm(struct nand_chip * chip,loff_t ofs)313 static int nand_isbad_bbm(struct nand_chip *chip, loff_t ofs)
314 {
315 struct mtd_info *mtd = nand_to_mtd(chip);
316
317 if (chip->options & NAND_NO_BBM_QUIRK)
318 return 0;
319
320 /* Check if the region is secured */
321 if (nand_region_is_secured(chip, ofs, mtd->erasesize))
322 return -EIO;
323
324 if (mtd_check_expert_analysis_mode())
325 return 0;
326
327 if (chip->legacy.block_bad)
328 return chip->legacy.block_bad(chip, ofs);
329
330 return nand_block_bad(chip, ofs);
331 }
332
333 /**
334 * nand_get_device - [GENERIC] Get chip for selected access
335 * @chip: NAND chip structure
336 *
337 * Lock the device and its controller for exclusive access
338 */
nand_get_device(struct nand_chip * chip)339 static void nand_get_device(struct nand_chip *chip)
340 {
341 /* Wait until the device is resumed. */
342 while (1) {
343 mutex_lock(&chip->lock);
344 if (!chip->suspended) {
345 mutex_lock(&chip->controller->lock);
346 return;
347 }
348 mutex_unlock(&chip->lock);
349
350 wait_event(chip->resume_wq, !chip->suspended);
351 }
352 }
353
354 /**
355 * nand_check_wp - [GENERIC] check if the chip is write protected
356 * @chip: NAND chip object
357 *
358 * Check, if the device is write protected. The function expects, that the
359 * device is already selected.
360 */
nand_check_wp(struct nand_chip * chip)361 static int nand_check_wp(struct nand_chip *chip)
362 {
363 u8 status;
364 int ret;
365
366 /* Broken xD cards report WP despite being writable */
367 if (chip->options & NAND_BROKEN_XD)
368 return 0;
369
370 /* Check the WP bit */
371 ret = nand_status_op(chip, &status);
372 if (ret)
373 return ret;
374
375 return status & NAND_STATUS_WP ? 0 : 1;
376 }
377
378 /**
379 * nand_fill_oob - [INTERN] Transfer client buffer to oob
380 * @chip: NAND chip object
381 * @oob: oob data buffer
382 * @len: oob data write length
383 * @ops: oob ops structure
384 */
nand_fill_oob(struct nand_chip * chip,uint8_t * oob,size_t len,struct mtd_oob_ops * ops)385 static uint8_t *nand_fill_oob(struct nand_chip *chip, uint8_t *oob, size_t len,
386 struct mtd_oob_ops *ops)
387 {
388 struct mtd_info *mtd = nand_to_mtd(chip);
389 int ret;
390
391 /*
392 * Initialise to all 0xFF, to avoid the possibility of left over OOB
393 * data from a previous OOB read.
394 */
395 memset(chip->oob_poi, 0xff, mtd->oobsize);
396
397 switch (ops->mode) {
398
399 case MTD_OPS_PLACE_OOB:
400 case MTD_OPS_RAW:
401 memcpy(chip->oob_poi + ops->ooboffs, oob, len);
402 return oob + len;
403
404 case MTD_OPS_AUTO_OOB:
405 ret = mtd_ooblayout_set_databytes(mtd, oob, chip->oob_poi,
406 ops->ooboffs, len);
407 BUG_ON(ret);
408 return oob + len;
409
410 default:
411 BUG();
412 }
413 return NULL;
414 }
415
416 /**
417 * nand_do_write_oob - [MTD Interface] NAND write out-of-band
418 * @chip: NAND chip object
419 * @to: offset to write to
420 * @ops: oob operation description structure
421 *
422 * NAND write out-of-band.
423 */
nand_do_write_oob(struct nand_chip * chip,loff_t to,struct mtd_oob_ops * ops)424 static int nand_do_write_oob(struct nand_chip *chip, loff_t to,
425 struct mtd_oob_ops *ops)
426 {
427 struct mtd_info *mtd = nand_to_mtd(chip);
428 int chipnr, page, status, len, ret;
429
430 pr_debug("%s: to = 0x%08x, len = %i\n",
431 __func__, (unsigned int)to, (int)ops->ooblen);
432
433 len = mtd_oobavail(mtd, ops);
434
435 /* Do not allow write past end of page */
436 if ((ops->ooboffs + ops->ooblen) > len) {
437 pr_debug("%s: attempt to write past end of page\n",
438 __func__);
439 return -EINVAL;
440 }
441
442 /* Check if the region is secured */
443 if (nand_region_is_secured(chip, to, ops->ooblen))
444 return -EIO;
445
446 chipnr = (int)(to >> chip->chip_shift);
447
448 /*
449 * Reset the chip. Some chips (like the Toshiba TC5832DC found in one
450 * of my DiskOnChip 2000 test units) will clear the whole data page too
451 * if we don't do this. I have no clue why, but I seem to have 'fixed'
452 * it in the doc2000 driver in August 1999. dwmw2.
453 */
454 ret = nand_reset(chip, chipnr);
455 if (ret)
456 return ret;
457
458 nand_select_target(chip, chipnr);
459
460 /* Shift to get page */
461 page = (int)(to >> chip->page_shift);
462
463 /* Check, if it is write protected */
464 if (nand_check_wp(chip)) {
465 nand_deselect_target(chip);
466 return -EROFS;
467 }
468
469 /* Invalidate the page cache, if we write to the cached page */
470 if (page == chip->pagecache.page)
471 chip->pagecache.page = -1;
472
473 nand_fill_oob(chip, ops->oobbuf, ops->ooblen, ops);
474
475 if (ops->mode == MTD_OPS_RAW)
476 status = chip->ecc.write_oob_raw(chip, page & chip->pagemask);
477 else
478 status = chip->ecc.write_oob(chip, page & chip->pagemask);
479
480 nand_deselect_target(chip);
481
482 if (status)
483 return status;
484
485 ops->oobretlen = ops->ooblen;
486
487 return 0;
488 }
489
490 /**
491 * nand_default_block_markbad - [DEFAULT] mark a block bad via bad block marker
492 * @chip: NAND chip object
493 * @ofs: offset from device start
494 *
495 * This is the default implementation, which can be overridden by a hardware
496 * specific driver. It provides the details for writing a bad block marker to a
497 * block.
498 */
nand_default_block_markbad(struct nand_chip * chip,loff_t ofs)499 static int nand_default_block_markbad(struct nand_chip *chip, loff_t ofs)
500 {
501 struct mtd_info *mtd = nand_to_mtd(chip);
502 struct mtd_oob_ops ops;
503 uint8_t buf[2] = { 0, 0 };
504 int ret = 0, res, page_offset;
505
506 memset(&ops, 0, sizeof(ops));
507 ops.oobbuf = buf;
508 ops.ooboffs = chip->badblockpos;
509 if (chip->options & NAND_BUSWIDTH_16) {
510 ops.ooboffs &= ~0x01;
511 ops.len = ops.ooblen = 2;
512 } else {
513 ops.len = ops.ooblen = 1;
514 }
515 ops.mode = MTD_OPS_PLACE_OOB;
516
517 page_offset = nand_bbm_get_next_page(chip, 0);
518
519 while (page_offset >= 0) {
520 res = nand_do_write_oob(chip,
521 ofs + (page_offset * mtd->writesize),
522 &ops);
523
524 if (!ret)
525 ret = res;
526
527 page_offset = nand_bbm_get_next_page(chip, page_offset + 1);
528 }
529
530 return ret;
531 }
532
533 /**
534 * nand_markbad_bbm - mark a block by updating the BBM
535 * @chip: NAND chip object
536 * @ofs: offset of the block to mark bad
537 */
nand_markbad_bbm(struct nand_chip * chip,loff_t ofs)538 int nand_markbad_bbm(struct nand_chip *chip, loff_t ofs)
539 {
540 if (chip->legacy.block_markbad)
541 return chip->legacy.block_markbad(chip, ofs);
542
543 return nand_default_block_markbad(chip, ofs);
544 }
545
546 /**
547 * nand_block_markbad_lowlevel - mark a block bad
548 * @chip: NAND chip object
549 * @ofs: offset from device start
550 *
551 * This function performs the generic NAND bad block marking steps (i.e., bad
552 * block table(s) and/or marker(s)). We only allow the hardware driver to
553 * specify how to write bad block markers to OOB (chip->legacy.block_markbad).
554 *
555 * We try operations in the following order:
556 *
557 * (1) erase the affected block, to allow OOB marker to be written cleanly
558 * (2) write bad block marker to OOB area of affected block (unless flag
559 * NAND_BBT_NO_OOB_BBM is present)
560 * (3) update the BBT
561 *
562 * Note that we retain the first error encountered in (2) or (3), finish the
563 * procedures, and dump the error in the end.
564 */
nand_block_markbad_lowlevel(struct nand_chip * chip,loff_t ofs)565 static int nand_block_markbad_lowlevel(struct nand_chip *chip, loff_t ofs)
566 {
567 struct mtd_info *mtd = nand_to_mtd(chip);
568 int res, ret = 0;
569
570 if (!(chip->bbt_options & NAND_BBT_NO_OOB_BBM)) {
571 struct erase_info einfo;
572
573 /* Attempt erase before marking OOB */
574 memset(&einfo, 0, sizeof(einfo));
575 einfo.addr = ofs;
576 einfo.len = 1ULL << chip->phys_erase_shift;
577 nand_erase_nand(chip, &einfo, 0);
578
579 /* Write bad block marker to OOB */
580 nand_get_device(chip);
581
582 ret = nand_markbad_bbm(chip, ofs);
583 nand_release_device(chip);
584 }
585
586 /* Mark block bad in BBT */
587 if (chip->bbt) {
588 res = nand_markbad_bbt(chip, ofs);
589 if (!ret)
590 ret = res;
591 }
592
593 if (!ret)
594 mtd->ecc_stats.badblocks++;
595
596 return ret;
597 }
598
599 /**
600 * nand_block_isreserved - [GENERIC] Check if a block is marked reserved.
601 * @mtd: MTD device structure
602 * @ofs: offset from device start
603 *
604 * Check if the block is marked as reserved.
605 */
nand_block_isreserved(struct mtd_info * mtd,loff_t ofs)606 static int nand_block_isreserved(struct mtd_info *mtd, loff_t ofs)
607 {
608 struct nand_chip *chip = mtd_to_nand(mtd);
609
610 if (!chip->bbt)
611 return 0;
612 /* Return info from the table */
613 return nand_isreserved_bbt(chip, ofs);
614 }
615
616 /**
617 * nand_block_checkbad - [GENERIC] Check if a block is marked bad
618 * @chip: NAND chip object
619 * @ofs: offset from device start
620 * @allowbbt: 1, if its allowed to access the bbt area
621 *
622 * Check, if the block is bad. Either by reading the bad block table or
623 * calling of the scan function.
624 */
nand_block_checkbad(struct nand_chip * chip,loff_t ofs,int allowbbt)625 static int nand_block_checkbad(struct nand_chip *chip, loff_t ofs, int allowbbt)
626 {
627 /* Return info from the table */
628 if (chip->bbt)
629 return nand_isbad_bbt(chip, ofs, allowbbt);
630
631 return nand_isbad_bbm(chip, ofs);
632 }
633
634 /**
635 * nand_soft_waitrdy - Poll STATUS reg until RDY bit is set to 1
636 * @chip: NAND chip structure
637 * @timeout_ms: Timeout in ms
638 *
639 * Poll the STATUS register using ->exec_op() until the RDY bit becomes 1.
640 * If that does not happen whitin the specified timeout, -ETIMEDOUT is
641 * returned.
642 *
643 * This helper is intended to be used when the controller does not have access
644 * to the NAND R/B pin.
645 *
646 * Be aware that calling this helper from an ->exec_op() implementation means
647 * ->exec_op() must be re-entrant.
648 *
649 * Return 0 if the NAND chip is ready, a negative error otherwise.
650 */
nand_soft_waitrdy(struct nand_chip * chip,unsigned long timeout_ms)651 int nand_soft_waitrdy(struct nand_chip *chip, unsigned long timeout_ms)
652 {
653 const struct nand_interface_config *conf;
654 u8 status = 0;
655 int ret;
656
657 if (!nand_has_exec_op(chip))
658 return -ENOTSUPP;
659
660 /* Wait tWB before polling the STATUS reg. */
661 conf = nand_get_interface_config(chip);
662 ndelay(NAND_COMMON_TIMING_NS(conf, tWB_max));
663
664 ret = nand_status_op(chip, NULL);
665 if (ret)
666 return ret;
667
668 /*
669 * +1 below is necessary because if we are now in the last fraction
670 * of jiffy and msecs_to_jiffies is 1 then we will wait only that
671 * small jiffy fraction - possibly leading to false timeout
672 */
673 timeout_ms = jiffies + msecs_to_jiffies(timeout_ms) + 1;
674 do {
675 ret = nand_read_data_op(chip, &status, sizeof(status), true,
676 false);
677 if (ret)
678 break;
679
680 if (status & NAND_STATUS_READY)
681 break;
682
683 /*
684 * Typical lowest execution time for a tR on most NANDs is 10us,
685 * use this as polling delay before doing something smarter (ie.
686 * deriving a delay from the timeout value, timeout_ms/ratio).
687 */
688 udelay(10);
689 } while (time_before(jiffies, timeout_ms));
690
691 /*
692 * We have to exit READ_STATUS mode in order to read real data on the
693 * bus in case the WAITRDY instruction is preceding a DATA_IN
694 * instruction.
695 */
696 nand_exit_status_op(chip);
697
698 if (ret)
699 return ret;
700
701 return status & NAND_STATUS_READY ? 0 : -ETIMEDOUT;
702 };
703 EXPORT_SYMBOL_GPL(nand_soft_waitrdy);
704
705 /**
706 * nand_gpio_waitrdy - Poll R/B GPIO pin until ready
707 * @chip: NAND chip structure
708 * @gpiod: GPIO descriptor of R/B pin
709 * @timeout_ms: Timeout in ms
710 *
711 * Poll the R/B GPIO pin until it becomes ready. If that does not happen
712 * whitin the specified timeout, -ETIMEDOUT is returned.
713 *
714 * This helper is intended to be used when the controller has access to the
715 * NAND R/B pin over GPIO.
716 *
717 * Return 0 if the R/B pin indicates chip is ready, a negative error otherwise.
718 */
nand_gpio_waitrdy(struct nand_chip * chip,struct gpio_desc * gpiod,unsigned long timeout_ms)719 int nand_gpio_waitrdy(struct nand_chip *chip, struct gpio_desc *gpiod,
720 unsigned long timeout_ms)
721 {
722
723 /*
724 * Wait until R/B pin indicates chip is ready or timeout occurs.
725 * +1 below is necessary because if we are now in the last fraction
726 * of jiffy and msecs_to_jiffies is 1 then we will wait only that
727 * small jiffy fraction - possibly leading to false timeout.
728 */
729 timeout_ms = jiffies + msecs_to_jiffies(timeout_ms) + 1;
730 do {
731 if (gpiod_get_value_cansleep(gpiod))
732 return 0;
733
734 cond_resched();
735 } while (time_before(jiffies, timeout_ms));
736
737 return gpiod_get_value_cansleep(gpiod) ? 0 : -ETIMEDOUT;
738 };
739 EXPORT_SYMBOL_GPL(nand_gpio_waitrdy);
740
741 /**
742 * panic_nand_wait - [GENERIC] wait until the command is done
743 * @chip: NAND chip structure
744 * @timeo: timeout
745 *
746 * Wait for command done. This is a helper function for nand_wait used when
747 * we are in interrupt context. May happen when in panic and trying to write
748 * an oops through mtdoops.
749 */
panic_nand_wait(struct nand_chip * chip,unsigned long timeo)750 void panic_nand_wait(struct nand_chip *chip, unsigned long timeo)
751 {
752 int i;
753 for (i = 0; i < timeo; i++) {
754 if (chip->legacy.dev_ready) {
755 if (chip->legacy.dev_ready(chip))
756 break;
757 } else {
758 int ret;
759 u8 status;
760
761 ret = nand_read_data_op(chip, &status, sizeof(status),
762 true, false);
763 if (ret)
764 return;
765
766 if (status & NAND_STATUS_READY)
767 break;
768 }
769 mdelay(1);
770 }
771 }
772
nand_supports_get_features(struct nand_chip * chip,int addr)773 static bool nand_supports_get_features(struct nand_chip *chip, int addr)
774 {
775 return (chip->parameters.supports_set_get_features &&
776 test_bit(addr, chip->parameters.get_feature_list));
777 }
778
nand_supports_set_features(struct nand_chip * chip,int addr)779 static bool nand_supports_set_features(struct nand_chip *chip, int addr)
780 {
781 return (chip->parameters.supports_set_get_features &&
782 test_bit(addr, chip->parameters.set_feature_list));
783 }
784
785 /**
786 * nand_reset_interface - Reset data interface and timings
787 * @chip: The NAND chip
788 * @chipnr: Internal die id
789 *
790 * Reset the Data interface and timings to ONFI mode 0.
791 *
792 * Returns 0 for success or negative error code otherwise.
793 */
nand_reset_interface(struct nand_chip * chip,int chipnr)794 static int nand_reset_interface(struct nand_chip *chip, int chipnr)
795 {
796 const struct nand_controller_ops *ops = chip->controller->ops;
797 int ret;
798
799 if (!nand_controller_can_setup_interface(chip))
800 return 0;
801
802 /*
803 * The ONFI specification says:
804 * "
805 * To transition from NV-DDR or NV-DDR2 to the SDR data
806 * interface, the host shall use the Reset (FFh) command
807 * using SDR timing mode 0. A device in any timing mode is
808 * required to recognize Reset (FFh) command issued in SDR
809 * timing mode 0.
810 * "
811 *
812 * Configure the data interface in SDR mode and set the
813 * timings to timing mode 0.
814 */
815
816 chip->current_interface_config = nand_get_reset_interface_config();
817 ret = ops->setup_interface(chip, chipnr,
818 chip->current_interface_config);
819 if (ret)
820 pr_err("Failed to configure data interface to SDR timing mode 0\n");
821
822 return ret;
823 }
824
825 /**
826 * nand_setup_interface - Setup the best data interface and timings
827 * @chip: The NAND chip
828 * @chipnr: Internal die id
829 *
830 * Configure what has been reported to be the best data interface and NAND
831 * timings supported by the chip and the driver.
832 *
833 * Returns 0 for success or negative error code otherwise.
834 */
nand_setup_interface(struct nand_chip * chip,int chipnr)835 static int nand_setup_interface(struct nand_chip *chip, int chipnr)
836 {
837 const struct nand_controller_ops *ops = chip->controller->ops;
838 u8 tmode_param[ONFI_SUBFEATURE_PARAM_LEN] = { }, request;
839 int ret;
840
841 if (!nand_controller_can_setup_interface(chip))
842 return 0;
843
844 /*
845 * A nand_reset_interface() put both the NAND chip and the NAND
846 * controller in timings mode 0. If the default mode for this chip is
847 * also 0, no need to proceed to the change again. Plus, at probe time,
848 * nand_setup_interface() uses ->set/get_features() which would
849 * fail anyway as the parameter page is not available yet.
850 */
851 if (!chip->best_interface_config)
852 return 0;
853
854 request = chip->best_interface_config->timings.mode;
855 if (nand_interface_is_sdr(chip->best_interface_config))
856 request |= ONFI_DATA_INTERFACE_SDR;
857 else
858 request |= ONFI_DATA_INTERFACE_NVDDR;
859 tmode_param[0] = request;
860
861 /* Change the mode on the chip side (if supported by the NAND chip) */
862 if (nand_supports_set_features(chip, ONFI_FEATURE_ADDR_TIMING_MODE)) {
863 nand_select_target(chip, chipnr);
864 ret = nand_set_features(chip, ONFI_FEATURE_ADDR_TIMING_MODE,
865 tmode_param);
866 nand_deselect_target(chip);
867 if (ret)
868 return ret;
869 }
870
871 /* Change the mode on the controller side */
872 ret = ops->setup_interface(chip, chipnr, chip->best_interface_config);
873 if (ret)
874 return ret;
875
876 /* Check the mode has been accepted by the chip, if supported */
877 if (!nand_supports_get_features(chip, ONFI_FEATURE_ADDR_TIMING_MODE))
878 goto update_interface_config;
879
880 memset(tmode_param, 0, ONFI_SUBFEATURE_PARAM_LEN);
881 nand_select_target(chip, chipnr);
882 ret = nand_get_features(chip, ONFI_FEATURE_ADDR_TIMING_MODE,
883 tmode_param);
884 nand_deselect_target(chip);
885 if (ret)
886 goto err_reset_chip;
887
888 if (request != tmode_param[0]) {
889 pr_warn("%s timing mode %d not acknowledged by the NAND chip\n",
890 nand_interface_is_nvddr(chip->best_interface_config) ? "NV-DDR" : "SDR",
891 chip->best_interface_config->timings.mode);
892 pr_debug("NAND chip would work in %s timing mode %d\n",
893 tmode_param[0] & ONFI_DATA_INTERFACE_NVDDR ? "NV-DDR" : "SDR",
894 (unsigned int)ONFI_TIMING_MODE_PARAM(tmode_param[0]));
895 goto err_reset_chip;
896 }
897
898 update_interface_config:
899 chip->current_interface_config = chip->best_interface_config;
900
901 return 0;
902
903 err_reset_chip:
904 /*
905 * Fallback to mode 0 if the chip explicitly did not ack the chosen
906 * timing mode.
907 */
908 nand_reset_interface(chip, chipnr);
909 nand_select_target(chip, chipnr);
910 nand_reset_op(chip);
911 nand_deselect_target(chip);
912
913 return ret;
914 }
915
916 /**
917 * nand_choose_best_sdr_timings - Pick up the best SDR timings that both the
918 * NAND controller and the NAND chip support
919 * @chip: the NAND chip
920 * @iface: the interface configuration (can eventually be updated)
921 * @spec_timings: specific timings, when not fitting the ONFI specification
922 *
923 * If specific timings are provided, use them. Otherwise, retrieve supported
924 * timing modes from ONFI information.
925 */
nand_choose_best_sdr_timings(struct nand_chip * chip,struct nand_interface_config * iface,struct nand_sdr_timings * spec_timings)926 int nand_choose_best_sdr_timings(struct nand_chip *chip,
927 struct nand_interface_config *iface,
928 struct nand_sdr_timings *spec_timings)
929 {
930 const struct nand_controller_ops *ops = chip->controller->ops;
931 int best_mode = 0, mode, ret = -EOPNOTSUPP;
932
933 iface->type = NAND_SDR_IFACE;
934
935 if (spec_timings) {
936 iface->timings.sdr = *spec_timings;
937 iface->timings.mode = onfi_find_closest_sdr_mode(spec_timings);
938
939 /* Verify the controller supports the requested interface */
940 ret = ops->setup_interface(chip, NAND_DATA_IFACE_CHECK_ONLY,
941 iface);
942 if (!ret) {
943 chip->best_interface_config = iface;
944 return ret;
945 }
946
947 /* Fallback to slower modes */
948 best_mode = iface->timings.mode;
949 } else if (chip->parameters.onfi) {
950 best_mode = fls(chip->parameters.onfi->sdr_timing_modes) - 1;
951 }
952
953 for (mode = best_mode; mode >= 0; mode--) {
954 onfi_fill_interface_config(chip, iface, NAND_SDR_IFACE, mode);
955
956 ret = ops->setup_interface(chip, NAND_DATA_IFACE_CHECK_ONLY,
957 iface);
958 if (!ret) {
959 chip->best_interface_config = iface;
960 break;
961 }
962 }
963
964 return ret;
965 }
966
967 /**
968 * nand_choose_best_nvddr_timings - Pick up the best NVDDR timings that both the
969 * NAND controller and the NAND chip support
970 * @chip: the NAND chip
971 * @iface: the interface configuration (can eventually be updated)
972 * @spec_timings: specific timings, when not fitting the ONFI specification
973 *
974 * If specific timings are provided, use them. Otherwise, retrieve supported
975 * timing modes from ONFI information.
976 */
nand_choose_best_nvddr_timings(struct nand_chip * chip,struct nand_interface_config * iface,struct nand_nvddr_timings * spec_timings)977 int nand_choose_best_nvddr_timings(struct nand_chip *chip,
978 struct nand_interface_config *iface,
979 struct nand_nvddr_timings *spec_timings)
980 {
981 const struct nand_controller_ops *ops = chip->controller->ops;
982 int best_mode = 0, mode, ret = -EOPNOTSUPP;
983
984 iface->type = NAND_NVDDR_IFACE;
985
986 if (spec_timings) {
987 iface->timings.nvddr = *spec_timings;
988 iface->timings.mode = onfi_find_closest_nvddr_mode(spec_timings);
989
990 /* Verify the controller supports the requested interface */
991 ret = ops->setup_interface(chip, NAND_DATA_IFACE_CHECK_ONLY,
992 iface);
993 if (!ret) {
994 chip->best_interface_config = iface;
995 return ret;
996 }
997
998 /* Fallback to slower modes */
999 best_mode = iface->timings.mode;
1000 } else if (chip->parameters.onfi) {
1001 best_mode = fls(chip->parameters.onfi->nvddr_timing_modes) - 1;
1002 }
1003
1004 for (mode = best_mode; mode >= 0; mode--) {
1005 onfi_fill_interface_config(chip, iface, NAND_NVDDR_IFACE, mode);
1006
1007 ret = ops->setup_interface(chip, NAND_DATA_IFACE_CHECK_ONLY,
1008 iface);
1009 if (!ret) {
1010 chip->best_interface_config = iface;
1011 break;
1012 }
1013 }
1014
1015 return ret;
1016 }
1017
1018 /**
1019 * nand_choose_best_timings - Pick up the best NVDDR or SDR timings that both
1020 * NAND controller and the NAND chip support
1021 * @chip: the NAND chip
1022 * @iface: the interface configuration (can eventually be updated)
1023 *
1024 * If specific timings are provided, use them. Otherwise, retrieve supported
1025 * timing modes from ONFI information.
1026 */
nand_choose_best_timings(struct nand_chip * chip,struct nand_interface_config * iface)1027 static int nand_choose_best_timings(struct nand_chip *chip,
1028 struct nand_interface_config *iface)
1029 {
1030 int ret;
1031
1032 /* Try the fastest timings: NV-DDR */
1033 ret = nand_choose_best_nvddr_timings(chip, iface, NULL);
1034 if (!ret)
1035 return 0;
1036
1037 /* Fallback to SDR timings otherwise */
1038 return nand_choose_best_sdr_timings(chip, iface, NULL);
1039 }
1040
1041 /**
1042 * nand_choose_interface_config - find the best data interface and timings
1043 * @chip: The NAND chip
1044 *
1045 * Find the best data interface and NAND timings supported by the chip
1046 * and the driver. Eventually let the NAND manufacturer driver propose his own
1047 * set of timings.
1048 *
1049 * After this function nand_chip->interface_config is initialized with the best
1050 * timing mode available.
1051 *
1052 * Returns 0 for success or negative error code otherwise.
1053 */
nand_choose_interface_config(struct nand_chip * chip)1054 static int nand_choose_interface_config(struct nand_chip *chip)
1055 {
1056 struct nand_interface_config *iface;
1057 int ret;
1058
1059 if (!nand_controller_can_setup_interface(chip))
1060 return 0;
1061
1062 iface = kzalloc(sizeof(*iface), GFP_KERNEL);
1063 if (!iface)
1064 return -ENOMEM;
1065
1066 if (chip->ops.choose_interface_config)
1067 ret = chip->ops.choose_interface_config(chip, iface);
1068 else
1069 ret = nand_choose_best_timings(chip, iface);
1070
1071 if (ret)
1072 kfree(iface);
1073
1074 return ret;
1075 }
1076
1077 /**
1078 * nand_fill_column_cycles - fill the column cycles of an address
1079 * @chip: The NAND chip
1080 * @addrs: Array of address cycles to fill
1081 * @offset_in_page: The offset in the page
1082 *
1083 * Fills the first or the first two bytes of the @addrs field depending
1084 * on the NAND bus width and the page size.
1085 *
1086 * Returns the number of cycles needed to encode the column, or a negative
1087 * error code in case one of the arguments is invalid.
1088 */
nand_fill_column_cycles(struct nand_chip * chip,u8 * addrs,unsigned int offset_in_page)1089 static int nand_fill_column_cycles(struct nand_chip *chip, u8 *addrs,
1090 unsigned int offset_in_page)
1091 {
1092 struct mtd_info *mtd = nand_to_mtd(chip);
1093
1094 /* Make sure the offset is less than the actual page size. */
1095 if (offset_in_page > mtd->writesize + mtd->oobsize)
1096 return -EINVAL;
1097
1098 /*
1099 * On small page NANDs, there's a dedicated command to access the OOB
1100 * area, and the column address is relative to the start of the OOB
1101 * area, not the start of the page. Asjust the address accordingly.
1102 */
1103 if (mtd->writesize <= 512 && offset_in_page >= mtd->writesize)
1104 offset_in_page -= mtd->writesize;
1105
1106 /*
1107 * The offset in page is expressed in bytes, if the NAND bus is 16-bit
1108 * wide, then it must be divided by 2.
1109 */
1110 if (chip->options & NAND_BUSWIDTH_16) {
1111 if (WARN_ON(offset_in_page % 2))
1112 return -EINVAL;
1113
1114 offset_in_page /= 2;
1115 }
1116
1117 addrs[0] = offset_in_page;
1118
1119 /*
1120 * Small page NANDs use 1 cycle for the columns, while large page NANDs
1121 * need 2
1122 */
1123 if (mtd->writesize <= 512)
1124 return 1;
1125
1126 addrs[1] = offset_in_page >> 8;
1127
1128 return 2;
1129 }
1130
nand_sp_exec_read_page_op(struct nand_chip * chip,unsigned int page,unsigned int offset_in_page,void * buf,unsigned int len)1131 static int nand_sp_exec_read_page_op(struct nand_chip *chip, unsigned int page,
1132 unsigned int offset_in_page, void *buf,
1133 unsigned int len)
1134 {
1135 const struct nand_interface_config *conf =
1136 nand_get_interface_config(chip);
1137 struct mtd_info *mtd = nand_to_mtd(chip);
1138 u8 addrs[4];
1139 struct nand_op_instr instrs[] = {
1140 NAND_OP_CMD(NAND_CMD_READ0, 0),
1141 NAND_OP_ADDR(3, addrs, NAND_COMMON_TIMING_NS(conf, tWB_max)),
1142 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tR_max),
1143 NAND_COMMON_TIMING_NS(conf, tRR_min)),
1144 NAND_OP_DATA_IN(len, buf, 0),
1145 };
1146 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1147 int ret;
1148
1149 /* Drop the DATA_IN instruction if len is set to 0. */
1150 if (!len)
1151 op.ninstrs--;
1152
1153 if (offset_in_page >= mtd->writesize)
1154 instrs[0].ctx.cmd.opcode = NAND_CMD_READOOB;
1155 else if (offset_in_page >= 256 &&
1156 !(chip->options & NAND_BUSWIDTH_16))
1157 instrs[0].ctx.cmd.opcode = NAND_CMD_READ1;
1158
1159 ret = nand_fill_column_cycles(chip, addrs, offset_in_page);
1160 if (ret < 0)
1161 return ret;
1162
1163 addrs[1] = page;
1164 addrs[2] = page >> 8;
1165
1166 if (chip->options & NAND_ROW_ADDR_3) {
1167 addrs[3] = page >> 16;
1168 instrs[1].ctx.addr.naddrs++;
1169 }
1170
1171 return nand_exec_op(chip, &op);
1172 }
1173
nand_lp_exec_read_page_op(struct nand_chip * chip,unsigned int page,unsigned int offset_in_page,void * buf,unsigned int len)1174 static int nand_lp_exec_read_page_op(struct nand_chip *chip, unsigned int page,
1175 unsigned int offset_in_page, void *buf,
1176 unsigned int len)
1177 {
1178 const struct nand_interface_config *conf =
1179 nand_get_interface_config(chip);
1180 u8 addrs[5];
1181 struct nand_op_instr instrs[] = {
1182 NAND_OP_CMD(NAND_CMD_READ0, 0),
1183 NAND_OP_ADDR(4, addrs, 0),
1184 NAND_OP_CMD(NAND_CMD_READSTART, NAND_COMMON_TIMING_NS(conf, tWB_max)),
1185 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tR_max),
1186 NAND_COMMON_TIMING_NS(conf, tRR_min)),
1187 NAND_OP_DATA_IN(len, buf, 0),
1188 };
1189 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1190 int ret;
1191
1192 /* Drop the DATA_IN instruction if len is set to 0. */
1193 if (!len)
1194 op.ninstrs--;
1195
1196 ret = nand_fill_column_cycles(chip, addrs, offset_in_page);
1197 if (ret < 0)
1198 return ret;
1199
1200 addrs[2] = page;
1201 addrs[3] = page >> 8;
1202
1203 if (chip->options & NAND_ROW_ADDR_3) {
1204 addrs[4] = page >> 16;
1205 instrs[1].ctx.addr.naddrs++;
1206 }
1207
1208 return nand_exec_op(chip, &op);
1209 }
1210
rawnand_cap_cont_reads(struct nand_chip * chip)1211 static void rawnand_cap_cont_reads(struct nand_chip *chip)
1212 {
1213 struct nand_memory_organization *memorg;
1214 unsigned int pages_per_lun, first_lun, last_lun;
1215
1216 memorg = nanddev_get_memorg(&chip->base);
1217 pages_per_lun = memorg->pages_per_eraseblock * memorg->eraseblocks_per_lun;
1218 first_lun = chip->cont_read.first_page / pages_per_lun;
1219 last_lun = chip->cont_read.last_page / pages_per_lun;
1220
1221 /* Prevent sequential cache reads across LUN boundaries */
1222 if (first_lun != last_lun)
1223 chip->cont_read.pause_page = first_lun * pages_per_lun + pages_per_lun - 1;
1224 else
1225 chip->cont_read.pause_page = chip->cont_read.last_page;
1226 }
1227
nand_lp_exec_cont_read_page_op(struct nand_chip * chip,unsigned int page,unsigned int offset_in_page,void * buf,unsigned int len,bool check_only)1228 static int nand_lp_exec_cont_read_page_op(struct nand_chip *chip, unsigned int page,
1229 unsigned int offset_in_page, void *buf,
1230 unsigned int len, bool check_only)
1231 {
1232 const struct nand_interface_config *conf =
1233 nand_get_interface_config(chip);
1234 u8 addrs[5];
1235 struct nand_op_instr start_instrs[] = {
1236 NAND_OP_CMD(NAND_CMD_READ0, 0),
1237 NAND_OP_ADDR(4, addrs, 0),
1238 NAND_OP_CMD(NAND_CMD_READSTART, NAND_COMMON_TIMING_NS(conf, tWB_max)),
1239 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tR_max), 0),
1240 NAND_OP_CMD(NAND_CMD_READCACHESEQ, NAND_COMMON_TIMING_NS(conf, tWB_max)),
1241 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tR_max),
1242 NAND_COMMON_TIMING_NS(conf, tRR_min)),
1243 NAND_OP_DATA_IN(len, buf, 0),
1244 };
1245 struct nand_op_instr cont_instrs[] = {
1246 NAND_OP_CMD(page == chip->cont_read.pause_page ?
1247 NAND_CMD_READCACHEEND : NAND_CMD_READCACHESEQ,
1248 NAND_COMMON_TIMING_NS(conf, tWB_max)),
1249 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tR_max),
1250 NAND_COMMON_TIMING_NS(conf, tRR_min)),
1251 NAND_OP_DATA_IN(len, buf, 0),
1252 };
1253 struct nand_operation start_op = NAND_OPERATION(chip->cur_cs, start_instrs);
1254 struct nand_operation cont_op = NAND_OPERATION(chip->cur_cs, cont_instrs);
1255 int ret;
1256
1257 if (!len) {
1258 start_op.ninstrs--;
1259 cont_op.ninstrs--;
1260 }
1261
1262 ret = nand_fill_column_cycles(chip, addrs, offset_in_page);
1263 if (ret < 0)
1264 return ret;
1265
1266 addrs[2] = page;
1267 addrs[3] = page >> 8;
1268
1269 if (chip->options & NAND_ROW_ADDR_3) {
1270 addrs[4] = page >> 16;
1271 start_instrs[1].ctx.addr.naddrs++;
1272 }
1273
1274 /* Check if cache reads are supported */
1275 if (check_only) {
1276 if (nand_check_op(chip, &start_op) || nand_check_op(chip, &cont_op))
1277 return -EOPNOTSUPP;
1278
1279 return 0;
1280 }
1281
1282 if (page == chip->cont_read.first_page)
1283 ret = nand_exec_op(chip, &start_op);
1284 else
1285 ret = nand_exec_op(chip, &cont_op);
1286 if (ret)
1287 return ret;
1288
1289 if (!chip->cont_read.ongoing)
1290 return 0;
1291
1292 if (page == chip->cont_read.pause_page &&
1293 page != chip->cont_read.last_page) {
1294 chip->cont_read.first_page = chip->cont_read.pause_page + 1;
1295 rawnand_cap_cont_reads(chip);
1296 } else if (page == chip->cont_read.last_page) {
1297 chip->cont_read.ongoing = false;
1298 }
1299
1300 return 0;
1301 }
1302
rawnand_cont_read_ongoing(struct nand_chip * chip,unsigned int page)1303 static bool rawnand_cont_read_ongoing(struct nand_chip *chip, unsigned int page)
1304 {
1305 return chip->cont_read.ongoing && page >= chip->cont_read.first_page;
1306 }
1307
1308 /**
1309 * nand_read_page_op - Do a READ PAGE operation
1310 * @chip: The NAND chip
1311 * @page: page to read
1312 * @offset_in_page: offset within the page
1313 * @buf: buffer used to store the data
1314 * @len: length of the buffer
1315 *
1316 * This function issues a READ PAGE operation.
1317 * This function does not select/unselect the CS line.
1318 *
1319 * Returns 0 on success, a negative error code otherwise.
1320 */
nand_read_page_op(struct nand_chip * chip,unsigned int page,unsigned int offset_in_page,void * buf,unsigned int len)1321 int nand_read_page_op(struct nand_chip *chip, unsigned int page,
1322 unsigned int offset_in_page, void *buf, unsigned int len)
1323 {
1324 struct mtd_info *mtd = nand_to_mtd(chip);
1325
1326 if (len && !buf)
1327 return -EINVAL;
1328
1329 if (offset_in_page + len > mtd->writesize + mtd->oobsize)
1330 return -EINVAL;
1331
1332 if (nand_has_exec_op(chip)) {
1333 if (mtd->writesize > 512) {
1334 if (rawnand_cont_read_ongoing(chip, page))
1335 return nand_lp_exec_cont_read_page_op(chip, page,
1336 offset_in_page,
1337 buf, len, false);
1338 else
1339 return nand_lp_exec_read_page_op(chip, page,
1340 offset_in_page, buf,
1341 len);
1342 }
1343
1344 return nand_sp_exec_read_page_op(chip, page, offset_in_page,
1345 buf, len);
1346 }
1347
1348 chip->legacy.cmdfunc(chip, NAND_CMD_READ0, offset_in_page, page);
1349 if (len)
1350 chip->legacy.read_buf(chip, buf, len);
1351
1352 return 0;
1353 }
1354 EXPORT_SYMBOL_GPL(nand_read_page_op);
1355
1356 /**
1357 * nand_read_param_page_op - Do a READ PARAMETER PAGE operation
1358 * @chip: The NAND chip
1359 * @page: parameter page to read
1360 * @buf: buffer used to store the data
1361 * @len: length of the buffer
1362 *
1363 * This function issues a READ PARAMETER PAGE operation.
1364 * This function does not select/unselect the CS line.
1365 *
1366 * Returns 0 on success, a negative error code otherwise.
1367 */
nand_read_param_page_op(struct nand_chip * chip,u8 page,void * buf,unsigned int len)1368 int nand_read_param_page_op(struct nand_chip *chip, u8 page, void *buf,
1369 unsigned int len)
1370 {
1371 unsigned int i;
1372 u8 *p = buf;
1373
1374 if (len && !buf)
1375 return -EINVAL;
1376
1377 if (nand_has_exec_op(chip)) {
1378 const struct nand_interface_config *conf =
1379 nand_get_interface_config(chip);
1380 struct nand_op_instr instrs[] = {
1381 NAND_OP_CMD(NAND_CMD_PARAM, 0),
1382 NAND_OP_ADDR(1, &page,
1383 NAND_COMMON_TIMING_NS(conf, tWB_max)),
1384 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tR_max),
1385 NAND_COMMON_TIMING_NS(conf, tRR_min)),
1386 NAND_OP_8BIT_DATA_IN(len, buf, 0),
1387 };
1388 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1389
1390 /* Drop the DATA_IN instruction if len is set to 0. */
1391 if (!len)
1392 op.ninstrs--;
1393
1394 return nand_exec_op(chip, &op);
1395 }
1396
1397 chip->legacy.cmdfunc(chip, NAND_CMD_PARAM, page, -1);
1398 for (i = 0; i < len; i++)
1399 p[i] = chip->legacy.read_byte(chip);
1400
1401 return 0;
1402 }
1403
1404 /**
1405 * nand_change_read_column_op - Do a CHANGE READ COLUMN operation
1406 * @chip: The NAND chip
1407 * @offset_in_page: offset within the page
1408 * @buf: buffer used to store the data
1409 * @len: length of the buffer
1410 * @force_8bit: force 8-bit bus access
1411 *
1412 * This function issues a CHANGE READ COLUMN operation.
1413 * This function does not select/unselect the CS line.
1414 *
1415 * Returns 0 on success, a negative error code otherwise.
1416 */
nand_change_read_column_op(struct nand_chip * chip,unsigned int offset_in_page,void * buf,unsigned int len,bool force_8bit)1417 int nand_change_read_column_op(struct nand_chip *chip,
1418 unsigned int offset_in_page, void *buf,
1419 unsigned int len, bool force_8bit)
1420 {
1421 struct mtd_info *mtd = nand_to_mtd(chip);
1422
1423 if (len && !buf)
1424 return -EINVAL;
1425
1426 if (offset_in_page + len > mtd->writesize + mtd->oobsize)
1427 return -EINVAL;
1428
1429 /* Small page NANDs do not support column change. */
1430 if (mtd->writesize <= 512)
1431 return -ENOTSUPP;
1432
1433 if (nand_has_exec_op(chip)) {
1434 const struct nand_interface_config *conf =
1435 nand_get_interface_config(chip);
1436 u8 addrs[2] = {};
1437 struct nand_op_instr instrs[] = {
1438 NAND_OP_CMD(NAND_CMD_RNDOUT, 0),
1439 NAND_OP_ADDR(2, addrs, 0),
1440 NAND_OP_CMD(NAND_CMD_RNDOUTSTART,
1441 NAND_COMMON_TIMING_NS(conf, tCCS_min)),
1442 NAND_OP_DATA_IN(len, buf, 0),
1443 };
1444 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1445 int ret;
1446
1447 ret = nand_fill_column_cycles(chip, addrs, offset_in_page);
1448 if (ret < 0)
1449 return ret;
1450
1451 /* Drop the DATA_IN instruction if len is set to 0. */
1452 if (!len)
1453 op.ninstrs--;
1454
1455 instrs[3].ctx.data.force_8bit = force_8bit;
1456
1457 return nand_exec_op(chip, &op);
1458 }
1459
1460 chip->legacy.cmdfunc(chip, NAND_CMD_RNDOUT, offset_in_page, -1);
1461 if (len)
1462 chip->legacy.read_buf(chip, buf, len);
1463
1464 return 0;
1465 }
1466 EXPORT_SYMBOL_GPL(nand_change_read_column_op);
1467
1468 /**
1469 * nand_read_oob_op - Do a READ OOB operation
1470 * @chip: The NAND chip
1471 * @page: page to read
1472 * @offset_in_oob: offset within the OOB area
1473 * @buf: buffer used to store the data
1474 * @len: length of the buffer
1475 *
1476 * This function issues a READ OOB operation.
1477 * This function does not select/unselect the CS line.
1478 *
1479 * Returns 0 on success, a negative error code otherwise.
1480 */
nand_read_oob_op(struct nand_chip * chip,unsigned int page,unsigned int offset_in_oob,void * buf,unsigned int len)1481 int nand_read_oob_op(struct nand_chip *chip, unsigned int page,
1482 unsigned int offset_in_oob, void *buf, unsigned int len)
1483 {
1484 struct mtd_info *mtd = nand_to_mtd(chip);
1485
1486 if (len && !buf)
1487 return -EINVAL;
1488
1489 if (offset_in_oob + len > mtd->oobsize)
1490 return -EINVAL;
1491
1492 if (nand_has_exec_op(chip))
1493 return nand_read_page_op(chip, page,
1494 mtd->writesize + offset_in_oob,
1495 buf, len);
1496
1497 chip->legacy.cmdfunc(chip, NAND_CMD_READOOB, offset_in_oob, page);
1498 if (len)
1499 chip->legacy.read_buf(chip, buf, len);
1500
1501 return 0;
1502 }
1503 EXPORT_SYMBOL_GPL(nand_read_oob_op);
1504
nand_exec_prog_page_op(struct nand_chip * chip,unsigned int page,unsigned int offset_in_page,const void * buf,unsigned int len,bool prog)1505 static int nand_exec_prog_page_op(struct nand_chip *chip, unsigned int page,
1506 unsigned int offset_in_page, const void *buf,
1507 unsigned int len, bool prog)
1508 {
1509 const struct nand_interface_config *conf =
1510 nand_get_interface_config(chip);
1511 struct mtd_info *mtd = nand_to_mtd(chip);
1512 u8 addrs[5] = {};
1513 struct nand_op_instr instrs[] = {
1514 /*
1515 * The first instruction will be dropped if we're dealing
1516 * with a large page NAND and adjusted if we're dealing
1517 * with a small page NAND and the page offset is > 255.
1518 */
1519 NAND_OP_CMD(NAND_CMD_READ0, 0),
1520 NAND_OP_CMD(NAND_CMD_SEQIN, 0),
1521 NAND_OP_ADDR(0, addrs, NAND_COMMON_TIMING_NS(conf, tADL_min)),
1522 NAND_OP_DATA_OUT(len, buf, 0),
1523 NAND_OP_CMD(NAND_CMD_PAGEPROG,
1524 NAND_COMMON_TIMING_NS(conf, tWB_max)),
1525 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tPROG_max), 0),
1526 };
1527 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1528 int naddrs = nand_fill_column_cycles(chip, addrs, offset_in_page);
1529
1530 if (naddrs < 0)
1531 return naddrs;
1532
1533 addrs[naddrs++] = page;
1534 addrs[naddrs++] = page >> 8;
1535 if (chip->options & NAND_ROW_ADDR_3)
1536 addrs[naddrs++] = page >> 16;
1537
1538 instrs[2].ctx.addr.naddrs = naddrs;
1539
1540 /* Drop the last two instructions if we're not programming the page. */
1541 if (!prog) {
1542 op.ninstrs -= 2;
1543 /* Also drop the DATA_OUT instruction if empty. */
1544 if (!len)
1545 op.ninstrs--;
1546 }
1547
1548 if (mtd->writesize <= 512) {
1549 /*
1550 * Small pages need some more tweaking: we have to adjust the
1551 * first instruction depending on the page offset we're trying
1552 * to access.
1553 */
1554 if (offset_in_page >= mtd->writesize)
1555 instrs[0].ctx.cmd.opcode = NAND_CMD_READOOB;
1556 else if (offset_in_page >= 256 &&
1557 !(chip->options & NAND_BUSWIDTH_16))
1558 instrs[0].ctx.cmd.opcode = NAND_CMD_READ1;
1559 } else {
1560 /*
1561 * Drop the first command if we're dealing with a large page
1562 * NAND.
1563 */
1564 op.instrs++;
1565 op.ninstrs--;
1566 }
1567
1568 return nand_exec_op(chip, &op);
1569 }
1570
1571 /**
1572 * nand_prog_page_begin_op - starts a PROG PAGE operation
1573 * @chip: The NAND chip
1574 * @page: page to write
1575 * @offset_in_page: offset within the page
1576 * @buf: buffer containing the data to write to the page
1577 * @len: length of the buffer
1578 *
1579 * This function issues the first half of a PROG PAGE operation.
1580 * This function does not select/unselect the CS line.
1581 *
1582 * Returns 0 on success, a negative error code otherwise.
1583 */
nand_prog_page_begin_op(struct nand_chip * chip,unsigned int page,unsigned int offset_in_page,const void * buf,unsigned int len)1584 int nand_prog_page_begin_op(struct nand_chip *chip, unsigned int page,
1585 unsigned int offset_in_page, const void *buf,
1586 unsigned int len)
1587 {
1588 struct mtd_info *mtd = nand_to_mtd(chip);
1589
1590 if (len && !buf)
1591 return -EINVAL;
1592
1593 if (offset_in_page + len > mtd->writesize + mtd->oobsize)
1594 return -EINVAL;
1595
1596 if (nand_has_exec_op(chip))
1597 return nand_exec_prog_page_op(chip, page, offset_in_page, buf,
1598 len, false);
1599
1600 chip->legacy.cmdfunc(chip, NAND_CMD_SEQIN, offset_in_page, page);
1601
1602 if (buf)
1603 chip->legacy.write_buf(chip, buf, len);
1604
1605 return 0;
1606 }
1607 EXPORT_SYMBOL_GPL(nand_prog_page_begin_op);
1608
1609 /**
1610 * nand_prog_page_end_op - ends a PROG PAGE operation
1611 * @chip: The NAND chip
1612 *
1613 * This function issues the second half of a PROG PAGE operation.
1614 * This function does not select/unselect the CS line.
1615 *
1616 * Returns 0 on success, a negative error code otherwise.
1617 */
nand_prog_page_end_op(struct nand_chip * chip)1618 int nand_prog_page_end_op(struct nand_chip *chip)
1619 {
1620 int ret;
1621 u8 status;
1622
1623 if (nand_has_exec_op(chip)) {
1624 const struct nand_interface_config *conf =
1625 nand_get_interface_config(chip);
1626 struct nand_op_instr instrs[] = {
1627 NAND_OP_CMD(NAND_CMD_PAGEPROG,
1628 NAND_COMMON_TIMING_NS(conf, tWB_max)),
1629 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tPROG_max),
1630 0),
1631 };
1632 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1633
1634 ret = nand_exec_op(chip, &op);
1635 if (ret)
1636 return ret;
1637
1638 ret = nand_status_op(chip, &status);
1639 if (ret)
1640 return ret;
1641 } else {
1642 chip->legacy.cmdfunc(chip, NAND_CMD_PAGEPROG, -1, -1);
1643 ret = chip->legacy.waitfunc(chip);
1644 if (ret < 0)
1645 return ret;
1646
1647 status = ret;
1648 }
1649
1650 if (status & NAND_STATUS_FAIL)
1651 return -EIO;
1652
1653 return 0;
1654 }
1655 EXPORT_SYMBOL_GPL(nand_prog_page_end_op);
1656
1657 /**
1658 * nand_prog_page_op - Do a full PROG PAGE operation
1659 * @chip: The NAND chip
1660 * @page: page to write
1661 * @offset_in_page: offset within the page
1662 * @buf: buffer containing the data to write to the page
1663 * @len: length of the buffer
1664 *
1665 * This function issues a full PROG PAGE operation.
1666 * This function does not select/unselect the CS line.
1667 *
1668 * Returns 0 on success, a negative error code otherwise.
1669 */
nand_prog_page_op(struct nand_chip * chip,unsigned int page,unsigned int offset_in_page,const void * buf,unsigned int len)1670 int nand_prog_page_op(struct nand_chip *chip, unsigned int page,
1671 unsigned int offset_in_page, const void *buf,
1672 unsigned int len)
1673 {
1674 struct mtd_info *mtd = nand_to_mtd(chip);
1675 u8 status;
1676 int ret;
1677
1678 if (!len || !buf)
1679 return -EINVAL;
1680
1681 if (offset_in_page + len > mtd->writesize + mtd->oobsize)
1682 return -EINVAL;
1683
1684 if (nand_has_exec_op(chip)) {
1685 ret = nand_exec_prog_page_op(chip, page, offset_in_page, buf,
1686 len, true);
1687 if (ret)
1688 return ret;
1689
1690 ret = nand_status_op(chip, &status);
1691 if (ret)
1692 return ret;
1693 } else {
1694 chip->legacy.cmdfunc(chip, NAND_CMD_SEQIN, offset_in_page,
1695 page);
1696 chip->legacy.write_buf(chip, buf, len);
1697 chip->legacy.cmdfunc(chip, NAND_CMD_PAGEPROG, -1, -1);
1698 ret = chip->legacy.waitfunc(chip);
1699 if (ret < 0)
1700 return ret;
1701
1702 status = ret;
1703 }
1704
1705 if (status & NAND_STATUS_FAIL)
1706 return -EIO;
1707
1708 return 0;
1709 }
1710 EXPORT_SYMBOL_GPL(nand_prog_page_op);
1711
1712 /**
1713 * nand_change_write_column_op - Do a CHANGE WRITE COLUMN operation
1714 * @chip: The NAND chip
1715 * @offset_in_page: offset within the page
1716 * @buf: buffer containing the data to send to the NAND
1717 * @len: length of the buffer
1718 * @force_8bit: force 8-bit bus access
1719 *
1720 * This function issues a CHANGE WRITE COLUMN operation.
1721 * This function does not select/unselect the CS line.
1722 *
1723 * Returns 0 on success, a negative error code otherwise.
1724 */
nand_change_write_column_op(struct nand_chip * chip,unsigned int offset_in_page,const void * buf,unsigned int len,bool force_8bit)1725 int nand_change_write_column_op(struct nand_chip *chip,
1726 unsigned int offset_in_page,
1727 const void *buf, unsigned int len,
1728 bool force_8bit)
1729 {
1730 struct mtd_info *mtd = nand_to_mtd(chip);
1731
1732 if (len && !buf)
1733 return -EINVAL;
1734
1735 if (offset_in_page + len > mtd->writesize + mtd->oobsize)
1736 return -EINVAL;
1737
1738 /* Small page NANDs do not support column change. */
1739 if (mtd->writesize <= 512)
1740 return -ENOTSUPP;
1741
1742 if (nand_has_exec_op(chip)) {
1743 const struct nand_interface_config *conf =
1744 nand_get_interface_config(chip);
1745 u8 addrs[2];
1746 struct nand_op_instr instrs[] = {
1747 NAND_OP_CMD(NAND_CMD_RNDIN, 0),
1748 NAND_OP_ADDR(2, addrs, NAND_COMMON_TIMING_NS(conf, tCCS_min)),
1749 NAND_OP_DATA_OUT(len, buf, 0),
1750 };
1751 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1752 int ret;
1753
1754 ret = nand_fill_column_cycles(chip, addrs, offset_in_page);
1755 if (ret < 0)
1756 return ret;
1757
1758 instrs[2].ctx.data.force_8bit = force_8bit;
1759
1760 /* Drop the DATA_OUT instruction if len is set to 0. */
1761 if (!len)
1762 op.ninstrs--;
1763
1764 return nand_exec_op(chip, &op);
1765 }
1766
1767 chip->legacy.cmdfunc(chip, NAND_CMD_RNDIN, offset_in_page, -1);
1768 if (len)
1769 chip->legacy.write_buf(chip, buf, len);
1770
1771 return 0;
1772 }
1773 EXPORT_SYMBOL_GPL(nand_change_write_column_op);
1774
1775 /**
1776 * nand_readid_op - Do a READID operation
1777 * @chip: The NAND chip
1778 * @addr: address cycle to pass after the READID command
1779 * @buf: buffer used to store the ID
1780 * @len: length of the buffer
1781 *
1782 * This function sends a READID command and reads back the ID returned by the
1783 * NAND.
1784 * This function does not select/unselect the CS line.
1785 *
1786 * Returns 0 on success, a negative error code otherwise.
1787 */
nand_readid_op(struct nand_chip * chip,u8 addr,void * buf,unsigned int len)1788 int nand_readid_op(struct nand_chip *chip, u8 addr, void *buf,
1789 unsigned int len)
1790 {
1791 unsigned int i;
1792 u8 *id = buf, *ddrbuf = NULL;
1793
1794 if (len && !buf)
1795 return -EINVAL;
1796
1797 if (nand_has_exec_op(chip)) {
1798 const struct nand_interface_config *conf =
1799 nand_get_interface_config(chip);
1800 struct nand_op_instr instrs[] = {
1801 NAND_OP_CMD(NAND_CMD_READID, 0),
1802 NAND_OP_ADDR(1, &addr,
1803 NAND_COMMON_TIMING_NS(conf, tADL_min)),
1804 NAND_OP_8BIT_DATA_IN(len, buf, 0),
1805 };
1806 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1807 int ret;
1808
1809 /* READ_ID data bytes are received twice in NV-DDR mode */
1810 if (len && nand_interface_is_nvddr(conf)) {
1811 ddrbuf = kzalloc(len * 2, GFP_KERNEL);
1812 if (!ddrbuf)
1813 return -ENOMEM;
1814
1815 instrs[2].ctx.data.len *= 2;
1816 instrs[2].ctx.data.buf.in = ddrbuf;
1817 }
1818
1819 /* Drop the DATA_IN instruction if len is set to 0. */
1820 if (!len)
1821 op.ninstrs--;
1822
1823 ret = nand_exec_op(chip, &op);
1824 if (!ret && len && nand_interface_is_nvddr(conf)) {
1825 for (i = 0; i < len; i++)
1826 id[i] = ddrbuf[i * 2];
1827 }
1828
1829 kfree(ddrbuf);
1830
1831 return ret;
1832 }
1833
1834 chip->legacy.cmdfunc(chip, NAND_CMD_READID, addr, -1);
1835
1836 for (i = 0; i < len; i++)
1837 id[i] = chip->legacy.read_byte(chip);
1838
1839 return 0;
1840 }
1841 EXPORT_SYMBOL_GPL(nand_readid_op);
1842
1843 /**
1844 * nand_status_op - Do a STATUS operation
1845 * @chip: The NAND chip
1846 * @status: out variable to store the NAND status
1847 *
1848 * This function sends a STATUS command and reads back the status returned by
1849 * the NAND.
1850 * This function does not select/unselect the CS line.
1851 *
1852 * Returns 0 on success, a negative error code otherwise.
1853 */
nand_status_op(struct nand_chip * chip,u8 * status)1854 int nand_status_op(struct nand_chip *chip, u8 *status)
1855 {
1856 if (nand_has_exec_op(chip)) {
1857 const struct nand_interface_config *conf =
1858 nand_get_interface_config(chip);
1859 u8 ddrstatus[2];
1860 struct nand_op_instr instrs[] = {
1861 NAND_OP_CMD(NAND_CMD_STATUS,
1862 NAND_COMMON_TIMING_NS(conf, tADL_min)),
1863 NAND_OP_8BIT_DATA_IN(1, status, 0),
1864 };
1865 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1866 int ret;
1867
1868 /* The status data byte will be received twice in NV-DDR mode */
1869 if (status && nand_interface_is_nvddr(conf)) {
1870 instrs[1].ctx.data.len *= 2;
1871 instrs[1].ctx.data.buf.in = ddrstatus;
1872 }
1873
1874 if (!status)
1875 op.ninstrs--;
1876
1877 ret = nand_exec_op(chip, &op);
1878 if (!ret && status && nand_interface_is_nvddr(conf))
1879 *status = ddrstatus[0];
1880
1881 return ret;
1882 }
1883
1884 chip->legacy.cmdfunc(chip, NAND_CMD_STATUS, -1, -1);
1885 if (status)
1886 *status = chip->legacy.read_byte(chip);
1887
1888 return 0;
1889 }
1890 EXPORT_SYMBOL_GPL(nand_status_op);
1891
1892 /**
1893 * nand_exit_status_op - Exit a STATUS operation
1894 * @chip: The NAND chip
1895 *
1896 * This function sends a READ0 command to cancel the effect of the STATUS
1897 * command to avoid reading only the status until a new read command is sent.
1898 *
1899 * This function does not select/unselect the CS line.
1900 *
1901 * Returns 0 on success, a negative error code otherwise.
1902 */
nand_exit_status_op(struct nand_chip * chip)1903 int nand_exit_status_op(struct nand_chip *chip)
1904 {
1905 if (nand_has_exec_op(chip)) {
1906 struct nand_op_instr instrs[] = {
1907 NAND_OP_CMD(NAND_CMD_READ0, 0),
1908 };
1909 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1910
1911 return nand_exec_op(chip, &op);
1912 }
1913
1914 chip->legacy.cmdfunc(chip, NAND_CMD_READ0, -1, -1);
1915
1916 return 0;
1917 }
1918 EXPORT_SYMBOL_GPL(nand_exit_status_op);
1919
1920 /**
1921 * nand_erase_op - Do an erase operation
1922 * @chip: The NAND chip
1923 * @eraseblock: block to erase
1924 *
1925 * This function sends an ERASE command and waits for the NAND to be ready
1926 * before returning.
1927 * This function does not select/unselect the CS line.
1928 *
1929 * Returns 0 on success, a negative error code otherwise.
1930 */
nand_erase_op(struct nand_chip * chip,unsigned int eraseblock)1931 int nand_erase_op(struct nand_chip *chip, unsigned int eraseblock)
1932 {
1933 unsigned int page = eraseblock <<
1934 (chip->phys_erase_shift - chip->page_shift);
1935 int ret;
1936 u8 status;
1937
1938 if (nand_has_exec_op(chip)) {
1939 const struct nand_interface_config *conf =
1940 nand_get_interface_config(chip);
1941 u8 addrs[3] = { page, page >> 8, page >> 16 };
1942 struct nand_op_instr instrs[] = {
1943 NAND_OP_CMD(NAND_CMD_ERASE1, 0),
1944 NAND_OP_ADDR(2, addrs, 0),
1945 NAND_OP_CMD(NAND_CMD_ERASE2,
1946 NAND_COMMON_TIMING_NS(conf, tWB_max)),
1947 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tBERS_max),
1948 0),
1949 };
1950 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1951
1952 if (chip->options & NAND_ROW_ADDR_3)
1953 instrs[1].ctx.addr.naddrs++;
1954
1955 ret = nand_exec_op(chip, &op);
1956 if (ret)
1957 return ret;
1958
1959 ret = nand_status_op(chip, &status);
1960 if (ret)
1961 return ret;
1962 } else {
1963 chip->legacy.cmdfunc(chip, NAND_CMD_ERASE1, -1, page);
1964 chip->legacy.cmdfunc(chip, NAND_CMD_ERASE2, -1, -1);
1965
1966 ret = chip->legacy.waitfunc(chip);
1967 if (ret < 0)
1968 return ret;
1969
1970 status = ret;
1971 }
1972
1973 if (status & NAND_STATUS_FAIL)
1974 return -EIO;
1975
1976 return 0;
1977 }
1978 EXPORT_SYMBOL_GPL(nand_erase_op);
1979
1980 /**
1981 * nand_set_features_op - Do a SET FEATURES operation
1982 * @chip: The NAND chip
1983 * @feature: feature id
1984 * @data: 4 bytes of data
1985 *
1986 * This function sends a SET FEATURES command and waits for the NAND to be
1987 * ready before returning.
1988 * This function does not select/unselect the CS line.
1989 *
1990 * Returns 0 on success, a negative error code otherwise.
1991 */
nand_set_features_op(struct nand_chip * chip,u8 feature,const void * data)1992 static int nand_set_features_op(struct nand_chip *chip, u8 feature,
1993 const void *data)
1994 {
1995 const u8 *params = data;
1996 int i, ret;
1997
1998 if (nand_has_exec_op(chip)) {
1999 const struct nand_interface_config *conf =
2000 nand_get_interface_config(chip);
2001 struct nand_op_instr instrs[] = {
2002 NAND_OP_CMD(NAND_CMD_SET_FEATURES, 0),
2003 NAND_OP_ADDR(1, &feature, NAND_COMMON_TIMING_NS(conf,
2004 tADL_min)),
2005 NAND_OP_8BIT_DATA_OUT(ONFI_SUBFEATURE_PARAM_LEN, data,
2006 NAND_COMMON_TIMING_NS(conf,
2007 tWB_max)),
2008 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tFEAT_max),
2009 0),
2010 };
2011 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
2012
2013 return nand_exec_op(chip, &op);
2014 }
2015
2016 chip->legacy.cmdfunc(chip, NAND_CMD_SET_FEATURES, feature, -1);
2017 for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; ++i)
2018 chip->legacy.write_byte(chip, params[i]);
2019
2020 ret = chip->legacy.waitfunc(chip);
2021 if (ret < 0)
2022 return ret;
2023
2024 if (ret & NAND_STATUS_FAIL)
2025 return -EIO;
2026
2027 return 0;
2028 }
2029
2030 /**
2031 * nand_get_features_op - Do a GET FEATURES operation
2032 * @chip: The NAND chip
2033 * @feature: feature id
2034 * @data: 4 bytes of data
2035 *
2036 * This function sends a GET FEATURES command and waits for the NAND to be
2037 * ready before returning.
2038 * This function does not select/unselect the CS line.
2039 *
2040 * Returns 0 on success, a negative error code otherwise.
2041 */
nand_get_features_op(struct nand_chip * chip,u8 feature,void * data)2042 static int nand_get_features_op(struct nand_chip *chip, u8 feature,
2043 void *data)
2044 {
2045 u8 *params = data, ddrbuf[ONFI_SUBFEATURE_PARAM_LEN * 2];
2046 int i;
2047
2048 if (nand_has_exec_op(chip)) {
2049 const struct nand_interface_config *conf =
2050 nand_get_interface_config(chip);
2051 struct nand_op_instr instrs[] = {
2052 NAND_OP_CMD(NAND_CMD_GET_FEATURES, 0),
2053 NAND_OP_ADDR(1, &feature,
2054 NAND_COMMON_TIMING_NS(conf, tWB_max)),
2055 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tFEAT_max),
2056 NAND_COMMON_TIMING_NS(conf, tRR_min)),
2057 NAND_OP_8BIT_DATA_IN(ONFI_SUBFEATURE_PARAM_LEN,
2058 data, 0),
2059 };
2060 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
2061 int ret;
2062
2063 /* GET_FEATURE data bytes are received twice in NV-DDR mode */
2064 if (nand_interface_is_nvddr(conf)) {
2065 instrs[3].ctx.data.len *= 2;
2066 instrs[3].ctx.data.buf.in = ddrbuf;
2067 }
2068
2069 ret = nand_exec_op(chip, &op);
2070 if (nand_interface_is_nvddr(conf)) {
2071 for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; i++)
2072 params[i] = ddrbuf[i * 2];
2073 }
2074
2075 return ret;
2076 }
2077
2078 chip->legacy.cmdfunc(chip, NAND_CMD_GET_FEATURES, feature, -1);
2079 for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; ++i)
2080 params[i] = chip->legacy.read_byte(chip);
2081
2082 return 0;
2083 }
2084
nand_wait_rdy_op(struct nand_chip * chip,unsigned int timeout_ms,unsigned int delay_ns)2085 static int nand_wait_rdy_op(struct nand_chip *chip, unsigned int timeout_ms,
2086 unsigned int delay_ns)
2087 {
2088 if (nand_has_exec_op(chip)) {
2089 struct nand_op_instr instrs[] = {
2090 NAND_OP_WAIT_RDY(PSEC_TO_MSEC(timeout_ms),
2091 PSEC_TO_NSEC(delay_ns)),
2092 };
2093 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
2094
2095 return nand_exec_op(chip, &op);
2096 }
2097
2098 /* Apply delay or wait for ready/busy pin */
2099 if (!chip->legacy.dev_ready)
2100 udelay(chip->legacy.chip_delay);
2101 else
2102 nand_wait_ready(chip);
2103
2104 return 0;
2105 }
2106
2107 /**
2108 * nand_reset_op - Do a reset operation
2109 * @chip: The NAND chip
2110 *
2111 * This function sends a RESET command and waits for the NAND to be ready
2112 * before returning.
2113 * This function does not select/unselect the CS line.
2114 *
2115 * Returns 0 on success, a negative error code otherwise.
2116 */
nand_reset_op(struct nand_chip * chip)2117 int nand_reset_op(struct nand_chip *chip)
2118 {
2119 if (nand_has_exec_op(chip)) {
2120 const struct nand_interface_config *conf =
2121 nand_get_interface_config(chip);
2122 struct nand_op_instr instrs[] = {
2123 NAND_OP_CMD(NAND_CMD_RESET,
2124 NAND_COMMON_TIMING_NS(conf, tWB_max)),
2125 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tRST_max),
2126 0),
2127 };
2128 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
2129
2130 return nand_exec_op(chip, &op);
2131 }
2132
2133 chip->legacy.cmdfunc(chip, NAND_CMD_RESET, -1, -1);
2134
2135 return 0;
2136 }
2137 EXPORT_SYMBOL_GPL(nand_reset_op);
2138
2139 /**
2140 * nand_read_data_op - Read data from the NAND
2141 * @chip: The NAND chip
2142 * @buf: buffer used to store the data
2143 * @len: length of the buffer
2144 * @force_8bit: force 8-bit bus access
2145 * @check_only: do not actually run the command, only checks if the
2146 * controller driver supports it
2147 *
2148 * This function does a raw data read on the bus. Usually used after launching
2149 * another NAND operation like nand_read_page_op().
2150 * This function does not select/unselect the CS line.
2151 *
2152 * Returns 0 on success, a negative error code otherwise.
2153 */
nand_read_data_op(struct nand_chip * chip,void * buf,unsigned int len,bool force_8bit,bool check_only)2154 int nand_read_data_op(struct nand_chip *chip, void *buf, unsigned int len,
2155 bool force_8bit, bool check_only)
2156 {
2157 if (!len || !buf)
2158 return -EINVAL;
2159
2160 if (nand_has_exec_op(chip)) {
2161 const struct nand_interface_config *conf =
2162 nand_get_interface_config(chip);
2163 struct nand_op_instr instrs[] = {
2164 NAND_OP_DATA_IN(len, buf, 0),
2165 };
2166 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
2167 u8 *ddrbuf = NULL;
2168 int ret, i;
2169
2170 instrs[0].ctx.data.force_8bit = force_8bit;
2171
2172 /*
2173 * Parameter payloads (ID, status, features, etc) do not go
2174 * through the same pipeline as regular data, hence the
2175 * force_8bit flag must be set and this also indicates that in
2176 * case NV-DDR timings are being used the data will be received
2177 * twice.
2178 */
2179 if (force_8bit && nand_interface_is_nvddr(conf)) {
2180 ddrbuf = kzalloc(len * 2, GFP_KERNEL);
2181 if (!ddrbuf)
2182 return -ENOMEM;
2183
2184 instrs[0].ctx.data.len *= 2;
2185 instrs[0].ctx.data.buf.in = ddrbuf;
2186 }
2187
2188 if (check_only) {
2189 ret = nand_check_op(chip, &op);
2190 kfree(ddrbuf);
2191 return ret;
2192 }
2193
2194 ret = nand_exec_op(chip, &op);
2195 if (!ret && force_8bit && nand_interface_is_nvddr(conf)) {
2196 u8 *dst = buf;
2197
2198 for (i = 0; i < len; i++)
2199 dst[i] = ddrbuf[i * 2];
2200 }
2201
2202 kfree(ddrbuf);
2203
2204 return ret;
2205 }
2206
2207 if (check_only)
2208 return 0;
2209
2210 if (force_8bit) {
2211 u8 *p = buf;
2212 unsigned int i;
2213
2214 for (i = 0; i < len; i++)
2215 p[i] = chip->legacy.read_byte(chip);
2216 } else {
2217 chip->legacy.read_buf(chip, buf, len);
2218 }
2219
2220 return 0;
2221 }
2222 EXPORT_SYMBOL_GPL(nand_read_data_op);
2223
2224 /**
2225 * nand_write_data_op - Write data from the NAND
2226 * @chip: The NAND chip
2227 * @buf: buffer containing the data to send on the bus
2228 * @len: length of the buffer
2229 * @force_8bit: force 8-bit bus access
2230 *
2231 * This function does a raw data write on the bus. Usually used after launching
2232 * another NAND operation like nand_write_page_begin_op().
2233 * This function does not select/unselect the CS line.
2234 *
2235 * Returns 0 on success, a negative error code otherwise.
2236 */
nand_write_data_op(struct nand_chip * chip,const void * buf,unsigned int len,bool force_8bit)2237 int nand_write_data_op(struct nand_chip *chip, const void *buf,
2238 unsigned int len, bool force_8bit)
2239 {
2240 if (!len || !buf)
2241 return -EINVAL;
2242
2243 if (nand_has_exec_op(chip)) {
2244 struct nand_op_instr instrs[] = {
2245 NAND_OP_DATA_OUT(len, buf, 0),
2246 };
2247 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
2248
2249 instrs[0].ctx.data.force_8bit = force_8bit;
2250
2251 return nand_exec_op(chip, &op);
2252 }
2253
2254 if (force_8bit) {
2255 const u8 *p = buf;
2256 unsigned int i;
2257
2258 for (i = 0; i < len; i++)
2259 chip->legacy.write_byte(chip, p[i]);
2260 } else {
2261 chip->legacy.write_buf(chip, buf, len);
2262 }
2263
2264 return 0;
2265 }
2266 EXPORT_SYMBOL_GPL(nand_write_data_op);
2267
2268 /**
2269 * struct nand_op_parser_ctx - Context used by the parser
2270 * @instrs: array of all the instructions that must be addressed
2271 * @ninstrs: length of the @instrs array
2272 * @subop: Sub-operation to be passed to the NAND controller
2273 *
2274 * This structure is used by the core to split NAND operations into
2275 * sub-operations that can be handled by the NAND controller.
2276 */
2277 struct nand_op_parser_ctx {
2278 const struct nand_op_instr *instrs;
2279 unsigned int ninstrs;
2280 struct nand_subop subop;
2281 };
2282
2283 /**
2284 * nand_op_parser_must_split_instr - Checks if an instruction must be split
2285 * @pat: the parser pattern element that matches @instr
2286 * @instr: pointer to the instruction to check
2287 * @start_offset: this is an in/out parameter. If @instr has already been
2288 * split, then @start_offset is the offset from which to start
2289 * (either an address cycle or an offset in the data buffer).
2290 * Conversely, if the function returns true (ie. instr must be
2291 * split), this parameter is updated to point to the first
2292 * data/address cycle that has not been taken care of.
2293 *
2294 * Some NAND controllers are limited and cannot send X address cycles with a
2295 * unique operation, or cannot read/write more than Y bytes at the same time.
2296 * In this case, split the instruction that does not fit in a single
2297 * controller-operation into two or more chunks.
2298 *
2299 * Returns true if the instruction must be split, false otherwise.
2300 * The @start_offset parameter is also updated to the offset at which the next
2301 * bundle of instruction must start (if an address or a data instruction).
2302 */
2303 static bool
nand_op_parser_must_split_instr(const struct nand_op_parser_pattern_elem * pat,const struct nand_op_instr * instr,unsigned int * start_offset)2304 nand_op_parser_must_split_instr(const struct nand_op_parser_pattern_elem *pat,
2305 const struct nand_op_instr *instr,
2306 unsigned int *start_offset)
2307 {
2308 switch (pat->type) {
2309 case NAND_OP_ADDR_INSTR:
2310 if (!pat->ctx.addr.maxcycles)
2311 break;
2312
2313 if (instr->ctx.addr.naddrs - *start_offset >
2314 pat->ctx.addr.maxcycles) {
2315 *start_offset += pat->ctx.addr.maxcycles;
2316 return true;
2317 }
2318 break;
2319
2320 case NAND_OP_DATA_IN_INSTR:
2321 case NAND_OP_DATA_OUT_INSTR:
2322 if (!pat->ctx.data.maxlen)
2323 break;
2324
2325 if (instr->ctx.data.len - *start_offset >
2326 pat->ctx.data.maxlen) {
2327 *start_offset += pat->ctx.data.maxlen;
2328 return true;
2329 }
2330 break;
2331
2332 default:
2333 break;
2334 }
2335
2336 return false;
2337 }
2338
2339 /**
2340 * nand_op_parser_match_pat - Checks if a pattern matches the instructions
2341 * remaining in the parser context
2342 * @pat: the pattern to test
2343 * @ctx: the parser context structure to match with the pattern @pat
2344 *
2345 * Check if @pat matches the set or a sub-set of instructions remaining in @ctx.
2346 * Returns true if this is the case, false ortherwise. When true is returned,
2347 * @ctx->subop is updated with the set of instructions to be passed to the
2348 * controller driver.
2349 */
2350 static bool
nand_op_parser_match_pat(const struct nand_op_parser_pattern * pat,struct nand_op_parser_ctx * ctx)2351 nand_op_parser_match_pat(const struct nand_op_parser_pattern *pat,
2352 struct nand_op_parser_ctx *ctx)
2353 {
2354 unsigned int instr_offset = ctx->subop.first_instr_start_off;
2355 const struct nand_op_instr *end = ctx->instrs + ctx->ninstrs;
2356 const struct nand_op_instr *instr = ctx->subop.instrs;
2357 unsigned int i, ninstrs;
2358
2359 for (i = 0, ninstrs = 0; i < pat->nelems && instr < end; i++) {
2360 /*
2361 * The pattern instruction does not match the operation
2362 * instruction. If the instruction is marked optional in the
2363 * pattern definition, we skip the pattern element and continue
2364 * to the next one. If the element is mandatory, there's no
2365 * match and we can return false directly.
2366 */
2367 if (instr->type != pat->elems[i].type) {
2368 if (!pat->elems[i].optional)
2369 return false;
2370
2371 continue;
2372 }
2373
2374 /*
2375 * Now check the pattern element constraints. If the pattern is
2376 * not able to handle the whole instruction in a single step,
2377 * we have to split it.
2378 * The last_instr_end_off value comes back updated to point to
2379 * the position where we have to split the instruction (the
2380 * start of the next subop chunk).
2381 */
2382 if (nand_op_parser_must_split_instr(&pat->elems[i], instr,
2383 &instr_offset)) {
2384 ninstrs++;
2385 i++;
2386 break;
2387 }
2388
2389 instr++;
2390 ninstrs++;
2391 instr_offset = 0;
2392 }
2393
2394 /*
2395 * This can happen if all instructions of a pattern are optional.
2396 * Still, if there's not at least one instruction handled by this
2397 * pattern, this is not a match, and we should try the next one (if
2398 * any).
2399 */
2400 if (!ninstrs)
2401 return false;
2402
2403 /*
2404 * We had a match on the pattern head, but the pattern may be longer
2405 * than the instructions we're asked to execute. We need to make sure
2406 * there's no mandatory elements in the pattern tail.
2407 */
2408 for (; i < pat->nelems; i++) {
2409 if (!pat->elems[i].optional)
2410 return false;
2411 }
2412
2413 /*
2414 * We have a match: update the subop structure accordingly and return
2415 * true.
2416 */
2417 ctx->subop.ninstrs = ninstrs;
2418 ctx->subop.last_instr_end_off = instr_offset;
2419
2420 return true;
2421 }
2422
2423 #if IS_ENABLED(CONFIG_DYNAMIC_DEBUG) || defined(DEBUG)
nand_op_parser_trace(const struct nand_op_parser_ctx * ctx)2424 static void nand_op_parser_trace(const struct nand_op_parser_ctx *ctx)
2425 {
2426 const struct nand_op_instr *instr;
2427 char *prefix = " ";
2428 unsigned int i;
2429
2430 pr_debug("executing subop (CS%d):\n", ctx->subop.cs);
2431
2432 for (i = 0; i < ctx->ninstrs; i++) {
2433 instr = &ctx->instrs[i];
2434
2435 if (instr == &ctx->subop.instrs[0])
2436 prefix = " ->";
2437
2438 nand_op_trace(prefix, instr);
2439
2440 if (instr == &ctx->subop.instrs[ctx->subop.ninstrs - 1])
2441 prefix = " ";
2442 }
2443 }
2444 #else
nand_op_parser_trace(const struct nand_op_parser_ctx * ctx)2445 static void nand_op_parser_trace(const struct nand_op_parser_ctx *ctx)
2446 {
2447 /* NOP */
2448 }
2449 #endif
2450
nand_op_parser_cmp_ctx(const struct nand_op_parser_ctx * a,const struct nand_op_parser_ctx * b)2451 static int nand_op_parser_cmp_ctx(const struct nand_op_parser_ctx *a,
2452 const struct nand_op_parser_ctx *b)
2453 {
2454 if (a->subop.ninstrs < b->subop.ninstrs)
2455 return -1;
2456 else if (a->subop.ninstrs > b->subop.ninstrs)
2457 return 1;
2458
2459 if (a->subop.last_instr_end_off < b->subop.last_instr_end_off)
2460 return -1;
2461 else if (a->subop.last_instr_end_off > b->subop.last_instr_end_off)
2462 return 1;
2463
2464 return 0;
2465 }
2466
2467 /**
2468 * nand_op_parser_exec_op - exec_op parser
2469 * @chip: the NAND chip
2470 * @parser: patterns description provided by the controller driver
2471 * @op: the NAND operation to address
2472 * @check_only: when true, the function only checks if @op can be handled but
2473 * does not execute the operation
2474 *
2475 * Helper function designed to ease integration of NAND controller drivers that
2476 * only support a limited set of instruction sequences. The supported sequences
2477 * are described in @parser, and the framework takes care of splitting @op into
2478 * multiple sub-operations (if required) and pass them back to the ->exec()
2479 * callback of the matching pattern if @check_only is set to false.
2480 *
2481 * NAND controller drivers should call this function from their own ->exec_op()
2482 * implementation.
2483 *
2484 * Returns 0 on success, a negative error code otherwise. A failure can be
2485 * caused by an unsupported operation (none of the supported patterns is able
2486 * to handle the requested operation), or an error returned by one of the
2487 * matching pattern->exec() hook.
2488 */
nand_op_parser_exec_op(struct nand_chip * chip,const struct nand_op_parser * parser,const struct nand_operation * op,bool check_only)2489 int nand_op_parser_exec_op(struct nand_chip *chip,
2490 const struct nand_op_parser *parser,
2491 const struct nand_operation *op, bool check_only)
2492 {
2493 struct nand_op_parser_ctx ctx = {
2494 .subop.cs = op->cs,
2495 .subop.instrs = op->instrs,
2496 .instrs = op->instrs,
2497 .ninstrs = op->ninstrs,
2498 };
2499 unsigned int i;
2500
2501 while (ctx.subop.instrs < op->instrs + op->ninstrs) {
2502 const struct nand_op_parser_pattern *pattern;
2503 struct nand_op_parser_ctx best_ctx;
2504 int ret, best_pattern = -1;
2505
2506 for (i = 0; i < parser->npatterns; i++) {
2507 struct nand_op_parser_ctx test_ctx = ctx;
2508
2509 pattern = &parser->patterns[i];
2510 if (!nand_op_parser_match_pat(pattern, &test_ctx))
2511 continue;
2512
2513 if (best_pattern >= 0 &&
2514 nand_op_parser_cmp_ctx(&test_ctx, &best_ctx) <= 0)
2515 continue;
2516
2517 best_pattern = i;
2518 best_ctx = test_ctx;
2519 }
2520
2521 if (best_pattern < 0) {
2522 pr_debug("->exec_op() parser: pattern not found!\n");
2523 return -ENOTSUPP;
2524 }
2525
2526 ctx = best_ctx;
2527 nand_op_parser_trace(&ctx);
2528
2529 if (!check_only) {
2530 pattern = &parser->patterns[best_pattern];
2531 ret = pattern->exec(chip, &ctx.subop);
2532 if (ret)
2533 return ret;
2534 }
2535
2536 /*
2537 * Update the context structure by pointing to the start of the
2538 * next subop.
2539 */
2540 ctx.subop.instrs = ctx.subop.instrs + ctx.subop.ninstrs;
2541 if (ctx.subop.last_instr_end_off)
2542 ctx.subop.instrs -= 1;
2543
2544 ctx.subop.first_instr_start_off = ctx.subop.last_instr_end_off;
2545 }
2546
2547 return 0;
2548 }
2549 EXPORT_SYMBOL_GPL(nand_op_parser_exec_op);
2550
nand_instr_is_data(const struct nand_op_instr * instr)2551 static bool nand_instr_is_data(const struct nand_op_instr *instr)
2552 {
2553 return instr && (instr->type == NAND_OP_DATA_IN_INSTR ||
2554 instr->type == NAND_OP_DATA_OUT_INSTR);
2555 }
2556
nand_subop_instr_is_valid(const struct nand_subop * subop,unsigned int instr_idx)2557 static bool nand_subop_instr_is_valid(const struct nand_subop *subop,
2558 unsigned int instr_idx)
2559 {
2560 return subop && instr_idx < subop->ninstrs;
2561 }
2562
nand_subop_get_start_off(const struct nand_subop * subop,unsigned int instr_idx)2563 static unsigned int nand_subop_get_start_off(const struct nand_subop *subop,
2564 unsigned int instr_idx)
2565 {
2566 if (instr_idx)
2567 return 0;
2568
2569 return subop->first_instr_start_off;
2570 }
2571
2572 /**
2573 * nand_subop_get_addr_start_off - Get the start offset in an address array
2574 * @subop: The entire sub-operation
2575 * @instr_idx: Index of the instruction inside the sub-operation
2576 *
2577 * During driver development, one could be tempted to directly use the
2578 * ->addr.addrs field of address instructions. This is wrong as address
2579 * instructions might be split.
2580 *
2581 * Given an address instruction, returns the offset of the first cycle to issue.
2582 */
nand_subop_get_addr_start_off(const struct nand_subop * subop,unsigned int instr_idx)2583 unsigned int nand_subop_get_addr_start_off(const struct nand_subop *subop,
2584 unsigned int instr_idx)
2585 {
2586 if (WARN_ON(!nand_subop_instr_is_valid(subop, instr_idx) ||
2587 subop->instrs[instr_idx].type != NAND_OP_ADDR_INSTR))
2588 return 0;
2589
2590 return nand_subop_get_start_off(subop, instr_idx);
2591 }
2592 EXPORT_SYMBOL_GPL(nand_subop_get_addr_start_off);
2593
2594 /**
2595 * nand_subop_get_num_addr_cyc - Get the remaining address cycles to assert
2596 * @subop: The entire sub-operation
2597 * @instr_idx: Index of the instruction inside the sub-operation
2598 *
2599 * During driver development, one could be tempted to directly use the
2600 * ->addr->naddrs field of a data instruction. This is wrong as instructions
2601 * might be split.
2602 *
2603 * Given an address instruction, returns the number of address cycle to issue.
2604 */
nand_subop_get_num_addr_cyc(const struct nand_subop * subop,unsigned int instr_idx)2605 unsigned int nand_subop_get_num_addr_cyc(const struct nand_subop *subop,
2606 unsigned int instr_idx)
2607 {
2608 int start_off, end_off;
2609
2610 if (WARN_ON(!nand_subop_instr_is_valid(subop, instr_idx) ||
2611 subop->instrs[instr_idx].type != NAND_OP_ADDR_INSTR))
2612 return 0;
2613
2614 start_off = nand_subop_get_addr_start_off(subop, instr_idx);
2615
2616 if (instr_idx == subop->ninstrs - 1 &&
2617 subop->last_instr_end_off)
2618 end_off = subop->last_instr_end_off;
2619 else
2620 end_off = subop->instrs[instr_idx].ctx.addr.naddrs;
2621
2622 return end_off - start_off;
2623 }
2624 EXPORT_SYMBOL_GPL(nand_subop_get_num_addr_cyc);
2625
2626 /**
2627 * nand_subop_get_data_start_off - Get the start offset in a data array
2628 * @subop: The entire sub-operation
2629 * @instr_idx: Index of the instruction inside the sub-operation
2630 *
2631 * During driver development, one could be tempted to directly use the
2632 * ->data->buf.{in,out} field of data instructions. This is wrong as data
2633 * instructions might be split.
2634 *
2635 * Given a data instruction, returns the offset to start from.
2636 */
nand_subop_get_data_start_off(const struct nand_subop * subop,unsigned int instr_idx)2637 unsigned int nand_subop_get_data_start_off(const struct nand_subop *subop,
2638 unsigned int instr_idx)
2639 {
2640 if (WARN_ON(!nand_subop_instr_is_valid(subop, instr_idx) ||
2641 !nand_instr_is_data(&subop->instrs[instr_idx])))
2642 return 0;
2643
2644 return nand_subop_get_start_off(subop, instr_idx);
2645 }
2646 EXPORT_SYMBOL_GPL(nand_subop_get_data_start_off);
2647
2648 /**
2649 * nand_subop_get_data_len - Get the number of bytes to retrieve
2650 * @subop: The entire sub-operation
2651 * @instr_idx: Index of the instruction inside the sub-operation
2652 *
2653 * During driver development, one could be tempted to directly use the
2654 * ->data->len field of a data instruction. This is wrong as data instructions
2655 * might be split.
2656 *
2657 * Returns the length of the chunk of data to send/receive.
2658 */
nand_subop_get_data_len(const struct nand_subop * subop,unsigned int instr_idx)2659 unsigned int nand_subop_get_data_len(const struct nand_subop *subop,
2660 unsigned int instr_idx)
2661 {
2662 int start_off = 0, end_off;
2663
2664 if (WARN_ON(!nand_subop_instr_is_valid(subop, instr_idx) ||
2665 !nand_instr_is_data(&subop->instrs[instr_idx])))
2666 return 0;
2667
2668 start_off = nand_subop_get_data_start_off(subop, instr_idx);
2669
2670 if (instr_idx == subop->ninstrs - 1 &&
2671 subop->last_instr_end_off)
2672 end_off = subop->last_instr_end_off;
2673 else
2674 end_off = subop->instrs[instr_idx].ctx.data.len;
2675
2676 return end_off - start_off;
2677 }
2678 EXPORT_SYMBOL_GPL(nand_subop_get_data_len);
2679
2680 /**
2681 * nand_reset - Reset and initialize a NAND device
2682 * @chip: The NAND chip
2683 * @chipnr: Internal die id
2684 *
2685 * Save the timings data structure, then apply SDR timings mode 0 (see
2686 * nand_reset_interface for details), do the reset operation, and apply
2687 * back the previous timings.
2688 *
2689 * Returns 0 on success, a negative error code otherwise.
2690 */
nand_reset(struct nand_chip * chip,int chipnr)2691 int nand_reset(struct nand_chip *chip, int chipnr)
2692 {
2693 int ret;
2694
2695 ret = nand_reset_interface(chip, chipnr);
2696 if (ret)
2697 return ret;
2698
2699 /*
2700 * The CS line has to be released before we can apply the new NAND
2701 * interface settings, hence this weird nand_select_target()
2702 * nand_deselect_target() dance.
2703 */
2704 nand_select_target(chip, chipnr);
2705 ret = nand_reset_op(chip);
2706 nand_deselect_target(chip);
2707 if (ret)
2708 return ret;
2709
2710 ret = nand_setup_interface(chip, chipnr);
2711 if (ret)
2712 return ret;
2713
2714 return 0;
2715 }
2716 EXPORT_SYMBOL_GPL(nand_reset);
2717
2718 /**
2719 * nand_get_features - wrapper to perform a GET_FEATURE
2720 * @chip: NAND chip info structure
2721 * @addr: feature address
2722 * @subfeature_param: the subfeature parameters, a four bytes array
2723 *
2724 * Returns 0 for success, a negative error otherwise. Returns -ENOTSUPP if the
2725 * operation cannot be handled.
2726 */
nand_get_features(struct nand_chip * chip,int addr,u8 * subfeature_param)2727 int nand_get_features(struct nand_chip *chip, int addr,
2728 u8 *subfeature_param)
2729 {
2730 if (!nand_supports_get_features(chip, addr))
2731 return -ENOTSUPP;
2732
2733 if (chip->legacy.get_features)
2734 return chip->legacy.get_features(chip, addr, subfeature_param);
2735
2736 return nand_get_features_op(chip, addr, subfeature_param);
2737 }
2738
2739 /**
2740 * nand_set_features - wrapper to perform a SET_FEATURE
2741 * @chip: NAND chip info structure
2742 * @addr: feature address
2743 * @subfeature_param: the subfeature parameters, a four bytes array
2744 *
2745 * Returns 0 for success, a negative error otherwise. Returns -ENOTSUPP if the
2746 * operation cannot be handled.
2747 */
nand_set_features(struct nand_chip * chip,int addr,u8 * subfeature_param)2748 int nand_set_features(struct nand_chip *chip, int addr,
2749 u8 *subfeature_param)
2750 {
2751 if (!nand_supports_set_features(chip, addr))
2752 return -ENOTSUPP;
2753
2754 if (chip->legacy.set_features)
2755 return chip->legacy.set_features(chip, addr, subfeature_param);
2756
2757 return nand_set_features_op(chip, addr, subfeature_param);
2758 }
2759
2760 /**
2761 * nand_check_erased_buf - check if a buffer contains (almost) only 0xff data
2762 * @buf: buffer to test
2763 * @len: buffer length
2764 * @bitflips_threshold: maximum number of bitflips
2765 *
2766 * Check if a buffer contains only 0xff, which means the underlying region
2767 * has been erased and is ready to be programmed.
2768 * The bitflips_threshold specify the maximum number of bitflips before
2769 * considering the region is not erased.
2770 * Note: The logic of this function has been extracted from the memweight
2771 * implementation, except that nand_check_erased_buf function exit before
2772 * testing the whole buffer if the number of bitflips exceed the
2773 * bitflips_threshold value.
2774 *
2775 * Returns a positive number of bitflips less than or equal to
2776 * bitflips_threshold, or -ERROR_CODE for bitflips in excess of the
2777 * threshold.
2778 */
nand_check_erased_buf(void * buf,int len,int bitflips_threshold)2779 static int nand_check_erased_buf(void *buf, int len, int bitflips_threshold)
2780 {
2781 const unsigned char *bitmap = buf;
2782 int bitflips = 0;
2783 int weight;
2784
2785 for (; len && ((uintptr_t)bitmap) % sizeof(long);
2786 len--, bitmap++) {
2787 weight = hweight8(*bitmap);
2788 bitflips += BITS_PER_BYTE - weight;
2789 if (unlikely(bitflips > bitflips_threshold))
2790 return -EBADMSG;
2791 }
2792
2793 for (; len >= sizeof(long);
2794 len -= sizeof(long), bitmap += sizeof(long)) {
2795 unsigned long d = *((unsigned long *)bitmap);
2796 if (d == ~0UL)
2797 continue;
2798 weight = hweight_long(d);
2799 bitflips += BITS_PER_LONG - weight;
2800 if (unlikely(bitflips > bitflips_threshold))
2801 return -EBADMSG;
2802 }
2803
2804 for (; len > 0; len--, bitmap++) {
2805 weight = hweight8(*bitmap);
2806 bitflips += BITS_PER_BYTE - weight;
2807 if (unlikely(bitflips > bitflips_threshold))
2808 return -EBADMSG;
2809 }
2810
2811 return bitflips;
2812 }
2813
2814 /**
2815 * nand_check_erased_ecc_chunk - check if an ECC chunk contains (almost) only
2816 * 0xff data
2817 * @data: data buffer to test
2818 * @datalen: data length
2819 * @ecc: ECC buffer
2820 * @ecclen: ECC length
2821 * @extraoob: extra OOB buffer
2822 * @extraooblen: extra OOB length
2823 * @bitflips_threshold: maximum number of bitflips
2824 *
2825 * Check if a data buffer and its associated ECC and OOB data contains only
2826 * 0xff pattern, which means the underlying region has been erased and is
2827 * ready to be programmed.
2828 * The bitflips_threshold specify the maximum number of bitflips before
2829 * considering the region as not erased.
2830 *
2831 * Note:
2832 * 1/ ECC algorithms are working on pre-defined block sizes which are usually
2833 * different from the NAND page size. When fixing bitflips, ECC engines will
2834 * report the number of errors per chunk, and the NAND core infrastructure
2835 * expect you to return the maximum number of bitflips for the whole page.
2836 * This is why you should always use this function on a single chunk and
2837 * not on the whole page. After checking each chunk you should update your
2838 * max_bitflips value accordingly.
2839 * 2/ When checking for bitflips in erased pages you should not only check
2840 * the payload data but also their associated ECC data, because a user might
2841 * have programmed almost all bits to 1 but a few. In this case, we
2842 * shouldn't consider the chunk as erased, and checking ECC bytes prevent
2843 * this case.
2844 * 3/ The extraoob argument is optional, and should be used if some of your OOB
2845 * data are protected by the ECC engine.
2846 * It could also be used if you support subpages and want to attach some
2847 * extra OOB data to an ECC chunk.
2848 *
2849 * Returns a positive number of bitflips less than or equal to
2850 * bitflips_threshold, or -ERROR_CODE for bitflips in excess of the
2851 * threshold. In case of success, the passed buffers are filled with 0xff.
2852 */
nand_check_erased_ecc_chunk(void * data,int datalen,void * ecc,int ecclen,void * extraoob,int extraooblen,int bitflips_threshold)2853 int nand_check_erased_ecc_chunk(void *data, int datalen,
2854 void *ecc, int ecclen,
2855 void *extraoob, int extraooblen,
2856 int bitflips_threshold)
2857 {
2858 int data_bitflips = 0, ecc_bitflips = 0, extraoob_bitflips = 0;
2859
2860 data_bitflips = nand_check_erased_buf(data, datalen,
2861 bitflips_threshold);
2862 if (data_bitflips < 0)
2863 return data_bitflips;
2864
2865 bitflips_threshold -= data_bitflips;
2866
2867 ecc_bitflips = nand_check_erased_buf(ecc, ecclen, bitflips_threshold);
2868 if (ecc_bitflips < 0)
2869 return ecc_bitflips;
2870
2871 bitflips_threshold -= ecc_bitflips;
2872
2873 extraoob_bitflips = nand_check_erased_buf(extraoob, extraooblen,
2874 bitflips_threshold);
2875 if (extraoob_bitflips < 0)
2876 return extraoob_bitflips;
2877
2878 if (data_bitflips)
2879 memset(data, 0xff, datalen);
2880
2881 if (ecc_bitflips)
2882 memset(ecc, 0xff, ecclen);
2883
2884 if (extraoob_bitflips)
2885 memset(extraoob, 0xff, extraooblen);
2886
2887 return data_bitflips + ecc_bitflips + extraoob_bitflips;
2888 }
2889 EXPORT_SYMBOL(nand_check_erased_ecc_chunk);
2890
2891 /**
2892 * nand_read_page_raw_notsupp - dummy read raw page function
2893 * @chip: nand chip info structure
2894 * @buf: buffer to store read data
2895 * @oob_required: caller requires OOB data read to chip->oob_poi
2896 * @page: page number to read
2897 *
2898 * Returns -ENOTSUPP unconditionally.
2899 */
nand_read_page_raw_notsupp(struct nand_chip * chip,u8 * buf,int oob_required,int page)2900 int nand_read_page_raw_notsupp(struct nand_chip *chip, u8 *buf,
2901 int oob_required, int page)
2902 {
2903 return -ENOTSUPP;
2904 }
2905
2906 /**
2907 * nand_read_page_raw - [INTERN] read raw page data without ecc
2908 * @chip: nand chip info structure
2909 * @buf: buffer to store read data
2910 * @oob_required: caller requires OOB data read to chip->oob_poi
2911 * @page: page number to read
2912 *
2913 * Not for syndrome calculating ECC controllers, which use a special oob layout.
2914 */
nand_read_page_raw(struct nand_chip * chip,uint8_t * buf,int oob_required,int page)2915 int nand_read_page_raw(struct nand_chip *chip, uint8_t *buf, int oob_required,
2916 int page)
2917 {
2918 struct mtd_info *mtd = nand_to_mtd(chip);
2919 int ret;
2920
2921 ret = nand_read_page_op(chip, page, 0, buf, mtd->writesize);
2922 if (ret)
2923 return ret;
2924
2925 if (oob_required) {
2926 ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize,
2927 false, false);
2928 if (ret)
2929 return ret;
2930 }
2931
2932 return 0;
2933 }
2934 EXPORT_SYMBOL(nand_read_page_raw);
2935
2936 /**
2937 * nand_monolithic_read_page_raw - Monolithic page read in raw mode
2938 * @chip: NAND chip info structure
2939 * @buf: buffer to store read data
2940 * @oob_required: caller requires OOB data read to chip->oob_poi
2941 * @page: page number to read
2942 *
2943 * This is a raw page read, ie. without any error detection/correction.
2944 * Monolithic means we are requesting all the relevant data (main plus
2945 * eventually OOB) to be loaded in the NAND cache and sent over the
2946 * bus (from the NAND chip to the NAND controller) in a single
2947 * operation. This is an alternative to nand_read_page_raw(), which
2948 * first reads the main data, and if the OOB data is requested too,
2949 * then reads more data on the bus.
2950 */
nand_monolithic_read_page_raw(struct nand_chip * chip,u8 * buf,int oob_required,int page)2951 int nand_monolithic_read_page_raw(struct nand_chip *chip, u8 *buf,
2952 int oob_required, int page)
2953 {
2954 struct mtd_info *mtd = nand_to_mtd(chip);
2955 unsigned int size = mtd->writesize;
2956 u8 *read_buf = buf;
2957 int ret;
2958
2959 if (oob_required) {
2960 size += mtd->oobsize;
2961
2962 if (buf != chip->data_buf)
2963 read_buf = nand_get_data_buf(chip);
2964 }
2965
2966 ret = nand_read_page_op(chip, page, 0, read_buf, size);
2967 if (ret)
2968 return ret;
2969
2970 if (buf != chip->data_buf)
2971 memcpy(buf, read_buf, mtd->writesize);
2972
2973 return 0;
2974 }
2975 EXPORT_SYMBOL(nand_monolithic_read_page_raw);
2976
2977 /**
2978 * nand_read_page_raw_syndrome - [INTERN] read raw page data without ecc
2979 * @chip: nand chip info structure
2980 * @buf: buffer to store read data
2981 * @oob_required: caller requires OOB data read to chip->oob_poi
2982 * @page: page number to read
2983 *
2984 * We need a special oob layout and handling even when OOB isn't used.
2985 */
nand_read_page_raw_syndrome(struct nand_chip * chip,uint8_t * buf,int oob_required,int page)2986 static int nand_read_page_raw_syndrome(struct nand_chip *chip, uint8_t *buf,
2987 int oob_required, int page)
2988 {
2989 struct mtd_info *mtd = nand_to_mtd(chip);
2990 int eccsize = chip->ecc.size;
2991 int eccbytes = chip->ecc.bytes;
2992 uint8_t *oob = chip->oob_poi;
2993 int steps, size, ret;
2994
2995 ret = nand_read_page_op(chip, page, 0, NULL, 0);
2996 if (ret)
2997 return ret;
2998
2999 for (steps = chip->ecc.steps; steps > 0; steps--) {
3000 ret = nand_read_data_op(chip, buf, eccsize, false, false);
3001 if (ret)
3002 return ret;
3003
3004 buf += eccsize;
3005
3006 if (chip->ecc.prepad) {
3007 ret = nand_read_data_op(chip, oob, chip->ecc.prepad,
3008 false, false);
3009 if (ret)
3010 return ret;
3011
3012 oob += chip->ecc.prepad;
3013 }
3014
3015 ret = nand_read_data_op(chip, oob, eccbytes, false, false);
3016 if (ret)
3017 return ret;
3018
3019 oob += eccbytes;
3020
3021 if (chip->ecc.postpad) {
3022 ret = nand_read_data_op(chip, oob, chip->ecc.postpad,
3023 false, false);
3024 if (ret)
3025 return ret;
3026
3027 oob += chip->ecc.postpad;
3028 }
3029 }
3030
3031 size = mtd->oobsize - (oob - chip->oob_poi);
3032 if (size) {
3033 ret = nand_read_data_op(chip, oob, size, false, false);
3034 if (ret)
3035 return ret;
3036 }
3037
3038 return 0;
3039 }
3040
3041 /**
3042 * nand_read_page_swecc - [REPLACEABLE] software ECC based page read function
3043 * @chip: nand chip info structure
3044 * @buf: buffer to store read data
3045 * @oob_required: caller requires OOB data read to chip->oob_poi
3046 * @page: page number to read
3047 */
nand_read_page_swecc(struct nand_chip * chip,uint8_t * buf,int oob_required,int page)3048 static int nand_read_page_swecc(struct nand_chip *chip, uint8_t *buf,
3049 int oob_required, int page)
3050 {
3051 struct mtd_info *mtd = nand_to_mtd(chip);
3052 int i, eccsize = chip->ecc.size, ret;
3053 int eccbytes = chip->ecc.bytes;
3054 int eccsteps = chip->ecc.steps;
3055 uint8_t *p = buf;
3056 uint8_t *ecc_calc = chip->ecc.calc_buf;
3057 uint8_t *ecc_code = chip->ecc.code_buf;
3058 unsigned int max_bitflips = 0;
3059
3060 chip->ecc.read_page_raw(chip, buf, 1, page);
3061
3062 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize)
3063 chip->ecc.calculate(chip, p, &ecc_calc[i]);
3064
3065 ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
3066 chip->ecc.total);
3067 if (ret)
3068 return ret;
3069
3070 eccsteps = chip->ecc.steps;
3071 p = buf;
3072
3073 for (i = 0 ; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
3074 int stat;
3075
3076 stat = chip->ecc.correct(chip, p, &ecc_code[i], &ecc_calc[i]);
3077 if (stat < 0) {
3078 mtd->ecc_stats.failed++;
3079 } else {
3080 mtd->ecc_stats.corrected += stat;
3081 max_bitflips = max_t(unsigned int, max_bitflips, stat);
3082 }
3083 }
3084 return max_bitflips;
3085 }
3086
3087 /**
3088 * nand_read_subpage - [REPLACEABLE] ECC based sub-page read function
3089 * @chip: nand chip info structure
3090 * @data_offs: offset of requested data within the page
3091 * @readlen: data length
3092 * @bufpoi: buffer to store read data
3093 * @page: page number to read
3094 */
nand_read_subpage(struct nand_chip * chip,uint32_t data_offs,uint32_t readlen,uint8_t * bufpoi,int page)3095 static int nand_read_subpage(struct nand_chip *chip, uint32_t data_offs,
3096 uint32_t readlen, uint8_t *bufpoi, int page)
3097 {
3098 struct mtd_info *mtd = nand_to_mtd(chip);
3099 int start_step, end_step, num_steps, ret;
3100 uint8_t *p;
3101 int data_col_addr, i, gaps = 0;
3102 int datafrag_len, eccfrag_len, aligned_len, aligned_pos;
3103 int busw = (chip->options & NAND_BUSWIDTH_16) ? 2 : 1;
3104 int index, section = 0;
3105 unsigned int max_bitflips = 0;
3106 struct mtd_oob_region oobregion = { };
3107
3108 /* Column address within the page aligned to ECC size (256bytes) */
3109 start_step = data_offs / chip->ecc.size;
3110 end_step = (data_offs + readlen - 1) / chip->ecc.size;
3111 num_steps = end_step - start_step + 1;
3112 index = start_step * chip->ecc.bytes;
3113
3114 /* Data size aligned to ECC ecc.size */
3115 datafrag_len = num_steps * chip->ecc.size;
3116 eccfrag_len = num_steps * chip->ecc.bytes;
3117
3118 data_col_addr = start_step * chip->ecc.size;
3119 /* If we read not a page aligned data */
3120 p = bufpoi + data_col_addr;
3121 ret = nand_read_page_op(chip, page, data_col_addr, p, datafrag_len);
3122 if (ret)
3123 return ret;
3124
3125 /* Calculate ECC */
3126 for (i = 0; i < eccfrag_len ; i += chip->ecc.bytes, p += chip->ecc.size)
3127 chip->ecc.calculate(chip, p, &chip->ecc.calc_buf[i]);
3128
3129 /*
3130 * The performance is faster if we position offsets according to
3131 * ecc.pos. Let's make sure that there are no gaps in ECC positions.
3132 */
3133 ret = mtd_ooblayout_find_eccregion(mtd, index, §ion, &oobregion);
3134 if (ret)
3135 return ret;
3136
3137 if (oobregion.length < eccfrag_len)
3138 gaps = 1;
3139
3140 if (gaps) {
3141 ret = nand_change_read_column_op(chip, mtd->writesize,
3142 chip->oob_poi, mtd->oobsize,
3143 false);
3144 if (ret)
3145 return ret;
3146 } else {
3147 /*
3148 * Send the command to read the particular ECC bytes take care
3149 * about buswidth alignment in read_buf.
3150 */
3151 aligned_pos = oobregion.offset & ~(busw - 1);
3152 aligned_len = eccfrag_len;
3153 if (oobregion.offset & (busw - 1))
3154 aligned_len++;
3155 if ((oobregion.offset + (num_steps * chip->ecc.bytes)) &
3156 (busw - 1))
3157 aligned_len++;
3158
3159 ret = nand_change_read_column_op(chip,
3160 mtd->writesize + aligned_pos,
3161 &chip->oob_poi[aligned_pos],
3162 aligned_len, false);
3163 if (ret)
3164 return ret;
3165 }
3166
3167 ret = mtd_ooblayout_get_eccbytes(mtd, chip->ecc.code_buf,
3168 chip->oob_poi, index, eccfrag_len);
3169 if (ret)
3170 return ret;
3171
3172 p = bufpoi + data_col_addr;
3173 for (i = 0; i < eccfrag_len ; i += chip->ecc.bytes, p += chip->ecc.size) {
3174 int stat;
3175
3176 stat = chip->ecc.correct(chip, p, &chip->ecc.code_buf[i],
3177 &chip->ecc.calc_buf[i]);
3178 if (stat == -EBADMSG &&
3179 (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) {
3180 /* check for empty pages with bitflips */
3181 stat = nand_check_erased_ecc_chunk(p, chip->ecc.size,
3182 &chip->ecc.code_buf[i],
3183 chip->ecc.bytes,
3184 NULL, 0,
3185 chip->ecc.strength);
3186 }
3187
3188 if (stat < 0) {
3189 mtd->ecc_stats.failed++;
3190 } else {
3191 mtd->ecc_stats.corrected += stat;
3192 max_bitflips = max_t(unsigned int, max_bitflips, stat);
3193 }
3194 }
3195 return max_bitflips;
3196 }
3197
3198 /**
3199 * nand_read_page_hwecc - [REPLACEABLE] hardware ECC based page read function
3200 * @chip: nand chip info structure
3201 * @buf: buffer to store read data
3202 * @oob_required: caller requires OOB data read to chip->oob_poi
3203 * @page: page number to read
3204 *
3205 * Not for syndrome calculating ECC controllers which need a special oob layout.
3206 */
nand_read_page_hwecc(struct nand_chip * chip,uint8_t * buf,int oob_required,int page)3207 static int nand_read_page_hwecc(struct nand_chip *chip, uint8_t *buf,
3208 int oob_required, int page)
3209 {
3210 struct mtd_info *mtd = nand_to_mtd(chip);
3211 int i, eccsize = chip->ecc.size, ret;
3212 int eccbytes = chip->ecc.bytes;
3213 int eccsteps = chip->ecc.steps;
3214 uint8_t *p = buf;
3215 uint8_t *ecc_calc = chip->ecc.calc_buf;
3216 uint8_t *ecc_code = chip->ecc.code_buf;
3217 unsigned int max_bitflips = 0;
3218
3219 ret = nand_read_page_op(chip, page, 0, NULL, 0);
3220 if (ret)
3221 return ret;
3222
3223 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
3224 chip->ecc.hwctl(chip, NAND_ECC_READ);
3225
3226 ret = nand_read_data_op(chip, p, eccsize, false, false);
3227 if (ret)
3228 return ret;
3229
3230 chip->ecc.calculate(chip, p, &ecc_calc[i]);
3231 }
3232
3233 ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, false,
3234 false);
3235 if (ret)
3236 return ret;
3237
3238 ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
3239 chip->ecc.total);
3240 if (ret)
3241 return ret;
3242
3243 eccsteps = chip->ecc.steps;
3244 p = buf;
3245
3246 for (i = 0 ; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
3247 int stat;
3248
3249 stat = chip->ecc.correct(chip, p, &ecc_code[i], &ecc_calc[i]);
3250 if (stat == -EBADMSG &&
3251 (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) {
3252 /* check for empty pages with bitflips */
3253 stat = nand_check_erased_ecc_chunk(p, eccsize,
3254 &ecc_code[i], eccbytes,
3255 NULL, 0,
3256 chip->ecc.strength);
3257 }
3258
3259 if (stat < 0) {
3260 mtd->ecc_stats.failed++;
3261 } else {
3262 mtd->ecc_stats.corrected += stat;
3263 max_bitflips = max_t(unsigned int, max_bitflips, stat);
3264 }
3265 }
3266 return max_bitflips;
3267 }
3268
3269 /**
3270 * nand_read_page_hwecc_oob_first - Hardware ECC page read with ECC
3271 * data read from OOB area
3272 * @chip: nand chip info structure
3273 * @buf: buffer to store read data
3274 * @oob_required: caller requires OOB data read to chip->oob_poi
3275 * @page: page number to read
3276 *
3277 * Hardware ECC for large page chips, which requires the ECC data to be
3278 * extracted from the OOB before the actual data is read.
3279 */
nand_read_page_hwecc_oob_first(struct nand_chip * chip,uint8_t * buf,int oob_required,int page)3280 int nand_read_page_hwecc_oob_first(struct nand_chip *chip, uint8_t *buf,
3281 int oob_required, int page)
3282 {
3283 struct mtd_info *mtd = nand_to_mtd(chip);
3284 int i, eccsize = chip->ecc.size, ret;
3285 int eccbytes = chip->ecc.bytes;
3286 int eccsteps = chip->ecc.steps;
3287 uint8_t *p = buf;
3288 uint8_t *ecc_code = chip->ecc.code_buf;
3289 unsigned int max_bitflips = 0;
3290
3291 /* Read the OOB area first */
3292 ret = nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize);
3293 if (ret)
3294 return ret;
3295
3296 /* Move read cursor to start of page */
3297 ret = nand_change_read_column_op(chip, 0, NULL, 0, false);
3298 if (ret)
3299 return ret;
3300
3301 ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
3302 chip->ecc.total);
3303 if (ret)
3304 return ret;
3305
3306 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
3307 int stat;
3308
3309 chip->ecc.hwctl(chip, NAND_ECC_READ);
3310
3311 ret = nand_read_data_op(chip, p, eccsize, false, false);
3312 if (ret)
3313 return ret;
3314
3315 stat = chip->ecc.correct(chip, p, &ecc_code[i], NULL);
3316 if (stat == -EBADMSG &&
3317 (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) {
3318 /* check for empty pages with bitflips */
3319 stat = nand_check_erased_ecc_chunk(p, eccsize,
3320 &ecc_code[i],
3321 eccbytes, NULL, 0,
3322 chip->ecc.strength);
3323 }
3324
3325 if (stat < 0) {
3326 mtd->ecc_stats.failed++;
3327 } else {
3328 mtd->ecc_stats.corrected += stat;
3329 max_bitflips = max_t(unsigned int, max_bitflips, stat);
3330 }
3331 }
3332 return max_bitflips;
3333 }
3334 EXPORT_SYMBOL_GPL(nand_read_page_hwecc_oob_first);
3335
3336 /**
3337 * nand_read_page_syndrome - [REPLACEABLE] hardware ECC syndrome based page read
3338 * @chip: nand chip info structure
3339 * @buf: buffer to store read data
3340 * @oob_required: caller requires OOB data read to chip->oob_poi
3341 * @page: page number to read
3342 *
3343 * The hw generator calculates the error syndrome automatically. Therefore we
3344 * need a special oob layout and handling.
3345 */
nand_read_page_syndrome(struct nand_chip * chip,uint8_t * buf,int oob_required,int page)3346 static int nand_read_page_syndrome(struct nand_chip *chip, uint8_t *buf,
3347 int oob_required, int page)
3348 {
3349 struct mtd_info *mtd = nand_to_mtd(chip);
3350 int ret, i, eccsize = chip->ecc.size;
3351 int eccbytes = chip->ecc.bytes;
3352 int eccsteps = chip->ecc.steps;
3353 int eccpadbytes = eccbytes + chip->ecc.prepad + chip->ecc.postpad;
3354 uint8_t *p = buf;
3355 uint8_t *oob = chip->oob_poi;
3356 unsigned int max_bitflips = 0;
3357
3358 ret = nand_read_page_op(chip, page, 0, NULL, 0);
3359 if (ret)
3360 return ret;
3361
3362 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
3363 int stat;
3364
3365 chip->ecc.hwctl(chip, NAND_ECC_READ);
3366
3367 ret = nand_read_data_op(chip, p, eccsize, false, false);
3368 if (ret)
3369 return ret;
3370
3371 if (chip->ecc.prepad) {
3372 ret = nand_read_data_op(chip, oob, chip->ecc.prepad,
3373 false, false);
3374 if (ret)
3375 return ret;
3376
3377 oob += chip->ecc.prepad;
3378 }
3379
3380 chip->ecc.hwctl(chip, NAND_ECC_READSYN);
3381
3382 ret = nand_read_data_op(chip, oob, eccbytes, false, false);
3383 if (ret)
3384 return ret;
3385
3386 stat = chip->ecc.correct(chip, p, oob, NULL);
3387
3388 oob += eccbytes;
3389
3390 if (chip->ecc.postpad) {
3391 ret = nand_read_data_op(chip, oob, chip->ecc.postpad,
3392 false, false);
3393 if (ret)
3394 return ret;
3395
3396 oob += chip->ecc.postpad;
3397 }
3398
3399 if (stat == -EBADMSG &&
3400 (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) {
3401 /* check for empty pages with bitflips */
3402 stat = nand_check_erased_ecc_chunk(p, chip->ecc.size,
3403 oob - eccpadbytes,
3404 eccpadbytes,
3405 NULL, 0,
3406 chip->ecc.strength);
3407 }
3408
3409 if (stat < 0) {
3410 mtd->ecc_stats.failed++;
3411 } else {
3412 mtd->ecc_stats.corrected += stat;
3413 max_bitflips = max_t(unsigned int, max_bitflips, stat);
3414 }
3415 }
3416
3417 /* Calculate remaining oob bytes */
3418 i = mtd->oobsize - (oob - chip->oob_poi);
3419 if (i) {
3420 ret = nand_read_data_op(chip, oob, i, false, false);
3421 if (ret)
3422 return ret;
3423 }
3424
3425 return max_bitflips;
3426 }
3427
3428 /**
3429 * nand_transfer_oob - [INTERN] Transfer oob to client buffer
3430 * @chip: NAND chip object
3431 * @oob: oob destination address
3432 * @ops: oob ops structure
3433 * @len: size of oob to transfer
3434 */
nand_transfer_oob(struct nand_chip * chip,uint8_t * oob,struct mtd_oob_ops * ops,size_t len)3435 static uint8_t *nand_transfer_oob(struct nand_chip *chip, uint8_t *oob,
3436 struct mtd_oob_ops *ops, size_t len)
3437 {
3438 struct mtd_info *mtd = nand_to_mtd(chip);
3439 int ret;
3440
3441 switch (ops->mode) {
3442
3443 case MTD_OPS_PLACE_OOB:
3444 case MTD_OPS_RAW:
3445 memcpy(oob, chip->oob_poi + ops->ooboffs, len);
3446 return oob + len;
3447
3448 case MTD_OPS_AUTO_OOB:
3449 ret = mtd_ooblayout_get_databytes(mtd, oob, chip->oob_poi,
3450 ops->ooboffs, len);
3451 BUG_ON(ret);
3452 return oob + len;
3453
3454 default:
3455 BUG();
3456 }
3457 return NULL;
3458 }
3459
rawnand_enable_cont_reads(struct nand_chip * chip,unsigned int page,u32 readlen,int col)3460 static void rawnand_enable_cont_reads(struct nand_chip *chip, unsigned int page,
3461 u32 readlen, int col)
3462 {
3463 struct mtd_info *mtd = nand_to_mtd(chip);
3464 unsigned int first_page, last_page;
3465
3466 chip->cont_read.ongoing = false;
3467
3468 if (!chip->controller->supported_op.cont_read)
3469 return;
3470
3471 /*
3472 * Don't bother making any calculations if the length is too small.
3473 * Side effect: avoids possible integer underflows below.
3474 */
3475 if (readlen < (2 * mtd->writesize))
3476 return;
3477
3478 /* Derive the page where continuous read should start (the first full page read) */
3479 first_page = page;
3480 if (col)
3481 first_page++;
3482
3483 /* Derive the page where continuous read should stop (the last full page read) */
3484 last_page = page + ((col + readlen) / mtd->writesize) - 1;
3485
3486 /* Configure and enable continuous read when suitable */
3487 if (first_page < last_page) {
3488 chip->cont_read.first_page = first_page;
3489 chip->cont_read.last_page = last_page;
3490 chip->cont_read.ongoing = true;
3491 /* May reset the ongoing flag */
3492 rawnand_cap_cont_reads(chip);
3493 }
3494 }
3495
rawnand_cont_read_skip_first_page(struct nand_chip * chip,unsigned int page)3496 static void rawnand_cont_read_skip_first_page(struct nand_chip *chip, unsigned int page)
3497 {
3498 if (!chip->cont_read.ongoing || page != chip->cont_read.first_page)
3499 return;
3500
3501 chip->cont_read.first_page++;
3502 if (chip->cont_read.first_page == chip->cont_read.pause_page)
3503 chip->cont_read.first_page++;
3504 if (chip->cont_read.first_page >= chip->cont_read.last_page)
3505 chip->cont_read.ongoing = false;
3506 }
3507
3508 /**
3509 * nand_setup_read_retry - [INTERN] Set the READ RETRY mode
3510 * @chip: NAND chip object
3511 * @retry_mode: the retry mode to use
3512 *
3513 * Some vendors supply a special command to shift the Vt threshold, to be used
3514 * when there are too many bitflips in a page (i.e., ECC error). After setting
3515 * a new threshold, the host should retry reading the page.
3516 */
nand_setup_read_retry(struct nand_chip * chip,int retry_mode)3517 static int nand_setup_read_retry(struct nand_chip *chip, int retry_mode)
3518 {
3519 pr_debug("setting READ RETRY mode %d\n", retry_mode);
3520
3521 if (retry_mode >= chip->read_retries)
3522 return -EINVAL;
3523
3524 if (!chip->ops.setup_read_retry)
3525 return -EOPNOTSUPP;
3526
3527 return chip->ops.setup_read_retry(chip, retry_mode);
3528 }
3529
nand_wait_readrdy(struct nand_chip * chip)3530 static void nand_wait_readrdy(struct nand_chip *chip)
3531 {
3532 const struct nand_interface_config *conf;
3533
3534 if (!(chip->options & NAND_NEED_READRDY))
3535 return;
3536
3537 conf = nand_get_interface_config(chip);
3538 WARN_ON(nand_wait_rdy_op(chip, NAND_COMMON_TIMING_MS(conf, tR_max), 0));
3539 }
3540
3541 /**
3542 * nand_do_read_ops - [INTERN] Read data with ECC
3543 * @chip: NAND chip object
3544 * @from: offset to read from
3545 * @ops: oob ops structure
3546 *
3547 * Internal function. Called with chip held.
3548 */
nand_do_read_ops(struct nand_chip * chip,loff_t from,struct mtd_oob_ops * ops)3549 static int nand_do_read_ops(struct nand_chip *chip, loff_t from,
3550 struct mtd_oob_ops *ops)
3551 {
3552 int chipnr, page, realpage, col, bytes, aligned, oob_required;
3553 struct mtd_info *mtd = nand_to_mtd(chip);
3554 int ret = 0;
3555 uint32_t readlen = ops->len;
3556 uint32_t oobreadlen = ops->ooblen;
3557 uint32_t max_oobsize = mtd_oobavail(mtd, ops);
3558
3559 uint8_t *bufpoi, *oob, *buf;
3560 int use_bounce_buf;
3561 unsigned int max_bitflips = 0;
3562 int retry_mode = 0;
3563 bool ecc_fail = false;
3564
3565 /* Check if the region is secured */
3566 if (nand_region_is_secured(chip, from, readlen))
3567 return -EIO;
3568
3569 chipnr = (int)(from >> chip->chip_shift);
3570 nand_select_target(chip, chipnr);
3571
3572 realpage = (int)(from >> chip->page_shift);
3573 page = realpage & chip->pagemask;
3574
3575 col = (int)(from & (mtd->writesize - 1));
3576
3577 buf = ops->datbuf;
3578 oob = ops->oobbuf;
3579 oob_required = oob ? 1 : 0;
3580
3581 if (likely(ops->mode != MTD_OPS_RAW))
3582 rawnand_enable_cont_reads(chip, page, readlen, col);
3583
3584 while (1) {
3585 struct mtd_ecc_stats ecc_stats = mtd->ecc_stats;
3586
3587 bytes = min(mtd->writesize - col, readlen);
3588 aligned = (bytes == mtd->writesize);
3589
3590 if (!aligned)
3591 use_bounce_buf = 1;
3592 else if (chip->options & NAND_USES_DMA)
3593 use_bounce_buf = !virt_addr_valid(buf) ||
3594 !IS_ALIGNED((unsigned long)buf,
3595 chip->buf_align);
3596 else
3597 use_bounce_buf = 0;
3598
3599 /* Is the current page in the buffer? */
3600 if (realpage != chip->pagecache.page || oob) {
3601 bufpoi = use_bounce_buf ? chip->data_buf : buf;
3602
3603 if (use_bounce_buf && aligned)
3604 pr_debug("%s: using read bounce buffer for buf@%p\n",
3605 __func__, buf);
3606
3607 read_retry:
3608 /*
3609 * Now read the page into the buffer. Absent an error,
3610 * the read methods return max bitflips per ecc step.
3611 */
3612 if (unlikely(ops->mode == MTD_OPS_RAW))
3613 ret = chip->ecc.read_page_raw(chip, bufpoi,
3614 oob_required,
3615 page);
3616 else if (!aligned && NAND_HAS_SUBPAGE_READ(chip) &&
3617 !oob)
3618 ret = chip->ecc.read_subpage(chip, col, bytes,
3619 bufpoi, page);
3620 else
3621 ret = chip->ecc.read_page(chip, bufpoi,
3622 oob_required, page);
3623 if (ret < 0) {
3624 if (use_bounce_buf)
3625 /* Invalidate page cache */
3626 chip->pagecache.page = -1;
3627 break;
3628 }
3629
3630 /*
3631 * Copy back the data in the initial buffer when reading
3632 * partial pages or when a bounce buffer is required.
3633 */
3634 if (use_bounce_buf) {
3635 if (!NAND_HAS_SUBPAGE_READ(chip) && !oob &&
3636 !(mtd->ecc_stats.failed - ecc_stats.failed) &&
3637 (ops->mode != MTD_OPS_RAW)) {
3638 chip->pagecache.page = realpage;
3639 chip->pagecache.bitflips = ret;
3640 } else {
3641 /* Invalidate page cache */
3642 chip->pagecache.page = -1;
3643 }
3644 memcpy(buf, bufpoi + col, bytes);
3645 }
3646
3647 if (unlikely(oob)) {
3648 int toread = min(oobreadlen, max_oobsize);
3649
3650 if (toread) {
3651 oob = nand_transfer_oob(chip, oob, ops,
3652 toread);
3653 oobreadlen -= toread;
3654 }
3655 }
3656
3657 nand_wait_readrdy(chip);
3658
3659 if (mtd->ecc_stats.failed - ecc_stats.failed) {
3660 if (retry_mode + 1 < chip->read_retries) {
3661 retry_mode++;
3662 ret = nand_setup_read_retry(chip,
3663 retry_mode);
3664 if (ret < 0)
3665 break;
3666
3667 /* Reset ecc_stats; retry */
3668 mtd->ecc_stats = ecc_stats;
3669 goto read_retry;
3670 } else {
3671 /* No more retry modes; real failure */
3672 ecc_fail = true;
3673 }
3674 }
3675
3676 buf += bytes;
3677 max_bitflips = max_t(unsigned int, max_bitflips, ret);
3678 } else {
3679 memcpy(buf, chip->data_buf + col, bytes);
3680 buf += bytes;
3681 max_bitflips = max_t(unsigned int, max_bitflips,
3682 chip->pagecache.bitflips);
3683
3684 rawnand_cont_read_skip_first_page(chip, page);
3685 }
3686
3687 readlen -= bytes;
3688
3689 /* Reset to retry mode 0 */
3690 if (retry_mode) {
3691 ret = nand_setup_read_retry(chip, 0);
3692 if (ret < 0)
3693 break;
3694 retry_mode = 0;
3695 }
3696
3697 if (!readlen)
3698 break;
3699
3700 /* For subsequent reads align to page boundary */
3701 col = 0;
3702 /* Increment page address */
3703 realpage++;
3704
3705 page = realpage & chip->pagemask;
3706 /* Check, if we cross a chip boundary */
3707 if (!page) {
3708 chipnr++;
3709 nand_deselect_target(chip);
3710 nand_select_target(chip, chipnr);
3711 }
3712 }
3713 nand_deselect_target(chip);
3714
3715 ops->retlen = ops->len - (size_t) readlen;
3716 if (oob)
3717 ops->oobretlen = ops->ooblen - oobreadlen;
3718
3719 if (ret < 0)
3720 return ret;
3721
3722 if (ecc_fail)
3723 return -EBADMSG;
3724
3725 return max_bitflips;
3726 }
3727
3728 /**
3729 * nand_read_oob_std - [REPLACEABLE] the most common OOB data read function
3730 * @chip: nand chip info structure
3731 * @page: page number to read
3732 */
nand_read_oob_std(struct nand_chip * chip,int page)3733 int nand_read_oob_std(struct nand_chip *chip, int page)
3734 {
3735 struct mtd_info *mtd = nand_to_mtd(chip);
3736
3737 return nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize);
3738 }
3739 EXPORT_SYMBOL(nand_read_oob_std);
3740
3741 /**
3742 * nand_read_oob_syndrome - [REPLACEABLE] OOB data read function for HW ECC
3743 * with syndromes
3744 * @chip: nand chip info structure
3745 * @page: page number to read
3746 */
nand_read_oob_syndrome(struct nand_chip * chip,int page)3747 static int nand_read_oob_syndrome(struct nand_chip *chip, int page)
3748 {
3749 struct mtd_info *mtd = nand_to_mtd(chip);
3750 int length = mtd->oobsize;
3751 int chunk = chip->ecc.bytes + chip->ecc.prepad + chip->ecc.postpad;
3752 int eccsize = chip->ecc.size;
3753 uint8_t *bufpoi = chip->oob_poi;
3754 int i, toread, sndrnd = 0, pos, ret;
3755
3756 ret = nand_read_page_op(chip, page, chip->ecc.size, NULL, 0);
3757 if (ret)
3758 return ret;
3759
3760 for (i = 0; i < chip->ecc.steps; i++) {
3761 if (sndrnd) {
3762 int ret;
3763
3764 pos = eccsize + i * (eccsize + chunk);
3765 if (mtd->writesize > 512)
3766 ret = nand_change_read_column_op(chip, pos,
3767 NULL, 0,
3768 false);
3769 else
3770 ret = nand_read_page_op(chip, page, pos, NULL,
3771 0);
3772
3773 if (ret)
3774 return ret;
3775 } else
3776 sndrnd = 1;
3777 toread = min_t(int, length, chunk);
3778
3779 ret = nand_read_data_op(chip, bufpoi, toread, false, false);
3780 if (ret)
3781 return ret;
3782
3783 bufpoi += toread;
3784 length -= toread;
3785 }
3786 if (length > 0) {
3787 ret = nand_read_data_op(chip, bufpoi, length, false, false);
3788 if (ret)
3789 return ret;
3790 }
3791
3792 return 0;
3793 }
3794
3795 /**
3796 * nand_write_oob_std - [REPLACEABLE] the most common OOB data write function
3797 * @chip: nand chip info structure
3798 * @page: page number to write
3799 */
nand_write_oob_std(struct nand_chip * chip,int page)3800 int nand_write_oob_std(struct nand_chip *chip, int page)
3801 {
3802 struct mtd_info *mtd = nand_to_mtd(chip);
3803
3804 return nand_prog_page_op(chip, page, mtd->writesize, chip->oob_poi,
3805 mtd->oobsize);
3806 }
3807 EXPORT_SYMBOL(nand_write_oob_std);
3808
3809 /**
3810 * nand_write_oob_syndrome - [REPLACEABLE] OOB data write function for HW ECC
3811 * with syndrome - only for large page flash
3812 * @chip: nand chip info structure
3813 * @page: page number to write
3814 */
nand_write_oob_syndrome(struct nand_chip * chip,int page)3815 static int nand_write_oob_syndrome(struct nand_chip *chip, int page)
3816 {
3817 struct mtd_info *mtd = nand_to_mtd(chip);
3818 int chunk = chip->ecc.bytes + chip->ecc.prepad + chip->ecc.postpad;
3819 int eccsize = chip->ecc.size, length = mtd->oobsize;
3820 int ret, i, len, pos, sndcmd = 0, steps = chip->ecc.steps;
3821 const uint8_t *bufpoi = chip->oob_poi;
3822
3823 /*
3824 * data-ecc-data-ecc ... ecc-oob
3825 * or
3826 * data-pad-ecc-pad-data-pad .... ecc-pad-oob
3827 */
3828 if (!chip->ecc.prepad && !chip->ecc.postpad) {
3829 pos = steps * (eccsize + chunk);
3830 steps = 0;
3831 } else
3832 pos = eccsize;
3833
3834 ret = nand_prog_page_begin_op(chip, page, pos, NULL, 0);
3835 if (ret)
3836 return ret;
3837
3838 for (i = 0; i < steps; i++) {
3839 if (sndcmd) {
3840 if (mtd->writesize <= 512) {
3841 uint32_t fill = 0xFFFFFFFF;
3842
3843 len = eccsize;
3844 while (len > 0) {
3845 int num = min_t(int, len, 4);
3846
3847 ret = nand_write_data_op(chip, &fill,
3848 num, false);
3849 if (ret)
3850 return ret;
3851
3852 len -= num;
3853 }
3854 } else {
3855 pos = eccsize + i * (eccsize + chunk);
3856 ret = nand_change_write_column_op(chip, pos,
3857 NULL, 0,
3858 false);
3859 if (ret)
3860 return ret;
3861 }
3862 } else
3863 sndcmd = 1;
3864 len = min_t(int, length, chunk);
3865
3866 ret = nand_write_data_op(chip, bufpoi, len, false);
3867 if (ret)
3868 return ret;
3869
3870 bufpoi += len;
3871 length -= len;
3872 }
3873 if (length > 0) {
3874 ret = nand_write_data_op(chip, bufpoi, length, false);
3875 if (ret)
3876 return ret;
3877 }
3878
3879 return nand_prog_page_end_op(chip);
3880 }
3881
3882 /**
3883 * nand_do_read_oob - [INTERN] NAND read out-of-band
3884 * @chip: NAND chip object
3885 * @from: offset to read from
3886 * @ops: oob operations description structure
3887 *
3888 * NAND read out-of-band data from the spare area.
3889 */
nand_do_read_oob(struct nand_chip * chip,loff_t from,struct mtd_oob_ops * ops)3890 static int nand_do_read_oob(struct nand_chip *chip, loff_t from,
3891 struct mtd_oob_ops *ops)
3892 {
3893 struct mtd_info *mtd = nand_to_mtd(chip);
3894 unsigned int max_bitflips = 0;
3895 int page, realpage, chipnr;
3896 struct mtd_ecc_stats stats;
3897 int readlen = ops->ooblen;
3898 int len;
3899 uint8_t *buf = ops->oobbuf;
3900 int ret = 0;
3901
3902 pr_debug("%s: from = 0x%08Lx, len = %i\n",
3903 __func__, (unsigned long long)from, readlen);
3904
3905 /* Check if the region is secured */
3906 if (nand_region_is_secured(chip, from, readlen))
3907 return -EIO;
3908
3909 stats = mtd->ecc_stats;
3910
3911 len = mtd_oobavail(mtd, ops);
3912
3913 chipnr = (int)(from >> chip->chip_shift);
3914 nand_select_target(chip, chipnr);
3915
3916 /* Shift to get page */
3917 realpage = (int)(from >> chip->page_shift);
3918 page = realpage & chip->pagemask;
3919
3920 while (1) {
3921 if (ops->mode == MTD_OPS_RAW)
3922 ret = chip->ecc.read_oob_raw(chip, page);
3923 else
3924 ret = chip->ecc.read_oob(chip, page);
3925
3926 if (ret < 0)
3927 break;
3928
3929 len = min(len, readlen);
3930 buf = nand_transfer_oob(chip, buf, ops, len);
3931
3932 nand_wait_readrdy(chip);
3933
3934 max_bitflips = max_t(unsigned int, max_bitflips, ret);
3935
3936 readlen -= len;
3937 if (!readlen)
3938 break;
3939
3940 /* Increment page address */
3941 realpage++;
3942
3943 page = realpage & chip->pagemask;
3944 /* Check, if we cross a chip boundary */
3945 if (!page) {
3946 chipnr++;
3947 nand_deselect_target(chip);
3948 nand_select_target(chip, chipnr);
3949 }
3950 }
3951 nand_deselect_target(chip);
3952
3953 ops->oobretlen = ops->ooblen - readlen;
3954
3955 if (ret < 0)
3956 return ret;
3957
3958 if (mtd->ecc_stats.failed - stats.failed)
3959 return -EBADMSG;
3960
3961 return max_bitflips;
3962 }
3963
3964 /**
3965 * nand_read_oob - [MTD Interface] NAND read data and/or out-of-band
3966 * @mtd: MTD device structure
3967 * @from: offset to read from
3968 * @ops: oob operation description structure
3969 *
3970 * NAND read data and/or out-of-band data.
3971 */
nand_read_oob(struct mtd_info * mtd,loff_t from,struct mtd_oob_ops * ops)3972 static int nand_read_oob(struct mtd_info *mtd, loff_t from,
3973 struct mtd_oob_ops *ops)
3974 {
3975 struct nand_chip *chip = mtd_to_nand(mtd);
3976 struct mtd_ecc_stats old_stats;
3977 int ret;
3978
3979 ops->retlen = 0;
3980
3981 if (ops->mode != MTD_OPS_PLACE_OOB &&
3982 ops->mode != MTD_OPS_AUTO_OOB &&
3983 ops->mode != MTD_OPS_RAW)
3984 return -ENOTSUPP;
3985
3986 nand_get_device(chip);
3987
3988 old_stats = mtd->ecc_stats;
3989
3990 if (!ops->datbuf)
3991 ret = nand_do_read_oob(chip, from, ops);
3992 else
3993 ret = nand_do_read_ops(chip, from, ops);
3994
3995 if (ops->stats) {
3996 ops->stats->uncorrectable_errors +=
3997 mtd->ecc_stats.failed - old_stats.failed;
3998 ops->stats->corrected_bitflips +=
3999 mtd->ecc_stats.corrected - old_stats.corrected;
4000 }
4001
4002 nand_release_device(chip);
4003 return ret;
4004 }
4005
4006 /**
4007 * nand_write_page_raw_notsupp - dummy raw page write function
4008 * @chip: nand chip info structure
4009 * @buf: data buffer
4010 * @oob_required: must write chip->oob_poi to OOB
4011 * @page: page number to write
4012 *
4013 * Returns -ENOTSUPP unconditionally.
4014 */
nand_write_page_raw_notsupp(struct nand_chip * chip,const u8 * buf,int oob_required,int page)4015 int nand_write_page_raw_notsupp(struct nand_chip *chip, const u8 *buf,
4016 int oob_required, int page)
4017 {
4018 return -ENOTSUPP;
4019 }
4020
4021 /**
4022 * nand_write_page_raw - [INTERN] raw page write function
4023 * @chip: nand chip info structure
4024 * @buf: data buffer
4025 * @oob_required: must write chip->oob_poi to OOB
4026 * @page: page number to write
4027 *
4028 * Not for syndrome calculating ECC controllers, which use a special oob layout.
4029 */
nand_write_page_raw(struct nand_chip * chip,const uint8_t * buf,int oob_required,int page)4030 int nand_write_page_raw(struct nand_chip *chip, const uint8_t *buf,
4031 int oob_required, int page)
4032 {
4033 struct mtd_info *mtd = nand_to_mtd(chip);
4034 int ret;
4035
4036 ret = nand_prog_page_begin_op(chip, page, 0, buf, mtd->writesize);
4037 if (ret)
4038 return ret;
4039
4040 if (oob_required) {
4041 ret = nand_write_data_op(chip, chip->oob_poi, mtd->oobsize,
4042 false);
4043 if (ret)
4044 return ret;
4045 }
4046
4047 return nand_prog_page_end_op(chip);
4048 }
4049 EXPORT_SYMBOL(nand_write_page_raw);
4050
4051 /**
4052 * nand_monolithic_write_page_raw - Monolithic page write in raw mode
4053 * @chip: NAND chip info structure
4054 * @buf: data buffer to write
4055 * @oob_required: must write chip->oob_poi to OOB
4056 * @page: page number to write
4057 *
4058 * This is a raw page write, ie. without any error detection/correction.
4059 * Monolithic means we are requesting all the relevant data (main plus
4060 * eventually OOB) to be sent over the bus and effectively programmed
4061 * into the NAND chip arrays in a single operation. This is an
4062 * alternative to nand_write_page_raw(), which first sends the main
4063 * data, then eventually send the OOB data by latching more data
4064 * cycles on the NAND bus, and finally sends the program command to
4065 * synchronyze the NAND chip cache.
4066 */
nand_monolithic_write_page_raw(struct nand_chip * chip,const u8 * buf,int oob_required,int page)4067 int nand_monolithic_write_page_raw(struct nand_chip *chip, const u8 *buf,
4068 int oob_required, int page)
4069 {
4070 struct mtd_info *mtd = nand_to_mtd(chip);
4071 unsigned int size = mtd->writesize;
4072 u8 *write_buf = (u8 *)buf;
4073
4074 if (oob_required) {
4075 size += mtd->oobsize;
4076
4077 if (buf != chip->data_buf) {
4078 write_buf = nand_get_data_buf(chip);
4079 memcpy(write_buf, buf, mtd->writesize);
4080 }
4081 }
4082
4083 return nand_prog_page_op(chip, page, 0, write_buf, size);
4084 }
4085 EXPORT_SYMBOL(nand_monolithic_write_page_raw);
4086
4087 /**
4088 * nand_write_page_raw_syndrome - [INTERN] raw page write function
4089 * @chip: nand chip info structure
4090 * @buf: data buffer
4091 * @oob_required: must write chip->oob_poi to OOB
4092 * @page: page number to write
4093 *
4094 * We need a special oob layout and handling even when ECC isn't checked.
4095 */
nand_write_page_raw_syndrome(struct nand_chip * chip,const uint8_t * buf,int oob_required,int page)4096 static int nand_write_page_raw_syndrome(struct nand_chip *chip,
4097 const uint8_t *buf, int oob_required,
4098 int page)
4099 {
4100 struct mtd_info *mtd = nand_to_mtd(chip);
4101 int eccsize = chip->ecc.size;
4102 int eccbytes = chip->ecc.bytes;
4103 uint8_t *oob = chip->oob_poi;
4104 int steps, size, ret;
4105
4106 ret = nand_prog_page_begin_op(chip, page, 0, NULL, 0);
4107 if (ret)
4108 return ret;
4109
4110 for (steps = chip->ecc.steps; steps > 0; steps--) {
4111 ret = nand_write_data_op(chip, buf, eccsize, false);
4112 if (ret)
4113 return ret;
4114
4115 buf += eccsize;
4116
4117 if (chip->ecc.prepad) {
4118 ret = nand_write_data_op(chip, oob, chip->ecc.prepad,
4119 false);
4120 if (ret)
4121 return ret;
4122
4123 oob += chip->ecc.prepad;
4124 }
4125
4126 ret = nand_write_data_op(chip, oob, eccbytes, false);
4127 if (ret)
4128 return ret;
4129
4130 oob += eccbytes;
4131
4132 if (chip->ecc.postpad) {
4133 ret = nand_write_data_op(chip, oob, chip->ecc.postpad,
4134 false);
4135 if (ret)
4136 return ret;
4137
4138 oob += chip->ecc.postpad;
4139 }
4140 }
4141
4142 size = mtd->oobsize - (oob - chip->oob_poi);
4143 if (size) {
4144 ret = nand_write_data_op(chip, oob, size, false);
4145 if (ret)
4146 return ret;
4147 }
4148
4149 return nand_prog_page_end_op(chip);
4150 }
4151 /**
4152 * nand_write_page_swecc - [REPLACEABLE] software ECC based page write function
4153 * @chip: nand chip info structure
4154 * @buf: data buffer
4155 * @oob_required: must write chip->oob_poi to OOB
4156 * @page: page number to write
4157 */
nand_write_page_swecc(struct nand_chip * chip,const uint8_t * buf,int oob_required,int page)4158 static int nand_write_page_swecc(struct nand_chip *chip, const uint8_t *buf,
4159 int oob_required, int page)
4160 {
4161 struct mtd_info *mtd = nand_to_mtd(chip);
4162 int i, eccsize = chip->ecc.size, ret;
4163 int eccbytes = chip->ecc.bytes;
4164 int eccsteps = chip->ecc.steps;
4165 uint8_t *ecc_calc = chip->ecc.calc_buf;
4166 const uint8_t *p = buf;
4167
4168 /* Software ECC calculation */
4169 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize)
4170 chip->ecc.calculate(chip, p, &ecc_calc[i]);
4171
4172 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
4173 chip->ecc.total);
4174 if (ret)
4175 return ret;
4176
4177 return chip->ecc.write_page_raw(chip, buf, 1, page);
4178 }
4179
4180 /**
4181 * nand_write_page_hwecc - [REPLACEABLE] hardware ECC based page write function
4182 * @chip: nand chip info structure
4183 * @buf: data buffer
4184 * @oob_required: must write chip->oob_poi to OOB
4185 * @page: page number to write
4186 */
nand_write_page_hwecc(struct nand_chip * chip,const uint8_t * buf,int oob_required,int page)4187 static int nand_write_page_hwecc(struct nand_chip *chip, const uint8_t *buf,
4188 int oob_required, int page)
4189 {
4190 struct mtd_info *mtd = nand_to_mtd(chip);
4191 int i, eccsize = chip->ecc.size, ret;
4192 int eccbytes = chip->ecc.bytes;
4193 int eccsteps = chip->ecc.steps;
4194 uint8_t *ecc_calc = chip->ecc.calc_buf;
4195 const uint8_t *p = buf;
4196
4197 ret = nand_prog_page_begin_op(chip, page, 0, NULL, 0);
4198 if (ret)
4199 return ret;
4200
4201 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
4202 chip->ecc.hwctl(chip, NAND_ECC_WRITE);
4203
4204 ret = nand_write_data_op(chip, p, eccsize, false);
4205 if (ret)
4206 return ret;
4207
4208 chip->ecc.calculate(chip, p, &ecc_calc[i]);
4209 }
4210
4211 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
4212 chip->ecc.total);
4213 if (ret)
4214 return ret;
4215
4216 ret = nand_write_data_op(chip, chip->oob_poi, mtd->oobsize, false);
4217 if (ret)
4218 return ret;
4219
4220 return nand_prog_page_end_op(chip);
4221 }
4222
4223
4224 /**
4225 * nand_write_subpage_hwecc - [REPLACEABLE] hardware ECC based subpage write
4226 * @chip: nand chip info structure
4227 * @offset: column address of subpage within the page
4228 * @data_len: data length
4229 * @buf: data buffer
4230 * @oob_required: must write chip->oob_poi to OOB
4231 * @page: page number to write
4232 */
nand_write_subpage_hwecc(struct nand_chip * chip,uint32_t offset,uint32_t data_len,const uint8_t * buf,int oob_required,int page)4233 static int nand_write_subpage_hwecc(struct nand_chip *chip, uint32_t offset,
4234 uint32_t data_len, const uint8_t *buf,
4235 int oob_required, int page)
4236 {
4237 struct mtd_info *mtd = nand_to_mtd(chip);
4238 uint8_t *oob_buf = chip->oob_poi;
4239 uint8_t *ecc_calc = chip->ecc.calc_buf;
4240 int ecc_size = chip->ecc.size;
4241 int ecc_bytes = chip->ecc.bytes;
4242 int ecc_steps = chip->ecc.steps;
4243 uint32_t start_step = offset / ecc_size;
4244 uint32_t end_step = (offset + data_len - 1) / ecc_size;
4245 int oob_bytes = mtd->oobsize / ecc_steps;
4246 int step, ret;
4247
4248 ret = nand_prog_page_begin_op(chip, page, 0, NULL, 0);
4249 if (ret)
4250 return ret;
4251
4252 for (step = 0; step < ecc_steps; step++) {
4253 /* configure controller for WRITE access */
4254 chip->ecc.hwctl(chip, NAND_ECC_WRITE);
4255
4256 /* write data (untouched subpages already masked by 0xFF) */
4257 ret = nand_write_data_op(chip, buf, ecc_size, false);
4258 if (ret)
4259 return ret;
4260
4261 /* mask ECC of un-touched subpages by padding 0xFF */
4262 if ((step < start_step) || (step > end_step))
4263 memset(ecc_calc, 0xff, ecc_bytes);
4264 else
4265 chip->ecc.calculate(chip, buf, ecc_calc);
4266
4267 /* mask OOB of un-touched subpages by padding 0xFF */
4268 /* if oob_required, preserve OOB metadata of written subpage */
4269 if (!oob_required || (step < start_step) || (step > end_step))
4270 memset(oob_buf, 0xff, oob_bytes);
4271
4272 buf += ecc_size;
4273 ecc_calc += ecc_bytes;
4274 oob_buf += oob_bytes;
4275 }
4276
4277 /* copy calculated ECC for whole page to chip->buffer->oob */
4278 /* this include masked-value(0xFF) for unwritten subpages */
4279 ecc_calc = chip->ecc.calc_buf;
4280 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
4281 chip->ecc.total);
4282 if (ret)
4283 return ret;
4284
4285 /* write OOB buffer to NAND device */
4286 ret = nand_write_data_op(chip, chip->oob_poi, mtd->oobsize, false);
4287 if (ret)
4288 return ret;
4289
4290 return nand_prog_page_end_op(chip);
4291 }
4292
4293
4294 /**
4295 * nand_write_page_syndrome - [REPLACEABLE] hardware ECC syndrome based page write
4296 * @chip: nand chip info structure
4297 * @buf: data buffer
4298 * @oob_required: must write chip->oob_poi to OOB
4299 * @page: page number to write
4300 *
4301 * The hw generator calculates the error syndrome automatically. Therefore we
4302 * need a special oob layout and handling.
4303 */
nand_write_page_syndrome(struct nand_chip * chip,const uint8_t * buf,int oob_required,int page)4304 static int nand_write_page_syndrome(struct nand_chip *chip, const uint8_t *buf,
4305 int oob_required, int page)
4306 {
4307 struct mtd_info *mtd = nand_to_mtd(chip);
4308 int i, eccsize = chip->ecc.size;
4309 int eccbytes = chip->ecc.bytes;
4310 int eccsteps = chip->ecc.steps;
4311 const uint8_t *p = buf;
4312 uint8_t *oob = chip->oob_poi;
4313 int ret;
4314
4315 ret = nand_prog_page_begin_op(chip, page, 0, NULL, 0);
4316 if (ret)
4317 return ret;
4318
4319 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
4320 chip->ecc.hwctl(chip, NAND_ECC_WRITE);
4321
4322 ret = nand_write_data_op(chip, p, eccsize, false);
4323 if (ret)
4324 return ret;
4325
4326 if (chip->ecc.prepad) {
4327 ret = nand_write_data_op(chip, oob, chip->ecc.prepad,
4328 false);
4329 if (ret)
4330 return ret;
4331
4332 oob += chip->ecc.prepad;
4333 }
4334
4335 chip->ecc.calculate(chip, p, oob);
4336
4337 ret = nand_write_data_op(chip, oob, eccbytes, false);
4338 if (ret)
4339 return ret;
4340
4341 oob += eccbytes;
4342
4343 if (chip->ecc.postpad) {
4344 ret = nand_write_data_op(chip, oob, chip->ecc.postpad,
4345 false);
4346 if (ret)
4347 return ret;
4348
4349 oob += chip->ecc.postpad;
4350 }
4351 }
4352
4353 /* Calculate remaining oob bytes */
4354 i = mtd->oobsize - (oob - chip->oob_poi);
4355 if (i) {
4356 ret = nand_write_data_op(chip, oob, i, false);
4357 if (ret)
4358 return ret;
4359 }
4360
4361 return nand_prog_page_end_op(chip);
4362 }
4363
4364 /**
4365 * nand_write_page - write one page
4366 * @chip: NAND chip descriptor
4367 * @offset: address offset within the page
4368 * @data_len: length of actual data to be written
4369 * @buf: the data to write
4370 * @oob_required: must write chip->oob_poi to OOB
4371 * @page: page number to write
4372 * @raw: use _raw version of write_page
4373 */
nand_write_page(struct nand_chip * chip,uint32_t offset,int data_len,const uint8_t * buf,int oob_required,int page,int raw)4374 static int nand_write_page(struct nand_chip *chip, uint32_t offset,
4375 int data_len, const uint8_t *buf, int oob_required,
4376 int page, int raw)
4377 {
4378 struct mtd_info *mtd = nand_to_mtd(chip);
4379 int status, subpage;
4380
4381 if (!(chip->options & NAND_NO_SUBPAGE_WRITE) &&
4382 chip->ecc.write_subpage)
4383 subpage = offset || (data_len < mtd->writesize);
4384 else
4385 subpage = 0;
4386
4387 if (unlikely(raw))
4388 status = chip->ecc.write_page_raw(chip, buf, oob_required,
4389 page);
4390 else if (subpage)
4391 status = chip->ecc.write_subpage(chip, offset, data_len, buf,
4392 oob_required, page);
4393 else
4394 status = chip->ecc.write_page(chip, buf, oob_required, page);
4395
4396 if (status < 0)
4397 return status;
4398
4399 return 0;
4400 }
4401
4402 #define NOTALIGNED(x) ((x & (chip->subpagesize - 1)) != 0)
4403
4404 /**
4405 * nand_do_write_ops - [INTERN] NAND write with ECC
4406 * @chip: NAND chip object
4407 * @to: offset to write to
4408 * @ops: oob operations description structure
4409 *
4410 * NAND write with ECC.
4411 */
nand_do_write_ops(struct nand_chip * chip,loff_t to,struct mtd_oob_ops * ops)4412 static int nand_do_write_ops(struct nand_chip *chip, loff_t to,
4413 struct mtd_oob_ops *ops)
4414 {
4415 struct mtd_info *mtd = nand_to_mtd(chip);
4416 int chipnr, realpage, page, column;
4417 uint32_t writelen = ops->len;
4418
4419 uint32_t oobwritelen = ops->ooblen;
4420 uint32_t oobmaxlen = mtd_oobavail(mtd, ops);
4421
4422 uint8_t *oob = ops->oobbuf;
4423 uint8_t *buf = ops->datbuf;
4424 int ret;
4425 int oob_required = oob ? 1 : 0;
4426
4427 ops->retlen = 0;
4428 if (!writelen)
4429 return 0;
4430
4431 /* Reject writes, which are not page aligned */
4432 if (NOTALIGNED(to) || NOTALIGNED(ops->len)) {
4433 pr_notice("%s: attempt to write non page aligned data\n",
4434 __func__);
4435 return -EINVAL;
4436 }
4437
4438 /* Check if the region is secured */
4439 if (nand_region_is_secured(chip, to, writelen))
4440 return -EIO;
4441
4442 column = to & (mtd->writesize - 1);
4443
4444 chipnr = (int)(to >> chip->chip_shift);
4445 nand_select_target(chip, chipnr);
4446
4447 /* Check, if it is write protected */
4448 if (nand_check_wp(chip)) {
4449 ret = -EIO;
4450 goto err_out;
4451 }
4452
4453 realpage = (int)(to >> chip->page_shift);
4454 page = realpage & chip->pagemask;
4455
4456 /* Invalidate the page cache, when we write to the cached page */
4457 if (to <= ((loff_t)chip->pagecache.page << chip->page_shift) &&
4458 ((loff_t)chip->pagecache.page << chip->page_shift) < (to + ops->len))
4459 chip->pagecache.page = -1;
4460
4461 /* Don't allow multipage oob writes with offset */
4462 if (oob && ops->ooboffs && (ops->ooboffs + ops->ooblen > oobmaxlen)) {
4463 ret = -EINVAL;
4464 goto err_out;
4465 }
4466
4467 while (1) {
4468 int bytes = mtd->writesize;
4469 uint8_t *wbuf = buf;
4470 int use_bounce_buf;
4471 int part_pagewr = (column || writelen < mtd->writesize);
4472
4473 if (part_pagewr)
4474 use_bounce_buf = 1;
4475 else if (chip->options & NAND_USES_DMA)
4476 use_bounce_buf = !virt_addr_valid(buf) ||
4477 !IS_ALIGNED((unsigned long)buf,
4478 chip->buf_align);
4479 else
4480 use_bounce_buf = 0;
4481
4482 /*
4483 * Copy the data from the initial buffer when doing partial page
4484 * writes or when a bounce buffer is required.
4485 */
4486 if (use_bounce_buf) {
4487 pr_debug("%s: using write bounce buffer for buf@%p\n",
4488 __func__, buf);
4489 if (part_pagewr)
4490 bytes = min_t(int, bytes - column, writelen);
4491 wbuf = nand_get_data_buf(chip);
4492 memset(wbuf, 0xff, mtd->writesize);
4493 memcpy(&wbuf[column], buf, bytes);
4494 }
4495
4496 if (unlikely(oob)) {
4497 size_t len = min(oobwritelen, oobmaxlen);
4498 oob = nand_fill_oob(chip, oob, len, ops);
4499 oobwritelen -= len;
4500 } else {
4501 /* We still need to erase leftover OOB data */
4502 memset(chip->oob_poi, 0xff, mtd->oobsize);
4503 }
4504
4505 ret = nand_write_page(chip, column, bytes, wbuf,
4506 oob_required, page,
4507 (ops->mode == MTD_OPS_RAW));
4508 if (ret)
4509 break;
4510
4511 writelen -= bytes;
4512 if (!writelen)
4513 break;
4514
4515 column = 0;
4516 buf += bytes;
4517 realpage++;
4518
4519 page = realpage & chip->pagemask;
4520 /* Check, if we cross a chip boundary */
4521 if (!page) {
4522 chipnr++;
4523 nand_deselect_target(chip);
4524 nand_select_target(chip, chipnr);
4525 }
4526 }
4527
4528 ops->retlen = ops->len - writelen;
4529 if (unlikely(oob))
4530 ops->oobretlen = ops->ooblen;
4531
4532 err_out:
4533 nand_deselect_target(chip);
4534 return ret;
4535 }
4536
4537 /**
4538 * panic_nand_write - [MTD Interface] NAND write with ECC
4539 * @mtd: MTD device structure
4540 * @to: offset to write to
4541 * @len: number of bytes to write
4542 * @retlen: pointer to variable to store the number of written bytes
4543 * @buf: the data to write
4544 *
4545 * NAND write with ECC. Used when performing writes in interrupt context, this
4546 * may for example be called by mtdoops when writing an oops while in panic.
4547 */
panic_nand_write(struct mtd_info * mtd,loff_t to,size_t len,size_t * retlen,const uint8_t * buf)4548 static int panic_nand_write(struct mtd_info *mtd, loff_t to, size_t len,
4549 size_t *retlen, const uint8_t *buf)
4550 {
4551 struct nand_chip *chip = mtd_to_nand(mtd);
4552 int chipnr = (int)(to >> chip->chip_shift);
4553 struct mtd_oob_ops ops;
4554 int ret;
4555
4556 nand_select_target(chip, chipnr);
4557
4558 /* Wait for the device to get ready */
4559 panic_nand_wait(chip, 400);
4560
4561 memset(&ops, 0, sizeof(ops));
4562 ops.len = len;
4563 ops.datbuf = (uint8_t *)buf;
4564 ops.mode = MTD_OPS_PLACE_OOB;
4565
4566 ret = nand_do_write_ops(chip, to, &ops);
4567
4568 *retlen = ops.retlen;
4569 return ret;
4570 }
4571
4572 /**
4573 * nand_write_oob - [MTD Interface] NAND write data and/or out-of-band
4574 * @mtd: MTD device structure
4575 * @to: offset to write to
4576 * @ops: oob operation description structure
4577 */
nand_write_oob(struct mtd_info * mtd,loff_t to,struct mtd_oob_ops * ops)4578 static int nand_write_oob(struct mtd_info *mtd, loff_t to,
4579 struct mtd_oob_ops *ops)
4580 {
4581 struct nand_chip *chip = mtd_to_nand(mtd);
4582 int ret = 0;
4583
4584 ops->retlen = 0;
4585
4586 nand_get_device(chip);
4587
4588 switch (ops->mode) {
4589 case MTD_OPS_PLACE_OOB:
4590 case MTD_OPS_AUTO_OOB:
4591 case MTD_OPS_RAW:
4592 break;
4593
4594 default:
4595 goto out;
4596 }
4597
4598 if (!ops->datbuf)
4599 ret = nand_do_write_oob(chip, to, ops);
4600 else
4601 ret = nand_do_write_ops(chip, to, ops);
4602
4603 out:
4604 nand_release_device(chip);
4605 return ret;
4606 }
4607
4608 /**
4609 * nand_erase - [MTD Interface] erase block(s)
4610 * @mtd: MTD device structure
4611 * @instr: erase instruction
4612 *
4613 * Erase one ore more blocks.
4614 */
nand_erase(struct mtd_info * mtd,struct erase_info * instr)4615 static int nand_erase(struct mtd_info *mtd, struct erase_info *instr)
4616 {
4617 return nand_erase_nand(mtd_to_nand(mtd), instr, 0);
4618 }
4619
4620 /**
4621 * nand_erase_nand - [INTERN] erase block(s)
4622 * @chip: NAND chip object
4623 * @instr: erase instruction
4624 * @allowbbt: allow erasing the bbt area
4625 *
4626 * Erase one ore more blocks.
4627 */
nand_erase_nand(struct nand_chip * chip,struct erase_info * instr,int allowbbt)4628 int nand_erase_nand(struct nand_chip *chip, struct erase_info *instr,
4629 int allowbbt)
4630 {
4631 int page, pages_per_block, ret, chipnr;
4632 loff_t len;
4633
4634 pr_debug("%s: start = 0x%012llx, len = %llu\n",
4635 __func__, (unsigned long long)instr->addr,
4636 (unsigned long long)instr->len);
4637
4638 if (check_offs_len(chip, instr->addr, instr->len))
4639 return -EINVAL;
4640
4641 /* Check if the region is secured */
4642 if (nand_region_is_secured(chip, instr->addr, instr->len))
4643 return -EIO;
4644
4645 /* Grab the lock and see if the device is available */
4646 nand_get_device(chip);
4647
4648 /* Shift to get first page */
4649 page = (int)(instr->addr >> chip->page_shift);
4650 chipnr = (int)(instr->addr >> chip->chip_shift);
4651
4652 /* Calculate pages in each block */
4653 pages_per_block = 1 << (chip->phys_erase_shift - chip->page_shift);
4654
4655 /* Select the NAND device */
4656 nand_select_target(chip, chipnr);
4657
4658 /* Check, if it is write protected */
4659 if (nand_check_wp(chip)) {
4660 pr_debug("%s: device is write protected!\n",
4661 __func__);
4662 ret = -EIO;
4663 goto erase_exit;
4664 }
4665
4666 /* Loop through the pages */
4667 len = instr->len;
4668
4669 while (len) {
4670 loff_t ofs = (loff_t)page << chip->page_shift;
4671
4672 /* Check if we have a bad block, we do not erase bad blocks! */
4673 if (nand_block_checkbad(chip, ((loff_t) page) <<
4674 chip->page_shift, allowbbt)) {
4675 pr_warn("%s: attempt to erase a bad block at 0x%08llx\n",
4676 __func__, (unsigned long long)ofs);
4677 ret = -EIO;
4678 goto erase_exit;
4679 }
4680
4681 /*
4682 * Invalidate the page cache, if we erase the block which
4683 * contains the current cached page.
4684 */
4685 if (page <= chip->pagecache.page && chip->pagecache.page <
4686 (page + pages_per_block))
4687 chip->pagecache.page = -1;
4688
4689 ret = nand_erase_op(chip, (page & chip->pagemask) >>
4690 (chip->phys_erase_shift - chip->page_shift));
4691 if (ret) {
4692 pr_debug("%s: failed erase, page 0x%08x\n",
4693 __func__, page);
4694 instr->fail_addr = ofs;
4695 goto erase_exit;
4696 }
4697
4698 /* Increment page address and decrement length */
4699 len -= (1ULL << chip->phys_erase_shift);
4700 page += pages_per_block;
4701
4702 /* Check, if we cross a chip boundary */
4703 if (len && !(page & chip->pagemask)) {
4704 chipnr++;
4705 nand_deselect_target(chip);
4706 nand_select_target(chip, chipnr);
4707 }
4708 }
4709
4710 ret = 0;
4711 erase_exit:
4712
4713 /* Deselect and wake up anyone waiting on the device */
4714 nand_deselect_target(chip);
4715 nand_release_device(chip);
4716
4717 /* Return more or less happy */
4718 return ret;
4719 }
4720
4721 /**
4722 * nand_sync - [MTD Interface] sync
4723 * @mtd: MTD device structure
4724 *
4725 * Sync is actually a wait for chip ready function.
4726 */
nand_sync(struct mtd_info * mtd)4727 static void nand_sync(struct mtd_info *mtd)
4728 {
4729 struct nand_chip *chip = mtd_to_nand(mtd);
4730
4731 pr_debug("%s: called\n", __func__);
4732
4733 /* Grab the lock and see if the device is available */
4734 nand_get_device(chip);
4735 /* Release it and go back */
4736 nand_release_device(chip);
4737 }
4738
4739 /**
4740 * nand_block_isbad - [MTD Interface] Check if block at offset is bad
4741 * @mtd: MTD device structure
4742 * @offs: offset relative to mtd start
4743 */
nand_block_isbad(struct mtd_info * mtd,loff_t offs)4744 static int nand_block_isbad(struct mtd_info *mtd, loff_t offs)
4745 {
4746 struct nand_chip *chip = mtd_to_nand(mtd);
4747 int chipnr = (int)(offs >> chip->chip_shift);
4748 int ret;
4749
4750 /* Select the NAND device */
4751 nand_get_device(chip);
4752
4753 nand_select_target(chip, chipnr);
4754
4755 ret = nand_block_checkbad(chip, offs, 0);
4756
4757 nand_deselect_target(chip);
4758 nand_release_device(chip);
4759
4760 return ret;
4761 }
4762
4763 /**
4764 * nand_block_markbad - [MTD Interface] Mark block at the given offset as bad
4765 * @mtd: MTD device structure
4766 * @ofs: offset relative to mtd start
4767 */
nand_block_markbad(struct mtd_info * mtd,loff_t ofs)4768 static int nand_block_markbad(struct mtd_info *mtd, loff_t ofs)
4769 {
4770 int ret;
4771
4772 ret = nand_block_isbad(mtd, ofs);
4773 if (ret) {
4774 /* If it was bad already, return success and do nothing */
4775 if (ret > 0)
4776 return 0;
4777 return ret;
4778 }
4779
4780 return nand_block_markbad_lowlevel(mtd_to_nand(mtd), ofs);
4781 }
4782
4783 /**
4784 * nand_suspend - [MTD Interface] Suspend the NAND flash
4785 * @mtd: MTD device structure
4786 *
4787 * Returns 0 for success or negative error code otherwise.
4788 */
nand_suspend(struct mtd_info * mtd)4789 static int nand_suspend(struct mtd_info *mtd)
4790 {
4791 struct nand_chip *chip = mtd_to_nand(mtd);
4792 int ret = 0;
4793
4794 mutex_lock(&chip->lock);
4795 if (chip->ops.suspend)
4796 ret = chip->ops.suspend(chip);
4797 if (!ret)
4798 chip->suspended = 1;
4799 mutex_unlock(&chip->lock);
4800
4801 return ret;
4802 }
4803
4804 /**
4805 * nand_resume - [MTD Interface] Resume the NAND flash
4806 * @mtd: MTD device structure
4807 */
nand_resume(struct mtd_info * mtd)4808 static void nand_resume(struct mtd_info *mtd)
4809 {
4810 struct nand_chip *chip = mtd_to_nand(mtd);
4811
4812 mutex_lock(&chip->lock);
4813 if (chip->suspended) {
4814 if (chip->ops.resume)
4815 chip->ops.resume(chip);
4816 chip->suspended = 0;
4817 } else {
4818 pr_err("%s called for a chip which is not in suspended state\n",
4819 __func__);
4820 }
4821 mutex_unlock(&chip->lock);
4822
4823 wake_up_all(&chip->resume_wq);
4824 }
4825
4826 /**
4827 * nand_shutdown - [MTD Interface] Finish the current NAND operation and
4828 * prevent further operations
4829 * @mtd: MTD device structure
4830 */
nand_shutdown(struct mtd_info * mtd)4831 static void nand_shutdown(struct mtd_info *mtd)
4832 {
4833 nand_suspend(mtd);
4834 }
4835
4836 /**
4837 * nand_lock - [MTD Interface] Lock the NAND flash
4838 * @mtd: MTD device structure
4839 * @ofs: offset byte address
4840 * @len: number of bytes to lock (must be a multiple of block/page size)
4841 */
nand_lock(struct mtd_info * mtd,loff_t ofs,uint64_t len)4842 static int nand_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
4843 {
4844 struct nand_chip *chip = mtd_to_nand(mtd);
4845
4846 if (!chip->ops.lock_area)
4847 return -ENOTSUPP;
4848
4849 return chip->ops.lock_area(chip, ofs, len);
4850 }
4851
4852 /**
4853 * nand_unlock - [MTD Interface] Unlock the NAND flash
4854 * @mtd: MTD device structure
4855 * @ofs: offset byte address
4856 * @len: number of bytes to unlock (must be a multiple of block/page size)
4857 */
nand_unlock(struct mtd_info * mtd,loff_t ofs,uint64_t len)4858 static int nand_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
4859 {
4860 struct nand_chip *chip = mtd_to_nand(mtd);
4861
4862 if (!chip->ops.unlock_area)
4863 return -ENOTSUPP;
4864
4865 return chip->ops.unlock_area(chip, ofs, len);
4866 }
4867
4868 /* Set default functions */
nand_set_defaults(struct nand_chip * chip)4869 static void nand_set_defaults(struct nand_chip *chip)
4870 {
4871 /* If no controller is provided, use the dummy, legacy one. */
4872 if (!chip->controller) {
4873 chip->controller = &chip->legacy.dummy_controller;
4874 nand_controller_init(chip->controller);
4875 }
4876
4877 nand_legacy_set_defaults(chip);
4878
4879 if (!chip->buf_align)
4880 chip->buf_align = 1;
4881 }
4882
4883 /* Sanitize ONFI strings so we can safely print them */
sanitize_string(uint8_t * s,size_t len)4884 void sanitize_string(uint8_t *s, size_t len)
4885 {
4886 ssize_t i;
4887
4888 /* Null terminate */
4889 s[len - 1] = 0;
4890
4891 /* Remove non printable chars */
4892 for (i = 0; i < len - 1; i++) {
4893 if (s[i] < ' ' || s[i] > 127)
4894 s[i] = '?';
4895 }
4896
4897 /* Remove trailing spaces */
4898 strim(s);
4899 }
4900
4901 /*
4902 * nand_id_has_period - Check if an ID string has a given wraparound period
4903 * @id_data: the ID string
4904 * @arrlen: the length of the @id_data array
4905 * @period: the period of repitition
4906 *
4907 * Check if an ID string is repeated within a given sequence of bytes at
4908 * specific repetition interval period (e.g., {0x20,0x01,0x7F,0x20} has a
4909 * period of 3). This is a helper function for nand_id_len(). Returns non-zero
4910 * if the repetition has a period of @period; otherwise, returns zero.
4911 */
nand_id_has_period(u8 * id_data,int arrlen,int period)4912 static int nand_id_has_period(u8 *id_data, int arrlen, int period)
4913 {
4914 int i, j;
4915 for (i = 0; i < period; i++)
4916 for (j = i + period; j < arrlen; j += period)
4917 if (id_data[i] != id_data[j])
4918 return 0;
4919 return 1;
4920 }
4921
4922 /*
4923 * nand_id_len - Get the length of an ID string returned by CMD_READID
4924 * @id_data: the ID string
4925 * @arrlen: the length of the @id_data array
4926
4927 * Returns the length of the ID string, according to known wraparound/trailing
4928 * zero patterns. If no pattern exists, returns the length of the array.
4929 */
nand_id_len(u8 * id_data,int arrlen)4930 static int nand_id_len(u8 *id_data, int arrlen)
4931 {
4932 int last_nonzero, period;
4933
4934 /* Find last non-zero byte */
4935 for (last_nonzero = arrlen - 1; last_nonzero >= 0; last_nonzero--)
4936 if (id_data[last_nonzero])
4937 break;
4938
4939 /* All zeros */
4940 if (last_nonzero < 0)
4941 return 0;
4942
4943 /* Calculate wraparound period */
4944 for (period = 1; period < arrlen; period++)
4945 if (nand_id_has_period(id_data, arrlen, period))
4946 break;
4947
4948 /* There's a repeated pattern */
4949 if (period < arrlen)
4950 return period;
4951
4952 /* There are trailing zeros */
4953 if (last_nonzero < arrlen - 1)
4954 return last_nonzero + 1;
4955
4956 /* No pattern detected */
4957 return arrlen;
4958 }
4959
4960 /* Extract the bits of per cell from the 3rd byte of the extended ID */
nand_get_bits_per_cell(u8 cellinfo)4961 static int nand_get_bits_per_cell(u8 cellinfo)
4962 {
4963 int bits;
4964
4965 bits = cellinfo & NAND_CI_CELLTYPE_MSK;
4966 bits >>= NAND_CI_CELLTYPE_SHIFT;
4967 return bits + 1;
4968 }
4969
4970 /*
4971 * Many new NAND share similar device ID codes, which represent the size of the
4972 * chip. The rest of the parameters must be decoded according to generic or
4973 * manufacturer-specific "extended ID" decoding patterns.
4974 */
nand_decode_ext_id(struct nand_chip * chip)4975 void nand_decode_ext_id(struct nand_chip *chip)
4976 {
4977 struct nand_memory_organization *memorg;
4978 struct mtd_info *mtd = nand_to_mtd(chip);
4979 int extid;
4980 u8 *id_data = chip->id.data;
4981
4982 memorg = nanddev_get_memorg(&chip->base);
4983
4984 /* The 3rd id byte holds MLC / multichip data */
4985 memorg->bits_per_cell = nand_get_bits_per_cell(id_data[2]);
4986 /* The 4th id byte is the important one */
4987 extid = id_data[3];
4988
4989 /* Calc pagesize */
4990 memorg->pagesize = 1024 << (extid & 0x03);
4991 mtd->writesize = memorg->pagesize;
4992 extid >>= 2;
4993 /* Calc oobsize */
4994 memorg->oobsize = (8 << (extid & 0x01)) * (mtd->writesize >> 9);
4995 mtd->oobsize = memorg->oobsize;
4996 extid >>= 2;
4997 /* Calc blocksize. Blocksize is multiples of 64KiB */
4998 memorg->pages_per_eraseblock = ((64 * 1024) << (extid & 0x03)) /
4999 memorg->pagesize;
5000 mtd->erasesize = (64 * 1024) << (extid & 0x03);
5001 extid >>= 2;
5002 /* Get buswidth information */
5003 if (extid & 0x1)
5004 chip->options |= NAND_BUSWIDTH_16;
5005 }
5006 EXPORT_SYMBOL_GPL(nand_decode_ext_id);
5007
5008 /*
5009 * Old devices have chip data hardcoded in the device ID table. nand_decode_id
5010 * decodes a matching ID table entry and assigns the MTD size parameters for
5011 * the chip.
5012 */
nand_decode_id(struct nand_chip * chip,struct nand_flash_dev * type)5013 static void nand_decode_id(struct nand_chip *chip, struct nand_flash_dev *type)
5014 {
5015 struct mtd_info *mtd = nand_to_mtd(chip);
5016 struct nand_memory_organization *memorg;
5017
5018 memorg = nanddev_get_memorg(&chip->base);
5019
5020 memorg->pages_per_eraseblock = type->erasesize / type->pagesize;
5021 mtd->erasesize = type->erasesize;
5022 memorg->pagesize = type->pagesize;
5023 mtd->writesize = memorg->pagesize;
5024 memorg->oobsize = memorg->pagesize / 32;
5025 mtd->oobsize = memorg->oobsize;
5026
5027 /* All legacy ID NAND are small-page, SLC */
5028 memorg->bits_per_cell = 1;
5029 }
5030
5031 /*
5032 * Set the bad block marker/indicator (BBM/BBI) patterns according to some
5033 * heuristic patterns using various detected parameters (e.g., manufacturer,
5034 * page size, cell-type information).
5035 */
nand_decode_bbm_options(struct nand_chip * chip)5036 static void nand_decode_bbm_options(struct nand_chip *chip)
5037 {
5038 struct mtd_info *mtd = nand_to_mtd(chip);
5039
5040 /* Set the bad block position */
5041 if (mtd->writesize > 512 || (chip->options & NAND_BUSWIDTH_16))
5042 chip->badblockpos = NAND_BBM_POS_LARGE;
5043 else
5044 chip->badblockpos = NAND_BBM_POS_SMALL;
5045 }
5046
is_full_id_nand(struct nand_flash_dev * type)5047 static inline bool is_full_id_nand(struct nand_flash_dev *type)
5048 {
5049 return type->id_len;
5050 }
5051
find_full_id_nand(struct nand_chip * chip,struct nand_flash_dev * type)5052 static bool find_full_id_nand(struct nand_chip *chip,
5053 struct nand_flash_dev *type)
5054 {
5055 struct nand_device *base = &chip->base;
5056 struct nand_ecc_props requirements;
5057 struct mtd_info *mtd = nand_to_mtd(chip);
5058 struct nand_memory_organization *memorg;
5059 u8 *id_data = chip->id.data;
5060
5061 memorg = nanddev_get_memorg(&chip->base);
5062
5063 if (!strncmp(type->id, id_data, type->id_len)) {
5064 memorg->pagesize = type->pagesize;
5065 mtd->writesize = memorg->pagesize;
5066 memorg->pages_per_eraseblock = type->erasesize /
5067 type->pagesize;
5068 mtd->erasesize = type->erasesize;
5069 memorg->oobsize = type->oobsize;
5070 mtd->oobsize = memorg->oobsize;
5071
5072 memorg->bits_per_cell = nand_get_bits_per_cell(id_data[2]);
5073 memorg->eraseblocks_per_lun =
5074 DIV_ROUND_DOWN_ULL((u64)type->chipsize << 20,
5075 memorg->pagesize *
5076 memorg->pages_per_eraseblock);
5077 chip->options |= type->options;
5078 requirements.strength = NAND_ECC_STRENGTH(type);
5079 requirements.step_size = NAND_ECC_STEP(type);
5080 nanddev_set_ecc_requirements(base, &requirements);
5081
5082 chip->parameters.model = kstrdup(type->name, GFP_KERNEL);
5083 if (!chip->parameters.model)
5084 return false;
5085
5086 return true;
5087 }
5088 return false;
5089 }
5090
5091 /*
5092 * Manufacturer detection. Only used when the NAND is not ONFI or JEDEC
5093 * compliant and does not have a full-id or legacy-id entry in the nand_ids
5094 * table.
5095 */
nand_manufacturer_detect(struct nand_chip * chip)5096 static void nand_manufacturer_detect(struct nand_chip *chip)
5097 {
5098 /*
5099 * Try manufacturer detection if available and use
5100 * nand_decode_ext_id() otherwise.
5101 */
5102 if (chip->manufacturer.desc && chip->manufacturer.desc->ops &&
5103 chip->manufacturer.desc->ops->detect) {
5104 struct nand_memory_organization *memorg;
5105
5106 memorg = nanddev_get_memorg(&chip->base);
5107
5108 /* The 3rd id byte holds MLC / multichip data */
5109 memorg->bits_per_cell = nand_get_bits_per_cell(chip->id.data[2]);
5110 chip->manufacturer.desc->ops->detect(chip);
5111 } else {
5112 nand_decode_ext_id(chip);
5113 }
5114 }
5115
5116 /*
5117 * Manufacturer initialization. This function is called for all NANDs including
5118 * ONFI and JEDEC compliant ones.
5119 * Manufacturer drivers should put all their specific initialization code in
5120 * their ->init() hook.
5121 */
nand_manufacturer_init(struct nand_chip * chip)5122 static int nand_manufacturer_init(struct nand_chip *chip)
5123 {
5124 if (!chip->manufacturer.desc || !chip->manufacturer.desc->ops ||
5125 !chip->manufacturer.desc->ops->init)
5126 return 0;
5127
5128 return chip->manufacturer.desc->ops->init(chip);
5129 }
5130
5131 /*
5132 * Manufacturer cleanup. This function is called for all NANDs including
5133 * ONFI and JEDEC compliant ones.
5134 * Manufacturer drivers should put all their specific cleanup code in their
5135 * ->cleanup() hook.
5136 */
nand_manufacturer_cleanup(struct nand_chip * chip)5137 static void nand_manufacturer_cleanup(struct nand_chip *chip)
5138 {
5139 /* Release manufacturer private data */
5140 if (chip->manufacturer.desc && chip->manufacturer.desc->ops &&
5141 chip->manufacturer.desc->ops->cleanup)
5142 chip->manufacturer.desc->ops->cleanup(chip);
5143 }
5144
5145 static const char *
nand_manufacturer_name(const struct nand_manufacturer_desc * manufacturer_desc)5146 nand_manufacturer_name(const struct nand_manufacturer_desc *manufacturer_desc)
5147 {
5148 return manufacturer_desc ? manufacturer_desc->name : "Unknown";
5149 }
5150
rawnand_check_data_only_read_support(struct nand_chip * chip)5151 static void rawnand_check_data_only_read_support(struct nand_chip *chip)
5152 {
5153 /* Use an arbitrary size for the check */
5154 if (!nand_read_data_op(chip, NULL, SZ_512, true, true))
5155 chip->controller->supported_op.data_only_read = 1;
5156 }
5157
rawnand_early_check_supported_ops(struct nand_chip * chip)5158 static void rawnand_early_check_supported_ops(struct nand_chip *chip)
5159 {
5160 /* The supported_op fields should not be set by individual drivers */
5161 WARN_ON_ONCE(chip->controller->supported_op.data_only_read);
5162
5163 if (!nand_has_exec_op(chip))
5164 return;
5165
5166 rawnand_check_data_only_read_support(chip);
5167 }
5168
rawnand_check_cont_read_support(struct nand_chip * chip)5169 static void rawnand_check_cont_read_support(struct nand_chip *chip)
5170 {
5171 struct mtd_info *mtd = nand_to_mtd(chip);
5172
5173 if (!chip->parameters.supports_read_cache)
5174 return;
5175
5176 if (chip->read_retries)
5177 return;
5178
5179 if (!nand_lp_exec_cont_read_page_op(chip, 0, 0, NULL,
5180 mtd->writesize, true))
5181 chip->controller->supported_op.cont_read = 1;
5182 }
5183
rawnand_late_check_supported_ops(struct nand_chip * chip)5184 static void rawnand_late_check_supported_ops(struct nand_chip *chip)
5185 {
5186 /* The supported_op fields should not be set by individual drivers */
5187 WARN_ON_ONCE(chip->controller->supported_op.cont_read);
5188
5189 /*
5190 * Too many devices do not support sequential cached reads with on-die
5191 * ECC correction enabled, so in this case refuse to perform the
5192 * automation.
5193 */
5194 if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE)
5195 return;
5196
5197 if (!nand_has_exec_op(chip))
5198 return;
5199
5200 /*
5201 * For now, continuous reads can only be used with the core page helpers.
5202 * This can be extended later.
5203 */
5204 if (!(chip->ecc.read_page == nand_read_page_hwecc ||
5205 chip->ecc.read_page == nand_read_page_syndrome ||
5206 chip->ecc.read_page == nand_read_page_swecc))
5207 return;
5208
5209 rawnand_check_cont_read_support(chip);
5210 }
5211
5212 /*
5213 * Get the flash and manufacturer id and lookup if the type is supported.
5214 */
nand_detect(struct nand_chip * chip,struct nand_flash_dev * type)5215 static int nand_detect(struct nand_chip *chip, struct nand_flash_dev *type)
5216 {
5217 const struct nand_manufacturer_desc *manufacturer_desc;
5218 struct mtd_info *mtd = nand_to_mtd(chip);
5219 struct nand_memory_organization *memorg;
5220 int busw, ret;
5221 u8 *id_data = chip->id.data;
5222 u8 maf_id, dev_id;
5223 u64 targetsize;
5224
5225 /*
5226 * Let's start by initializing memorg fields that might be left
5227 * unassigned by the ID-based detection logic.
5228 */
5229 memorg = nanddev_get_memorg(&chip->base);
5230 memorg->planes_per_lun = 1;
5231 memorg->luns_per_target = 1;
5232
5233 /*
5234 * Reset the chip, required by some chips (e.g. Micron MT29FxGxxxxx)
5235 * after power-up.
5236 */
5237 ret = nand_reset(chip, 0);
5238 if (ret)
5239 return ret;
5240
5241 /* Select the device */
5242 nand_select_target(chip, 0);
5243
5244 rawnand_early_check_supported_ops(chip);
5245
5246 /* Send the command for reading device ID */
5247 ret = nand_readid_op(chip, 0, id_data, 2);
5248 if (ret)
5249 return ret;
5250
5251 /* Read manufacturer and device IDs */
5252 maf_id = id_data[0];
5253 dev_id = id_data[1];
5254
5255 /*
5256 * Try again to make sure, as some systems the bus-hold or other
5257 * interface concerns can cause random data which looks like a
5258 * possibly credible NAND flash to appear. If the two results do
5259 * not match, ignore the device completely.
5260 */
5261
5262 /* Read entire ID string */
5263 ret = nand_readid_op(chip, 0, id_data, sizeof(chip->id.data));
5264 if (ret)
5265 return ret;
5266
5267 if (id_data[0] != maf_id || id_data[1] != dev_id) {
5268 pr_info("second ID read did not match %02x,%02x against %02x,%02x\n",
5269 maf_id, dev_id, id_data[0], id_data[1]);
5270 return -ENODEV;
5271 }
5272
5273 chip->id.len = nand_id_len(id_data, ARRAY_SIZE(chip->id.data));
5274
5275 /* Try to identify manufacturer */
5276 manufacturer_desc = nand_get_manufacturer_desc(maf_id);
5277 chip->manufacturer.desc = manufacturer_desc;
5278
5279 if (!type)
5280 type = nand_flash_ids;
5281
5282 /*
5283 * Save the NAND_BUSWIDTH_16 flag before letting auto-detection logic
5284 * override it.
5285 * This is required to make sure initial NAND bus width set by the
5286 * NAND controller driver is coherent with the real NAND bus width
5287 * (extracted by auto-detection code).
5288 */
5289 busw = chip->options & NAND_BUSWIDTH_16;
5290
5291 /*
5292 * The flag is only set (never cleared), reset it to its default value
5293 * before starting auto-detection.
5294 */
5295 chip->options &= ~NAND_BUSWIDTH_16;
5296
5297 for (; type->name != NULL; type++) {
5298 if (is_full_id_nand(type)) {
5299 if (find_full_id_nand(chip, type))
5300 goto ident_done;
5301 } else if (dev_id == type->dev_id) {
5302 break;
5303 }
5304 }
5305
5306 if (!type->name || !type->pagesize) {
5307 /* Check if the chip is ONFI compliant */
5308 ret = nand_onfi_detect(chip);
5309 if (ret < 0)
5310 return ret;
5311 else if (ret)
5312 goto ident_done;
5313
5314 /* Check if the chip is JEDEC compliant */
5315 ret = nand_jedec_detect(chip);
5316 if (ret < 0)
5317 return ret;
5318 else if (ret)
5319 goto ident_done;
5320 }
5321
5322 if (!type->name)
5323 return -ENODEV;
5324
5325 chip->parameters.model = kstrdup(type->name, GFP_KERNEL);
5326 if (!chip->parameters.model)
5327 return -ENOMEM;
5328
5329 if (!type->pagesize)
5330 nand_manufacturer_detect(chip);
5331 else
5332 nand_decode_id(chip, type);
5333
5334 /* Get chip options */
5335 chip->options |= type->options;
5336
5337 memorg->eraseblocks_per_lun =
5338 DIV_ROUND_DOWN_ULL((u64)type->chipsize << 20,
5339 memorg->pagesize *
5340 memorg->pages_per_eraseblock);
5341
5342 ident_done:
5343 if (!mtd->name)
5344 mtd->name = chip->parameters.model;
5345
5346 if (chip->options & NAND_BUSWIDTH_AUTO) {
5347 WARN_ON(busw & NAND_BUSWIDTH_16);
5348 nand_set_defaults(chip);
5349 } else if (busw != (chip->options & NAND_BUSWIDTH_16)) {
5350 /*
5351 * Check, if buswidth is correct. Hardware drivers should set
5352 * chip correct!
5353 */
5354 pr_info("device found, Manufacturer ID: 0x%02x, Chip ID: 0x%02x\n",
5355 maf_id, dev_id);
5356 pr_info("%s %s\n", nand_manufacturer_name(manufacturer_desc),
5357 mtd->name);
5358 pr_warn("bus width %d instead of %d bits\n", busw ? 16 : 8,
5359 (chip->options & NAND_BUSWIDTH_16) ? 16 : 8);
5360 ret = -EINVAL;
5361
5362 goto free_detect_allocation;
5363 }
5364
5365 nand_decode_bbm_options(chip);
5366
5367 /* Calculate the address shift from the page size */
5368 chip->page_shift = ffs(mtd->writesize) - 1;
5369 /* Convert chipsize to number of pages per chip -1 */
5370 targetsize = nanddev_target_size(&chip->base);
5371 chip->pagemask = (targetsize >> chip->page_shift) - 1;
5372
5373 chip->bbt_erase_shift = chip->phys_erase_shift =
5374 ffs(mtd->erasesize) - 1;
5375 if (targetsize & 0xffffffff)
5376 chip->chip_shift = ffs((unsigned)targetsize) - 1;
5377 else {
5378 chip->chip_shift = ffs((unsigned)(targetsize >> 32));
5379 chip->chip_shift += 32 - 1;
5380 }
5381
5382 if (chip->chip_shift - chip->page_shift > 16)
5383 chip->options |= NAND_ROW_ADDR_3;
5384
5385 chip->badblockbits = 8;
5386
5387 nand_legacy_adjust_cmdfunc(chip);
5388
5389 pr_info("device found, Manufacturer ID: 0x%02x, Chip ID: 0x%02x\n",
5390 maf_id, dev_id);
5391 pr_info("%s %s\n", nand_manufacturer_name(manufacturer_desc),
5392 chip->parameters.model);
5393 pr_info("%d MiB, %s, erase size: %d KiB, page size: %d, OOB size: %d\n",
5394 (int)(targetsize >> 20), nand_is_slc(chip) ? "SLC" : "MLC",
5395 mtd->erasesize >> 10, mtd->writesize, mtd->oobsize);
5396 return 0;
5397
5398 free_detect_allocation:
5399 kfree(chip->parameters.model);
5400
5401 return ret;
5402 }
5403
5404 static enum nand_ecc_engine_type
of_get_rawnand_ecc_engine_type_legacy(struct device_node * np)5405 of_get_rawnand_ecc_engine_type_legacy(struct device_node *np)
5406 {
5407 enum nand_ecc_legacy_mode {
5408 NAND_ECC_INVALID,
5409 NAND_ECC_NONE,
5410 NAND_ECC_SOFT,
5411 NAND_ECC_SOFT_BCH,
5412 NAND_ECC_HW,
5413 NAND_ECC_HW_SYNDROME,
5414 NAND_ECC_ON_DIE,
5415 };
5416 const char * const nand_ecc_legacy_modes[] = {
5417 [NAND_ECC_NONE] = "none",
5418 [NAND_ECC_SOFT] = "soft",
5419 [NAND_ECC_SOFT_BCH] = "soft_bch",
5420 [NAND_ECC_HW] = "hw",
5421 [NAND_ECC_HW_SYNDROME] = "hw_syndrome",
5422 [NAND_ECC_ON_DIE] = "on-die",
5423 };
5424 enum nand_ecc_legacy_mode eng_type;
5425 const char *pm;
5426 int err;
5427
5428 err = of_property_read_string(np, "nand-ecc-mode", &pm);
5429 if (err)
5430 return NAND_ECC_ENGINE_TYPE_INVALID;
5431
5432 for (eng_type = NAND_ECC_NONE;
5433 eng_type < ARRAY_SIZE(nand_ecc_legacy_modes); eng_type++) {
5434 if (!strcasecmp(pm, nand_ecc_legacy_modes[eng_type])) {
5435 switch (eng_type) {
5436 case NAND_ECC_NONE:
5437 return NAND_ECC_ENGINE_TYPE_NONE;
5438 case NAND_ECC_SOFT:
5439 case NAND_ECC_SOFT_BCH:
5440 return NAND_ECC_ENGINE_TYPE_SOFT;
5441 case NAND_ECC_HW:
5442 case NAND_ECC_HW_SYNDROME:
5443 return NAND_ECC_ENGINE_TYPE_ON_HOST;
5444 case NAND_ECC_ON_DIE:
5445 return NAND_ECC_ENGINE_TYPE_ON_DIE;
5446 default:
5447 break;
5448 }
5449 }
5450 }
5451
5452 return NAND_ECC_ENGINE_TYPE_INVALID;
5453 }
5454
5455 static enum nand_ecc_placement
of_get_rawnand_ecc_placement_legacy(struct device_node * np)5456 of_get_rawnand_ecc_placement_legacy(struct device_node *np)
5457 {
5458 const char *pm;
5459 int err;
5460
5461 err = of_property_read_string(np, "nand-ecc-mode", &pm);
5462 if (!err) {
5463 if (!strcasecmp(pm, "hw_syndrome"))
5464 return NAND_ECC_PLACEMENT_INTERLEAVED;
5465 }
5466
5467 return NAND_ECC_PLACEMENT_UNKNOWN;
5468 }
5469
of_get_rawnand_ecc_algo_legacy(struct device_node * np)5470 static enum nand_ecc_algo of_get_rawnand_ecc_algo_legacy(struct device_node *np)
5471 {
5472 const char *pm;
5473 int err;
5474
5475 err = of_property_read_string(np, "nand-ecc-mode", &pm);
5476 if (!err) {
5477 if (!strcasecmp(pm, "soft"))
5478 return NAND_ECC_ALGO_HAMMING;
5479 else if (!strcasecmp(pm, "soft_bch"))
5480 return NAND_ECC_ALGO_BCH;
5481 }
5482
5483 return NAND_ECC_ALGO_UNKNOWN;
5484 }
5485
of_get_nand_ecc_legacy_user_config(struct nand_chip * chip)5486 static void of_get_nand_ecc_legacy_user_config(struct nand_chip *chip)
5487 {
5488 struct device_node *dn = nand_get_flash_node(chip);
5489 struct nand_ecc_props *user_conf = &chip->base.ecc.user_conf;
5490
5491 if (user_conf->engine_type == NAND_ECC_ENGINE_TYPE_INVALID)
5492 user_conf->engine_type = of_get_rawnand_ecc_engine_type_legacy(dn);
5493
5494 if (user_conf->algo == NAND_ECC_ALGO_UNKNOWN)
5495 user_conf->algo = of_get_rawnand_ecc_algo_legacy(dn);
5496
5497 if (user_conf->placement == NAND_ECC_PLACEMENT_UNKNOWN)
5498 user_conf->placement = of_get_rawnand_ecc_placement_legacy(dn);
5499 }
5500
of_get_nand_bus_width(struct nand_chip * chip)5501 static int of_get_nand_bus_width(struct nand_chip *chip)
5502 {
5503 struct device_node *dn = nand_get_flash_node(chip);
5504 u32 val;
5505 int ret;
5506
5507 ret = of_property_read_u32(dn, "nand-bus-width", &val);
5508 if (ret == -EINVAL)
5509 /* Buswidth defaults to 8 if the property does not exist .*/
5510 return 0;
5511 else if (ret)
5512 return ret;
5513
5514 if (val == 16)
5515 chip->options |= NAND_BUSWIDTH_16;
5516 else if (val != 8)
5517 return -EINVAL;
5518 return 0;
5519 }
5520
of_get_nand_secure_regions(struct nand_chip * chip)5521 static int of_get_nand_secure_regions(struct nand_chip *chip)
5522 {
5523 struct device_node *dn = nand_get_flash_node(chip);
5524 struct property *prop;
5525 int nr_elem, i, j;
5526
5527 /* Only proceed if the "secure-regions" property is present in DT */
5528 prop = of_find_property(dn, "secure-regions", NULL);
5529 if (!prop)
5530 return 0;
5531
5532 nr_elem = of_property_count_elems_of_size(dn, "secure-regions", sizeof(u64));
5533 if (nr_elem <= 0)
5534 return nr_elem;
5535
5536 chip->nr_secure_regions = nr_elem / 2;
5537 chip->secure_regions = kcalloc(chip->nr_secure_regions, sizeof(*chip->secure_regions),
5538 GFP_KERNEL);
5539 if (!chip->secure_regions)
5540 return -ENOMEM;
5541
5542 for (i = 0, j = 0; i < chip->nr_secure_regions; i++, j += 2) {
5543 of_property_read_u64_index(dn, "secure-regions", j,
5544 &chip->secure_regions[i].offset);
5545 of_property_read_u64_index(dn, "secure-regions", j + 1,
5546 &chip->secure_regions[i].size);
5547 }
5548
5549 return 0;
5550 }
5551
5552 /**
5553 * rawnand_dt_parse_gpio_cs - Parse the gpio-cs property of a controller
5554 * @dev: Device that will be parsed. Also used for managed allocations.
5555 * @cs_array: Array of GPIO desc pointers allocated on success
5556 * @ncs_array: Number of entries in @cs_array updated on success.
5557 * @return 0 on success, an error otherwise.
5558 */
rawnand_dt_parse_gpio_cs(struct device * dev,struct gpio_desc *** cs_array,unsigned int * ncs_array)5559 int rawnand_dt_parse_gpio_cs(struct device *dev, struct gpio_desc ***cs_array,
5560 unsigned int *ncs_array)
5561 {
5562 struct gpio_desc **descs;
5563 int ndescs, i;
5564
5565 ndescs = gpiod_count(dev, "cs");
5566 if (ndescs < 0) {
5567 dev_dbg(dev, "No valid cs-gpios property\n");
5568 return 0;
5569 }
5570
5571 descs = devm_kcalloc(dev, ndescs, sizeof(*descs), GFP_KERNEL);
5572 if (!descs)
5573 return -ENOMEM;
5574
5575 for (i = 0; i < ndescs; i++) {
5576 descs[i] = gpiod_get_index_optional(dev, "cs", i,
5577 GPIOD_OUT_HIGH);
5578 if (IS_ERR(descs[i]))
5579 return PTR_ERR(descs[i]);
5580 }
5581
5582 *ncs_array = ndescs;
5583 *cs_array = descs;
5584
5585 return 0;
5586 }
5587 EXPORT_SYMBOL(rawnand_dt_parse_gpio_cs);
5588
rawnand_dt_init(struct nand_chip * chip)5589 static int rawnand_dt_init(struct nand_chip *chip)
5590 {
5591 struct nand_device *nand = mtd_to_nanddev(nand_to_mtd(chip));
5592 struct device_node *dn = nand_get_flash_node(chip);
5593 int ret;
5594
5595 if (!dn)
5596 return 0;
5597
5598 ret = of_get_nand_bus_width(chip);
5599 if (ret)
5600 return ret;
5601
5602 if (of_property_read_bool(dn, "nand-is-boot-medium"))
5603 chip->options |= NAND_IS_BOOT_MEDIUM;
5604
5605 if (of_property_read_bool(dn, "nand-on-flash-bbt"))
5606 chip->bbt_options |= NAND_BBT_USE_FLASH;
5607
5608 of_get_nand_ecc_user_config(nand);
5609 of_get_nand_ecc_legacy_user_config(chip);
5610
5611 /*
5612 * If neither the user nor the NAND controller have requested a specific
5613 * ECC engine type, we will default to NAND_ECC_ENGINE_TYPE_ON_HOST.
5614 */
5615 nand->ecc.defaults.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
5616
5617 /*
5618 * Use the user requested engine type, unless there is none, in this
5619 * case default to the NAND controller choice, otherwise fallback to
5620 * the raw NAND default one.
5621 */
5622 if (nand->ecc.user_conf.engine_type != NAND_ECC_ENGINE_TYPE_INVALID)
5623 chip->ecc.engine_type = nand->ecc.user_conf.engine_type;
5624 if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_INVALID)
5625 chip->ecc.engine_type = nand->ecc.defaults.engine_type;
5626
5627 chip->ecc.placement = nand->ecc.user_conf.placement;
5628 chip->ecc.algo = nand->ecc.user_conf.algo;
5629 chip->ecc.strength = nand->ecc.user_conf.strength;
5630 chip->ecc.size = nand->ecc.user_conf.step_size;
5631
5632 return 0;
5633 }
5634
5635 /**
5636 * nand_scan_ident - Scan for the NAND device
5637 * @chip: NAND chip object
5638 * @maxchips: number of chips to scan for
5639 * @table: alternative NAND ID table
5640 *
5641 * This is the first phase of the normal nand_scan() function. It reads the
5642 * flash ID and sets up MTD fields accordingly.
5643 *
5644 * This helper used to be called directly from controller drivers that needed
5645 * to tweak some ECC-related parameters before nand_scan_tail(). This separation
5646 * prevented dynamic allocations during this phase which was unconvenient and
5647 * as been banned for the benefit of the ->init_ecc()/cleanup_ecc() hooks.
5648 */
nand_scan_ident(struct nand_chip * chip,unsigned int maxchips,struct nand_flash_dev * table)5649 static int nand_scan_ident(struct nand_chip *chip, unsigned int maxchips,
5650 struct nand_flash_dev *table)
5651 {
5652 struct mtd_info *mtd = nand_to_mtd(chip);
5653 struct nand_memory_organization *memorg;
5654 int nand_maf_id, nand_dev_id;
5655 unsigned int i;
5656 int ret;
5657
5658 memorg = nanddev_get_memorg(&chip->base);
5659
5660 /* Assume all dies are deselected when we enter nand_scan_ident(). */
5661 chip->cur_cs = -1;
5662
5663 mutex_init(&chip->lock);
5664 init_waitqueue_head(&chip->resume_wq);
5665
5666 /* Enforce the right timings for reset/detection */
5667 chip->current_interface_config = nand_get_reset_interface_config();
5668
5669 ret = rawnand_dt_init(chip);
5670 if (ret)
5671 return ret;
5672
5673 if (!mtd->name && mtd->dev.parent)
5674 mtd->name = dev_name(mtd->dev.parent);
5675
5676 /* Set the default functions */
5677 nand_set_defaults(chip);
5678
5679 ret = nand_legacy_check_hooks(chip);
5680 if (ret)
5681 return ret;
5682
5683 memorg->ntargets = maxchips;
5684
5685 /* Read the flash type */
5686 ret = nand_detect(chip, table);
5687 if (ret) {
5688 if (!(chip->options & NAND_SCAN_SILENT_NODEV))
5689 pr_warn("No NAND device found\n");
5690 nand_deselect_target(chip);
5691 return ret;
5692 }
5693
5694 nand_maf_id = chip->id.data[0];
5695 nand_dev_id = chip->id.data[1];
5696
5697 nand_deselect_target(chip);
5698
5699 /* Check for a chip array */
5700 for (i = 1; i < maxchips; i++) {
5701 u8 id[2];
5702
5703 /* See comment in nand_get_flash_type for reset */
5704 ret = nand_reset(chip, i);
5705 if (ret)
5706 break;
5707
5708 nand_select_target(chip, i);
5709 /* Send the command for reading device ID */
5710 ret = nand_readid_op(chip, 0, id, sizeof(id));
5711 if (ret)
5712 break;
5713 /* Read manufacturer and device IDs */
5714 if (nand_maf_id != id[0] || nand_dev_id != id[1]) {
5715 nand_deselect_target(chip);
5716 break;
5717 }
5718 nand_deselect_target(chip);
5719 }
5720 if (i > 1)
5721 pr_info("%d chips detected\n", i);
5722
5723 /* Store the number of chips and calc total size for mtd */
5724 memorg->ntargets = i;
5725 mtd->size = i * nanddev_target_size(&chip->base);
5726
5727 return 0;
5728 }
5729
nand_scan_ident_cleanup(struct nand_chip * chip)5730 static void nand_scan_ident_cleanup(struct nand_chip *chip)
5731 {
5732 kfree(chip->parameters.model);
5733 kfree(chip->parameters.onfi);
5734 }
5735
rawnand_sw_hamming_init(struct nand_chip * chip)5736 int rawnand_sw_hamming_init(struct nand_chip *chip)
5737 {
5738 struct nand_ecc_sw_hamming_conf *engine_conf;
5739 struct nand_device *base = &chip->base;
5740 int ret;
5741
5742 base->ecc.user_conf.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
5743 base->ecc.user_conf.algo = NAND_ECC_ALGO_HAMMING;
5744 base->ecc.user_conf.strength = chip->ecc.strength;
5745 base->ecc.user_conf.step_size = chip->ecc.size;
5746
5747 ret = nand_ecc_sw_hamming_init_ctx(base);
5748 if (ret)
5749 return ret;
5750
5751 engine_conf = base->ecc.ctx.priv;
5752
5753 if (chip->ecc.options & NAND_ECC_SOFT_HAMMING_SM_ORDER)
5754 engine_conf->sm_order = true;
5755
5756 chip->ecc.size = base->ecc.ctx.conf.step_size;
5757 chip->ecc.strength = base->ecc.ctx.conf.strength;
5758 chip->ecc.total = base->ecc.ctx.total;
5759 chip->ecc.steps = nanddev_get_ecc_nsteps(base);
5760 chip->ecc.bytes = base->ecc.ctx.total / nanddev_get_ecc_nsteps(base);
5761
5762 return 0;
5763 }
5764 EXPORT_SYMBOL(rawnand_sw_hamming_init);
5765
rawnand_sw_hamming_calculate(struct nand_chip * chip,const unsigned char * buf,unsigned char * code)5766 int rawnand_sw_hamming_calculate(struct nand_chip *chip,
5767 const unsigned char *buf,
5768 unsigned char *code)
5769 {
5770 struct nand_device *base = &chip->base;
5771
5772 return nand_ecc_sw_hamming_calculate(base, buf, code);
5773 }
5774 EXPORT_SYMBOL(rawnand_sw_hamming_calculate);
5775
rawnand_sw_hamming_correct(struct nand_chip * chip,unsigned char * buf,unsigned char * read_ecc,unsigned char * calc_ecc)5776 int rawnand_sw_hamming_correct(struct nand_chip *chip,
5777 unsigned char *buf,
5778 unsigned char *read_ecc,
5779 unsigned char *calc_ecc)
5780 {
5781 struct nand_device *base = &chip->base;
5782
5783 return nand_ecc_sw_hamming_correct(base, buf, read_ecc, calc_ecc);
5784 }
5785 EXPORT_SYMBOL(rawnand_sw_hamming_correct);
5786
rawnand_sw_hamming_cleanup(struct nand_chip * chip)5787 void rawnand_sw_hamming_cleanup(struct nand_chip *chip)
5788 {
5789 struct nand_device *base = &chip->base;
5790
5791 nand_ecc_sw_hamming_cleanup_ctx(base);
5792 }
5793 EXPORT_SYMBOL(rawnand_sw_hamming_cleanup);
5794
rawnand_sw_bch_init(struct nand_chip * chip)5795 int rawnand_sw_bch_init(struct nand_chip *chip)
5796 {
5797 struct nand_device *base = &chip->base;
5798 const struct nand_ecc_props *ecc_conf = nanddev_get_ecc_conf(base);
5799 int ret;
5800
5801 base->ecc.user_conf.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
5802 base->ecc.user_conf.algo = NAND_ECC_ALGO_BCH;
5803 base->ecc.user_conf.step_size = chip->ecc.size;
5804 base->ecc.user_conf.strength = chip->ecc.strength;
5805
5806 ret = nand_ecc_sw_bch_init_ctx(base);
5807 if (ret)
5808 return ret;
5809
5810 chip->ecc.size = ecc_conf->step_size;
5811 chip->ecc.strength = ecc_conf->strength;
5812 chip->ecc.total = base->ecc.ctx.total;
5813 chip->ecc.steps = nanddev_get_ecc_nsteps(base);
5814 chip->ecc.bytes = base->ecc.ctx.total / nanddev_get_ecc_nsteps(base);
5815
5816 return 0;
5817 }
5818 EXPORT_SYMBOL(rawnand_sw_bch_init);
5819
rawnand_sw_bch_calculate(struct nand_chip * chip,const unsigned char * buf,unsigned char * code)5820 static int rawnand_sw_bch_calculate(struct nand_chip *chip,
5821 const unsigned char *buf,
5822 unsigned char *code)
5823 {
5824 struct nand_device *base = &chip->base;
5825
5826 return nand_ecc_sw_bch_calculate(base, buf, code);
5827 }
5828
rawnand_sw_bch_correct(struct nand_chip * chip,unsigned char * buf,unsigned char * read_ecc,unsigned char * calc_ecc)5829 int rawnand_sw_bch_correct(struct nand_chip *chip, unsigned char *buf,
5830 unsigned char *read_ecc, unsigned char *calc_ecc)
5831 {
5832 struct nand_device *base = &chip->base;
5833
5834 return nand_ecc_sw_bch_correct(base, buf, read_ecc, calc_ecc);
5835 }
5836 EXPORT_SYMBOL(rawnand_sw_bch_correct);
5837
rawnand_sw_bch_cleanup(struct nand_chip * chip)5838 void rawnand_sw_bch_cleanup(struct nand_chip *chip)
5839 {
5840 struct nand_device *base = &chip->base;
5841
5842 nand_ecc_sw_bch_cleanup_ctx(base);
5843 }
5844 EXPORT_SYMBOL(rawnand_sw_bch_cleanup);
5845
nand_set_ecc_on_host_ops(struct nand_chip * chip)5846 static int nand_set_ecc_on_host_ops(struct nand_chip *chip)
5847 {
5848 struct nand_ecc_ctrl *ecc = &chip->ecc;
5849
5850 switch (ecc->placement) {
5851 case NAND_ECC_PLACEMENT_UNKNOWN:
5852 case NAND_ECC_PLACEMENT_OOB:
5853 /* Use standard hwecc read page function? */
5854 if (!ecc->read_page)
5855 ecc->read_page = nand_read_page_hwecc;
5856 if (!ecc->write_page)
5857 ecc->write_page = nand_write_page_hwecc;
5858 if (!ecc->read_page_raw)
5859 ecc->read_page_raw = nand_read_page_raw;
5860 if (!ecc->write_page_raw)
5861 ecc->write_page_raw = nand_write_page_raw;
5862 if (!ecc->read_oob)
5863 ecc->read_oob = nand_read_oob_std;
5864 if (!ecc->write_oob)
5865 ecc->write_oob = nand_write_oob_std;
5866 if (!ecc->read_subpage)
5867 ecc->read_subpage = nand_read_subpage;
5868 if (!ecc->write_subpage && ecc->hwctl && ecc->calculate)
5869 ecc->write_subpage = nand_write_subpage_hwecc;
5870 fallthrough;
5871
5872 case NAND_ECC_PLACEMENT_INTERLEAVED:
5873 if ((!ecc->calculate || !ecc->correct || !ecc->hwctl) &&
5874 (!ecc->read_page ||
5875 ecc->read_page == nand_read_page_hwecc ||
5876 !ecc->write_page ||
5877 ecc->write_page == nand_write_page_hwecc)) {
5878 WARN(1, "No ECC functions supplied; hardware ECC not possible\n");
5879 return -EINVAL;
5880 }
5881 /* Use standard syndrome read/write page function? */
5882 if (!ecc->read_page)
5883 ecc->read_page = nand_read_page_syndrome;
5884 if (!ecc->write_page)
5885 ecc->write_page = nand_write_page_syndrome;
5886 if (!ecc->read_page_raw)
5887 ecc->read_page_raw = nand_read_page_raw_syndrome;
5888 if (!ecc->write_page_raw)
5889 ecc->write_page_raw = nand_write_page_raw_syndrome;
5890 if (!ecc->read_oob)
5891 ecc->read_oob = nand_read_oob_syndrome;
5892 if (!ecc->write_oob)
5893 ecc->write_oob = nand_write_oob_syndrome;
5894 break;
5895
5896 default:
5897 pr_warn("Invalid NAND_ECC_PLACEMENT %d\n",
5898 ecc->placement);
5899 return -EINVAL;
5900 }
5901
5902 return 0;
5903 }
5904
nand_set_ecc_soft_ops(struct nand_chip * chip)5905 static int nand_set_ecc_soft_ops(struct nand_chip *chip)
5906 {
5907 struct mtd_info *mtd = nand_to_mtd(chip);
5908 struct nand_device *nanddev = mtd_to_nanddev(mtd);
5909 struct nand_ecc_ctrl *ecc = &chip->ecc;
5910 int ret;
5911
5912 if (WARN_ON(ecc->engine_type != NAND_ECC_ENGINE_TYPE_SOFT))
5913 return -EINVAL;
5914
5915 switch (ecc->algo) {
5916 case NAND_ECC_ALGO_HAMMING:
5917 ecc->calculate = rawnand_sw_hamming_calculate;
5918 ecc->correct = rawnand_sw_hamming_correct;
5919 ecc->read_page = nand_read_page_swecc;
5920 ecc->read_subpage = nand_read_subpage;
5921 ecc->write_page = nand_write_page_swecc;
5922 if (!ecc->read_page_raw)
5923 ecc->read_page_raw = nand_read_page_raw;
5924 if (!ecc->write_page_raw)
5925 ecc->write_page_raw = nand_write_page_raw;
5926 ecc->read_oob = nand_read_oob_std;
5927 ecc->write_oob = nand_write_oob_std;
5928 if (!ecc->size)
5929 ecc->size = 256;
5930 ecc->bytes = 3;
5931 ecc->strength = 1;
5932
5933 if (IS_ENABLED(CONFIG_MTD_NAND_ECC_SW_HAMMING_SMC))
5934 ecc->options |= NAND_ECC_SOFT_HAMMING_SM_ORDER;
5935
5936 ret = rawnand_sw_hamming_init(chip);
5937 if (ret) {
5938 WARN(1, "Hamming ECC initialization failed!\n");
5939 return ret;
5940 }
5941
5942 return 0;
5943 case NAND_ECC_ALGO_BCH:
5944 if (!IS_ENABLED(CONFIG_MTD_NAND_ECC_SW_BCH)) {
5945 WARN(1, "CONFIG_MTD_NAND_ECC_SW_BCH not enabled\n");
5946 return -EINVAL;
5947 }
5948 ecc->calculate = rawnand_sw_bch_calculate;
5949 ecc->correct = rawnand_sw_bch_correct;
5950 ecc->read_page = nand_read_page_swecc;
5951 ecc->read_subpage = nand_read_subpage;
5952 ecc->write_page = nand_write_page_swecc;
5953 if (!ecc->read_page_raw)
5954 ecc->read_page_raw = nand_read_page_raw;
5955 if (!ecc->write_page_raw)
5956 ecc->write_page_raw = nand_write_page_raw;
5957 ecc->read_oob = nand_read_oob_std;
5958 ecc->write_oob = nand_write_oob_std;
5959
5960 /*
5961 * We can only maximize ECC config when the default layout is
5962 * used, otherwise we don't know how many bytes can really be
5963 * used.
5964 */
5965 if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH &&
5966 mtd->ooblayout != nand_get_large_page_ooblayout())
5967 nanddev->ecc.user_conf.flags &= ~NAND_ECC_MAXIMIZE_STRENGTH;
5968
5969 ret = rawnand_sw_bch_init(chip);
5970 if (ret) {
5971 WARN(1, "BCH ECC initialization failed!\n");
5972 return ret;
5973 }
5974
5975 return 0;
5976 default:
5977 WARN(1, "Unsupported ECC algorithm!\n");
5978 return -EINVAL;
5979 }
5980 }
5981
5982 /**
5983 * nand_check_ecc_caps - check the sanity of preset ECC settings
5984 * @chip: nand chip info structure
5985 * @caps: ECC caps info structure
5986 * @oobavail: OOB size that the ECC engine can use
5987 *
5988 * When ECC step size and strength are already set, check if they are supported
5989 * by the controller and the calculated ECC bytes fit within the chip's OOB.
5990 * On success, the calculated ECC bytes is set.
5991 */
5992 static int
nand_check_ecc_caps(struct nand_chip * chip,const struct nand_ecc_caps * caps,int oobavail)5993 nand_check_ecc_caps(struct nand_chip *chip,
5994 const struct nand_ecc_caps *caps, int oobavail)
5995 {
5996 struct mtd_info *mtd = nand_to_mtd(chip);
5997 const struct nand_ecc_step_info *stepinfo;
5998 int preset_step = chip->ecc.size;
5999 int preset_strength = chip->ecc.strength;
6000 int ecc_bytes, nsteps = mtd->writesize / preset_step;
6001 int i, j;
6002
6003 for (i = 0; i < caps->nstepinfos; i++) {
6004 stepinfo = &caps->stepinfos[i];
6005
6006 if (stepinfo->stepsize != preset_step)
6007 continue;
6008
6009 for (j = 0; j < stepinfo->nstrengths; j++) {
6010 if (stepinfo->strengths[j] != preset_strength)
6011 continue;
6012
6013 ecc_bytes = caps->calc_ecc_bytes(preset_step,
6014 preset_strength);
6015 if (WARN_ON_ONCE(ecc_bytes < 0))
6016 return ecc_bytes;
6017
6018 if (ecc_bytes * nsteps > oobavail) {
6019 pr_err("ECC (step, strength) = (%d, %d) does not fit in OOB",
6020 preset_step, preset_strength);
6021 return -ENOSPC;
6022 }
6023
6024 chip->ecc.bytes = ecc_bytes;
6025
6026 return 0;
6027 }
6028 }
6029
6030 pr_err("ECC (step, strength) = (%d, %d) not supported on this controller",
6031 preset_step, preset_strength);
6032
6033 return -ENOTSUPP;
6034 }
6035
6036 /**
6037 * nand_match_ecc_req - meet the chip's requirement with least ECC bytes
6038 * @chip: nand chip info structure
6039 * @caps: ECC engine caps info structure
6040 * @oobavail: OOB size that the ECC engine can use
6041 *
6042 * If a chip's ECC requirement is provided, try to meet it with the least
6043 * number of ECC bytes (i.e. with the largest number of OOB-free bytes).
6044 * On success, the chosen ECC settings are set.
6045 */
6046 static int
nand_match_ecc_req(struct nand_chip * chip,const struct nand_ecc_caps * caps,int oobavail)6047 nand_match_ecc_req(struct nand_chip *chip,
6048 const struct nand_ecc_caps *caps, int oobavail)
6049 {
6050 const struct nand_ecc_props *requirements =
6051 nanddev_get_ecc_requirements(&chip->base);
6052 struct mtd_info *mtd = nand_to_mtd(chip);
6053 const struct nand_ecc_step_info *stepinfo;
6054 int req_step = requirements->step_size;
6055 int req_strength = requirements->strength;
6056 int req_corr, step_size, strength, nsteps, ecc_bytes, ecc_bytes_total;
6057 int best_step = 0, best_strength = 0, best_ecc_bytes = 0;
6058 int best_ecc_bytes_total = INT_MAX;
6059 int i, j;
6060
6061 /* No information provided by the NAND chip */
6062 if (!req_step || !req_strength)
6063 return -ENOTSUPP;
6064
6065 /* number of correctable bits the chip requires in a page */
6066 req_corr = mtd->writesize / req_step * req_strength;
6067
6068 for (i = 0; i < caps->nstepinfos; i++) {
6069 stepinfo = &caps->stepinfos[i];
6070 step_size = stepinfo->stepsize;
6071
6072 for (j = 0; j < stepinfo->nstrengths; j++) {
6073 strength = stepinfo->strengths[j];
6074
6075 /*
6076 * If both step size and strength are smaller than the
6077 * chip's requirement, it is not easy to compare the
6078 * resulted reliability.
6079 */
6080 if (step_size < req_step && strength < req_strength)
6081 continue;
6082
6083 if (mtd->writesize % step_size)
6084 continue;
6085
6086 nsteps = mtd->writesize / step_size;
6087
6088 ecc_bytes = caps->calc_ecc_bytes(step_size, strength);
6089 if (WARN_ON_ONCE(ecc_bytes < 0))
6090 continue;
6091 ecc_bytes_total = ecc_bytes * nsteps;
6092
6093 if (ecc_bytes_total > oobavail ||
6094 strength * nsteps < req_corr)
6095 continue;
6096
6097 /*
6098 * We assume the best is to meet the chip's requrement
6099 * with the least number of ECC bytes.
6100 */
6101 if (ecc_bytes_total < best_ecc_bytes_total) {
6102 best_ecc_bytes_total = ecc_bytes_total;
6103 best_step = step_size;
6104 best_strength = strength;
6105 best_ecc_bytes = ecc_bytes;
6106 }
6107 }
6108 }
6109
6110 if (best_ecc_bytes_total == INT_MAX)
6111 return -ENOTSUPP;
6112
6113 chip->ecc.size = best_step;
6114 chip->ecc.strength = best_strength;
6115 chip->ecc.bytes = best_ecc_bytes;
6116
6117 return 0;
6118 }
6119
6120 /**
6121 * nand_maximize_ecc - choose the max ECC strength available
6122 * @chip: nand chip info structure
6123 * @caps: ECC engine caps info structure
6124 * @oobavail: OOB size that the ECC engine can use
6125 *
6126 * Choose the max ECC strength that is supported on the controller, and can fit
6127 * within the chip's OOB. On success, the chosen ECC settings are set.
6128 */
6129 static int
nand_maximize_ecc(struct nand_chip * chip,const struct nand_ecc_caps * caps,int oobavail)6130 nand_maximize_ecc(struct nand_chip *chip,
6131 const struct nand_ecc_caps *caps, int oobavail)
6132 {
6133 struct mtd_info *mtd = nand_to_mtd(chip);
6134 const struct nand_ecc_step_info *stepinfo;
6135 int step_size, strength, nsteps, ecc_bytes, corr;
6136 int best_corr = 0;
6137 int best_step = 0;
6138 int best_strength = 0, best_ecc_bytes = 0;
6139 int i, j;
6140
6141 for (i = 0; i < caps->nstepinfos; i++) {
6142 stepinfo = &caps->stepinfos[i];
6143 step_size = stepinfo->stepsize;
6144
6145 /* If chip->ecc.size is already set, respect it */
6146 if (chip->ecc.size && step_size != chip->ecc.size)
6147 continue;
6148
6149 for (j = 0; j < stepinfo->nstrengths; j++) {
6150 strength = stepinfo->strengths[j];
6151
6152 if (mtd->writesize % step_size)
6153 continue;
6154
6155 nsteps = mtd->writesize / step_size;
6156
6157 ecc_bytes = caps->calc_ecc_bytes(step_size, strength);
6158 if (WARN_ON_ONCE(ecc_bytes < 0))
6159 continue;
6160
6161 if (ecc_bytes * nsteps > oobavail)
6162 continue;
6163
6164 corr = strength * nsteps;
6165
6166 /*
6167 * If the number of correctable bits is the same,
6168 * bigger step_size has more reliability.
6169 */
6170 if (corr > best_corr ||
6171 (corr == best_corr && step_size > best_step)) {
6172 best_corr = corr;
6173 best_step = step_size;
6174 best_strength = strength;
6175 best_ecc_bytes = ecc_bytes;
6176 }
6177 }
6178 }
6179
6180 if (!best_corr)
6181 return -ENOTSUPP;
6182
6183 chip->ecc.size = best_step;
6184 chip->ecc.strength = best_strength;
6185 chip->ecc.bytes = best_ecc_bytes;
6186
6187 return 0;
6188 }
6189
6190 /**
6191 * nand_ecc_choose_conf - Set the ECC strength and ECC step size
6192 * @chip: nand chip info structure
6193 * @caps: ECC engine caps info structure
6194 * @oobavail: OOB size that the ECC engine can use
6195 *
6196 * Choose the ECC configuration according to following logic.
6197 *
6198 * 1. If both ECC step size and ECC strength are already set (usually by DT)
6199 * then check if it is supported by this controller.
6200 * 2. If the user provided the nand-ecc-maximize property, then select maximum
6201 * ECC strength.
6202 * 3. Otherwise, try to match the ECC step size and ECC strength closest
6203 * to the chip's requirement. If available OOB size can't fit the chip
6204 * requirement then fallback to the maximum ECC step size and ECC strength.
6205 *
6206 * On success, the chosen ECC settings are set.
6207 */
nand_ecc_choose_conf(struct nand_chip * chip,const struct nand_ecc_caps * caps,int oobavail)6208 int nand_ecc_choose_conf(struct nand_chip *chip,
6209 const struct nand_ecc_caps *caps, int oobavail)
6210 {
6211 struct mtd_info *mtd = nand_to_mtd(chip);
6212 struct nand_device *nanddev = mtd_to_nanddev(mtd);
6213
6214 if (WARN_ON(oobavail < 0 || oobavail > mtd->oobsize))
6215 return -EINVAL;
6216
6217 if (chip->ecc.size && chip->ecc.strength)
6218 return nand_check_ecc_caps(chip, caps, oobavail);
6219
6220 if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH)
6221 return nand_maximize_ecc(chip, caps, oobavail);
6222
6223 if (!nand_match_ecc_req(chip, caps, oobavail))
6224 return 0;
6225
6226 return nand_maximize_ecc(chip, caps, oobavail);
6227 }
6228 EXPORT_SYMBOL_GPL(nand_ecc_choose_conf);
6229
rawnand_erase(struct nand_device * nand,const struct nand_pos * pos)6230 static int rawnand_erase(struct nand_device *nand, const struct nand_pos *pos)
6231 {
6232 struct nand_chip *chip = container_of(nand, struct nand_chip,
6233 base);
6234 unsigned int eb = nanddev_pos_to_row(nand, pos);
6235 int ret;
6236
6237 eb >>= nand->rowconv.eraseblock_addr_shift;
6238
6239 nand_select_target(chip, pos->target);
6240 ret = nand_erase_op(chip, eb);
6241 nand_deselect_target(chip);
6242
6243 return ret;
6244 }
6245
rawnand_markbad(struct nand_device * nand,const struct nand_pos * pos)6246 static int rawnand_markbad(struct nand_device *nand,
6247 const struct nand_pos *pos)
6248 {
6249 struct nand_chip *chip = container_of(nand, struct nand_chip,
6250 base);
6251
6252 return nand_markbad_bbm(chip, nanddev_pos_to_offs(nand, pos));
6253 }
6254
rawnand_isbad(struct nand_device * nand,const struct nand_pos * pos)6255 static bool rawnand_isbad(struct nand_device *nand, const struct nand_pos *pos)
6256 {
6257 struct nand_chip *chip = container_of(nand, struct nand_chip,
6258 base);
6259 int ret;
6260
6261 nand_select_target(chip, pos->target);
6262 ret = nand_isbad_bbm(chip, nanddev_pos_to_offs(nand, pos));
6263 nand_deselect_target(chip);
6264
6265 return ret;
6266 }
6267
6268 static const struct nand_ops rawnand_ops = {
6269 .erase = rawnand_erase,
6270 .markbad = rawnand_markbad,
6271 .isbad = rawnand_isbad,
6272 };
6273
6274 /**
6275 * nand_scan_tail - Scan for the NAND device
6276 * @chip: NAND chip object
6277 *
6278 * This is the second phase of the normal nand_scan() function. It fills out
6279 * all the uninitialized function pointers with the defaults and scans for a
6280 * bad block table if appropriate.
6281 */
nand_scan_tail(struct nand_chip * chip)6282 static int nand_scan_tail(struct nand_chip *chip)
6283 {
6284 struct mtd_info *mtd = nand_to_mtd(chip);
6285 struct nand_ecc_ctrl *ecc = &chip->ecc;
6286 int ret, i;
6287
6288 /* New bad blocks should be marked in OOB, flash-based BBT, or both */
6289 if (WARN_ON((chip->bbt_options & NAND_BBT_NO_OOB_BBM) &&
6290 !(chip->bbt_options & NAND_BBT_USE_FLASH))) {
6291 return -EINVAL;
6292 }
6293
6294 chip->data_buf = kmalloc(mtd->writesize + mtd->oobsize, GFP_KERNEL);
6295 if (!chip->data_buf)
6296 return -ENOMEM;
6297
6298 /*
6299 * FIXME: some NAND manufacturer drivers expect the first die to be
6300 * selected when manufacturer->init() is called. They should be fixed
6301 * to explictly select the relevant die when interacting with the NAND
6302 * chip.
6303 */
6304 nand_select_target(chip, 0);
6305 ret = nand_manufacturer_init(chip);
6306 nand_deselect_target(chip);
6307 if (ret)
6308 goto err_free_buf;
6309
6310 /* Set the internal oob buffer location, just after the page data */
6311 chip->oob_poi = chip->data_buf + mtd->writesize;
6312
6313 /*
6314 * If no default placement scheme is given, select an appropriate one.
6315 */
6316 if (!mtd->ooblayout &&
6317 !(ecc->engine_type == NAND_ECC_ENGINE_TYPE_SOFT &&
6318 ecc->algo == NAND_ECC_ALGO_BCH) &&
6319 !(ecc->engine_type == NAND_ECC_ENGINE_TYPE_SOFT &&
6320 ecc->algo == NAND_ECC_ALGO_HAMMING)) {
6321 switch (mtd->oobsize) {
6322 case 8:
6323 case 16:
6324 mtd_set_ooblayout(mtd, nand_get_small_page_ooblayout());
6325 break;
6326 case 64:
6327 case 128:
6328 mtd_set_ooblayout(mtd,
6329 nand_get_large_page_hamming_ooblayout());
6330 break;
6331 default:
6332 /*
6333 * Expose the whole OOB area to users if ECC_NONE
6334 * is passed. We could do that for all kind of
6335 * ->oobsize, but we must keep the old large/small
6336 * page with ECC layout when ->oobsize <= 128 for
6337 * compatibility reasons.
6338 */
6339 if (ecc->engine_type == NAND_ECC_ENGINE_TYPE_NONE) {
6340 mtd_set_ooblayout(mtd,
6341 nand_get_large_page_ooblayout());
6342 break;
6343 }
6344
6345 WARN(1, "No oob scheme defined for oobsize %d\n",
6346 mtd->oobsize);
6347 ret = -EINVAL;
6348 goto err_nand_manuf_cleanup;
6349 }
6350 }
6351
6352 /*
6353 * Check ECC mode, default to software if 3byte/512byte hardware ECC is
6354 * selected and we have 256 byte pagesize fallback to software ECC
6355 */
6356
6357 switch (ecc->engine_type) {
6358 case NAND_ECC_ENGINE_TYPE_ON_HOST:
6359 ret = nand_set_ecc_on_host_ops(chip);
6360 if (ret)
6361 goto err_nand_manuf_cleanup;
6362
6363 if (mtd->writesize >= ecc->size) {
6364 if (!ecc->strength) {
6365 WARN(1, "Driver must set ecc.strength when using hardware ECC\n");
6366 ret = -EINVAL;
6367 goto err_nand_manuf_cleanup;
6368 }
6369 break;
6370 }
6371 pr_warn("%d byte HW ECC not possible on %d byte page size, fallback to SW ECC\n",
6372 ecc->size, mtd->writesize);
6373 ecc->engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
6374 ecc->algo = NAND_ECC_ALGO_HAMMING;
6375 fallthrough;
6376
6377 case NAND_ECC_ENGINE_TYPE_SOFT:
6378 ret = nand_set_ecc_soft_ops(chip);
6379 if (ret)
6380 goto err_nand_manuf_cleanup;
6381 break;
6382
6383 case NAND_ECC_ENGINE_TYPE_ON_DIE:
6384 if (!ecc->read_page || !ecc->write_page) {
6385 WARN(1, "No ECC functions supplied; on-die ECC not possible\n");
6386 ret = -EINVAL;
6387 goto err_nand_manuf_cleanup;
6388 }
6389 if (!ecc->read_oob)
6390 ecc->read_oob = nand_read_oob_std;
6391 if (!ecc->write_oob)
6392 ecc->write_oob = nand_write_oob_std;
6393 break;
6394
6395 case NAND_ECC_ENGINE_TYPE_NONE:
6396 pr_warn("NAND_ECC_ENGINE_TYPE_NONE selected by board driver. This is not recommended!\n");
6397 ecc->read_page = nand_read_page_raw;
6398 ecc->write_page = nand_write_page_raw;
6399 ecc->read_oob = nand_read_oob_std;
6400 ecc->read_page_raw = nand_read_page_raw;
6401 ecc->write_page_raw = nand_write_page_raw;
6402 ecc->write_oob = nand_write_oob_std;
6403 ecc->size = mtd->writesize;
6404 ecc->bytes = 0;
6405 ecc->strength = 0;
6406 break;
6407
6408 default:
6409 WARN(1, "Invalid NAND_ECC_MODE %d\n", ecc->engine_type);
6410 ret = -EINVAL;
6411 goto err_nand_manuf_cleanup;
6412 }
6413
6414 if (ecc->correct || ecc->calculate) {
6415 ecc->calc_buf = kmalloc(mtd->oobsize, GFP_KERNEL);
6416 ecc->code_buf = kmalloc(mtd->oobsize, GFP_KERNEL);
6417 if (!ecc->calc_buf || !ecc->code_buf) {
6418 ret = -ENOMEM;
6419 goto err_nand_manuf_cleanup;
6420 }
6421 }
6422
6423 /* For many systems, the standard OOB write also works for raw */
6424 if (!ecc->read_oob_raw)
6425 ecc->read_oob_raw = ecc->read_oob;
6426 if (!ecc->write_oob_raw)
6427 ecc->write_oob_raw = ecc->write_oob;
6428
6429 /* propagate ecc info to mtd_info */
6430 mtd->ecc_strength = ecc->strength;
6431 mtd->ecc_step_size = ecc->size;
6432
6433 /*
6434 * Set the number of read / write steps for one page depending on ECC
6435 * mode.
6436 */
6437 if (!ecc->steps)
6438 ecc->steps = mtd->writesize / ecc->size;
6439 if (ecc->steps * ecc->size != mtd->writesize) {
6440 WARN(1, "Invalid ECC parameters\n");
6441 ret = -EINVAL;
6442 goto err_nand_manuf_cleanup;
6443 }
6444
6445 if (!ecc->total) {
6446 ecc->total = ecc->steps * ecc->bytes;
6447 chip->base.ecc.ctx.total = ecc->total;
6448 }
6449
6450 if (ecc->total > mtd->oobsize) {
6451 WARN(1, "Total number of ECC bytes exceeded oobsize\n");
6452 ret = -EINVAL;
6453 goto err_nand_manuf_cleanup;
6454 }
6455
6456 /*
6457 * The number of bytes available for a client to place data into
6458 * the out of band area.
6459 */
6460 ret = mtd_ooblayout_count_freebytes(mtd);
6461 if (ret < 0)
6462 ret = 0;
6463
6464 mtd->oobavail = ret;
6465
6466 /* ECC sanity check: warn if it's too weak */
6467 if (!nand_ecc_is_strong_enough(&chip->base))
6468 pr_warn("WARNING: %s: the ECC used on your system (%db/%dB) is too weak compared to the one required by the NAND chip (%db/%dB)\n",
6469 mtd->name, chip->ecc.strength, chip->ecc.size,
6470 nanddev_get_ecc_requirements(&chip->base)->strength,
6471 nanddev_get_ecc_requirements(&chip->base)->step_size);
6472
6473 /* Allow subpage writes up to ecc.steps. Not possible for MLC flash */
6474 if (!(chip->options & NAND_NO_SUBPAGE_WRITE) && nand_is_slc(chip)) {
6475 switch (ecc->steps) {
6476 case 2:
6477 mtd->subpage_sft = 1;
6478 break;
6479 case 4:
6480 case 8:
6481 case 16:
6482 mtd->subpage_sft = 2;
6483 break;
6484 }
6485 }
6486 chip->subpagesize = mtd->writesize >> mtd->subpage_sft;
6487
6488 /* Invalidate the pagebuffer reference */
6489 chip->pagecache.page = -1;
6490
6491 /* Large page NAND with SOFT_ECC should support subpage reads */
6492 switch (ecc->engine_type) {
6493 case NAND_ECC_ENGINE_TYPE_SOFT:
6494 if (chip->page_shift > 9)
6495 chip->options |= NAND_SUBPAGE_READ;
6496 break;
6497
6498 default:
6499 break;
6500 }
6501
6502 ret = nanddev_init(&chip->base, &rawnand_ops, mtd->owner);
6503 if (ret)
6504 goto err_nand_manuf_cleanup;
6505
6506 /* Adjust the MTD_CAP_ flags when NAND_ROM is set. */
6507 if (chip->options & NAND_ROM)
6508 mtd->flags = MTD_CAP_ROM;
6509
6510 /* Fill in remaining MTD driver data */
6511 mtd->_erase = nand_erase;
6512 mtd->_point = NULL;
6513 mtd->_unpoint = NULL;
6514 mtd->_panic_write = panic_nand_write;
6515 mtd->_read_oob = nand_read_oob;
6516 mtd->_write_oob = nand_write_oob;
6517 mtd->_sync = nand_sync;
6518 mtd->_lock = nand_lock;
6519 mtd->_unlock = nand_unlock;
6520 mtd->_suspend = nand_suspend;
6521 mtd->_resume = nand_resume;
6522 mtd->_reboot = nand_shutdown;
6523 mtd->_block_isreserved = nand_block_isreserved;
6524 mtd->_block_isbad = nand_block_isbad;
6525 mtd->_block_markbad = nand_block_markbad;
6526 mtd->_max_bad_blocks = nanddev_mtd_max_bad_blocks;
6527
6528 /*
6529 * Initialize bitflip_threshold to its default prior scan_bbt() call.
6530 * scan_bbt() might invoke mtd_read(), thus bitflip_threshold must be
6531 * properly set.
6532 */
6533 if (!mtd->bitflip_threshold)
6534 mtd->bitflip_threshold = DIV_ROUND_UP(mtd->ecc_strength * 3, 4);
6535
6536 /* Find the fastest data interface for this chip */
6537 ret = nand_choose_interface_config(chip);
6538 if (ret)
6539 goto err_nanddev_cleanup;
6540
6541 /* Enter fastest possible mode on all dies. */
6542 for (i = 0; i < nanddev_ntargets(&chip->base); i++) {
6543 ret = nand_setup_interface(chip, i);
6544 if (ret)
6545 goto err_free_interface_config;
6546 }
6547
6548 rawnand_late_check_supported_ops(chip);
6549
6550 /*
6551 * Look for secure regions in the NAND chip. These regions are supposed
6552 * to be protected by a secure element like Trustzone. So the read/write
6553 * accesses to these regions will be blocked in the runtime by this
6554 * driver.
6555 */
6556 ret = of_get_nand_secure_regions(chip);
6557 if (ret)
6558 goto err_free_interface_config;
6559
6560 /* Check, if we should skip the bad block table scan */
6561 if (chip->options & NAND_SKIP_BBTSCAN)
6562 return 0;
6563
6564 /* Build bad block table */
6565 ret = nand_create_bbt(chip);
6566 if (ret)
6567 goto err_free_secure_regions;
6568
6569 return 0;
6570
6571 err_free_secure_regions:
6572 kfree(chip->secure_regions);
6573
6574 err_free_interface_config:
6575 kfree(chip->best_interface_config);
6576
6577 err_nanddev_cleanup:
6578 nanddev_cleanup(&chip->base);
6579
6580 err_nand_manuf_cleanup:
6581 nand_manufacturer_cleanup(chip);
6582
6583 err_free_buf:
6584 kfree(chip->data_buf);
6585 kfree(ecc->code_buf);
6586 kfree(ecc->calc_buf);
6587
6588 return ret;
6589 }
6590
nand_attach(struct nand_chip * chip)6591 static int nand_attach(struct nand_chip *chip)
6592 {
6593 if (chip->controller->ops && chip->controller->ops->attach_chip)
6594 return chip->controller->ops->attach_chip(chip);
6595
6596 return 0;
6597 }
6598
nand_detach(struct nand_chip * chip)6599 static void nand_detach(struct nand_chip *chip)
6600 {
6601 if (chip->controller->ops && chip->controller->ops->detach_chip)
6602 chip->controller->ops->detach_chip(chip);
6603 }
6604
6605 /**
6606 * nand_scan_with_ids - [NAND Interface] Scan for the NAND device
6607 * @chip: NAND chip object
6608 * @maxchips: number of chips to scan for.
6609 * @ids: optional flash IDs table
6610 *
6611 * This fills out all the uninitialized function pointers with the defaults.
6612 * The flash ID is read and the mtd/chip structures are filled with the
6613 * appropriate values.
6614 */
nand_scan_with_ids(struct nand_chip * chip,unsigned int maxchips,struct nand_flash_dev * ids)6615 int nand_scan_with_ids(struct nand_chip *chip, unsigned int maxchips,
6616 struct nand_flash_dev *ids)
6617 {
6618 int ret;
6619
6620 if (!maxchips)
6621 return -EINVAL;
6622
6623 ret = nand_scan_ident(chip, maxchips, ids);
6624 if (ret)
6625 return ret;
6626
6627 ret = nand_attach(chip);
6628 if (ret)
6629 goto cleanup_ident;
6630
6631 ret = nand_scan_tail(chip);
6632 if (ret)
6633 goto detach_chip;
6634
6635 return 0;
6636
6637 detach_chip:
6638 nand_detach(chip);
6639 cleanup_ident:
6640 nand_scan_ident_cleanup(chip);
6641
6642 return ret;
6643 }
6644 EXPORT_SYMBOL(nand_scan_with_ids);
6645
6646 /**
6647 * nand_cleanup - [NAND Interface] Free resources held by the NAND device
6648 * @chip: NAND chip object
6649 */
nand_cleanup(struct nand_chip * chip)6650 void nand_cleanup(struct nand_chip *chip)
6651 {
6652 if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_SOFT) {
6653 if (chip->ecc.algo == NAND_ECC_ALGO_HAMMING)
6654 rawnand_sw_hamming_cleanup(chip);
6655 else if (chip->ecc.algo == NAND_ECC_ALGO_BCH)
6656 rawnand_sw_bch_cleanup(chip);
6657 }
6658
6659 nanddev_cleanup(&chip->base);
6660
6661 /* Free secure regions data */
6662 kfree(chip->secure_regions);
6663
6664 /* Free bad block table memory */
6665 kfree(chip->bbt);
6666 kfree(chip->data_buf);
6667 kfree(chip->ecc.code_buf);
6668 kfree(chip->ecc.calc_buf);
6669
6670 /* Free bad block descriptor memory */
6671 if (chip->badblock_pattern && chip->badblock_pattern->options
6672 & NAND_BBT_DYNAMICSTRUCT)
6673 kfree(chip->badblock_pattern);
6674
6675 /* Free the data interface */
6676 kfree(chip->best_interface_config);
6677
6678 /* Free manufacturer priv data. */
6679 nand_manufacturer_cleanup(chip);
6680
6681 /* Free controller specific allocations after chip identification */
6682 nand_detach(chip);
6683
6684 /* Free identification phase allocations */
6685 nand_scan_ident_cleanup(chip);
6686 }
6687
6688 EXPORT_SYMBOL_GPL(nand_cleanup);
6689
6690 MODULE_LICENSE("GPL");
6691 MODULE_AUTHOR("Steven J. Hill <sjhill@realitydiluted.com>");
6692 MODULE_AUTHOR("Thomas Gleixner <tglx@linutronix.de>");
6693 MODULE_DESCRIPTION("Generic NAND flash driver code");
6694