xref: /openbmc/linux/drivers/mtd/nand/raw/nand_base.c (revision 8365a898)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  Overview:
4  *   This is the generic MTD driver for NAND flash devices. It should be
5  *   capable of working with almost all NAND chips currently available.
6  *
7  *	Additional technical information is available on
8  *	http://www.linux-mtd.infradead.org/doc/nand.html
9  *
10  *  Copyright (C) 2000 Steven J. Hill (sjhill@realitydiluted.com)
11  *		  2002-2006 Thomas Gleixner (tglx@linutronix.de)
12  *
13  *  Credits:
14  *	David Woodhouse for adding multichip support
15  *
16  *	Aleph One Ltd. and Toby Churchill Ltd. for supporting the
17  *	rework for 2K page size chips
18  *
19  *  TODO:
20  *	Enable cached programming for 2k page size chips
21  *	Check, if mtd->ecctype should be set to MTD_ECC_HW
22  *	if we have HW ECC support.
23  *	BBT table is not serialized, has to be fixed
24  */
25 
26 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
27 
28 #include <linux/module.h>
29 #include <linux/delay.h>
30 #include <linux/errno.h>
31 #include <linux/err.h>
32 #include <linux/sched.h>
33 #include <linux/slab.h>
34 #include <linux/mm.h>
35 #include <linux/types.h>
36 #include <linux/mtd/mtd.h>
37 #include <linux/mtd/nand_ecc.h>
38 #include <linux/mtd/nand_bch.h>
39 #include <linux/interrupt.h>
40 #include <linux/bitops.h>
41 #include <linux/io.h>
42 #include <linux/mtd/partitions.h>
43 #include <linux/of.h>
44 #include <linux/gpio/consumer.h>
45 
46 #include "internals.h"
47 
48 /* Define default oob placement schemes for large and small page devices */
49 static int nand_ooblayout_ecc_sp(struct mtd_info *mtd, int section,
50 				 struct mtd_oob_region *oobregion)
51 {
52 	struct nand_chip *chip = mtd_to_nand(mtd);
53 	struct nand_ecc_ctrl *ecc = &chip->ecc;
54 
55 	if (section > 1)
56 		return -ERANGE;
57 
58 	if (!section) {
59 		oobregion->offset = 0;
60 		if (mtd->oobsize == 16)
61 			oobregion->length = 4;
62 		else
63 			oobregion->length = 3;
64 	} else {
65 		if (mtd->oobsize == 8)
66 			return -ERANGE;
67 
68 		oobregion->offset = 6;
69 		oobregion->length = ecc->total - 4;
70 	}
71 
72 	return 0;
73 }
74 
75 static int nand_ooblayout_free_sp(struct mtd_info *mtd, int section,
76 				  struct mtd_oob_region *oobregion)
77 {
78 	if (section > 1)
79 		return -ERANGE;
80 
81 	if (mtd->oobsize == 16) {
82 		if (section)
83 			return -ERANGE;
84 
85 		oobregion->length = 8;
86 		oobregion->offset = 8;
87 	} else {
88 		oobregion->length = 2;
89 		if (!section)
90 			oobregion->offset = 3;
91 		else
92 			oobregion->offset = 6;
93 	}
94 
95 	return 0;
96 }
97 
98 const struct mtd_ooblayout_ops nand_ooblayout_sp_ops = {
99 	.ecc = nand_ooblayout_ecc_sp,
100 	.free = nand_ooblayout_free_sp,
101 };
102 EXPORT_SYMBOL_GPL(nand_ooblayout_sp_ops);
103 
104 static int nand_ooblayout_ecc_lp(struct mtd_info *mtd, int section,
105 				 struct mtd_oob_region *oobregion)
106 {
107 	struct nand_chip *chip = mtd_to_nand(mtd);
108 	struct nand_ecc_ctrl *ecc = &chip->ecc;
109 
110 	if (section || !ecc->total)
111 		return -ERANGE;
112 
113 	oobregion->length = ecc->total;
114 	oobregion->offset = mtd->oobsize - oobregion->length;
115 
116 	return 0;
117 }
118 
119 static int nand_ooblayout_free_lp(struct mtd_info *mtd, int section,
120 				  struct mtd_oob_region *oobregion)
121 {
122 	struct nand_chip *chip = mtd_to_nand(mtd);
123 	struct nand_ecc_ctrl *ecc = &chip->ecc;
124 
125 	if (section)
126 		return -ERANGE;
127 
128 	oobregion->length = mtd->oobsize - ecc->total - 2;
129 	oobregion->offset = 2;
130 
131 	return 0;
132 }
133 
134 const struct mtd_ooblayout_ops nand_ooblayout_lp_ops = {
135 	.ecc = nand_ooblayout_ecc_lp,
136 	.free = nand_ooblayout_free_lp,
137 };
138 EXPORT_SYMBOL_GPL(nand_ooblayout_lp_ops);
139 
140 /*
141  * Support the old "large page" layout used for 1-bit Hamming ECC where ECC
142  * are placed at a fixed offset.
143  */
144 static int nand_ooblayout_ecc_lp_hamming(struct mtd_info *mtd, int section,
145 					 struct mtd_oob_region *oobregion)
146 {
147 	struct nand_chip *chip = mtd_to_nand(mtd);
148 	struct nand_ecc_ctrl *ecc = &chip->ecc;
149 
150 	if (section)
151 		return -ERANGE;
152 
153 	switch (mtd->oobsize) {
154 	case 64:
155 		oobregion->offset = 40;
156 		break;
157 	case 128:
158 		oobregion->offset = 80;
159 		break;
160 	default:
161 		return -EINVAL;
162 	}
163 
164 	oobregion->length = ecc->total;
165 	if (oobregion->offset + oobregion->length > mtd->oobsize)
166 		return -ERANGE;
167 
168 	return 0;
169 }
170 
171 static int nand_ooblayout_free_lp_hamming(struct mtd_info *mtd, int section,
172 					  struct mtd_oob_region *oobregion)
173 {
174 	struct nand_chip *chip = mtd_to_nand(mtd);
175 	struct nand_ecc_ctrl *ecc = &chip->ecc;
176 	int ecc_offset = 0;
177 
178 	if (section < 0 || section > 1)
179 		return -ERANGE;
180 
181 	switch (mtd->oobsize) {
182 	case 64:
183 		ecc_offset = 40;
184 		break;
185 	case 128:
186 		ecc_offset = 80;
187 		break;
188 	default:
189 		return -EINVAL;
190 	}
191 
192 	if (section == 0) {
193 		oobregion->offset = 2;
194 		oobregion->length = ecc_offset - 2;
195 	} else {
196 		oobregion->offset = ecc_offset + ecc->total;
197 		oobregion->length = mtd->oobsize - oobregion->offset;
198 	}
199 
200 	return 0;
201 }
202 
203 static const struct mtd_ooblayout_ops nand_ooblayout_lp_hamming_ops = {
204 	.ecc = nand_ooblayout_ecc_lp_hamming,
205 	.free = nand_ooblayout_free_lp_hamming,
206 };
207 
208 static int nand_pairing_dist3_get_info(struct mtd_info *mtd, int page,
209 				       struct mtd_pairing_info *info)
210 {
211 	int lastpage = (mtd->erasesize / mtd->writesize) - 1;
212 	int dist = 3;
213 
214 	if (page == lastpage)
215 		dist = 2;
216 
217 	if (!page || (page & 1)) {
218 		info->group = 0;
219 		info->pair = (page + 1) / 2;
220 	} else {
221 		info->group = 1;
222 		info->pair = (page + 1 - dist) / 2;
223 	}
224 
225 	return 0;
226 }
227 
228 static int nand_pairing_dist3_get_wunit(struct mtd_info *mtd,
229 					const struct mtd_pairing_info *info)
230 {
231 	int lastpair = ((mtd->erasesize / mtd->writesize) - 1) / 2;
232 	int page = info->pair * 2;
233 	int dist = 3;
234 
235 	if (!info->group && !info->pair)
236 		return 0;
237 
238 	if (info->pair == lastpair && info->group)
239 		dist = 2;
240 
241 	if (!info->group)
242 		page--;
243 	else if (info->pair)
244 		page += dist - 1;
245 
246 	if (page >= mtd->erasesize / mtd->writesize)
247 		return -EINVAL;
248 
249 	return page;
250 }
251 
252 const struct mtd_pairing_scheme dist3_pairing_scheme = {
253 	.ngroups = 2,
254 	.get_info = nand_pairing_dist3_get_info,
255 	.get_wunit = nand_pairing_dist3_get_wunit,
256 };
257 
258 static int check_offs_len(struct nand_chip *chip, loff_t ofs, uint64_t len)
259 {
260 	int ret = 0;
261 
262 	/* Start address must align on block boundary */
263 	if (ofs & ((1ULL << chip->phys_erase_shift) - 1)) {
264 		pr_debug("%s: unaligned address\n", __func__);
265 		ret = -EINVAL;
266 	}
267 
268 	/* Length must align on block boundary */
269 	if (len & ((1ULL << chip->phys_erase_shift) - 1)) {
270 		pr_debug("%s: length not block aligned\n", __func__);
271 		ret = -EINVAL;
272 	}
273 
274 	return ret;
275 }
276 
277 /**
278  * nand_extract_bits - Copy unaligned bits from one buffer to another one
279  * @dst: destination buffer
280  * @dst_off: bit offset at which the writing starts
281  * @src: source buffer
282  * @src_off: bit offset at which the reading starts
283  * @nbits: number of bits to copy from @src to @dst
284  *
285  * Copy bits from one memory region to another (overlap authorized).
286  */
287 void nand_extract_bits(u8 *dst, unsigned int dst_off, const u8 *src,
288 		       unsigned int src_off, unsigned int nbits)
289 {
290 	unsigned int tmp, n;
291 
292 	dst += dst_off / 8;
293 	dst_off %= 8;
294 	src += src_off / 8;
295 	src_off %= 8;
296 
297 	while (nbits) {
298 		n = min3(8 - dst_off, 8 - src_off, nbits);
299 
300 		tmp = (*src >> src_off) & GENMASK(n - 1, 0);
301 		*dst &= ~GENMASK(n - 1 + dst_off, dst_off);
302 		*dst |= tmp << dst_off;
303 
304 		dst_off += n;
305 		if (dst_off >= 8) {
306 			dst++;
307 			dst_off -= 8;
308 		}
309 
310 		src_off += n;
311 		if (src_off >= 8) {
312 			src++;
313 			src_off -= 8;
314 		}
315 
316 		nbits -= n;
317 	}
318 }
319 EXPORT_SYMBOL_GPL(nand_extract_bits);
320 
321 /**
322  * nand_select_target() - Select a NAND target (A.K.A. die)
323  * @chip: NAND chip object
324  * @cs: the CS line to select. Note that this CS id is always from the chip
325  *	PoV, not the controller one
326  *
327  * Select a NAND target so that further operations executed on @chip go to the
328  * selected NAND target.
329  */
330 void nand_select_target(struct nand_chip *chip, unsigned int cs)
331 {
332 	/*
333 	 * cs should always lie between 0 and nanddev_ntargets(), when that's
334 	 * not the case it's a bug and the caller should be fixed.
335 	 */
336 	if (WARN_ON(cs > nanddev_ntargets(&chip->base)))
337 		return;
338 
339 	chip->cur_cs = cs;
340 
341 	if (chip->legacy.select_chip)
342 		chip->legacy.select_chip(chip, cs);
343 }
344 EXPORT_SYMBOL_GPL(nand_select_target);
345 
346 /**
347  * nand_deselect_target() - Deselect the currently selected target
348  * @chip: NAND chip object
349  *
350  * Deselect the currently selected NAND target. The result of operations
351  * executed on @chip after the target has been deselected is undefined.
352  */
353 void nand_deselect_target(struct nand_chip *chip)
354 {
355 	if (chip->legacy.select_chip)
356 		chip->legacy.select_chip(chip, -1);
357 
358 	chip->cur_cs = -1;
359 }
360 EXPORT_SYMBOL_GPL(nand_deselect_target);
361 
362 /**
363  * nand_release_device - [GENERIC] release chip
364  * @chip: NAND chip object
365  *
366  * Release chip lock and wake up anyone waiting on the device.
367  */
368 static void nand_release_device(struct nand_chip *chip)
369 {
370 	/* Release the controller and the chip */
371 	mutex_unlock(&chip->controller->lock);
372 	mutex_unlock(&chip->lock);
373 }
374 
375 /**
376  * nand_bbm_get_next_page - Get the next page for bad block markers
377  * @chip: NAND chip object
378  * @page: First page to start checking for bad block marker usage
379  *
380  * Returns an integer that corresponds to the page offset within a block, for
381  * a page that is used to store bad block markers. If no more pages are
382  * available, -EINVAL is returned.
383  */
384 int nand_bbm_get_next_page(struct nand_chip *chip, int page)
385 {
386 	struct mtd_info *mtd = nand_to_mtd(chip);
387 	int last_page = ((mtd->erasesize - mtd->writesize) >>
388 			 chip->page_shift) & chip->pagemask;
389 	unsigned int bbm_flags = NAND_BBM_FIRSTPAGE | NAND_BBM_SECONDPAGE
390 		| NAND_BBM_LASTPAGE;
391 
392 	if (page == 0 && !(chip->options & bbm_flags))
393 		return 0;
394 	if (page == 0 && chip->options & NAND_BBM_FIRSTPAGE)
395 		return 0;
396 	if (page <= 1 && chip->options & NAND_BBM_SECONDPAGE)
397 		return 1;
398 	if (page <= last_page && chip->options & NAND_BBM_LASTPAGE)
399 		return last_page;
400 
401 	return -EINVAL;
402 }
403 
404 /**
405  * nand_block_bad - [DEFAULT] Read bad block marker from the chip
406  * @chip: NAND chip object
407  * @ofs: offset from device start
408  *
409  * Check, if the block is bad.
410  */
411 static int nand_block_bad(struct nand_chip *chip, loff_t ofs)
412 {
413 	int first_page, page_offset;
414 	int res;
415 	u8 bad;
416 
417 	first_page = (int)(ofs >> chip->page_shift) & chip->pagemask;
418 	page_offset = nand_bbm_get_next_page(chip, 0);
419 
420 	while (page_offset >= 0) {
421 		res = chip->ecc.read_oob(chip, first_page + page_offset);
422 		if (res < 0)
423 			return res;
424 
425 		bad = chip->oob_poi[chip->badblockpos];
426 
427 		if (likely(chip->badblockbits == 8))
428 			res = bad != 0xFF;
429 		else
430 			res = hweight8(bad) < chip->badblockbits;
431 		if (res)
432 			return res;
433 
434 		page_offset = nand_bbm_get_next_page(chip, page_offset + 1);
435 	}
436 
437 	return 0;
438 }
439 
440 static int nand_isbad_bbm(struct nand_chip *chip, loff_t ofs)
441 {
442 	if (chip->options & NAND_NO_BBM_QUIRK)
443 		return 0;
444 
445 	if (chip->legacy.block_bad)
446 		return chip->legacy.block_bad(chip, ofs);
447 
448 	return nand_block_bad(chip, ofs);
449 }
450 
451 /**
452  * nand_get_device - [GENERIC] Get chip for selected access
453  * @chip: NAND chip structure
454  *
455  * Lock the device and its controller for exclusive access
456  *
457  * Return: -EBUSY if the chip has been suspended, 0 otherwise
458  */
459 static int nand_get_device(struct nand_chip *chip)
460 {
461 	mutex_lock(&chip->lock);
462 	if (chip->suspended) {
463 		mutex_unlock(&chip->lock);
464 		return -EBUSY;
465 	}
466 	mutex_lock(&chip->controller->lock);
467 
468 	return 0;
469 }
470 
471 /**
472  * nand_check_wp - [GENERIC] check if the chip is write protected
473  * @chip: NAND chip object
474  *
475  * Check, if the device is write protected. The function expects, that the
476  * device is already selected.
477  */
478 static int nand_check_wp(struct nand_chip *chip)
479 {
480 	u8 status;
481 	int ret;
482 
483 	/* Broken xD cards report WP despite being writable */
484 	if (chip->options & NAND_BROKEN_XD)
485 		return 0;
486 
487 	/* Check the WP bit */
488 	ret = nand_status_op(chip, &status);
489 	if (ret)
490 		return ret;
491 
492 	return status & NAND_STATUS_WP ? 0 : 1;
493 }
494 
495 /**
496  * nand_fill_oob - [INTERN] Transfer client buffer to oob
497  * @chip: NAND chip object
498  * @oob: oob data buffer
499  * @len: oob data write length
500  * @ops: oob ops structure
501  */
502 static uint8_t *nand_fill_oob(struct nand_chip *chip, uint8_t *oob, size_t len,
503 			      struct mtd_oob_ops *ops)
504 {
505 	struct mtd_info *mtd = nand_to_mtd(chip);
506 	int ret;
507 
508 	/*
509 	 * Initialise to all 0xFF, to avoid the possibility of left over OOB
510 	 * data from a previous OOB read.
511 	 */
512 	memset(chip->oob_poi, 0xff, mtd->oobsize);
513 
514 	switch (ops->mode) {
515 
516 	case MTD_OPS_PLACE_OOB:
517 	case MTD_OPS_RAW:
518 		memcpy(chip->oob_poi + ops->ooboffs, oob, len);
519 		return oob + len;
520 
521 	case MTD_OPS_AUTO_OOB:
522 		ret = mtd_ooblayout_set_databytes(mtd, oob, chip->oob_poi,
523 						  ops->ooboffs, len);
524 		BUG_ON(ret);
525 		return oob + len;
526 
527 	default:
528 		BUG();
529 	}
530 	return NULL;
531 }
532 
533 /**
534  * nand_do_write_oob - [MTD Interface] NAND write out-of-band
535  * @chip: NAND chip object
536  * @to: offset to write to
537  * @ops: oob operation description structure
538  *
539  * NAND write out-of-band.
540  */
541 static int nand_do_write_oob(struct nand_chip *chip, loff_t to,
542 			     struct mtd_oob_ops *ops)
543 {
544 	struct mtd_info *mtd = nand_to_mtd(chip);
545 	int chipnr, page, status, len, ret;
546 
547 	pr_debug("%s: to = 0x%08x, len = %i\n",
548 			 __func__, (unsigned int)to, (int)ops->ooblen);
549 
550 	len = mtd_oobavail(mtd, ops);
551 
552 	/* Do not allow write past end of page */
553 	if ((ops->ooboffs + ops->ooblen) > len) {
554 		pr_debug("%s: attempt to write past end of page\n",
555 				__func__);
556 		return -EINVAL;
557 	}
558 
559 	chipnr = (int)(to >> chip->chip_shift);
560 
561 	/*
562 	 * Reset the chip. Some chips (like the Toshiba TC5832DC found in one
563 	 * of my DiskOnChip 2000 test units) will clear the whole data page too
564 	 * if we don't do this. I have no clue why, but I seem to have 'fixed'
565 	 * it in the doc2000 driver in August 1999.  dwmw2.
566 	 */
567 	ret = nand_reset(chip, chipnr);
568 	if (ret)
569 		return ret;
570 
571 	nand_select_target(chip, chipnr);
572 
573 	/* Shift to get page */
574 	page = (int)(to >> chip->page_shift);
575 
576 	/* Check, if it is write protected */
577 	if (nand_check_wp(chip)) {
578 		nand_deselect_target(chip);
579 		return -EROFS;
580 	}
581 
582 	/* Invalidate the page cache, if we write to the cached page */
583 	if (page == chip->pagecache.page)
584 		chip->pagecache.page = -1;
585 
586 	nand_fill_oob(chip, ops->oobbuf, ops->ooblen, ops);
587 
588 	if (ops->mode == MTD_OPS_RAW)
589 		status = chip->ecc.write_oob_raw(chip, page & chip->pagemask);
590 	else
591 		status = chip->ecc.write_oob(chip, page & chip->pagemask);
592 
593 	nand_deselect_target(chip);
594 
595 	if (status)
596 		return status;
597 
598 	ops->oobretlen = ops->ooblen;
599 
600 	return 0;
601 }
602 
603 /**
604  * nand_default_block_markbad - [DEFAULT] mark a block bad via bad block marker
605  * @chip: NAND chip object
606  * @ofs: offset from device start
607  *
608  * This is the default implementation, which can be overridden by a hardware
609  * specific driver. It provides the details for writing a bad block marker to a
610  * block.
611  */
612 static int nand_default_block_markbad(struct nand_chip *chip, loff_t ofs)
613 {
614 	struct mtd_info *mtd = nand_to_mtd(chip);
615 	struct mtd_oob_ops ops;
616 	uint8_t buf[2] = { 0, 0 };
617 	int ret = 0, res, page_offset;
618 
619 	memset(&ops, 0, sizeof(ops));
620 	ops.oobbuf = buf;
621 	ops.ooboffs = chip->badblockpos;
622 	if (chip->options & NAND_BUSWIDTH_16) {
623 		ops.ooboffs &= ~0x01;
624 		ops.len = ops.ooblen = 2;
625 	} else {
626 		ops.len = ops.ooblen = 1;
627 	}
628 	ops.mode = MTD_OPS_PLACE_OOB;
629 
630 	page_offset = nand_bbm_get_next_page(chip, 0);
631 
632 	while (page_offset >= 0) {
633 		res = nand_do_write_oob(chip,
634 					ofs + (page_offset * mtd->writesize),
635 					&ops);
636 
637 		if (!ret)
638 			ret = res;
639 
640 		page_offset = nand_bbm_get_next_page(chip, page_offset + 1);
641 	}
642 
643 	return ret;
644 }
645 
646 /**
647  * nand_markbad_bbm - mark a block by updating the BBM
648  * @chip: NAND chip object
649  * @ofs: offset of the block to mark bad
650  */
651 int nand_markbad_bbm(struct nand_chip *chip, loff_t ofs)
652 {
653 	if (chip->legacy.block_markbad)
654 		return chip->legacy.block_markbad(chip, ofs);
655 
656 	return nand_default_block_markbad(chip, ofs);
657 }
658 
659 /**
660  * nand_block_markbad_lowlevel - mark a block bad
661  * @chip: NAND chip object
662  * @ofs: offset from device start
663  *
664  * This function performs the generic NAND bad block marking steps (i.e., bad
665  * block table(s) and/or marker(s)). We only allow the hardware driver to
666  * specify how to write bad block markers to OOB (chip->legacy.block_markbad).
667  *
668  * We try operations in the following order:
669  *
670  *  (1) erase the affected block, to allow OOB marker to be written cleanly
671  *  (2) write bad block marker to OOB area of affected block (unless flag
672  *      NAND_BBT_NO_OOB_BBM is present)
673  *  (3) update the BBT
674  *
675  * Note that we retain the first error encountered in (2) or (3), finish the
676  * procedures, and dump the error in the end.
677 */
678 static int nand_block_markbad_lowlevel(struct nand_chip *chip, loff_t ofs)
679 {
680 	struct mtd_info *mtd = nand_to_mtd(chip);
681 	int res, ret = 0;
682 
683 	if (!(chip->bbt_options & NAND_BBT_NO_OOB_BBM)) {
684 		struct erase_info einfo;
685 
686 		/* Attempt erase before marking OOB */
687 		memset(&einfo, 0, sizeof(einfo));
688 		einfo.addr = ofs;
689 		einfo.len = 1ULL << chip->phys_erase_shift;
690 		nand_erase_nand(chip, &einfo, 0);
691 
692 		/* Write bad block marker to OOB */
693 		ret = nand_get_device(chip);
694 		if (ret)
695 			return ret;
696 
697 		ret = nand_markbad_bbm(chip, ofs);
698 		nand_release_device(chip);
699 	}
700 
701 	/* Mark block bad in BBT */
702 	if (chip->bbt) {
703 		res = nand_markbad_bbt(chip, ofs);
704 		if (!ret)
705 			ret = res;
706 	}
707 
708 	if (!ret)
709 		mtd->ecc_stats.badblocks++;
710 
711 	return ret;
712 }
713 
714 /**
715  * nand_block_isreserved - [GENERIC] Check if a block is marked reserved.
716  * @mtd: MTD device structure
717  * @ofs: offset from device start
718  *
719  * Check if the block is marked as reserved.
720  */
721 static int nand_block_isreserved(struct mtd_info *mtd, loff_t ofs)
722 {
723 	struct nand_chip *chip = mtd_to_nand(mtd);
724 
725 	if (!chip->bbt)
726 		return 0;
727 	/* Return info from the table */
728 	return nand_isreserved_bbt(chip, ofs);
729 }
730 
731 /**
732  * nand_block_checkbad - [GENERIC] Check if a block is marked bad
733  * @chip: NAND chip object
734  * @ofs: offset from device start
735  * @allowbbt: 1, if its allowed to access the bbt area
736  *
737  * Check, if the block is bad. Either by reading the bad block table or
738  * calling of the scan function.
739  */
740 static int nand_block_checkbad(struct nand_chip *chip, loff_t ofs, int allowbbt)
741 {
742 	/* Return info from the table */
743 	if (chip->bbt)
744 		return nand_isbad_bbt(chip, ofs, allowbbt);
745 
746 	return nand_isbad_bbm(chip, ofs);
747 }
748 
749 /**
750  * nand_soft_waitrdy - Poll STATUS reg until RDY bit is set to 1
751  * @chip: NAND chip structure
752  * @timeout_ms: Timeout in ms
753  *
754  * Poll the STATUS register using ->exec_op() until the RDY bit becomes 1.
755  * If that does not happen whitin the specified timeout, -ETIMEDOUT is
756  * returned.
757  *
758  * This helper is intended to be used when the controller does not have access
759  * to the NAND R/B pin.
760  *
761  * Be aware that calling this helper from an ->exec_op() implementation means
762  * ->exec_op() must be re-entrant.
763  *
764  * Return 0 if the NAND chip is ready, a negative error otherwise.
765  */
766 int nand_soft_waitrdy(struct nand_chip *chip, unsigned long timeout_ms)
767 {
768 	const struct nand_sdr_timings *timings;
769 	u8 status = 0;
770 	int ret;
771 
772 	if (!nand_has_exec_op(chip))
773 		return -ENOTSUPP;
774 
775 	/* Wait tWB before polling the STATUS reg. */
776 	timings = nand_get_sdr_timings(&chip->data_interface);
777 	ndelay(PSEC_TO_NSEC(timings->tWB_max));
778 
779 	ret = nand_status_op(chip, NULL);
780 	if (ret)
781 		return ret;
782 
783 	/*
784 	 * +1 below is necessary because if we are now in the last fraction
785 	 * of jiffy and msecs_to_jiffies is 1 then we will wait only that
786 	 * small jiffy fraction - possibly leading to false timeout
787 	 */
788 	timeout_ms = jiffies + msecs_to_jiffies(timeout_ms) + 1;
789 	do {
790 		ret = nand_read_data_op(chip, &status, sizeof(status), true,
791 					false);
792 		if (ret)
793 			break;
794 
795 		if (status & NAND_STATUS_READY)
796 			break;
797 
798 		/*
799 		 * Typical lowest execution time for a tR on most NANDs is 10us,
800 		 * use this as polling delay before doing something smarter (ie.
801 		 * deriving a delay from the timeout value, timeout_ms/ratio).
802 		 */
803 		udelay(10);
804 	} while	(time_before(jiffies, timeout_ms));
805 
806 	/*
807 	 * We have to exit READ_STATUS mode in order to read real data on the
808 	 * bus in case the WAITRDY instruction is preceding a DATA_IN
809 	 * instruction.
810 	 */
811 	nand_exit_status_op(chip);
812 
813 	if (ret)
814 		return ret;
815 
816 	return status & NAND_STATUS_READY ? 0 : -ETIMEDOUT;
817 };
818 EXPORT_SYMBOL_GPL(nand_soft_waitrdy);
819 
820 /**
821  * nand_gpio_waitrdy - Poll R/B GPIO pin until ready
822  * @chip: NAND chip structure
823  * @gpiod: GPIO descriptor of R/B pin
824  * @timeout_ms: Timeout in ms
825  *
826  * Poll the R/B GPIO pin until it becomes ready. If that does not happen
827  * whitin the specified timeout, -ETIMEDOUT is returned.
828  *
829  * This helper is intended to be used when the controller has access to the
830  * NAND R/B pin over GPIO.
831  *
832  * Return 0 if the R/B pin indicates chip is ready, a negative error otherwise.
833  */
834 int nand_gpio_waitrdy(struct nand_chip *chip, struct gpio_desc *gpiod,
835 		      unsigned long timeout_ms)
836 {
837 
838 	/*
839 	 * Wait until R/B pin indicates chip is ready or timeout occurs.
840 	 * +1 below is necessary because if we are now in the last fraction
841 	 * of jiffy and msecs_to_jiffies is 1 then we will wait only that
842 	 * small jiffy fraction - possibly leading to false timeout.
843 	 */
844 	timeout_ms = jiffies + msecs_to_jiffies(timeout_ms) + 1;
845 	do {
846 		if (gpiod_get_value_cansleep(gpiod))
847 			return 0;
848 
849 		cond_resched();
850 	} while	(time_before(jiffies, timeout_ms));
851 
852 	return gpiod_get_value_cansleep(gpiod) ? 0 : -ETIMEDOUT;
853 };
854 EXPORT_SYMBOL_GPL(nand_gpio_waitrdy);
855 
856 /**
857  * panic_nand_wait - [GENERIC] wait until the command is done
858  * @chip: NAND chip structure
859  * @timeo: timeout
860  *
861  * Wait for command done. This is a helper function for nand_wait used when
862  * we are in interrupt context. May happen when in panic and trying to write
863  * an oops through mtdoops.
864  */
865 void panic_nand_wait(struct nand_chip *chip, unsigned long timeo)
866 {
867 	int i;
868 	for (i = 0; i < timeo; i++) {
869 		if (chip->legacy.dev_ready) {
870 			if (chip->legacy.dev_ready(chip))
871 				break;
872 		} else {
873 			int ret;
874 			u8 status;
875 
876 			ret = nand_read_data_op(chip, &status, sizeof(status),
877 						true, false);
878 			if (ret)
879 				return;
880 
881 			if (status & NAND_STATUS_READY)
882 				break;
883 		}
884 		mdelay(1);
885 	}
886 }
887 
888 static bool nand_supports_get_features(struct nand_chip *chip, int addr)
889 {
890 	return (chip->parameters.supports_set_get_features &&
891 		test_bit(addr, chip->parameters.get_feature_list));
892 }
893 
894 static bool nand_supports_set_features(struct nand_chip *chip, int addr)
895 {
896 	return (chip->parameters.supports_set_get_features &&
897 		test_bit(addr, chip->parameters.set_feature_list));
898 }
899 
900 /**
901  * nand_reset_data_interface - Reset data interface and timings
902  * @chip: The NAND chip
903  * @chipnr: Internal die id
904  *
905  * Reset the Data interface and timings to ONFI mode 0.
906  *
907  * Returns 0 for success or negative error code otherwise.
908  */
909 static int nand_reset_data_interface(struct nand_chip *chip, int chipnr)
910 {
911 	int ret;
912 
913 	if (!nand_has_setup_data_iface(chip))
914 		return 0;
915 
916 	/*
917 	 * The ONFI specification says:
918 	 * "
919 	 * To transition from NV-DDR or NV-DDR2 to the SDR data
920 	 * interface, the host shall use the Reset (FFh) command
921 	 * using SDR timing mode 0. A device in any timing mode is
922 	 * required to recognize Reset (FFh) command issued in SDR
923 	 * timing mode 0.
924 	 * "
925 	 *
926 	 * Configure the data interface in SDR mode and set the
927 	 * timings to timing mode 0.
928 	 */
929 
930 	onfi_fill_data_interface(chip, NAND_SDR_IFACE, 0);
931 	ret = chip->controller->ops->setup_data_interface(chip, chipnr,
932 							&chip->data_interface);
933 	if (ret)
934 		pr_err("Failed to configure data interface to SDR timing mode 0\n");
935 
936 	return ret;
937 }
938 
939 /**
940  * nand_setup_data_interface - Setup the best data interface and timings
941  * @chip: The NAND chip
942  * @chipnr: Internal die id
943  *
944  * Find and configure the best data interface and NAND timings supported by
945  * the chip and the driver.
946  * First tries to retrieve supported timing modes from ONFI information,
947  * and if the NAND chip does not support ONFI, relies on the
948  * ->onfi_timing_mode_default specified in the nand_ids table.
949  *
950  * Returns 0 for success or negative error code otherwise.
951  */
952 static int nand_setup_data_interface(struct nand_chip *chip, int chipnr)
953 {
954 	u8 tmode_param[ONFI_SUBFEATURE_PARAM_LEN] = {
955 		chip->onfi_timing_mode_default,
956 	};
957 	int ret;
958 
959 	if (!nand_has_setup_data_iface(chip))
960 		return 0;
961 
962 	/* Change the mode on the chip side (if supported by the NAND chip) */
963 	if (nand_supports_set_features(chip, ONFI_FEATURE_ADDR_TIMING_MODE)) {
964 		nand_select_target(chip, chipnr);
965 		ret = nand_set_features(chip, ONFI_FEATURE_ADDR_TIMING_MODE,
966 					tmode_param);
967 		nand_deselect_target(chip);
968 		if (ret)
969 			return ret;
970 	}
971 
972 	/* Change the mode on the controller side */
973 	ret = chip->controller->ops->setup_data_interface(chip, chipnr,
974 							&chip->data_interface);
975 	if (ret)
976 		return ret;
977 
978 	/* Check the mode has been accepted by the chip, if supported */
979 	if (!nand_supports_get_features(chip, ONFI_FEATURE_ADDR_TIMING_MODE))
980 		return 0;
981 
982 	memset(tmode_param, 0, ONFI_SUBFEATURE_PARAM_LEN);
983 	nand_select_target(chip, chipnr);
984 	ret = nand_get_features(chip, ONFI_FEATURE_ADDR_TIMING_MODE,
985 				tmode_param);
986 	nand_deselect_target(chip);
987 	if (ret)
988 		goto err_reset_chip;
989 
990 	if (tmode_param[0] != chip->onfi_timing_mode_default) {
991 		pr_warn("timing mode %d not acknowledged by the NAND chip\n",
992 			chip->onfi_timing_mode_default);
993 		goto err_reset_chip;
994 	}
995 
996 	return 0;
997 
998 err_reset_chip:
999 	/*
1000 	 * Fallback to mode 0 if the chip explicitly did not ack the chosen
1001 	 * timing mode.
1002 	 */
1003 	nand_reset_data_interface(chip, chipnr);
1004 	nand_select_target(chip, chipnr);
1005 	nand_reset_op(chip);
1006 	nand_deselect_target(chip);
1007 
1008 	return ret;
1009 }
1010 
1011 /**
1012  * nand_init_data_interface - find the best data interface and timings
1013  * @chip: The NAND chip
1014  *
1015  * Find the best data interface and NAND timings supported by the chip
1016  * and the driver.
1017  * First tries to retrieve supported timing modes from ONFI information,
1018  * and if the NAND chip does not support ONFI, relies on the
1019  * ->onfi_timing_mode_default specified in the nand_ids table. After this
1020  * function nand_chip->data_interface is initialized with the best timing mode
1021  * available.
1022  *
1023  * Returns 0 for success or negative error code otherwise.
1024  */
1025 static int nand_init_data_interface(struct nand_chip *chip)
1026 {
1027 	int modes, mode, ret;
1028 
1029 	if (!nand_has_setup_data_iface(chip))
1030 		return 0;
1031 
1032 	/*
1033 	 * First try to identify the best timings from ONFI parameters and
1034 	 * if the NAND does not support ONFI, fallback to the default ONFI
1035 	 * timing mode.
1036 	 */
1037 	if (chip->parameters.onfi) {
1038 		modes = chip->parameters.onfi->async_timing_mode;
1039 	} else {
1040 		if (!chip->onfi_timing_mode_default)
1041 			return 0;
1042 
1043 		modes = GENMASK(chip->onfi_timing_mode_default, 0);
1044 	}
1045 
1046 	for (mode = fls(modes) - 1; mode >= 0; mode--) {
1047 		ret = onfi_fill_data_interface(chip, NAND_SDR_IFACE, mode);
1048 		if (ret)
1049 			continue;
1050 
1051 		/*
1052 		 * Pass NAND_DATA_IFACE_CHECK_ONLY to only check if the
1053 		 * controller supports the requested timings.
1054 		 */
1055 		ret = chip->controller->ops->setup_data_interface(chip,
1056 						 NAND_DATA_IFACE_CHECK_ONLY,
1057 						 &chip->data_interface);
1058 		if (!ret) {
1059 			chip->onfi_timing_mode_default = mode;
1060 			break;
1061 		}
1062 	}
1063 
1064 	return 0;
1065 }
1066 
1067 /**
1068  * nand_fill_column_cycles - fill the column cycles of an address
1069  * @chip: The NAND chip
1070  * @addrs: Array of address cycles to fill
1071  * @offset_in_page: The offset in the page
1072  *
1073  * Fills the first or the first two bytes of the @addrs field depending
1074  * on the NAND bus width and the page size.
1075  *
1076  * Returns the number of cycles needed to encode the column, or a negative
1077  * error code in case one of the arguments is invalid.
1078  */
1079 static int nand_fill_column_cycles(struct nand_chip *chip, u8 *addrs,
1080 				   unsigned int offset_in_page)
1081 {
1082 	struct mtd_info *mtd = nand_to_mtd(chip);
1083 
1084 	/* Make sure the offset is less than the actual page size. */
1085 	if (offset_in_page > mtd->writesize + mtd->oobsize)
1086 		return -EINVAL;
1087 
1088 	/*
1089 	 * On small page NANDs, there's a dedicated command to access the OOB
1090 	 * area, and the column address is relative to the start of the OOB
1091 	 * area, not the start of the page. Asjust the address accordingly.
1092 	 */
1093 	if (mtd->writesize <= 512 && offset_in_page >= mtd->writesize)
1094 		offset_in_page -= mtd->writesize;
1095 
1096 	/*
1097 	 * The offset in page is expressed in bytes, if the NAND bus is 16-bit
1098 	 * wide, then it must be divided by 2.
1099 	 */
1100 	if (chip->options & NAND_BUSWIDTH_16) {
1101 		if (WARN_ON(offset_in_page % 2))
1102 			return -EINVAL;
1103 
1104 		offset_in_page /= 2;
1105 	}
1106 
1107 	addrs[0] = offset_in_page;
1108 
1109 	/*
1110 	 * Small page NANDs use 1 cycle for the columns, while large page NANDs
1111 	 * need 2
1112 	 */
1113 	if (mtd->writesize <= 512)
1114 		return 1;
1115 
1116 	addrs[1] = offset_in_page >> 8;
1117 
1118 	return 2;
1119 }
1120 
1121 static int nand_sp_exec_read_page_op(struct nand_chip *chip, unsigned int page,
1122 				     unsigned int offset_in_page, void *buf,
1123 				     unsigned int len)
1124 {
1125 	struct mtd_info *mtd = nand_to_mtd(chip);
1126 	const struct nand_sdr_timings *sdr =
1127 		nand_get_sdr_timings(&chip->data_interface);
1128 	u8 addrs[4];
1129 	struct nand_op_instr instrs[] = {
1130 		NAND_OP_CMD(NAND_CMD_READ0, 0),
1131 		NAND_OP_ADDR(3, addrs, PSEC_TO_NSEC(sdr->tWB_max)),
1132 		NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tR_max),
1133 				 PSEC_TO_NSEC(sdr->tRR_min)),
1134 		NAND_OP_DATA_IN(len, buf, 0),
1135 	};
1136 	struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1137 	int ret;
1138 
1139 	/* Drop the DATA_IN instruction if len is set to 0. */
1140 	if (!len)
1141 		op.ninstrs--;
1142 
1143 	if (offset_in_page >= mtd->writesize)
1144 		instrs[0].ctx.cmd.opcode = NAND_CMD_READOOB;
1145 	else if (offset_in_page >= 256 &&
1146 		 !(chip->options & NAND_BUSWIDTH_16))
1147 		instrs[0].ctx.cmd.opcode = NAND_CMD_READ1;
1148 
1149 	ret = nand_fill_column_cycles(chip, addrs, offset_in_page);
1150 	if (ret < 0)
1151 		return ret;
1152 
1153 	addrs[1] = page;
1154 	addrs[2] = page >> 8;
1155 
1156 	if (chip->options & NAND_ROW_ADDR_3) {
1157 		addrs[3] = page >> 16;
1158 		instrs[1].ctx.addr.naddrs++;
1159 	}
1160 
1161 	return nand_exec_op(chip, &op);
1162 }
1163 
1164 static int nand_lp_exec_read_page_op(struct nand_chip *chip, unsigned int page,
1165 				     unsigned int offset_in_page, void *buf,
1166 				     unsigned int len)
1167 {
1168 	const struct nand_sdr_timings *sdr =
1169 		nand_get_sdr_timings(&chip->data_interface);
1170 	u8 addrs[5];
1171 	struct nand_op_instr instrs[] = {
1172 		NAND_OP_CMD(NAND_CMD_READ0, 0),
1173 		NAND_OP_ADDR(4, addrs, 0),
1174 		NAND_OP_CMD(NAND_CMD_READSTART, PSEC_TO_NSEC(sdr->tWB_max)),
1175 		NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tR_max),
1176 				 PSEC_TO_NSEC(sdr->tRR_min)),
1177 		NAND_OP_DATA_IN(len, buf, 0),
1178 	};
1179 	struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1180 	int ret;
1181 
1182 	/* Drop the DATA_IN instruction if len is set to 0. */
1183 	if (!len)
1184 		op.ninstrs--;
1185 
1186 	ret = nand_fill_column_cycles(chip, addrs, offset_in_page);
1187 	if (ret < 0)
1188 		return ret;
1189 
1190 	addrs[2] = page;
1191 	addrs[3] = page >> 8;
1192 
1193 	if (chip->options & NAND_ROW_ADDR_3) {
1194 		addrs[4] = page >> 16;
1195 		instrs[1].ctx.addr.naddrs++;
1196 	}
1197 
1198 	return nand_exec_op(chip, &op);
1199 }
1200 
1201 /**
1202  * nand_read_page_op - Do a READ PAGE operation
1203  * @chip: The NAND chip
1204  * @page: page to read
1205  * @offset_in_page: offset within the page
1206  * @buf: buffer used to store the data
1207  * @len: length of the buffer
1208  *
1209  * This function issues a READ PAGE operation.
1210  * This function does not select/unselect the CS line.
1211  *
1212  * Returns 0 on success, a negative error code otherwise.
1213  */
1214 int nand_read_page_op(struct nand_chip *chip, unsigned int page,
1215 		      unsigned int offset_in_page, void *buf, unsigned int len)
1216 {
1217 	struct mtd_info *mtd = nand_to_mtd(chip);
1218 
1219 	if (len && !buf)
1220 		return -EINVAL;
1221 
1222 	if (offset_in_page + len > mtd->writesize + mtd->oobsize)
1223 		return -EINVAL;
1224 
1225 	if (nand_has_exec_op(chip)) {
1226 		if (mtd->writesize > 512)
1227 			return nand_lp_exec_read_page_op(chip, page,
1228 							 offset_in_page, buf,
1229 							 len);
1230 
1231 		return nand_sp_exec_read_page_op(chip, page, offset_in_page,
1232 						 buf, len);
1233 	}
1234 
1235 	chip->legacy.cmdfunc(chip, NAND_CMD_READ0, offset_in_page, page);
1236 	if (len)
1237 		chip->legacy.read_buf(chip, buf, len);
1238 
1239 	return 0;
1240 }
1241 EXPORT_SYMBOL_GPL(nand_read_page_op);
1242 
1243 /**
1244  * nand_read_param_page_op - Do a READ PARAMETER PAGE operation
1245  * @chip: The NAND chip
1246  * @page: parameter page to read
1247  * @buf: buffer used to store the data
1248  * @len: length of the buffer
1249  *
1250  * This function issues a READ PARAMETER PAGE operation.
1251  * This function does not select/unselect the CS line.
1252  *
1253  * Returns 0 on success, a negative error code otherwise.
1254  */
1255 int nand_read_param_page_op(struct nand_chip *chip, u8 page, void *buf,
1256 			    unsigned int len)
1257 {
1258 	unsigned int i;
1259 	u8 *p = buf;
1260 
1261 	if (len && !buf)
1262 		return -EINVAL;
1263 
1264 	if (nand_has_exec_op(chip)) {
1265 		const struct nand_sdr_timings *sdr =
1266 			nand_get_sdr_timings(&chip->data_interface);
1267 		struct nand_op_instr instrs[] = {
1268 			NAND_OP_CMD(NAND_CMD_PARAM, 0),
1269 			NAND_OP_ADDR(1, &page, PSEC_TO_NSEC(sdr->tWB_max)),
1270 			NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tR_max),
1271 					 PSEC_TO_NSEC(sdr->tRR_min)),
1272 			NAND_OP_8BIT_DATA_IN(len, buf, 0),
1273 		};
1274 		struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1275 
1276 		/* Drop the DATA_IN instruction if len is set to 0. */
1277 		if (!len)
1278 			op.ninstrs--;
1279 
1280 		return nand_exec_op(chip, &op);
1281 	}
1282 
1283 	chip->legacy.cmdfunc(chip, NAND_CMD_PARAM, page, -1);
1284 	for (i = 0; i < len; i++)
1285 		p[i] = chip->legacy.read_byte(chip);
1286 
1287 	return 0;
1288 }
1289 
1290 /**
1291  * nand_change_read_column_op - Do a CHANGE READ COLUMN operation
1292  * @chip: The NAND chip
1293  * @offset_in_page: offset within the page
1294  * @buf: buffer used to store the data
1295  * @len: length of the buffer
1296  * @force_8bit: force 8-bit bus access
1297  *
1298  * This function issues a CHANGE READ COLUMN operation.
1299  * This function does not select/unselect the CS line.
1300  *
1301  * Returns 0 on success, a negative error code otherwise.
1302  */
1303 int nand_change_read_column_op(struct nand_chip *chip,
1304 			       unsigned int offset_in_page, void *buf,
1305 			       unsigned int len, bool force_8bit)
1306 {
1307 	struct mtd_info *mtd = nand_to_mtd(chip);
1308 
1309 	if (len && !buf)
1310 		return -EINVAL;
1311 
1312 	if (offset_in_page + len > mtd->writesize + mtd->oobsize)
1313 		return -EINVAL;
1314 
1315 	/* Small page NANDs do not support column change. */
1316 	if (mtd->writesize <= 512)
1317 		return -ENOTSUPP;
1318 
1319 	if (nand_has_exec_op(chip)) {
1320 		const struct nand_sdr_timings *sdr =
1321 			nand_get_sdr_timings(&chip->data_interface);
1322 		u8 addrs[2] = {};
1323 		struct nand_op_instr instrs[] = {
1324 			NAND_OP_CMD(NAND_CMD_RNDOUT, 0),
1325 			NAND_OP_ADDR(2, addrs, 0),
1326 			NAND_OP_CMD(NAND_CMD_RNDOUTSTART,
1327 				    PSEC_TO_NSEC(sdr->tCCS_min)),
1328 			NAND_OP_DATA_IN(len, buf, 0),
1329 		};
1330 		struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1331 		int ret;
1332 
1333 		ret = nand_fill_column_cycles(chip, addrs, offset_in_page);
1334 		if (ret < 0)
1335 			return ret;
1336 
1337 		/* Drop the DATA_IN instruction if len is set to 0. */
1338 		if (!len)
1339 			op.ninstrs--;
1340 
1341 		instrs[3].ctx.data.force_8bit = force_8bit;
1342 
1343 		return nand_exec_op(chip, &op);
1344 	}
1345 
1346 	chip->legacy.cmdfunc(chip, NAND_CMD_RNDOUT, offset_in_page, -1);
1347 	if (len)
1348 		chip->legacy.read_buf(chip, buf, len);
1349 
1350 	return 0;
1351 }
1352 EXPORT_SYMBOL_GPL(nand_change_read_column_op);
1353 
1354 /**
1355  * nand_read_oob_op - Do a READ OOB operation
1356  * @chip: The NAND chip
1357  * @page: page to read
1358  * @offset_in_oob: offset within the OOB area
1359  * @buf: buffer used to store the data
1360  * @len: length of the buffer
1361  *
1362  * This function issues a READ OOB operation.
1363  * This function does not select/unselect the CS line.
1364  *
1365  * Returns 0 on success, a negative error code otherwise.
1366  */
1367 int nand_read_oob_op(struct nand_chip *chip, unsigned int page,
1368 		     unsigned int offset_in_oob, void *buf, unsigned int len)
1369 {
1370 	struct mtd_info *mtd = nand_to_mtd(chip);
1371 
1372 	if (len && !buf)
1373 		return -EINVAL;
1374 
1375 	if (offset_in_oob + len > mtd->oobsize)
1376 		return -EINVAL;
1377 
1378 	if (nand_has_exec_op(chip))
1379 		return nand_read_page_op(chip, page,
1380 					 mtd->writesize + offset_in_oob,
1381 					 buf, len);
1382 
1383 	chip->legacy.cmdfunc(chip, NAND_CMD_READOOB, offset_in_oob, page);
1384 	if (len)
1385 		chip->legacy.read_buf(chip, buf, len);
1386 
1387 	return 0;
1388 }
1389 EXPORT_SYMBOL_GPL(nand_read_oob_op);
1390 
1391 static int nand_exec_prog_page_op(struct nand_chip *chip, unsigned int page,
1392 				  unsigned int offset_in_page, const void *buf,
1393 				  unsigned int len, bool prog)
1394 {
1395 	struct mtd_info *mtd = nand_to_mtd(chip);
1396 	const struct nand_sdr_timings *sdr =
1397 		nand_get_sdr_timings(&chip->data_interface);
1398 	u8 addrs[5] = {};
1399 	struct nand_op_instr instrs[] = {
1400 		/*
1401 		 * The first instruction will be dropped if we're dealing
1402 		 * with a large page NAND and adjusted if we're dealing
1403 		 * with a small page NAND and the page offset is > 255.
1404 		 */
1405 		NAND_OP_CMD(NAND_CMD_READ0, 0),
1406 		NAND_OP_CMD(NAND_CMD_SEQIN, 0),
1407 		NAND_OP_ADDR(0, addrs, PSEC_TO_NSEC(sdr->tADL_min)),
1408 		NAND_OP_DATA_OUT(len, buf, 0),
1409 		NAND_OP_CMD(NAND_CMD_PAGEPROG, PSEC_TO_NSEC(sdr->tWB_max)),
1410 		NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tPROG_max), 0),
1411 	};
1412 	struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1413 	int naddrs = nand_fill_column_cycles(chip, addrs, offset_in_page);
1414 	int ret;
1415 	u8 status;
1416 
1417 	if (naddrs < 0)
1418 		return naddrs;
1419 
1420 	addrs[naddrs++] = page;
1421 	addrs[naddrs++] = page >> 8;
1422 	if (chip->options & NAND_ROW_ADDR_3)
1423 		addrs[naddrs++] = page >> 16;
1424 
1425 	instrs[2].ctx.addr.naddrs = naddrs;
1426 
1427 	/* Drop the last two instructions if we're not programming the page. */
1428 	if (!prog) {
1429 		op.ninstrs -= 2;
1430 		/* Also drop the DATA_OUT instruction if empty. */
1431 		if (!len)
1432 			op.ninstrs--;
1433 	}
1434 
1435 	if (mtd->writesize <= 512) {
1436 		/*
1437 		 * Small pages need some more tweaking: we have to adjust the
1438 		 * first instruction depending on the page offset we're trying
1439 		 * to access.
1440 		 */
1441 		if (offset_in_page >= mtd->writesize)
1442 			instrs[0].ctx.cmd.opcode = NAND_CMD_READOOB;
1443 		else if (offset_in_page >= 256 &&
1444 			 !(chip->options & NAND_BUSWIDTH_16))
1445 			instrs[0].ctx.cmd.opcode = NAND_CMD_READ1;
1446 	} else {
1447 		/*
1448 		 * Drop the first command if we're dealing with a large page
1449 		 * NAND.
1450 		 */
1451 		op.instrs++;
1452 		op.ninstrs--;
1453 	}
1454 
1455 	ret = nand_exec_op(chip, &op);
1456 	if (!prog || ret)
1457 		return ret;
1458 
1459 	ret = nand_status_op(chip, &status);
1460 	if (ret)
1461 		return ret;
1462 
1463 	return status;
1464 }
1465 
1466 /**
1467  * nand_prog_page_begin_op - starts a PROG PAGE operation
1468  * @chip: The NAND chip
1469  * @page: page to write
1470  * @offset_in_page: offset within the page
1471  * @buf: buffer containing the data to write to the page
1472  * @len: length of the buffer
1473  *
1474  * This function issues the first half of a PROG PAGE operation.
1475  * This function does not select/unselect the CS line.
1476  *
1477  * Returns 0 on success, a negative error code otherwise.
1478  */
1479 int nand_prog_page_begin_op(struct nand_chip *chip, unsigned int page,
1480 			    unsigned int offset_in_page, const void *buf,
1481 			    unsigned int len)
1482 {
1483 	struct mtd_info *mtd = nand_to_mtd(chip);
1484 
1485 	if (len && !buf)
1486 		return -EINVAL;
1487 
1488 	if (offset_in_page + len > mtd->writesize + mtd->oobsize)
1489 		return -EINVAL;
1490 
1491 	if (nand_has_exec_op(chip))
1492 		return nand_exec_prog_page_op(chip, page, offset_in_page, buf,
1493 					      len, false);
1494 
1495 	chip->legacy.cmdfunc(chip, NAND_CMD_SEQIN, offset_in_page, page);
1496 
1497 	if (buf)
1498 		chip->legacy.write_buf(chip, buf, len);
1499 
1500 	return 0;
1501 }
1502 EXPORT_SYMBOL_GPL(nand_prog_page_begin_op);
1503 
1504 /**
1505  * nand_prog_page_end_op - ends a PROG PAGE operation
1506  * @chip: The NAND chip
1507  *
1508  * This function issues the second half of a PROG PAGE operation.
1509  * This function does not select/unselect the CS line.
1510  *
1511  * Returns 0 on success, a negative error code otherwise.
1512  */
1513 int nand_prog_page_end_op(struct nand_chip *chip)
1514 {
1515 	int ret;
1516 	u8 status;
1517 
1518 	if (nand_has_exec_op(chip)) {
1519 		const struct nand_sdr_timings *sdr =
1520 			nand_get_sdr_timings(&chip->data_interface);
1521 		struct nand_op_instr instrs[] = {
1522 			NAND_OP_CMD(NAND_CMD_PAGEPROG,
1523 				    PSEC_TO_NSEC(sdr->tWB_max)),
1524 			NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tPROG_max), 0),
1525 		};
1526 		struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1527 
1528 		ret = nand_exec_op(chip, &op);
1529 		if (ret)
1530 			return ret;
1531 
1532 		ret = nand_status_op(chip, &status);
1533 		if (ret)
1534 			return ret;
1535 	} else {
1536 		chip->legacy.cmdfunc(chip, NAND_CMD_PAGEPROG, -1, -1);
1537 		ret = chip->legacy.waitfunc(chip);
1538 		if (ret < 0)
1539 			return ret;
1540 
1541 		status = ret;
1542 	}
1543 
1544 	if (status & NAND_STATUS_FAIL)
1545 		return -EIO;
1546 
1547 	return 0;
1548 }
1549 EXPORT_SYMBOL_GPL(nand_prog_page_end_op);
1550 
1551 /**
1552  * nand_prog_page_op - Do a full PROG PAGE operation
1553  * @chip: The NAND chip
1554  * @page: page to write
1555  * @offset_in_page: offset within the page
1556  * @buf: buffer containing the data to write to the page
1557  * @len: length of the buffer
1558  *
1559  * This function issues a full PROG PAGE operation.
1560  * This function does not select/unselect the CS line.
1561  *
1562  * Returns 0 on success, a negative error code otherwise.
1563  */
1564 int nand_prog_page_op(struct nand_chip *chip, unsigned int page,
1565 		      unsigned int offset_in_page, const void *buf,
1566 		      unsigned int len)
1567 {
1568 	struct mtd_info *mtd = nand_to_mtd(chip);
1569 	int status;
1570 
1571 	if (!len || !buf)
1572 		return -EINVAL;
1573 
1574 	if (offset_in_page + len > mtd->writesize + mtd->oobsize)
1575 		return -EINVAL;
1576 
1577 	if (nand_has_exec_op(chip)) {
1578 		status = nand_exec_prog_page_op(chip, page, offset_in_page, buf,
1579 						len, true);
1580 	} else {
1581 		chip->legacy.cmdfunc(chip, NAND_CMD_SEQIN, offset_in_page,
1582 				     page);
1583 		chip->legacy.write_buf(chip, buf, len);
1584 		chip->legacy.cmdfunc(chip, NAND_CMD_PAGEPROG, -1, -1);
1585 		status = chip->legacy.waitfunc(chip);
1586 	}
1587 
1588 	if (status & NAND_STATUS_FAIL)
1589 		return -EIO;
1590 
1591 	return 0;
1592 }
1593 EXPORT_SYMBOL_GPL(nand_prog_page_op);
1594 
1595 /**
1596  * nand_change_write_column_op - Do a CHANGE WRITE COLUMN operation
1597  * @chip: The NAND chip
1598  * @offset_in_page: offset within the page
1599  * @buf: buffer containing the data to send to the NAND
1600  * @len: length of the buffer
1601  * @force_8bit: force 8-bit bus access
1602  *
1603  * This function issues a CHANGE WRITE COLUMN operation.
1604  * This function does not select/unselect the CS line.
1605  *
1606  * Returns 0 on success, a negative error code otherwise.
1607  */
1608 int nand_change_write_column_op(struct nand_chip *chip,
1609 				unsigned int offset_in_page,
1610 				const void *buf, unsigned int len,
1611 				bool force_8bit)
1612 {
1613 	struct mtd_info *mtd = nand_to_mtd(chip);
1614 
1615 	if (len && !buf)
1616 		return -EINVAL;
1617 
1618 	if (offset_in_page + len > mtd->writesize + mtd->oobsize)
1619 		return -EINVAL;
1620 
1621 	/* Small page NANDs do not support column change. */
1622 	if (mtd->writesize <= 512)
1623 		return -ENOTSUPP;
1624 
1625 	if (nand_has_exec_op(chip)) {
1626 		const struct nand_sdr_timings *sdr =
1627 			nand_get_sdr_timings(&chip->data_interface);
1628 		u8 addrs[2];
1629 		struct nand_op_instr instrs[] = {
1630 			NAND_OP_CMD(NAND_CMD_RNDIN, 0),
1631 			NAND_OP_ADDR(2, addrs, PSEC_TO_NSEC(sdr->tCCS_min)),
1632 			NAND_OP_DATA_OUT(len, buf, 0),
1633 		};
1634 		struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1635 		int ret;
1636 
1637 		ret = nand_fill_column_cycles(chip, addrs, offset_in_page);
1638 		if (ret < 0)
1639 			return ret;
1640 
1641 		instrs[2].ctx.data.force_8bit = force_8bit;
1642 
1643 		/* Drop the DATA_OUT instruction if len is set to 0. */
1644 		if (!len)
1645 			op.ninstrs--;
1646 
1647 		return nand_exec_op(chip, &op);
1648 	}
1649 
1650 	chip->legacy.cmdfunc(chip, NAND_CMD_RNDIN, offset_in_page, -1);
1651 	if (len)
1652 		chip->legacy.write_buf(chip, buf, len);
1653 
1654 	return 0;
1655 }
1656 EXPORT_SYMBOL_GPL(nand_change_write_column_op);
1657 
1658 /**
1659  * nand_readid_op - Do a READID operation
1660  * @chip: The NAND chip
1661  * @addr: address cycle to pass after the READID command
1662  * @buf: buffer used to store the ID
1663  * @len: length of the buffer
1664  *
1665  * This function sends a READID command and reads back the ID returned by the
1666  * NAND.
1667  * This function does not select/unselect the CS line.
1668  *
1669  * Returns 0 on success, a negative error code otherwise.
1670  */
1671 int nand_readid_op(struct nand_chip *chip, u8 addr, void *buf,
1672 		   unsigned int len)
1673 {
1674 	unsigned int i;
1675 	u8 *id = buf;
1676 
1677 	if (len && !buf)
1678 		return -EINVAL;
1679 
1680 	if (nand_has_exec_op(chip)) {
1681 		const struct nand_sdr_timings *sdr =
1682 			nand_get_sdr_timings(&chip->data_interface);
1683 		struct nand_op_instr instrs[] = {
1684 			NAND_OP_CMD(NAND_CMD_READID, 0),
1685 			NAND_OP_ADDR(1, &addr, PSEC_TO_NSEC(sdr->tADL_min)),
1686 			NAND_OP_8BIT_DATA_IN(len, buf, 0),
1687 		};
1688 		struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1689 
1690 		/* Drop the DATA_IN instruction if len is set to 0. */
1691 		if (!len)
1692 			op.ninstrs--;
1693 
1694 		return nand_exec_op(chip, &op);
1695 	}
1696 
1697 	chip->legacy.cmdfunc(chip, NAND_CMD_READID, addr, -1);
1698 
1699 	for (i = 0; i < len; i++)
1700 		id[i] = chip->legacy.read_byte(chip);
1701 
1702 	return 0;
1703 }
1704 EXPORT_SYMBOL_GPL(nand_readid_op);
1705 
1706 /**
1707  * nand_status_op - Do a STATUS operation
1708  * @chip: The NAND chip
1709  * @status: out variable to store the NAND status
1710  *
1711  * This function sends a STATUS command and reads back the status returned by
1712  * the NAND.
1713  * This function does not select/unselect the CS line.
1714  *
1715  * Returns 0 on success, a negative error code otherwise.
1716  */
1717 int nand_status_op(struct nand_chip *chip, u8 *status)
1718 {
1719 	if (nand_has_exec_op(chip)) {
1720 		const struct nand_sdr_timings *sdr =
1721 			nand_get_sdr_timings(&chip->data_interface);
1722 		struct nand_op_instr instrs[] = {
1723 			NAND_OP_CMD(NAND_CMD_STATUS,
1724 				    PSEC_TO_NSEC(sdr->tADL_min)),
1725 			NAND_OP_8BIT_DATA_IN(1, status, 0),
1726 		};
1727 		struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1728 
1729 		if (!status)
1730 			op.ninstrs--;
1731 
1732 		return nand_exec_op(chip, &op);
1733 	}
1734 
1735 	chip->legacy.cmdfunc(chip, NAND_CMD_STATUS, -1, -1);
1736 	if (status)
1737 		*status = chip->legacy.read_byte(chip);
1738 
1739 	return 0;
1740 }
1741 EXPORT_SYMBOL_GPL(nand_status_op);
1742 
1743 /**
1744  * nand_exit_status_op - Exit a STATUS operation
1745  * @chip: The NAND chip
1746  *
1747  * This function sends a READ0 command to cancel the effect of the STATUS
1748  * command to avoid reading only the status until a new read command is sent.
1749  *
1750  * This function does not select/unselect the CS line.
1751  *
1752  * Returns 0 on success, a negative error code otherwise.
1753  */
1754 int nand_exit_status_op(struct nand_chip *chip)
1755 {
1756 	if (nand_has_exec_op(chip)) {
1757 		struct nand_op_instr instrs[] = {
1758 			NAND_OP_CMD(NAND_CMD_READ0, 0),
1759 		};
1760 		struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1761 
1762 		return nand_exec_op(chip, &op);
1763 	}
1764 
1765 	chip->legacy.cmdfunc(chip, NAND_CMD_READ0, -1, -1);
1766 
1767 	return 0;
1768 }
1769 
1770 /**
1771  * nand_erase_op - Do an erase operation
1772  * @chip: The NAND chip
1773  * @eraseblock: block to erase
1774  *
1775  * This function sends an ERASE command and waits for the NAND to be ready
1776  * before returning.
1777  * This function does not select/unselect the CS line.
1778  *
1779  * Returns 0 on success, a negative error code otherwise.
1780  */
1781 int nand_erase_op(struct nand_chip *chip, unsigned int eraseblock)
1782 {
1783 	unsigned int page = eraseblock <<
1784 			    (chip->phys_erase_shift - chip->page_shift);
1785 	int ret;
1786 	u8 status;
1787 
1788 	if (nand_has_exec_op(chip)) {
1789 		const struct nand_sdr_timings *sdr =
1790 			nand_get_sdr_timings(&chip->data_interface);
1791 		u8 addrs[3] = {	page, page >> 8, page >> 16 };
1792 		struct nand_op_instr instrs[] = {
1793 			NAND_OP_CMD(NAND_CMD_ERASE1, 0),
1794 			NAND_OP_ADDR(2, addrs, 0),
1795 			NAND_OP_CMD(NAND_CMD_ERASE2,
1796 				    PSEC_TO_MSEC(sdr->tWB_max)),
1797 			NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tBERS_max), 0),
1798 		};
1799 		struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1800 
1801 		if (chip->options & NAND_ROW_ADDR_3)
1802 			instrs[1].ctx.addr.naddrs++;
1803 
1804 		ret = nand_exec_op(chip, &op);
1805 		if (ret)
1806 			return ret;
1807 
1808 		ret = nand_status_op(chip, &status);
1809 		if (ret)
1810 			return ret;
1811 	} else {
1812 		chip->legacy.cmdfunc(chip, NAND_CMD_ERASE1, -1, page);
1813 		chip->legacy.cmdfunc(chip, NAND_CMD_ERASE2, -1, -1);
1814 
1815 		ret = chip->legacy.waitfunc(chip);
1816 		if (ret < 0)
1817 			return ret;
1818 
1819 		status = ret;
1820 	}
1821 
1822 	if (status & NAND_STATUS_FAIL)
1823 		return -EIO;
1824 
1825 	return 0;
1826 }
1827 EXPORT_SYMBOL_GPL(nand_erase_op);
1828 
1829 /**
1830  * nand_set_features_op - Do a SET FEATURES operation
1831  * @chip: The NAND chip
1832  * @feature: feature id
1833  * @data: 4 bytes of data
1834  *
1835  * This function sends a SET FEATURES command and waits for the NAND to be
1836  * ready before returning.
1837  * This function does not select/unselect the CS line.
1838  *
1839  * Returns 0 on success, a negative error code otherwise.
1840  */
1841 static int nand_set_features_op(struct nand_chip *chip, u8 feature,
1842 				const void *data)
1843 {
1844 	const u8 *params = data;
1845 	int i, ret;
1846 
1847 	if (nand_has_exec_op(chip)) {
1848 		const struct nand_sdr_timings *sdr =
1849 			nand_get_sdr_timings(&chip->data_interface);
1850 		struct nand_op_instr instrs[] = {
1851 			NAND_OP_CMD(NAND_CMD_SET_FEATURES, 0),
1852 			NAND_OP_ADDR(1, &feature, PSEC_TO_NSEC(sdr->tADL_min)),
1853 			NAND_OP_8BIT_DATA_OUT(ONFI_SUBFEATURE_PARAM_LEN, data,
1854 					      PSEC_TO_NSEC(sdr->tWB_max)),
1855 			NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tFEAT_max), 0),
1856 		};
1857 		struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1858 
1859 		return nand_exec_op(chip, &op);
1860 	}
1861 
1862 	chip->legacy.cmdfunc(chip, NAND_CMD_SET_FEATURES, feature, -1);
1863 	for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; ++i)
1864 		chip->legacy.write_byte(chip, params[i]);
1865 
1866 	ret = chip->legacy.waitfunc(chip);
1867 	if (ret < 0)
1868 		return ret;
1869 
1870 	if (ret & NAND_STATUS_FAIL)
1871 		return -EIO;
1872 
1873 	return 0;
1874 }
1875 
1876 /**
1877  * nand_get_features_op - Do a GET FEATURES operation
1878  * @chip: The NAND chip
1879  * @feature: feature id
1880  * @data: 4 bytes of data
1881  *
1882  * This function sends a GET FEATURES command and waits for the NAND to be
1883  * ready before returning.
1884  * This function does not select/unselect the CS line.
1885  *
1886  * Returns 0 on success, a negative error code otherwise.
1887  */
1888 static int nand_get_features_op(struct nand_chip *chip, u8 feature,
1889 				void *data)
1890 {
1891 	u8 *params = data;
1892 	int i;
1893 
1894 	if (nand_has_exec_op(chip)) {
1895 		const struct nand_sdr_timings *sdr =
1896 			nand_get_sdr_timings(&chip->data_interface);
1897 		struct nand_op_instr instrs[] = {
1898 			NAND_OP_CMD(NAND_CMD_GET_FEATURES, 0),
1899 			NAND_OP_ADDR(1, &feature, PSEC_TO_NSEC(sdr->tWB_max)),
1900 			NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tFEAT_max),
1901 					 PSEC_TO_NSEC(sdr->tRR_min)),
1902 			NAND_OP_8BIT_DATA_IN(ONFI_SUBFEATURE_PARAM_LEN,
1903 					     data, 0),
1904 		};
1905 		struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1906 
1907 		return nand_exec_op(chip, &op);
1908 	}
1909 
1910 	chip->legacy.cmdfunc(chip, NAND_CMD_GET_FEATURES, feature, -1);
1911 	for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; ++i)
1912 		params[i] = chip->legacy.read_byte(chip);
1913 
1914 	return 0;
1915 }
1916 
1917 static int nand_wait_rdy_op(struct nand_chip *chip, unsigned int timeout_ms,
1918 			    unsigned int delay_ns)
1919 {
1920 	if (nand_has_exec_op(chip)) {
1921 		struct nand_op_instr instrs[] = {
1922 			NAND_OP_WAIT_RDY(PSEC_TO_MSEC(timeout_ms),
1923 					 PSEC_TO_NSEC(delay_ns)),
1924 		};
1925 		struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1926 
1927 		return nand_exec_op(chip, &op);
1928 	}
1929 
1930 	/* Apply delay or wait for ready/busy pin */
1931 	if (!chip->legacy.dev_ready)
1932 		udelay(chip->legacy.chip_delay);
1933 	else
1934 		nand_wait_ready(chip);
1935 
1936 	return 0;
1937 }
1938 
1939 /**
1940  * nand_reset_op - Do a reset operation
1941  * @chip: The NAND chip
1942  *
1943  * This function sends a RESET command and waits for the NAND to be ready
1944  * before returning.
1945  * This function does not select/unselect the CS line.
1946  *
1947  * Returns 0 on success, a negative error code otherwise.
1948  */
1949 int nand_reset_op(struct nand_chip *chip)
1950 {
1951 	if (nand_has_exec_op(chip)) {
1952 		const struct nand_sdr_timings *sdr =
1953 			nand_get_sdr_timings(&chip->data_interface);
1954 		struct nand_op_instr instrs[] = {
1955 			NAND_OP_CMD(NAND_CMD_RESET, PSEC_TO_NSEC(sdr->tWB_max)),
1956 			NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tRST_max), 0),
1957 		};
1958 		struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1959 
1960 		return nand_exec_op(chip, &op);
1961 	}
1962 
1963 	chip->legacy.cmdfunc(chip, NAND_CMD_RESET, -1, -1);
1964 
1965 	return 0;
1966 }
1967 EXPORT_SYMBOL_GPL(nand_reset_op);
1968 
1969 /**
1970  * nand_read_data_op - Read data from the NAND
1971  * @chip: The NAND chip
1972  * @buf: buffer used to store the data
1973  * @len: length of the buffer
1974  * @force_8bit: force 8-bit bus access
1975  * @check_only: do not actually run the command, only checks if the
1976  *              controller driver supports it
1977  *
1978  * This function does a raw data read on the bus. Usually used after launching
1979  * another NAND operation like nand_read_page_op().
1980  * This function does not select/unselect the CS line.
1981  *
1982  * Returns 0 on success, a negative error code otherwise.
1983  */
1984 int nand_read_data_op(struct nand_chip *chip, void *buf, unsigned int len,
1985 		      bool force_8bit, bool check_only)
1986 {
1987 	if (!len || !buf)
1988 		return -EINVAL;
1989 
1990 	if (nand_has_exec_op(chip)) {
1991 		struct nand_op_instr instrs[] = {
1992 			NAND_OP_DATA_IN(len, buf, 0),
1993 		};
1994 		struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1995 
1996 		instrs[0].ctx.data.force_8bit = force_8bit;
1997 
1998 		if (check_only)
1999 			return nand_check_op(chip, &op);
2000 
2001 		return nand_exec_op(chip, &op);
2002 	}
2003 
2004 	if (check_only)
2005 		return 0;
2006 
2007 	if (force_8bit) {
2008 		u8 *p = buf;
2009 		unsigned int i;
2010 
2011 		for (i = 0; i < len; i++)
2012 			p[i] = chip->legacy.read_byte(chip);
2013 	} else {
2014 		chip->legacy.read_buf(chip, buf, len);
2015 	}
2016 
2017 	return 0;
2018 }
2019 EXPORT_SYMBOL_GPL(nand_read_data_op);
2020 
2021 /**
2022  * nand_write_data_op - Write data from the NAND
2023  * @chip: The NAND chip
2024  * @buf: buffer containing the data to send on the bus
2025  * @len: length of the buffer
2026  * @force_8bit: force 8-bit bus access
2027  *
2028  * This function does a raw data write on the bus. Usually used after launching
2029  * another NAND operation like nand_write_page_begin_op().
2030  * This function does not select/unselect the CS line.
2031  *
2032  * Returns 0 on success, a negative error code otherwise.
2033  */
2034 int nand_write_data_op(struct nand_chip *chip, const void *buf,
2035 		       unsigned int len, bool force_8bit)
2036 {
2037 	if (!len || !buf)
2038 		return -EINVAL;
2039 
2040 	if (nand_has_exec_op(chip)) {
2041 		struct nand_op_instr instrs[] = {
2042 			NAND_OP_DATA_OUT(len, buf, 0),
2043 		};
2044 		struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
2045 
2046 		instrs[0].ctx.data.force_8bit = force_8bit;
2047 
2048 		return nand_exec_op(chip, &op);
2049 	}
2050 
2051 	if (force_8bit) {
2052 		const u8 *p = buf;
2053 		unsigned int i;
2054 
2055 		for (i = 0; i < len; i++)
2056 			chip->legacy.write_byte(chip, p[i]);
2057 	} else {
2058 		chip->legacy.write_buf(chip, buf, len);
2059 	}
2060 
2061 	return 0;
2062 }
2063 EXPORT_SYMBOL_GPL(nand_write_data_op);
2064 
2065 /**
2066  * struct nand_op_parser_ctx - Context used by the parser
2067  * @instrs: array of all the instructions that must be addressed
2068  * @ninstrs: length of the @instrs array
2069  * @subop: Sub-operation to be passed to the NAND controller
2070  *
2071  * This structure is used by the core to split NAND operations into
2072  * sub-operations that can be handled by the NAND controller.
2073  */
2074 struct nand_op_parser_ctx {
2075 	const struct nand_op_instr *instrs;
2076 	unsigned int ninstrs;
2077 	struct nand_subop subop;
2078 };
2079 
2080 /**
2081  * nand_op_parser_must_split_instr - Checks if an instruction must be split
2082  * @pat: the parser pattern element that matches @instr
2083  * @instr: pointer to the instruction to check
2084  * @start_offset: this is an in/out parameter. If @instr has already been
2085  *		  split, then @start_offset is the offset from which to start
2086  *		  (either an address cycle or an offset in the data buffer).
2087  *		  Conversely, if the function returns true (ie. instr must be
2088  *		  split), this parameter is updated to point to the first
2089  *		  data/address cycle that has not been taken care of.
2090  *
2091  * Some NAND controllers are limited and cannot send X address cycles with a
2092  * unique operation, or cannot read/write more than Y bytes at the same time.
2093  * In this case, split the instruction that does not fit in a single
2094  * controller-operation into two or more chunks.
2095  *
2096  * Returns true if the instruction must be split, false otherwise.
2097  * The @start_offset parameter is also updated to the offset at which the next
2098  * bundle of instruction must start (if an address or a data instruction).
2099  */
2100 static bool
2101 nand_op_parser_must_split_instr(const struct nand_op_parser_pattern_elem *pat,
2102 				const struct nand_op_instr *instr,
2103 				unsigned int *start_offset)
2104 {
2105 	switch (pat->type) {
2106 	case NAND_OP_ADDR_INSTR:
2107 		if (!pat->ctx.addr.maxcycles)
2108 			break;
2109 
2110 		if (instr->ctx.addr.naddrs - *start_offset >
2111 		    pat->ctx.addr.maxcycles) {
2112 			*start_offset += pat->ctx.addr.maxcycles;
2113 			return true;
2114 		}
2115 		break;
2116 
2117 	case NAND_OP_DATA_IN_INSTR:
2118 	case NAND_OP_DATA_OUT_INSTR:
2119 		if (!pat->ctx.data.maxlen)
2120 			break;
2121 
2122 		if (instr->ctx.data.len - *start_offset >
2123 		    pat->ctx.data.maxlen) {
2124 			*start_offset += pat->ctx.data.maxlen;
2125 			return true;
2126 		}
2127 		break;
2128 
2129 	default:
2130 		break;
2131 	}
2132 
2133 	return false;
2134 }
2135 
2136 /**
2137  * nand_op_parser_match_pat - Checks if a pattern matches the instructions
2138  *			      remaining in the parser context
2139  * @pat: the pattern to test
2140  * @ctx: the parser context structure to match with the pattern @pat
2141  *
2142  * Check if @pat matches the set or a sub-set of instructions remaining in @ctx.
2143  * Returns true if this is the case, false ortherwise. When true is returned,
2144  * @ctx->subop is updated with the set of instructions to be passed to the
2145  * controller driver.
2146  */
2147 static bool
2148 nand_op_parser_match_pat(const struct nand_op_parser_pattern *pat,
2149 			 struct nand_op_parser_ctx *ctx)
2150 {
2151 	unsigned int instr_offset = ctx->subop.first_instr_start_off;
2152 	const struct nand_op_instr *end = ctx->instrs + ctx->ninstrs;
2153 	const struct nand_op_instr *instr = ctx->subop.instrs;
2154 	unsigned int i, ninstrs;
2155 
2156 	for (i = 0, ninstrs = 0; i < pat->nelems && instr < end; i++) {
2157 		/*
2158 		 * The pattern instruction does not match the operation
2159 		 * instruction. If the instruction is marked optional in the
2160 		 * pattern definition, we skip the pattern element and continue
2161 		 * to the next one. If the element is mandatory, there's no
2162 		 * match and we can return false directly.
2163 		 */
2164 		if (instr->type != pat->elems[i].type) {
2165 			if (!pat->elems[i].optional)
2166 				return false;
2167 
2168 			continue;
2169 		}
2170 
2171 		/*
2172 		 * Now check the pattern element constraints. If the pattern is
2173 		 * not able to handle the whole instruction in a single step,
2174 		 * we have to split it.
2175 		 * The last_instr_end_off value comes back updated to point to
2176 		 * the position where we have to split the instruction (the
2177 		 * start of the next subop chunk).
2178 		 */
2179 		if (nand_op_parser_must_split_instr(&pat->elems[i], instr,
2180 						    &instr_offset)) {
2181 			ninstrs++;
2182 			i++;
2183 			break;
2184 		}
2185 
2186 		instr++;
2187 		ninstrs++;
2188 		instr_offset = 0;
2189 	}
2190 
2191 	/*
2192 	 * This can happen if all instructions of a pattern are optional.
2193 	 * Still, if there's not at least one instruction handled by this
2194 	 * pattern, this is not a match, and we should try the next one (if
2195 	 * any).
2196 	 */
2197 	if (!ninstrs)
2198 		return false;
2199 
2200 	/*
2201 	 * We had a match on the pattern head, but the pattern may be longer
2202 	 * than the instructions we're asked to execute. We need to make sure
2203 	 * there's no mandatory elements in the pattern tail.
2204 	 */
2205 	for (; i < pat->nelems; i++) {
2206 		if (!pat->elems[i].optional)
2207 			return false;
2208 	}
2209 
2210 	/*
2211 	 * We have a match: update the subop structure accordingly and return
2212 	 * true.
2213 	 */
2214 	ctx->subop.ninstrs = ninstrs;
2215 	ctx->subop.last_instr_end_off = instr_offset;
2216 
2217 	return true;
2218 }
2219 
2220 #if IS_ENABLED(CONFIG_DYNAMIC_DEBUG) || defined(DEBUG)
2221 static void nand_op_parser_trace(const struct nand_op_parser_ctx *ctx)
2222 {
2223 	const struct nand_op_instr *instr;
2224 	char *prefix = "      ";
2225 	unsigned int i;
2226 
2227 	pr_debug("executing subop (CS%d):\n", ctx->subop.cs);
2228 
2229 	for (i = 0; i < ctx->ninstrs; i++) {
2230 		instr = &ctx->instrs[i];
2231 
2232 		if (instr == &ctx->subop.instrs[0])
2233 			prefix = "    ->";
2234 
2235 		nand_op_trace(prefix, instr);
2236 
2237 		if (instr == &ctx->subop.instrs[ctx->subop.ninstrs - 1])
2238 			prefix = "      ";
2239 	}
2240 }
2241 #else
2242 static void nand_op_parser_trace(const struct nand_op_parser_ctx *ctx)
2243 {
2244 	/* NOP */
2245 }
2246 #endif
2247 
2248 static int nand_op_parser_cmp_ctx(const struct nand_op_parser_ctx *a,
2249 				  const struct nand_op_parser_ctx *b)
2250 {
2251 	if (a->subop.ninstrs < b->subop.ninstrs)
2252 		return -1;
2253 	else if (a->subop.ninstrs > b->subop.ninstrs)
2254 		return 1;
2255 
2256 	if (a->subop.last_instr_end_off < b->subop.last_instr_end_off)
2257 		return -1;
2258 	else if (a->subop.last_instr_end_off > b->subop.last_instr_end_off)
2259 		return 1;
2260 
2261 	return 0;
2262 }
2263 
2264 /**
2265  * nand_op_parser_exec_op - exec_op parser
2266  * @chip: the NAND chip
2267  * @parser: patterns description provided by the controller driver
2268  * @op: the NAND operation to address
2269  * @check_only: when true, the function only checks if @op can be handled but
2270  *		does not execute the operation
2271  *
2272  * Helper function designed to ease integration of NAND controller drivers that
2273  * only support a limited set of instruction sequences. The supported sequences
2274  * are described in @parser, and the framework takes care of splitting @op into
2275  * multiple sub-operations (if required) and pass them back to the ->exec()
2276  * callback of the matching pattern if @check_only is set to false.
2277  *
2278  * NAND controller drivers should call this function from their own ->exec_op()
2279  * implementation.
2280  *
2281  * Returns 0 on success, a negative error code otherwise. A failure can be
2282  * caused by an unsupported operation (none of the supported patterns is able
2283  * to handle the requested operation), or an error returned by one of the
2284  * matching pattern->exec() hook.
2285  */
2286 int nand_op_parser_exec_op(struct nand_chip *chip,
2287 			   const struct nand_op_parser *parser,
2288 			   const struct nand_operation *op, bool check_only)
2289 {
2290 	struct nand_op_parser_ctx ctx = {
2291 		.subop.cs = op->cs,
2292 		.subop.instrs = op->instrs,
2293 		.instrs = op->instrs,
2294 		.ninstrs = op->ninstrs,
2295 	};
2296 	unsigned int i;
2297 
2298 	while (ctx.subop.instrs < op->instrs + op->ninstrs) {
2299 		const struct nand_op_parser_pattern *pattern;
2300 		struct nand_op_parser_ctx best_ctx;
2301 		int ret, best_pattern = -1;
2302 
2303 		for (i = 0; i < parser->npatterns; i++) {
2304 			struct nand_op_parser_ctx test_ctx = ctx;
2305 
2306 			pattern = &parser->patterns[i];
2307 			if (!nand_op_parser_match_pat(pattern, &test_ctx))
2308 				continue;
2309 
2310 			if (best_pattern >= 0 &&
2311 			    nand_op_parser_cmp_ctx(&test_ctx, &best_ctx) <= 0)
2312 				continue;
2313 
2314 			best_pattern = i;
2315 			best_ctx = test_ctx;
2316 		}
2317 
2318 		if (best_pattern < 0) {
2319 			pr_debug("->exec_op() parser: pattern not found!\n");
2320 			return -ENOTSUPP;
2321 		}
2322 
2323 		ctx = best_ctx;
2324 		nand_op_parser_trace(&ctx);
2325 
2326 		if (!check_only) {
2327 			pattern = &parser->patterns[best_pattern];
2328 			ret = pattern->exec(chip, &ctx.subop);
2329 			if (ret)
2330 				return ret;
2331 		}
2332 
2333 		/*
2334 		 * Update the context structure by pointing to the start of the
2335 		 * next subop.
2336 		 */
2337 		ctx.subop.instrs = ctx.subop.instrs + ctx.subop.ninstrs;
2338 		if (ctx.subop.last_instr_end_off)
2339 			ctx.subop.instrs -= 1;
2340 
2341 		ctx.subop.first_instr_start_off = ctx.subop.last_instr_end_off;
2342 	}
2343 
2344 	return 0;
2345 }
2346 EXPORT_SYMBOL_GPL(nand_op_parser_exec_op);
2347 
2348 static bool nand_instr_is_data(const struct nand_op_instr *instr)
2349 {
2350 	return instr && (instr->type == NAND_OP_DATA_IN_INSTR ||
2351 			 instr->type == NAND_OP_DATA_OUT_INSTR);
2352 }
2353 
2354 static bool nand_subop_instr_is_valid(const struct nand_subop *subop,
2355 				      unsigned int instr_idx)
2356 {
2357 	return subop && instr_idx < subop->ninstrs;
2358 }
2359 
2360 static unsigned int nand_subop_get_start_off(const struct nand_subop *subop,
2361 					     unsigned int instr_idx)
2362 {
2363 	if (instr_idx)
2364 		return 0;
2365 
2366 	return subop->first_instr_start_off;
2367 }
2368 
2369 /**
2370  * nand_subop_get_addr_start_off - Get the start offset in an address array
2371  * @subop: The entire sub-operation
2372  * @instr_idx: Index of the instruction inside the sub-operation
2373  *
2374  * During driver development, one could be tempted to directly use the
2375  * ->addr.addrs field of address instructions. This is wrong as address
2376  * instructions might be split.
2377  *
2378  * Given an address instruction, returns the offset of the first cycle to issue.
2379  */
2380 unsigned int nand_subop_get_addr_start_off(const struct nand_subop *subop,
2381 					   unsigned int instr_idx)
2382 {
2383 	if (WARN_ON(!nand_subop_instr_is_valid(subop, instr_idx) ||
2384 		    subop->instrs[instr_idx].type != NAND_OP_ADDR_INSTR))
2385 		return 0;
2386 
2387 	return nand_subop_get_start_off(subop, instr_idx);
2388 }
2389 EXPORT_SYMBOL_GPL(nand_subop_get_addr_start_off);
2390 
2391 /**
2392  * nand_subop_get_num_addr_cyc - Get the remaining address cycles to assert
2393  * @subop: The entire sub-operation
2394  * @instr_idx: Index of the instruction inside the sub-operation
2395  *
2396  * During driver development, one could be tempted to directly use the
2397  * ->addr->naddrs field of a data instruction. This is wrong as instructions
2398  * might be split.
2399  *
2400  * Given an address instruction, returns the number of address cycle to issue.
2401  */
2402 unsigned int nand_subop_get_num_addr_cyc(const struct nand_subop *subop,
2403 					 unsigned int instr_idx)
2404 {
2405 	int start_off, end_off;
2406 
2407 	if (WARN_ON(!nand_subop_instr_is_valid(subop, instr_idx) ||
2408 		    subop->instrs[instr_idx].type != NAND_OP_ADDR_INSTR))
2409 		return 0;
2410 
2411 	start_off = nand_subop_get_addr_start_off(subop, instr_idx);
2412 
2413 	if (instr_idx == subop->ninstrs - 1 &&
2414 	    subop->last_instr_end_off)
2415 		end_off = subop->last_instr_end_off;
2416 	else
2417 		end_off = subop->instrs[instr_idx].ctx.addr.naddrs;
2418 
2419 	return end_off - start_off;
2420 }
2421 EXPORT_SYMBOL_GPL(nand_subop_get_num_addr_cyc);
2422 
2423 /**
2424  * nand_subop_get_data_start_off - Get the start offset in a data array
2425  * @subop: The entire sub-operation
2426  * @instr_idx: Index of the instruction inside the sub-operation
2427  *
2428  * During driver development, one could be tempted to directly use the
2429  * ->data->buf.{in,out} field of data instructions. This is wrong as data
2430  * instructions might be split.
2431  *
2432  * Given a data instruction, returns the offset to start from.
2433  */
2434 unsigned int nand_subop_get_data_start_off(const struct nand_subop *subop,
2435 					   unsigned int instr_idx)
2436 {
2437 	if (WARN_ON(!nand_subop_instr_is_valid(subop, instr_idx) ||
2438 		    !nand_instr_is_data(&subop->instrs[instr_idx])))
2439 		return 0;
2440 
2441 	return nand_subop_get_start_off(subop, instr_idx);
2442 }
2443 EXPORT_SYMBOL_GPL(nand_subop_get_data_start_off);
2444 
2445 /**
2446  * nand_subop_get_data_len - Get the number of bytes to retrieve
2447  * @subop: The entire sub-operation
2448  * @instr_idx: Index of the instruction inside the sub-operation
2449  *
2450  * During driver development, one could be tempted to directly use the
2451  * ->data->len field of a data instruction. This is wrong as data instructions
2452  * might be split.
2453  *
2454  * Returns the length of the chunk of data to send/receive.
2455  */
2456 unsigned int nand_subop_get_data_len(const struct nand_subop *subop,
2457 				     unsigned int instr_idx)
2458 {
2459 	int start_off = 0, end_off;
2460 
2461 	if (WARN_ON(!nand_subop_instr_is_valid(subop, instr_idx) ||
2462 		    !nand_instr_is_data(&subop->instrs[instr_idx])))
2463 		return 0;
2464 
2465 	start_off = nand_subop_get_data_start_off(subop, instr_idx);
2466 
2467 	if (instr_idx == subop->ninstrs - 1 &&
2468 	    subop->last_instr_end_off)
2469 		end_off = subop->last_instr_end_off;
2470 	else
2471 		end_off = subop->instrs[instr_idx].ctx.data.len;
2472 
2473 	return end_off - start_off;
2474 }
2475 EXPORT_SYMBOL_GPL(nand_subop_get_data_len);
2476 
2477 /**
2478  * nand_reset - Reset and initialize a NAND device
2479  * @chip: The NAND chip
2480  * @chipnr: Internal die id
2481  *
2482  * Save the timings data structure, then apply SDR timings mode 0 (see
2483  * nand_reset_data_interface for details), do the reset operation, and
2484  * apply back the previous timings.
2485  *
2486  * Returns 0 on success, a negative error code otherwise.
2487  */
2488 int nand_reset(struct nand_chip *chip, int chipnr)
2489 {
2490 	struct nand_data_interface saved_data_intf = chip->data_interface;
2491 	int ret;
2492 
2493 	ret = nand_reset_data_interface(chip, chipnr);
2494 	if (ret)
2495 		return ret;
2496 
2497 	/*
2498 	 * The CS line has to be released before we can apply the new NAND
2499 	 * interface settings, hence this weird nand_select_target()
2500 	 * nand_deselect_target() dance.
2501 	 */
2502 	nand_select_target(chip, chipnr);
2503 	ret = nand_reset_op(chip);
2504 	nand_deselect_target(chip);
2505 	if (ret)
2506 		return ret;
2507 
2508 	/*
2509 	 * A nand_reset_data_interface() put both the NAND chip and the NAND
2510 	 * controller in timings mode 0. If the default mode for this chip is
2511 	 * also 0, no need to proceed to the change again. Plus, at probe time,
2512 	 * nand_setup_data_interface() uses ->set/get_features() which would
2513 	 * fail anyway as the parameter page is not available yet.
2514 	 */
2515 	if (!chip->onfi_timing_mode_default)
2516 		return 0;
2517 
2518 	chip->data_interface = saved_data_intf;
2519 	ret = nand_setup_data_interface(chip, chipnr);
2520 	if (ret)
2521 		return ret;
2522 
2523 	return 0;
2524 }
2525 EXPORT_SYMBOL_GPL(nand_reset);
2526 
2527 /**
2528  * nand_get_features - wrapper to perform a GET_FEATURE
2529  * @chip: NAND chip info structure
2530  * @addr: feature address
2531  * @subfeature_param: the subfeature parameters, a four bytes array
2532  *
2533  * Returns 0 for success, a negative error otherwise. Returns -ENOTSUPP if the
2534  * operation cannot be handled.
2535  */
2536 int nand_get_features(struct nand_chip *chip, int addr,
2537 		      u8 *subfeature_param)
2538 {
2539 	if (!nand_supports_get_features(chip, addr))
2540 		return -ENOTSUPP;
2541 
2542 	if (chip->legacy.get_features)
2543 		return chip->legacy.get_features(chip, addr, subfeature_param);
2544 
2545 	return nand_get_features_op(chip, addr, subfeature_param);
2546 }
2547 
2548 /**
2549  * nand_set_features - wrapper to perform a SET_FEATURE
2550  * @chip: NAND chip info structure
2551  * @addr: feature address
2552  * @subfeature_param: the subfeature parameters, a four bytes array
2553  *
2554  * Returns 0 for success, a negative error otherwise. Returns -ENOTSUPP if the
2555  * operation cannot be handled.
2556  */
2557 int nand_set_features(struct nand_chip *chip, int addr,
2558 		      u8 *subfeature_param)
2559 {
2560 	if (!nand_supports_set_features(chip, addr))
2561 		return -ENOTSUPP;
2562 
2563 	if (chip->legacy.set_features)
2564 		return chip->legacy.set_features(chip, addr, subfeature_param);
2565 
2566 	return nand_set_features_op(chip, addr, subfeature_param);
2567 }
2568 
2569 /**
2570  * nand_check_erased_buf - check if a buffer contains (almost) only 0xff data
2571  * @buf: buffer to test
2572  * @len: buffer length
2573  * @bitflips_threshold: maximum number of bitflips
2574  *
2575  * Check if a buffer contains only 0xff, which means the underlying region
2576  * has been erased and is ready to be programmed.
2577  * The bitflips_threshold specify the maximum number of bitflips before
2578  * considering the region is not erased.
2579  * Note: The logic of this function has been extracted from the memweight
2580  * implementation, except that nand_check_erased_buf function exit before
2581  * testing the whole buffer if the number of bitflips exceed the
2582  * bitflips_threshold value.
2583  *
2584  * Returns a positive number of bitflips less than or equal to
2585  * bitflips_threshold, or -ERROR_CODE for bitflips in excess of the
2586  * threshold.
2587  */
2588 static int nand_check_erased_buf(void *buf, int len, int bitflips_threshold)
2589 {
2590 	const unsigned char *bitmap = buf;
2591 	int bitflips = 0;
2592 	int weight;
2593 
2594 	for (; len && ((uintptr_t)bitmap) % sizeof(long);
2595 	     len--, bitmap++) {
2596 		weight = hweight8(*bitmap);
2597 		bitflips += BITS_PER_BYTE - weight;
2598 		if (unlikely(bitflips > bitflips_threshold))
2599 			return -EBADMSG;
2600 	}
2601 
2602 	for (; len >= sizeof(long);
2603 	     len -= sizeof(long), bitmap += sizeof(long)) {
2604 		unsigned long d = *((unsigned long *)bitmap);
2605 		if (d == ~0UL)
2606 			continue;
2607 		weight = hweight_long(d);
2608 		bitflips += BITS_PER_LONG - weight;
2609 		if (unlikely(bitflips > bitflips_threshold))
2610 			return -EBADMSG;
2611 	}
2612 
2613 	for (; len > 0; len--, bitmap++) {
2614 		weight = hweight8(*bitmap);
2615 		bitflips += BITS_PER_BYTE - weight;
2616 		if (unlikely(bitflips > bitflips_threshold))
2617 			return -EBADMSG;
2618 	}
2619 
2620 	return bitflips;
2621 }
2622 
2623 /**
2624  * nand_check_erased_ecc_chunk - check if an ECC chunk contains (almost) only
2625  *				 0xff data
2626  * @data: data buffer to test
2627  * @datalen: data length
2628  * @ecc: ECC buffer
2629  * @ecclen: ECC length
2630  * @extraoob: extra OOB buffer
2631  * @extraooblen: extra OOB length
2632  * @bitflips_threshold: maximum number of bitflips
2633  *
2634  * Check if a data buffer and its associated ECC and OOB data contains only
2635  * 0xff pattern, which means the underlying region has been erased and is
2636  * ready to be programmed.
2637  * The bitflips_threshold specify the maximum number of bitflips before
2638  * considering the region as not erased.
2639  *
2640  * Note:
2641  * 1/ ECC algorithms are working on pre-defined block sizes which are usually
2642  *    different from the NAND page size. When fixing bitflips, ECC engines will
2643  *    report the number of errors per chunk, and the NAND core infrastructure
2644  *    expect you to return the maximum number of bitflips for the whole page.
2645  *    This is why you should always use this function on a single chunk and
2646  *    not on the whole page. After checking each chunk you should update your
2647  *    max_bitflips value accordingly.
2648  * 2/ When checking for bitflips in erased pages you should not only check
2649  *    the payload data but also their associated ECC data, because a user might
2650  *    have programmed almost all bits to 1 but a few. In this case, we
2651  *    shouldn't consider the chunk as erased, and checking ECC bytes prevent
2652  *    this case.
2653  * 3/ The extraoob argument is optional, and should be used if some of your OOB
2654  *    data are protected by the ECC engine.
2655  *    It could also be used if you support subpages and want to attach some
2656  *    extra OOB data to an ECC chunk.
2657  *
2658  * Returns a positive number of bitflips less than or equal to
2659  * bitflips_threshold, or -ERROR_CODE for bitflips in excess of the
2660  * threshold. In case of success, the passed buffers are filled with 0xff.
2661  */
2662 int nand_check_erased_ecc_chunk(void *data, int datalen,
2663 				void *ecc, int ecclen,
2664 				void *extraoob, int extraooblen,
2665 				int bitflips_threshold)
2666 {
2667 	int data_bitflips = 0, ecc_bitflips = 0, extraoob_bitflips = 0;
2668 
2669 	data_bitflips = nand_check_erased_buf(data, datalen,
2670 					      bitflips_threshold);
2671 	if (data_bitflips < 0)
2672 		return data_bitflips;
2673 
2674 	bitflips_threshold -= data_bitflips;
2675 
2676 	ecc_bitflips = nand_check_erased_buf(ecc, ecclen, bitflips_threshold);
2677 	if (ecc_bitflips < 0)
2678 		return ecc_bitflips;
2679 
2680 	bitflips_threshold -= ecc_bitflips;
2681 
2682 	extraoob_bitflips = nand_check_erased_buf(extraoob, extraooblen,
2683 						  bitflips_threshold);
2684 	if (extraoob_bitflips < 0)
2685 		return extraoob_bitflips;
2686 
2687 	if (data_bitflips)
2688 		memset(data, 0xff, datalen);
2689 
2690 	if (ecc_bitflips)
2691 		memset(ecc, 0xff, ecclen);
2692 
2693 	if (extraoob_bitflips)
2694 		memset(extraoob, 0xff, extraooblen);
2695 
2696 	return data_bitflips + ecc_bitflips + extraoob_bitflips;
2697 }
2698 EXPORT_SYMBOL(nand_check_erased_ecc_chunk);
2699 
2700 /**
2701  * nand_read_page_raw_notsupp - dummy read raw page function
2702  * @chip: nand chip info structure
2703  * @buf: buffer to store read data
2704  * @oob_required: caller requires OOB data read to chip->oob_poi
2705  * @page: page number to read
2706  *
2707  * Returns -ENOTSUPP unconditionally.
2708  */
2709 int nand_read_page_raw_notsupp(struct nand_chip *chip, u8 *buf,
2710 			       int oob_required, int page)
2711 {
2712 	return -ENOTSUPP;
2713 }
2714 
2715 /**
2716  * nand_read_page_raw - [INTERN] read raw page data without ecc
2717  * @chip: nand chip info structure
2718  * @buf: buffer to store read data
2719  * @oob_required: caller requires OOB data read to chip->oob_poi
2720  * @page: page number to read
2721  *
2722  * Not for syndrome calculating ECC controllers, which use a special oob layout.
2723  */
2724 int nand_read_page_raw(struct nand_chip *chip, uint8_t *buf, int oob_required,
2725 		       int page)
2726 {
2727 	struct mtd_info *mtd = nand_to_mtd(chip);
2728 	int ret;
2729 
2730 	ret = nand_read_page_op(chip, page, 0, buf, mtd->writesize);
2731 	if (ret)
2732 		return ret;
2733 
2734 	if (oob_required) {
2735 		ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize,
2736 					false, false);
2737 		if (ret)
2738 			return ret;
2739 	}
2740 
2741 	return 0;
2742 }
2743 EXPORT_SYMBOL(nand_read_page_raw);
2744 
2745 /**
2746  * nand_monolithic_read_page_raw - Monolithic page read in raw mode
2747  * @chip: NAND chip info structure
2748  * @buf: buffer to store read data
2749  * @oob_required: caller requires OOB data read to chip->oob_poi
2750  * @page: page number to read
2751  *
2752  * This is a raw page read, ie. without any error detection/correction.
2753  * Monolithic means we are requesting all the relevant data (main plus
2754  * eventually OOB) to be loaded in the NAND cache and sent over the
2755  * bus (from the NAND chip to the NAND controller) in a single
2756  * operation. This is an alternative to nand_read_page_raw(), which
2757  * first reads the main data, and if the OOB data is requested too,
2758  * then reads more data on the bus.
2759  */
2760 int nand_monolithic_read_page_raw(struct nand_chip *chip, u8 *buf,
2761 				  int oob_required, int page)
2762 {
2763 	struct mtd_info *mtd = nand_to_mtd(chip);
2764 	unsigned int size = mtd->writesize;
2765 	u8 *read_buf = buf;
2766 	int ret;
2767 
2768 	if (oob_required) {
2769 		size += mtd->oobsize;
2770 
2771 		if (buf != chip->data_buf)
2772 			read_buf = nand_get_data_buf(chip);
2773 	}
2774 
2775 	ret = nand_read_page_op(chip, page, 0, read_buf, size);
2776 	if (ret)
2777 		return ret;
2778 
2779 	if (buf != chip->data_buf)
2780 		memcpy(buf, read_buf, mtd->writesize);
2781 
2782 	return 0;
2783 }
2784 EXPORT_SYMBOL(nand_monolithic_read_page_raw);
2785 
2786 /**
2787  * nand_read_page_raw_syndrome - [INTERN] read raw page data without ecc
2788  * @chip: nand chip info structure
2789  * @buf: buffer to store read data
2790  * @oob_required: caller requires OOB data read to chip->oob_poi
2791  * @page: page number to read
2792  *
2793  * We need a special oob layout and handling even when OOB isn't used.
2794  */
2795 static int nand_read_page_raw_syndrome(struct nand_chip *chip, uint8_t *buf,
2796 				       int oob_required, int page)
2797 {
2798 	struct mtd_info *mtd = nand_to_mtd(chip);
2799 	int eccsize = chip->ecc.size;
2800 	int eccbytes = chip->ecc.bytes;
2801 	uint8_t *oob = chip->oob_poi;
2802 	int steps, size, ret;
2803 
2804 	ret = nand_read_page_op(chip, page, 0, NULL, 0);
2805 	if (ret)
2806 		return ret;
2807 
2808 	for (steps = chip->ecc.steps; steps > 0; steps--) {
2809 		ret = nand_read_data_op(chip, buf, eccsize, false, false);
2810 		if (ret)
2811 			return ret;
2812 
2813 		buf += eccsize;
2814 
2815 		if (chip->ecc.prepad) {
2816 			ret = nand_read_data_op(chip, oob, chip->ecc.prepad,
2817 						false, false);
2818 			if (ret)
2819 				return ret;
2820 
2821 			oob += chip->ecc.prepad;
2822 		}
2823 
2824 		ret = nand_read_data_op(chip, oob, eccbytes, false, false);
2825 		if (ret)
2826 			return ret;
2827 
2828 		oob += eccbytes;
2829 
2830 		if (chip->ecc.postpad) {
2831 			ret = nand_read_data_op(chip, oob, chip->ecc.postpad,
2832 						false, false);
2833 			if (ret)
2834 				return ret;
2835 
2836 			oob += chip->ecc.postpad;
2837 		}
2838 	}
2839 
2840 	size = mtd->oobsize - (oob - chip->oob_poi);
2841 	if (size) {
2842 		ret = nand_read_data_op(chip, oob, size, false, false);
2843 		if (ret)
2844 			return ret;
2845 	}
2846 
2847 	return 0;
2848 }
2849 
2850 /**
2851  * nand_read_page_swecc - [REPLACEABLE] software ECC based page read function
2852  * @chip: nand chip info structure
2853  * @buf: buffer to store read data
2854  * @oob_required: caller requires OOB data read to chip->oob_poi
2855  * @page: page number to read
2856  */
2857 static int nand_read_page_swecc(struct nand_chip *chip, uint8_t *buf,
2858 				int oob_required, int page)
2859 {
2860 	struct mtd_info *mtd = nand_to_mtd(chip);
2861 	int i, eccsize = chip->ecc.size, ret;
2862 	int eccbytes = chip->ecc.bytes;
2863 	int eccsteps = chip->ecc.steps;
2864 	uint8_t *p = buf;
2865 	uint8_t *ecc_calc = chip->ecc.calc_buf;
2866 	uint8_t *ecc_code = chip->ecc.code_buf;
2867 	unsigned int max_bitflips = 0;
2868 
2869 	chip->ecc.read_page_raw(chip, buf, 1, page);
2870 
2871 	for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize)
2872 		chip->ecc.calculate(chip, p, &ecc_calc[i]);
2873 
2874 	ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
2875 					 chip->ecc.total);
2876 	if (ret)
2877 		return ret;
2878 
2879 	eccsteps = chip->ecc.steps;
2880 	p = buf;
2881 
2882 	for (i = 0 ; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
2883 		int stat;
2884 
2885 		stat = chip->ecc.correct(chip, p, &ecc_code[i], &ecc_calc[i]);
2886 		if (stat < 0) {
2887 			mtd->ecc_stats.failed++;
2888 		} else {
2889 			mtd->ecc_stats.corrected += stat;
2890 			max_bitflips = max_t(unsigned int, max_bitflips, stat);
2891 		}
2892 	}
2893 	return max_bitflips;
2894 }
2895 
2896 /**
2897  * nand_read_subpage - [REPLACEABLE] ECC based sub-page read function
2898  * @chip: nand chip info structure
2899  * @data_offs: offset of requested data within the page
2900  * @readlen: data length
2901  * @bufpoi: buffer to store read data
2902  * @page: page number to read
2903  */
2904 static int nand_read_subpage(struct nand_chip *chip, uint32_t data_offs,
2905 			     uint32_t readlen, uint8_t *bufpoi, int page)
2906 {
2907 	struct mtd_info *mtd = nand_to_mtd(chip);
2908 	int start_step, end_step, num_steps, ret;
2909 	uint8_t *p;
2910 	int data_col_addr, i, gaps = 0;
2911 	int datafrag_len, eccfrag_len, aligned_len, aligned_pos;
2912 	int busw = (chip->options & NAND_BUSWIDTH_16) ? 2 : 1;
2913 	int index, section = 0;
2914 	unsigned int max_bitflips = 0;
2915 	struct mtd_oob_region oobregion = { };
2916 
2917 	/* Column address within the page aligned to ECC size (256bytes) */
2918 	start_step = data_offs / chip->ecc.size;
2919 	end_step = (data_offs + readlen - 1) / chip->ecc.size;
2920 	num_steps = end_step - start_step + 1;
2921 	index = start_step * chip->ecc.bytes;
2922 
2923 	/* Data size aligned to ECC ecc.size */
2924 	datafrag_len = num_steps * chip->ecc.size;
2925 	eccfrag_len = num_steps * chip->ecc.bytes;
2926 
2927 	data_col_addr = start_step * chip->ecc.size;
2928 	/* If we read not a page aligned data */
2929 	p = bufpoi + data_col_addr;
2930 	ret = nand_read_page_op(chip, page, data_col_addr, p, datafrag_len);
2931 	if (ret)
2932 		return ret;
2933 
2934 	/* Calculate ECC */
2935 	for (i = 0; i < eccfrag_len ; i += chip->ecc.bytes, p += chip->ecc.size)
2936 		chip->ecc.calculate(chip, p, &chip->ecc.calc_buf[i]);
2937 
2938 	/*
2939 	 * The performance is faster if we position offsets according to
2940 	 * ecc.pos. Let's make sure that there are no gaps in ECC positions.
2941 	 */
2942 	ret = mtd_ooblayout_find_eccregion(mtd, index, &section, &oobregion);
2943 	if (ret)
2944 		return ret;
2945 
2946 	if (oobregion.length < eccfrag_len)
2947 		gaps = 1;
2948 
2949 	if (gaps) {
2950 		ret = nand_change_read_column_op(chip, mtd->writesize,
2951 						 chip->oob_poi, mtd->oobsize,
2952 						 false);
2953 		if (ret)
2954 			return ret;
2955 	} else {
2956 		/*
2957 		 * Send the command to read the particular ECC bytes take care
2958 		 * about buswidth alignment in read_buf.
2959 		 */
2960 		aligned_pos = oobregion.offset & ~(busw - 1);
2961 		aligned_len = eccfrag_len;
2962 		if (oobregion.offset & (busw - 1))
2963 			aligned_len++;
2964 		if ((oobregion.offset + (num_steps * chip->ecc.bytes)) &
2965 		    (busw - 1))
2966 			aligned_len++;
2967 
2968 		ret = nand_change_read_column_op(chip,
2969 						 mtd->writesize + aligned_pos,
2970 						 &chip->oob_poi[aligned_pos],
2971 						 aligned_len, false);
2972 		if (ret)
2973 			return ret;
2974 	}
2975 
2976 	ret = mtd_ooblayout_get_eccbytes(mtd, chip->ecc.code_buf,
2977 					 chip->oob_poi, index, eccfrag_len);
2978 	if (ret)
2979 		return ret;
2980 
2981 	p = bufpoi + data_col_addr;
2982 	for (i = 0; i < eccfrag_len ; i += chip->ecc.bytes, p += chip->ecc.size) {
2983 		int stat;
2984 
2985 		stat = chip->ecc.correct(chip, p, &chip->ecc.code_buf[i],
2986 					 &chip->ecc.calc_buf[i]);
2987 		if (stat == -EBADMSG &&
2988 		    (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) {
2989 			/* check for empty pages with bitflips */
2990 			stat = nand_check_erased_ecc_chunk(p, chip->ecc.size,
2991 						&chip->ecc.code_buf[i],
2992 						chip->ecc.bytes,
2993 						NULL, 0,
2994 						chip->ecc.strength);
2995 		}
2996 
2997 		if (stat < 0) {
2998 			mtd->ecc_stats.failed++;
2999 		} else {
3000 			mtd->ecc_stats.corrected += stat;
3001 			max_bitflips = max_t(unsigned int, max_bitflips, stat);
3002 		}
3003 	}
3004 	return max_bitflips;
3005 }
3006 
3007 /**
3008  * nand_read_page_hwecc - [REPLACEABLE] hardware ECC based page read function
3009  * @chip: nand chip info structure
3010  * @buf: buffer to store read data
3011  * @oob_required: caller requires OOB data read to chip->oob_poi
3012  * @page: page number to read
3013  *
3014  * Not for syndrome calculating ECC controllers which need a special oob layout.
3015  */
3016 static int nand_read_page_hwecc(struct nand_chip *chip, uint8_t *buf,
3017 				int oob_required, int page)
3018 {
3019 	struct mtd_info *mtd = nand_to_mtd(chip);
3020 	int i, eccsize = chip->ecc.size, ret;
3021 	int eccbytes = chip->ecc.bytes;
3022 	int eccsteps = chip->ecc.steps;
3023 	uint8_t *p = buf;
3024 	uint8_t *ecc_calc = chip->ecc.calc_buf;
3025 	uint8_t *ecc_code = chip->ecc.code_buf;
3026 	unsigned int max_bitflips = 0;
3027 
3028 	ret = nand_read_page_op(chip, page, 0, NULL, 0);
3029 	if (ret)
3030 		return ret;
3031 
3032 	for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
3033 		chip->ecc.hwctl(chip, NAND_ECC_READ);
3034 
3035 		ret = nand_read_data_op(chip, p, eccsize, false, false);
3036 		if (ret)
3037 			return ret;
3038 
3039 		chip->ecc.calculate(chip, p, &ecc_calc[i]);
3040 	}
3041 
3042 	ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, false,
3043 				false);
3044 	if (ret)
3045 		return ret;
3046 
3047 	ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
3048 					 chip->ecc.total);
3049 	if (ret)
3050 		return ret;
3051 
3052 	eccsteps = chip->ecc.steps;
3053 	p = buf;
3054 
3055 	for (i = 0 ; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
3056 		int stat;
3057 
3058 		stat = chip->ecc.correct(chip, p, &ecc_code[i], &ecc_calc[i]);
3059 		if (stat == -EBADMSG &&
3060 		    (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) {
3061 			/* check for empty pages with bitflips */
3062 			stat = nand_check_erased_ecc_chunk(p, eccsize,
3063 						&ecc_code[i], eccbytes,
3064 						NULL, 0,
3065 						chip->ecc.strength);
3066 		}
3067 
3068 		if (stat < 0) {
3069 			mtd->ecc_stats.failed++;
3070 		} else {
3071 			mtd->ecc_stats.corrected += stat;
3072 			max_bitflips = max_t(unsigned int, max_bitflips, stat);
3073 		}
3074 	}
3075 	return max_bitflips;
3076 }
3077 
3078 /**
3079  * nand_read_page_syndrome - [REPLACEABLE] hardware ECC syndrome based page read
3080  * @chip: nand chip info structure
3081  * @buf: buffer to store read data
3082  * @oob_required: caller requires OOB data read to chip->oob_poi
3083  * @page: page number to read
3084  *
3085  * The hw generator calculates the error syndrome automatically. Therefore we
3086  * need a special oob layout and handling.
3087  */
3088 static int nand_read_page_syndrome(struct nand_chip *chip, uint8_t *buf,
3089 				   int oob_required, int page)
3090 {
3091 	struct mtd_info *mtd = nand_to_mtd(chip);
3092 	int ret, i, eccsize = chip->ecc.size;
3093 	int eccbytes = chip->ecc.bytes;
3094 	int eccsteps = chip->ecc.steps;
3095 	int eccpadbytes = eccbytes + chip->ecc.prepad + chip->ecc.postpad;
3096 	uint8_t *p = buf;
3097 	uint8_t *oob = chip->oob_poi;
3098 	unsigned int max_bitflips = 0;
3099 
3100 	ret = nand_read_page_op(chip, page, 0, NULL, 0);
3101 	if (ret)
3102 		return ret;
3103 
3104 	for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
3105 		int stat;
3106 
3107 		chip->ecc.hwctl(chip, NAND_ECC_READ);
3108 
3109 		ret = nand_read_data_op(chip, p, eccsize, false, false);
3110 		if (ret)
3111 			return ret;
3112 
3113 		if (chip->ecc.prepad) {
3114 			ret = nand_read_data_op(chip, oob, chip->ecc.prepad,
3115 						false, false);
3116 			if (ret)
3117 				return ret;
3118 
3119 			oob += chip->ecc.prepad;
3120 		}
3121 
3122 		chip->ecc.hwctl(chip, NAND_ECC_READSYN);
3123 
3124 		ret = nand_read_data_op(chip, oob, eccbytes, false, false);
3125 		if (ret)
3126 			return ret;
3127 
3128 		stat = chip->ecc.correct(chip, p, oob, NULL);
3129 
3130 		oob += eccbytes;
3131 
3132 		if (chip->ecc.postpad) {
3133 			ret = nand_read_data_op(chip, oob, chip->ecc.postpad,
3134 						false, false);
3135 			if (ret)
3136 				return ret;
3137 
3138 			oob += chip->ecc.postpad;
3139 		}
3140 
3141 		if (stat == -EBADMSG &&
3142 		    (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) {
3143 			/* check for empty pages with bitflips */
3144 			stat = nand_check_erased_ecc_chunk(p, chip->ecc.size,
3145 							   oob - eccpadbytes,
3146 							   eccpadbytes,
3147 							   NULL, 0,
3148 							   chip->ecc.strength);
3149 		}
3150 
3151 		if (stat < 0) {
3152 			mtd->ecc_stats.failed++;
3153 		} else {
3154 			mtd->ecc_stats.corrected += stat;
3155 			max_bitflips = max_t(unsigned int, max_bitflips, stat);
3156 		}
3157 	}
3158 
3159 	/* Calculate remaining oob bytes */
3160 	i = mtd->oobsize - (oob - chip->oob_poi);
3161 	if (i) {
3162 		ret = nand_read_data_op(chip, oob, i, false, false);
3163 		if (ret)
3164 			return ret;
3165 	}
3166 
3167 	return max_bitflips;
3168 }
3169 
3170 /**
3171  * nand_transfer_oob - [INTERN] Transfer oob to client buffer
3172  * @chip: NAND chip object
3173  * @oob: oob destination address
3174  * @ops: oob ops structure
3175  * @len: size of oob to transfer
3176  */
3177 static uint8_t *nand_transfer_oob(struct nand_chip *chip, uint8_t *oob,
3178 				  struct mtd_oob_ops *ops, size_t len)
3179 {
3180 	struct mtd_info *mtd = nand_to_mtd(chip);
3181 	int ret;
3182 
3183 	switch (ops->mode) {
3184 
3185 	case MTD_OPS_PLACE_OOB:
3186 	case MTD_OPS_RAW:
3187 		memcpy(oob, chip->oob_poi + ops->ooboffs, len);
3188 		return oob + len;
3189 
3190 	case MTD_OPS_AUTO_OOB:
3191 		ret = mtd_ooblayout_get_databytes(mtd, oob, chip->oob_poi,
3192 						  ops->ooboffs, len);
3193 		BUG_ON(ret);
3194 		return oob + len;
3195 
3196 	default:
3197 		BUG();
3198 	}
3199 	return NULL;
3200 }
3201 
3202 /**
3203  * nand_setup_read_retry - [INTERN] Set the READ RETRY mode
3204  * @chip: NAND chip object
3205  * @retry_mode: the retry mode to use
3206  *
3207  * Some vendors supply a special command to shift the Vt threshold, to be used
3208  * when there are too many bitflips in a page (i.e., ECC error). After setting
3209  * a new threshold, the host should retry reading the page.
3210  */
3211 static int nand_setup_read_retry(struct nand_chip *chip, int retry_mode)
3212 {
3213 	pr_debug("setting READ RETRY mode %d\n", retry_mode);
3214 
3215 	if (retry_mode >= chip->read_retries)
3216 		return -EINVAL;
3217 
3218 	if (!chip->setup_read_retry)
3219 		return -EOPNOTSUPP;
3220 
3221 	return chip->setup_read_retry(chip, retry_mode);
3222 }
3223 
3224 static void nand_wait_readrdy(struct nand_chip *chip)
3225 {
3226 	const struct nand_sdr_timings *sdr;
3227 
3228 	if (!(chip->options & NAND_NEED_READRDY))
3229 		return;
3230 
3231 	sdr = nand_get_sdr_timings(&chip->data_interface);
3232 	WARN_ON(nand_wait_rdy_op(chip, PSEC_TO_MSEC(sdr->tR_max), 0));
3233 }
3234 
3235 /**
3236  * nand_do_read_ops - [INTERN] Read data with ECC
3237  * @chip: NAND chip object
3238  * @from: offset to read from
3239  * @ops: oob ops structure
3240  *
3241  * Internal function. Called with chip held.
3242  */
3243 static int nand_do_read_ops(struct nand_chip *chip, loff_t from,
3244 			    struct mtd_oob_ops *ops)
3245 {
3246 	int chipnr, page, realpage, col, bytes, aligned, oob_required;
3247 	struct mtd_info *mtd = nand_to_mtd(chip);
3248 	int ret = 0;
3249 	uint32_t readlen = ops->len;
3250 	uint32_t oobreadlen = ops->ooblen;
3251 	uint32_t max_oobsize = mtd_oobavail(mtd, ops);
3252 
3253 	uint8_t *bufpoi, *oob, *buf;
3254 	int use_bounce_buf;
3255 	unsigned int max_bitflips = 0;
3256 	int retry_mode = 0;
3257 	bool ecc_fail = false;
3258 
3259 	chipnr = (int)(from >> chip->chip_shift);
3260 	nand_select_target(chip, chipnr);
3261 
3262 	realpage = (int)(from >> chip->page_shift);
3263 	page = realpage & chip->pagemask;
3264 
3265 	col = (int)(from & (mtd->writesize - 1));
3266 
3267 	buf = ops->datbuf;
3268 	oob = ops->oobbuf;
3269 	oob_required = oob ? 1 : 0;
3270 
3271 	while (1) {
3272 		struct mtd_ecc_stats ecc_stats = mtd->ecc_stats;
3273 
3274 		bytes = min(mtd->writesize - col, readlen);
3275 		aligned = (bytes == mtd->writesize);
3276 
3277 		if (!aligned)
3278 			use_bounce_buf = 1;
3279 		else if (chip->options & NAND_USES_DMA)
3280 			use_bounce_buf = !virt_addr_valid(buf) ||
3281 					 !IS_ALIGNED((unsigned long)buf,
3282 						     chip->buf_align);
3283 		else
3284 			use_bounce_buf = 0;
3285 
3286 		/* Is the current page in the buffer? */
3287 		if (realpage != chip->pagecache.page || oob) {
3288 			bufpoi = use_bounce_buf ? chip->data_buf : buf;
3289 
3290 			if (use_bounce_buf && aligned)
3291 				pr_debug("%s: using read bounce buffer for buf@%p\n",
3292 						 __func__, buf);
3293 
3294 read_retry:
3295 			/*
3296 			 * Now read the page into the buffer.  Absent an error,
3297 			 * the read methods return max bitflips per ecc step.
3298 			 */
3299 			if (unlikely(ops->mode == MTD_OPS_RAW))
3300 				ret = chip->ecc.read_page_raw(chip, bufpoi,
3301 							      oob_required,
3302 							      page);
3303 			else if (!aligned && NAND_HAS_SUBPAGE_READ(chip) &&
3304 				 !oob)
3305 				ret = chip->ecc.read_subpage(chip, col, bytes,
3306 							     bufpoi, page);
3307 			else
3308 				ret = chip->ecc.read_page(chip, bufpoi,
3309 							  oob_required, page);
3310 			if (ret < 0) {
3311 				if (use_bounce_buf)
3312 					/* Invalidate page cache */
3313 					chip->pagecache.page = -1;
3314 				break;
3315 			}
3316 
3317 			/*
3318 			 * Copy back the data in the initial buffer when reading
3319 			 * partial pages or when a bounce buffer is required.
3320 			 */
3321 			if (use_bounce_buf) {
3322 				if (!NAND_HAS_SUBPAGE_READ(chip) && !oob &&
3323 				    !(mtd->ecc_stats.failed - ecc_stats.failed) &&
3324 				    (ops->mode != MTD_OPS_RAW)) {
3325 					chip->pagecache.page = realpage;
3326 					chip->pagecache.bitflips = ret;
3327 				} else {
3328 					/* Invalidate page cache */
3329 					chip->pagecache.page = -1;
3330 				}
3331 				memcpy(buf, bufpoi + col, bytes);
3332 			}
3333 
3334 			if (unlikely(oob)) {
3335 				int toread = min(oobreadlen, max_oobsize);
3336 
3337 				if (toread) {
3338 					oob = nand_transfer_oob(chip, oob, ops,
3339 								toread);
3340 					oobreadlen -= toread;
3341 				}
3342 			}
3343 
3344 			nand_wait_readrdy(chip);
3345 
3346 			if (mtd->ecc_stats.failed - ecc_stats.failed) {
3347 				if (retry_mode + 1 < chip->read_retries) {
3348 					retry_mode++;
3349 					ret = nand_setup_read_retry(chip,
3350 							retry_mode);
3351 					if (ret < 0)
3352 						break;
3353 
3354 					/* Reset ecc_stats; retry */
3355 					mtd->ecc_stats = ecc_stats;
3356 					goto read_retry;
3357 				} else {
3358 					/* No more retry modes; real failure */
3359 					ecc_fail = true;
3360 				}
3361 			}
3362 
3363 			buf += bytes;
3364 			max_bitflips = max_t(unsigned int, max_bitflips, ret);
3365 		} else {
3366 			memcpy(buf, chip->data_buf + col, bytes);
3367 			buf += bytes;
3368 			max_bitflips = max_t(unsigned int, max_bitflips,
3369 					     chip->pagecache.bitflips);
3370 		}
3371 
3372 		readlen -= bytes;
3373 
3374 		/* Reset to retry mode 0 */
3375 		if (retry_mode) {
3376 			ret = nand_setup_read_retry(chip, 0);
3377 			if (ret < 0)
3378 				break;
3379 			retry_mode = 0;
3380 		}
3381 
3382 		if (!readlen)
3383 			break;
3384 
3385 		/* For subsequent reads align to page boundary */
3386 		col = 0;
3387 		/* Increment page address */
3388 		realpage++;
3389 
3390 		page = realpage & chip->pagemask;
3391 		/* Check, if we cross a chip boundary */
3392 		if (!page) {
3393 			chipnr++;
3394 			nand_deselect_target(chip);
3395 			nand_select_target(chip, chipnr);
3396 		}
3397 	}
3398 	nand_deselect_target(chip);
3399 
3400 	ops->retlen = ops->len - (size_t) readlen;
3401 	if (oob)
3402 		ops->oobretlen = ops->ooblen - oobreadlen;
3403 
3404 	if (ret < 0)
3405 		return ret;
3406 
3407 	if (ecc_fail)
3408 		return -EBADMSG;
3409 
3410 	return max_bitflips;
3411 }
3412 
3413 /**
3414  * nand_read_oob_std - [REPLACEABLE] the most common OOB data read function
3415  * @chip: nand chip info structure
3416  * @page: page number to read
3417  */
3418 int nand_read_oob_std(struct nand_chip *chip, int page)
3419 {
3420 	struct mtd_info *mtd = nand_to_mtd(chip);
3421 
3422 	return nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize);
3423 }
3424 EXPORT_SYMBOL(nand_read_oob_std);
3425 
3426 /**
3427  * nand_read_oob_syndrome - [REPLACEABLE] OOB data read function for HW ECC
3428  *			    with syndromes
3429  * @chip: nand chip info structure
3430  * @page: page number to read
3431  */
3432 static int nand_read_oob_syndrome(struct nand_chip *chip, int page)
3433 {
3434 	struct mtd_info *mtd = nand_to_mtd(chip);
3435 	int length = mtd->oobsize;
3436 	int chunk = chip->ecc.bytes + chip->ecc.prepad + chip->ecc.postpad;
3437 	int eccsize = chip->ecc.size;
3438 	uint8_t *bufpoi = chip->oob_poi;
3439 	int i, toread, sndrnd = 0, pos, ret;
3440 
3441 	ret = nand_read_page_op(chip, page, chip->ecc.size, NULL, 0);
3442 	if (ret)
3443 		return ret;
3444 
3445 	for (i = 0; i < chip->ecc.steps; i++) {
3446 		if (sndrnd) {
3447 			int ret;
3448 
3449 			pos = eccsize + i * (eccsize + chunk);
3450 			if (mtd->writesize > 512)
3451 				ret = nand_change_read_column_op(chip, pos,
3452 								 NULL, 0,
3453 								 false);
3454 			else
3455 				ret = nand_read_page_op(chip, page, pos, NULL,
3456 							0);
3457 
3458 			if (ret)
3459 				return ret;
3460 		} else
3461 			sndrnd = 1;
3462 		toread = min_t(int, length, chunk);
3463 
3464 		ret = nand_read_data_op(chip, bufpoi, toread, false, false);
3465 		if (ret)
3466 			return ret;
3467 
3468 		bufpoi += toread;
3469 		length -= toread;
3470 	}
3471 	if (length > 0) {
3472 		ret = nand_read_data_op(chip, bufpoi, length, false, false);
3473 		if (ret)
3474 			return ret;
3475 	}
3476 
3477 	return 0;
3478 }
3479 
3480 /**
3481  * nand_write_oob_std - [REPLACEABLE] the most common OOB data write function
3482  * @chip: nand chip info structure
3483  * @page: page number to write
3484  */
3485 int nand_write_oob_std(struct nand_chip *chip, int page)
3486 {
3487 	struct mtd_info *mtd = nand_to_mtd(chip);
3488 
3489 	return nand_prog_page_op(chip, page, mtd->writesize, chip->oob_poi,
3490 				 mtd->oobsize);
3491 }
3492 EXPORT_SYMBOL(nand_write_oob_std);
3493 
3494 /**
3495  * nand_write_oob_syndrome - [REPLACEABLE] OOB data write function for HW ECC
3496  *			     with syndrome - only for large page flash
3497  * @chip: nand chip info structure
3498  * @page: page number to write
3499  */
3500 static int nand_write_oob_syndrome(struct nand_chip *chip, int page)
3501 {
3502 	struct mtd_info *mtd = nand_to_mtd(chip);
3503 	int chunk = chip->ecc.bytes + chip->ecc.prepad + chip->ecc.postpad;
3504 	int eccsize = chip->ecc.size, length = mtd->oobsize;
3505 	int ret, i, len, pos, sndcmd = 0, steps = chip->ecc.steps;
3506 	const uint8_t *bufpoi = chip->oob_poi;
3507 
3508 	/*
3509 	 * data-ecc-data-ecc ... ecc-oob
3510 	 * or
3511 	 * data-pad-ecc-pad-data-pad .... ecc-pad-oob
3512 	 */
3513 	if (!chip->ecc.prepad && !chip->ecc.postpad) {
3514 		pos = steps * (eccsize + chunk);
3515 		steps = 0;
3516 	} else
3517 		pos = eccsize;
3518 
3519 	ret = nand_prog_page_begin_op(chip, page, pos, NULL, 0);
3520 	if (ret)
3521 		return ret;
3522 
3523 	for (i = 0; i < steps; i++) {
3524 		if (sndcmd) {
3525 			if (mtd->writesize <= 512) {
3526 				uint32_t fill = 0xFFFFFFFF;
3527 
3528 				len = eccsize;
3529 				while (len > 0) {
3530 					int num = min_t(int, len, 4);
3531 
3532 					ret = nand_write_data_op(chip, &fill,
3533 								 num, false);
3534 					if (ret)
3535 						return ret;
3536 
3537 					len -= num;
3538 				}
3539 			} else {
3540 				pos = eccsize + i * (eccsize + chunk);
3541 				ret = nand_change_write_column_op(chip, pos,
3542 								  NULL, 0,
3543 								  false);
3544 				if (ret)
3545 					return ret;
3546 			}
3547 		} else
3548 			sndcmd = 1;
3549 		len = min_t(int, length, chunk);
3550 
3551 		ret = nand_write_data_op(chip, bufpoi, len, false);
3552 		if (ret)
3553 			return ret;
3554 
3555 		bufpoi += len;
3556 		length -= len;
3557 	}
3558 	if (length > 0) {
3559 		ret = nand_write_data_op(chip, bufpoi, length, false);
3560 		if (ret)
3561 			return ret;
3562 	}
3563 
3564 	return nand_prog_page_end_op(chip);
3565 }
3566 
3567 /**
3568  * nand_do_read_oob - [INTERN] NAND read out-of-band
3569  * @chip: NAND chip object
3570  * @from: offset to read from
3571  * @ops: oob operations description structure
3572  *
3573  * NAND read out-of-band data from the spare area.
3574  */
3575 static int nand_do_read_oob(struct nand_chip *chip, loff_t from,
3576 			    struct mtd_oob_ops *ops)
3577 {
3578 	struct mtd_info *mtd = nand_to_mtd(chip);
3579 	unsigned int max_bitflips = 0;
3580 	int page, realpage, chipnr;
3581 	struct mtd_ecc_stats stats;
3582 	int readlen = ops->ooblen;
3583 	int len;
3584 	uint8_t *buf = ops->oobbuf;
3585 	int ret = 0;
3586 
3587 	pr_debug("%s: from = 0x%08Lx, len = %i\n",
3588 			__func__, (unsigned long long)from, readlen);
3589 
3590 	stats = mtd->ecc_stats;
3591 
3592 	len = mtd_oobavail(mtd, ops);
3593 
3594 	chipnr = (int)(from >> chip->chip_shift);
3595 	nand_select_target(chip, chipnr);
3596 
3597 	/* Shift to get page */
3598 	realpage = (int)(from >> chip->page_shift);
3599 	page = realpage & chip->pagemask;
3600 
3601 	while (1) {
3602 		if (ops->mode == MTD_OPS_RAW)
3603 			ret = chip->ecc.read_oob_raw(chip, page);
3604 		else
3605 			ret = chip->ecc.read_oob(chip, page);
3606 
3607 		if (ret < 0)
3608 			break;
3609 
3610 		len = min(len, readlen);
3611 		buf = nand_transfer_oob(chip, buf, ops, len);
3612 
3613 		nand_wait_readrdy(chip);
3614 
3615 		max_bitflips = max_t(unsigned int, max_bitflips, ret);
3616 
3617 		readlen -= len;
3618 		if (!readlen)
3619 			break;
3620 
3621 		/* Increment page address */
3622 		realpage++;
3623 
3624 		page = realpage & chip->pagemask;
3625 		/* Check, if we cross a chip boundary */
3626 		if (!page) {
3627 			chipnr++;
3628 			nand_deselect_target(chip);
3629 			nand_select_target(chip, chipnr);
3630 		}
3631 	}
3632 	nand_deselect_target(chip);
3633 
3634 	ops->oobretlen = ops->ooblen - readlen;
3635 
3636 	if (ret < 0)
3637 		return ret;
3638 
3639 	if (mtd->ecc_stats.failed - stats.failed)
3640 		return -EBADMSG;
3641 
3642 	return max_bitflips;
3643 }
3644 
3645 /**
3646  * nand_read_oob - [MTD Interface] NAND read data and/or out-of-band
3647  * @mtd: MTD device structure
3648  * @from: offset to read from
3649  * @ops: oob operation description structure
3650  *
3651  * NAND read data and/or out-of-band data.
3652  */
3653 static int nand_read_oob(struct mtd_info *mtd, loff_t from,
3654 			 struct mtd_oob_ops *ops)
3655 {
3656 	struct nand_chip *chip = mtd_to_nand(mtd);
3657 	int ret;
3658 
3659 	ops->retlen = 0;
3660 
3661 	if (ops->mode != MTD_OPS_PLACE_OOB &&
3662 	    ops->mode != MTD_OPS_AUTO_OOB &&
3663 	    ops->mode != MTD_OPS_RAW)
3664 		return -ENOTSUPP;
3665 
3666 	ret = nand_get_device(chip);
3667 	if (ret)
3668 		return ret;
3669 
3670 	if (!ops->datbuf)
3671 		ret = nand_do_read_oob(chip, from, ops);
3672 	else
3673 		ret = nand_do_read_ops(chip, from, ops);
3674 
3675 	nand_release_device(chip);
3676 	return ret;
3677 }
3678 
3679 /**
3680  * nand_write_page_raw_notsupp - dummy raw page write function
3681  * @chip: nand chip info structure
3682  * @buf: data buffer
3683  * @oob_required: must write chip->oob_poi to OOB
3684  * @page: page number to write
3685  *
3686  * Returns -ENOTSUPP unconditionally.
3687  */
3688 int nand_write_page_raw_notsupp(struct nand_chip *chip, const u8 *buf,
3689 				int oob_required, int page)
3690 {
3691 	return -ENOTSUPP;
3692 }
3693 
3694 /**
3695  * nand_write_page_raw - [INTERN] raw page write function
3696  * @chip: nand chip info structure
3697  * @buf: data buffer
3698  * @oob_required: must write chip->oob_poi to OOB
3699  * @page: page number to write
3700  *
3701  * Not for syndrome calculating ECC controllers, which use a special oob layout.
3702  */
3703 int nand_write_page_raw(struct nand_chip *chip, const uint8_t *buf,
3704 			int oob_required, int page)
3705 {
3706 	struct mtd_info *mtd = nand_to_mtd(chip);
3707 	int ret;
3708 
3709 	ret = nand_prog_page_begin_op(chip, page, 0, buf, mtd->writesize);
3710 	if (ret)
3711 		return ret;
3712 
3713 	if (oob_required) {
3714 		ret = nand_write_data_op(chip, chip->oob_poi, mtd->oobsize,
3715 					 false);
3716 		if (ret)
3717 			return ret;
3718 	}
3719 
3720 	return nand_prog_page_end_op(chip);
3721 }
3722 EXPORT_SYMBOL(nand_write_page_raw);
3723 
3724 /**
3725  * nand_monolithic_write_page_raw - Monolithic page write in raw mode
3726  * @chip: NAND chip info structure
3727  * @buf: data buffer to write
3728  * @oob_required: must write chip->oob_poi to OOB
3729  * @page: page number to write
3730  *
3731  * This is a raw page write, ie. without any error detection/correction.
3732  * Monolithic means we are requesting all the relevant data (main plus
3733  * eventually OOB) to be sent over the bus and effectively programmed
3734  * into the NAND chip arrays in a single operation. This is an
3735  * alternative to nand_write_page_raw(), which first sends the main
3736  * data, then eventually send the OOB data by latching more data
3737  * cycles on the NAND bus, and finally sends the program command to
3738  * synchronyze the NAND chip cache.
3739  */
3740 int nand_monolithic_write_page_raw(struct nand_chip *chip, const u8 *buf,
3741 				   int oob_required, int page)
3742 {
3743 	struct mtd_info *mtd = nand_to_mtd(chip);
3744 	unsigned int size = mtd->writesize;
3745 	u8 *write_buf = (u8 *)buf;
3746 
3747 	if (oob_required) {
3748 		size += mtd->oobsize;
3749 
3750 		if (buf != chip->data_buf) {
3751 			write_buf = nand_get_data_buf(chip);
3752 			memcpy(write_buf, buf, mtd->writesize);
3753 		}
3754 	}
3755 
3756 	return nand_prog_page_op(chip, page, 0, write_buf, size);
3757 }
3758 EXPORT_SYMBOL(nand_monolithic_write_page_raw);
3759 
3760 /**
3761  * nand_write_page_raw_syndrome - [INTERN] raw page write function
3762  * @chip: nand chip info structure
3763  * @buf: data buffer
3764  * @oob_required: must write chip->oob_poi to OOB
3765  * @page: page number to write
3766  *
3767  * We need a special oob layout and handling even when ECC isn't checked.
3768  */
3769 static int nand_write_page_raw_syndrome(struct nand_chip *chip,
3770 					const uint8_t *buf, int oob_required,
3771 					int page)
3772 {
3773 	struct mtd_info *mtd = nand_to_mtd(chip);
3774 	int eccsize = chip->ecc.size;
3775 	int eccbytes = chip->ecc.bytes;
3776 	uint8_t *oob = chip->oob_poi;
3777 	int steps, size, ret;
3778 
3779 	ret = nand_prog_page_begin_op(chip, page, 0, NULL, 0);
3780 	if (ret)
3781 		return ret;
3782 
3783 	for (steps = chip->ecc.steps; steps > 0; steps--) {
3784 		ret = nand_write_data_op(chip, buf, eccsize, false);
3785 		if (ret)
3786 			return ret;
3787 
3788 		buf += eccsize;
3789 
3790 		if (chip->ecc.prepad) {
3791 			ret = nand_write_data_op(chip, oob, chip->ecc.prepad,
3792 						 false);
3793 			if (ret)
3794 				return ret;
3795 
3796 			oob += chip->ecc.prepad;
3797 		}
3798 
3799 		ret = nand_write_data_op(chip, oob, eccbytes, false);
3800 		if (ret)
3801 			return ret;
3802 
3803 		oob += eccbytes;
3804 
3805 		if (chip->ecc.postpad) {
3806 			ret = nand_write_data_op(chip, oob, chip->ecc.postpad,
3807 						 false);
3808 			if (ret)
3809 				return ret;
3810 
3811 			oob += chip->ecc.postpad;
3812 		}
3813 	}
3814 
3815 	size = mtd->oobsize - (oob - chip->oob_poi);
3816 	if (size) {
3817 		ret = nand_write_data_op(chip, oob, size, false);
3818 		if (ret)
3819 			return ret;
3820 	}
3821 
3822 	return nand_prog_page_end_op(chip);
3823 }
3824 /**
3825  * nand_write_page_swecc - [REPLACEABLE] software ECC based page write function
3826  * @chip: nand chip info structure
3827  * @buf: data buffer
3828  * @oob_required: must write chip->oob_poi to OOB
3829  * @page: page number to write
3830  */
3831 static int nand_write_page_swecc(struct nand_chip *chip, const uint8_t *buf,
3832 				 int oob_required, int page)
3833 {
3834 	struct mtd_info *mtd = nand_to_mtd(chip);
3835 	int i, eccsize = chip->ecc.size, ret;
3836 	int eccbytes = chip->ecc.bytes;
3837 	int eccsteps = chip->ecc.steps;
3838 	uint8_t *ecc_calc = chip->ecc.calc_buf;
3839 	const uint8_t *p = buf;
3840 
3841 	/* Software ECC calculation */
3842 	for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize)
3843 		chip->ecc.calculate(chip, p, &ecc_calc[i]);
3844 
3845 	ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
3846 					 chip->ecc.total);
3847 	if (ret)
3848 		return ret;
3849 
3850 	return chip->ecc.write_page_raw(chip, buf, 1, page);
3851 }
3852 
3853 /**
3854  * nand_write_page_hwecc - [REPLACEABLE] hardware ECC based page write function
3855  * @chip: nand chip info structure
3856  * @buf: data buffer
3857  * @oob_required: must write chip->oob_poi to OOB
3858  * @page: page number to write
3859  */
3860 static int nand_write_page_hwecc(struct nand_chip *chip, const uint8_t *buf,
3861 				 int oob_required, int page)
3862 {
3863 	struct mtd_info *mtd = nand_to_mtd(chip);
3864 	int i, eccsize = chip->ecc.size, ret;
3865 	int eccbytes = chip->ecc.bytes;
3866 	int eccsteps = chip->ecc.steps;
3867 	uint8_t *ecc_calc = chip->ecc.calc_buf;
3868 	const uint8_t *p = buf;
3869 
3870 	ret = nand_prog_page_begin_op(chip, page, 0, NULL, 0);
3871 	if (ret)
3872 		return ret;
3873 
3874 	for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
3875 		chip->ecc.hwctl(chip, NAND_ECC_WRITE);
3876 
3877 		ret = nand_write_data_op(chip, p, eccsize, false);
3878 		if (ret)
3879 			return ret;
3880 
3881 		chip->ecc.calculate(chip, p, &ecc_calc[i]);
3882 	}
3883 
3884 	ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
3885 					 chip->ecc.total);
3886 	if (ret)
3887 		return ret;
3888 
3889 	ret = nand_write_data_op(chip, chip->oob_poi, mtd->oobsize, false);
3890 	if (ret)
3891 		return ret;
3892 
3893 	return nand_prog_page_end_op(chip);
3894 }
3895 
3896 
3897 /**
3898  * nand_write_subpage_hwecc - [REPLACEABLE] hardware ECC based subpage write
3899  * @chip:	nand chip info structure
3900  * @offset:	column address of subpage within the page
3901  * @data_len:	data length
3902  * @buf:	data buffer
3903  * @oob_required: must write chip->oob_poi to OOB
3904  * @page: page number to write
3905  */
3906 static int nand_write_subpage_hwecc(struct nand_chip *chip, uint32_t offset,
3907 				    uint32_t data_len, const uint8_t *buf,
3908 				    int oob_required, int page)
3909 {
3910 	struct mtd_info *mtd = nand_to_mtd(chip);
3911 	uint8_t *oob_buf  = chip->oob_poi;
3912 	uint8_t *ecc_calc = chip->ecc.calc_buf;
3913 	int ecc_size      = chip->ecc.size;
3914 	int ecc_bytes     = chip->ecc.bytes;
3915 	int ecc_steps     = chip->ecc.steps;
3916 	uint32_t start_step = offset / ecc_size;
3917 	uint32_t end_step   = (offset + data_len - 1) / ecc_size;
3918 	int oob_bytes       = mtd->oobsize / ecc_steps;
3919 	int step, ret;
3920 
3921 	ret = nand_prog_page_begin_op(chip, page, 0, NULL, 0);
3922 	if (ret)
3923 		return ret;
3924 
3925 	for (step = 0; step < ecc_steps; step++) {
3926 		/* configure controller for WRITE access */
3927 		chip->ecc.hwctl(chip, NAND_ECC_WRITE);
3928 
3929 		/* write data (untouched subpages already masked by 0xFF) */
3930 		ret = nand_write_data_op(chip, buf, ecc_size, false);
3931 		if (ret)
3932 			return ret;
3933 
3934 		/* mask ECC of un-touched subpages by padding 0xFF */
3935 		if ((step < start_step) || (step > end_step))
3936 			memset(ecc_calc, 0xff, ecc_bytes);
3937 		else
3938 			chip->ecc.calculate(chip, buf, ecc_calc);
3939 
3940 		/* mask OOB of un-touched subpages by padding 0xFF */
3941 		/* if oob_required, preserve OOB metadata of written subpage */
3942 		if (!oob_required || (step < start_step) || (step > end_step))
3943 			memset(oob_buf, 0xff, oob_bytes);
3944 
3945 		buf += ecc_size;
3946 		ecc_calc += ecc_bytes;
3947 		oob_buf  += oob_bytes;
3948 	}
3949 
3950 	/* copy calculated ECC for whole page to chip->buffer->oob */
3951 	/* this include masked-value(0xFF) for unwritten subpages */
3952 	ecc_calc = chip->ecc.calc_buf;
3953 	ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
3954 					 chip->ecc.total);
3955 	if (ret)
3956 		return ret;
3957 
3958 	/* write OOB buffer to NAND device */
3959 	ret = nand_write_data_op(chip, chip->oob_poi, mtd->oobsize, false);
3960 	if (ret)
3961 		return ret;
3962 
3963 	return nand_prog_page_end_op(chip);
3964 }
3965 
3966 
3967 /**
3968  * nand_write_page_syndrome - [REPLACEABLE] hardware ECC syndrome based page write
3969  * @chip: nand chip info structure
3970  * @buf: data buffer
3971  * @oob_required: must write chip->oob_poi to OOB
3972  * @page: page number to write
3973  *
3974  * The hw generator calculates the error syndrome automatically. Therefore we
3975  * need a special oob layout and handling.
3976  */
3977 static int nand_write_page_syndrome(struct nand_chip *chip, const uint8_t *buf,
3978 				    int oob_required, int page)
3979 {
3980 	struct mtd_info *mtd = nand_to_mtd(chip);
3981 	int i, eccsize = chip->ecc.size;
3982 	int eccbytes = chip->ecc.bytes;
3983 	int eccsteps = chip->ecc.steps;
3984 	const uint8_t *p = buf;
3985 	uint8_t *oob = chip->oob_poi;
3986 	int ret;
3987 
3988 	ret = nand_prog_page_begin_op(chip, page, 0, NULL, 0);
3989 	if (ret)
3990 		return ret;
3991 
3992 	for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
3993 		chip->ecc.hwctl(chip, NAND_ECC_WRITE);
3994 
3995 		ret = nand_write_data_op(chip, p, eccsize, false);
3996 		if (ret)
3997 			return ret;
3998 
3999 		if (chip->ecc.prepad) {
4000 			ret = nand_write_data_op(chip, oob, chip->ecc.prepad,
4001 						 false);
4002 			if (ret)
4003 				return ret;
4004 
4005 			oob += chip->ecc.prepad;
4006 		}
4007 
4008 		chip->ecc.calculate(chip, p, oob);
4009 
4010 		ret = nand_write_data_op(chip, oob, eccbytes, false);
4011 		if (ret)
4012 			return ret;
4013 
4014 		oob += eccbytes;
4015 
4016 		if (chip->ecc.postpad) {
4017 			ret = nand_write_data_op(chip, oob, chip->ecc.postpad,
4018 						 false);
4019 			if (ret)
4020 				return ret;
4021 
4022 			oob += chip->ecc.postpad;
4023 		}
4024 	}
4025 
4026 	/* Calculate remaining oob bytes */
4027 	i = mtd->oobsize - (oob - chip->oob_poi);
4028 	if (i) {
4029 		ret = nand_write_data_op(chip, oob, i, false);
4030 		if (ret)
4031 			return ret;
4032 	}
4033 
4034 	return nand_prog_page_end_op(chip);
4035 }
4036 
4037 /**
4038  * nand_write_page - write one page
4039  * @chip: NAND chip descriptor
4040  * @offset: address offset within the page
4041  * @data_len: length of actual data to be written
4042  * @buf: the data to write
4043  * @oob_required: must write chip->oob_poi to OOB
4044  * @page: page number to write
4045  * @raw: use _raw version of write_page
4046  */
4047 static int nand_write_page(struct nand_chip *chip, uint32_t offset,
4048 			   int data_len, const uint8_t *buf, int oob_required,
4049 			   int page, int raw)
4050 {
4051 	struct mtd_info *mtd = nand_to_mtd(chip);
4052 	int status, subpage;
4053 
4054 	if (!(chip->options & NAND_NO_SUBPAGE_WRITE) &&
4055 		chip->ecc.write_subpage)
4056 		subpage = offset || (data_len < mtd->writesize);
4057 	else
4058 		subpage = 0;
4059 
4060 	if (unlikely(raw))
4061 		status = chip->ecc.write_page_raw(chip, buf, oob_required,
4062 						  page);
4063 	else if (subpage)
4064 		status = chip->ecc.write_subpage(chip, offset, data_len, buf,
4065 						 oob_required, page);
4066 	else
4067 		status = chip->ecc.write_page(chip, buf, oob_required, page);
4068 
4069 	if (status < 0)
4070 		return status;
4071 
4072 	return 0;
4073 }
4074 
4075 #define NOTALIGNED(x)	((x & (chip->subpagesize - 1)) != 0)
4076 
4077 /**
4078  * nand_do_write_ops - [INTERN] NAND write with ECC
4079  * @chip: NAND chip object
4080  * @to: offset to write to
4081  * @ops: oob operations description structure
4082  *
4083  * NAND write with ECC.
4084  */
4085 static int nand_do_write_ops(struct nand_chip *chip, loff_t to,
4086 			     struct mtd_oob_ops *ops)
4087 {
4088 	struct mtd_info *mtd = nand_to_mtd(chip);
4089 	int chipnr, realpage, page, column;
4090 	uint32_t writelen = ops->len;
4091 
4092 	uint32_t oobwritelen = ops->ooblen;
4093 	uint32_t oobmaxlen = mtd_oobavail(mtd, ops);
4094 
4095 	uint8_t *oob = ops->oobbuf;
4096 	uint8_t *buf = ops->datbuf;
4097 	int ret;
4098 	int oob_required = oob ? 1 : 0;
4099 
4100 	ops->retlen = 0;
4101 	if (!writelen)
4102 		return 0;
4103 
4104 	/* Reject writes, which are not page aligned */
4105 	if (NOTALIGNED(to) || NOTALIGNED(ops->len)) {
4106 		pr_notice("%s: attempt to write non page aligned data\n",
4107 			   __func__);
4108 		return -EINVAL;
4109 	}
4110 
4111 	column = to & (mtd->writesize - 1);
4112 
4113 	chipnr = (int)(to >> chip->chip_shift);
4114 	nand_select_target(chip, chipnr);
4115 
4116 	/* Check, if it is write protected */
4117 	if (nand_check_wp(chip)) {
4118 		ret = -EIO;
4119 		goto err_out;
4120 	}
4121 
4122 	realpage = (int)(to >> chip->page_shift);
4123 	page = realpage & chip->pagemask;
4124 
4125 	/* Invalidate the page cache, when we write to the cached page */
4126 	if (to <= ((loff_t)chip->pagecache.page << chip->page_shift) &&
4127 	    ((loff_t)chip->pagecache.page << chip->page_shift) < (to + ops->len))
4128 		chip->pagecache.page = -1;
4129 
4130 	/* Don't allow multipage oob writes with offset */
4131 	if (oob && ops->ooboffs && (ops->ooboffs + ops->ooblen > oobmaxlen)) {
4132 		ret = -EINVAL;
4133 		goto err_out;
4134 	}
4135 
4136 	while (1) {
4137 		int bytes = mtd->writesize;
4138 		uint8_t *wbuf = buf;
4139 		int use_bounce_buf;
4140 		int part_pagewr = (column || writelen < mtd->writesize);
4141 
4142 		if (part_pagewr)
4143 			use_bounce_buf = 1;
4144 		else if (chip->options & NAND_USES_DMA)
4145 			use_bounce_buf = !virt_addr_valid(buf) ||
4146 					 !IS_ALIGNED((unsigned long)buf,
4147 						     chip->buf_align);
4148 		else
4149 			use_bounce_buf = 0;
4150 
4151 		/*
4152 		 * Copy the data from the initial buffer when doing partial page
4153 		 * writes or when a bounce buffer is required.
4154 		 */
4155 		if (use_bounce_buf) {
4156 			pr_debug("%s: using write bounce buffer for buf@%p\n",
4157 					 __func__, buf);
4158 			if (part_pagewr)
4159 				bytes = min_t(int, bytes - column, writelen);
4160 			wbuf = nand_get_data_buf(chip);
4161 			memset(wbuf, 0xff, mtd->writesize);
4162 			memcpy(&wbuf[column], buf, bytes);
4163 		}
4164 
4165 		if (unlikely(oob)) {
4166 			size_t len = min(oobwritelen, oobmaxlen);
4167 			oob = nand_fill_oob(chip, oob, len, ops);
4168 			oobwritelen -= len;
4169 		} else {
4170 			/* We still need to erase leftover OOB data */
4171 			memset(chip->oob_poi, 0xff, mtd->oobsize);
4172 		}
4173 
4174 		ret = nand_write_page(chip, column, bytes, wbuf,
4175 				      oob_required, page,
4176 				      (ops->mode == MTD_OPS_RAW));
4177 		if (ret)
4178 			break;
4179 
4180 		writelen -= bytes;
4181 		if (!writelen)
4182 			break;
4183 
4184 		column = 0;
4185 		buf += bytes;
4186 		realpage++;
4187 
4188 		page = realpage & chip->pagemask;
4189 		/* Check, if we cross a chip boundary */
4190 		if (!page) {
4191 			chipnr++;
4192 			nand_deselect_target(chip);
4193 			nand_select_target(chip, chipnr);
4194 		}
4195 	}
4196 
4197 	ops->retlen = ops->len - writelen;
4198 	if (unlikely(oob))
4199 		ops->oobretlen = ops->ooblen;
4200 
4201 err_out:
4202 	nand_deselect_target(chip);
4203 	return ret;
4204 }
4205 
4206 /**
4207  * panic_nand_write - [MTD Interface] NAND write with ECC
4208  * @mtd: MTD device structure
4209  * @to: offset to write to
4210  * @len: number of bytes to write
4211  * @retlen: pointer to variable to store the number of written bytes
4212  * @buf: the data to write
4213  *
4214  * NAND write with ECC. Used when performing writes in interrupt context, this
4215  * may for example be called by mtdoops when writing an oops while in panic.
4216  */
4217 static int panic_nand_write(struct mtd_info *mtd, loff_t to, size_t len,
4218 			    size_t *retlen, const uint8_t *buf)
4219 {
4220 	struct nand_chip *chip = mtd_to_nand(mtd);
4221 	int chipnr = (int)(to >> chip->chip_shift);
4222 	struct mtd_oob_ops ops;
4223 	int ret;
4224 
4225 	nand_select_target(chip, chipnr);
4226 
4227 	/* Wait for the device to get ready */
4228 	panic_nand_wait(chip, 400);
4229 
4230 	memset(&ops, 0, sizeof(ops));
4231 	ops.len = len;
4232 	ops.datbuf = (uint8_t *)buf;
4233 	ops.mode = MTD_OPS_PLACE_OOB;
4234 
4235 	ret = nand_do_write_ops(chip, to, &ops);
4236 
4237 	*retlen = ops.retlen;
4238 	return ret;
4239 }
4240 
4241 /**
4242  * nand_write_oob - [MTD Interface] NAND write data and/or out-of-band
4243  * @mtd: MTD device structure
4244  * @to: offset to write to
4245  * @ops: oob operation description structure
4246  */
4247 static int nand_write_oob(struct mtd_info *mtd, loff_t to,
4248 			  struct mtd_oob_ops *ops)
4249 {
4250 	struct nand_chip *chip = mtd_to_nand(mtd);
4251 	int ret;
4252 
4253 	ops->retlen = 0;
4254 
4255 	ret = nand_get_device(chip);
4256 	if (ret)
4257 		return ret;
4258 
4259 	switch (ops->mode) {
4260 	case MTD_OPS_PLACE_OOB:
4261 	case MTD_OPS_AUTO_OOB:
4262 	case MTD_OPS_RAW:
4263 		break;
4264 
4265 	default:
4266 		goto out;
4267 	}
4268 
4269 	if (!ops->datbuf)
4270 		ret = nand_do_write_oob(chip, to, ops);
4271 	else
4272 		ret = nand_do_write_ops(chip, to, ops);
4273 
4274 out:
4275 	nand_release_device(chip);
4276 	return ret;
4277 }
4278 
4279 /**
4280  * nand_erase - [MTD Interface] erase block(s)
4281  * @mtd: MTD device structure
4282  * @instr: erase instruction
4283  *
4284  * Erase one ore more blocks.
4285  */
4286 static int nand_erase(struct mtd_info *mtd, struct erase_info *instr)
4287 {
4288 	return nand_erase_nand(mtd_to_nand(mtd), instr, 0);
4289 }
4290 
4291 /**
4292  * nand_erase_nand - [INTERN] erase block(s)
4293  * @chip: NAND chip object
4294  * @instr: erase instruction
4295  * @allowbbt: allow erasing the bbt area
4296  *
4297  * Erase one ore more blocks.
4298  */
4299 int nand_erase_nand(struct nand_chip *chip, struct erase_info *instr,
4300 		    int allowbbt)
4301 {
4302 	int page, pages_per_block, ret, chipnr;
4303 	loff_t len;
4304 
4305 	pr_debug("%s: start = 0x%012llx, len = %llu\n",
4306 			__func__, (unsigned long long)instr->addr,
4307 			(unsigned long long)instr->len);
4308 
4309 	if (check_offs_len(chip, instr->addr, instr->len))
4310 		return -EINVAL;
4311 
4312 	/* Grab the lock and see if the device is available */
4313 	ret = nand_get_device(chip);
4314 	if (ret)
4315 		return ret;
4316 
4317 	/* Shift to get first page */
4318 	page = (int)(instr->addr >> chip->page_shift);
4319 	chipnr = (int)(instr->addr >> chip->chip_shift);
4320 
4321 	/* Calculate pages in each block */
4322 	pages_per_block = 1 << (chip->phys_erase_shift - chip->page_shift);
4323 
4324 	/* Select the NAND device */
4325 	nand_select_target(chip, chipnr);
4326 
4327 	/* Check, if it is write protected */
4328 	if (nand_check_wp(chip)) {
4329 		pr_debug("%s: device is write protected!\n",
4330 				__func__);
4331 		ret = -EIO;
4332 		goto erase_exit;
4333 	}
4334 
4335 	/* Loop through the pages */
4336 	len = instr->len;
4337 
4338 	while (len) {
4339 		/* Check if we have a bad block, we do not erase bad blocks! */
4340 		if (nand_block_checkbad(chip, ((loff_t) page) <<
4341 					chip->page_shift, allowbbt)) {
4342 			pr_warn("%s: attempt to erase a bad block at page 0x%08x\n",
4343 				    __func__, page);
4344 			ret = -EIO;
4345 			goto erase_exit;
4346 		}
4347 
4348 		/*
4349 		 * Invalidate the page cache, if we erase the block which
4350 		 * contains the current cached page.
4351 		 */
4352 		if (page <= chip->pagecache.page && chip->pagecache.page <
4353 		    (page + pages_per_block))
4354 			chip->pagecache.page = -1;
4355 
4356 		ret = nand_erase_op(chip, (page & chip->pagemask) >>
4357 				    (chip->phys_erase_shift - chip->page_shift));
4358 		if (ret) {
4359 			pr_debug("%s: failed erase, page 0x%08x\n",
4360 					__func__, page);
4361 			instr->fail_addr =
4362 				((loff_t)page << chip->page_shift);
4363 			goto erase_exit;
4364 		}
4365 
4366 		/* Increment page address and decrement length */
4367 		len -= (1ULL << chip->phys_erase_shift);
4368 		page += pages_per_block;
4369 
4370 		/* Check, if we cross a chip boundary */
4371 		if (len && !(page & chip->pagemask)) {
4372 			chipnr++;
4373 			nand_deselect_target(chip);
4374 			nand_select_target(chip, chipnr);
4375 		}
4376 	}
4377 
4378 	ret = 0;
4379 erase_exit:
4380 
4381 	/* Deselect and wake up anyone waiting on the device */
4382 	nand_deselect_target(chip);
4383 	nand_release_device(chip);
4384 
4385 	/* Return more or less happy */
4386 	return ret;
4387 }
4388 
4389 /**
4390  * nand_sync - [MTD Interface] sync
4391  * @mtd: MTD device structure
4392  *
4393  * Sync is actually a wait for chip ready function.
4394  */
4395 static void nand_sync(struct mtd_info *mtd)
4396 {
4397 	struct nand_chip *chip = mtd_to_nand(mtd);
4398 
4399 	pr_debug("%s: called\n", __func__);
4400 
4401 	/* Grab the lock and see if the device is available */
4402 	WARN_ON(nand_get_device(chip));
4403 	/* Release it and go back */
4404 	nand_release_device(chip);
4405 }
4406 
4407 /**
4408  * nand_block_isbad - [MTD Interface] Check if block at offset is bad
4409  * @mtd: MTD device structure
4410  * @offs: offset relative to mtd start
4411  */
4412 static int nand_block_isbad(struct mtd_info *mtd, loff_t offs)
4413 {
4414 	struct nand_chip *chip = mtd_to_nand(mtd);
4415 	int chipnr = (int)(offs >> chip->chip_shift);
4416 	int ret;
4417 
4418 	/* Select the NAND device */
4419 	ret = nand_get_device(chip);
4420 	if (ret)
4421 		return ret;
4422 
4423 	nand_select_target(chip, chipnr);
4424 
4425 	ret = nand_block_checkbad(chip, offs, 0);
4426 
4427 	nand_deselect_target(chip);
4428 	nand_release_device(chip);
4429 
4430 	return ret;
4431 }
4432 
4433 /**
4434  * nand_block_markbad - [MTD Interface] Mark block at the given offset as bad
4435  * @mtd: MTD device structure
4436  * @ofs: offset relative to mtd start
4437  */
4438 static int nand_block_markbad(struct mtd_info *mtd, loff_t ofs)
4439 {
4440 	int ret;
4441 
4442 	ret = nand_block_isbad(mtd, ofs);
4443 	if (ret) {
4444 		/* If it was bad already, return success and do nothing */
4445 		if (ret > 0)
4446 			return 0;
4447 		return ret;
4448 	}
4449 
4450 	return nand_block_markbad_lowlevel(mtd_to_nand(mtd), ofs);
4451 }
4452 
4453 /**
4454  * nand_suspend - [MTD Interface] Suspend the NAND flash
4455  * @mtd: MTD device structure
4456  *
4457  * Returns 0 for success or negative error code otherwise.
4458  */
4459 static int nand_suspend(struct mtd_info *mtd)
4460 {
4461 	struct nand_chip *chip = mtd_to_nand(mtd);
4462 	int ret = 0;
4463 
4464 	mutex_lock(&chip->lock);
4465 	if (chip->suspend)
4466 		ret = chip->suspend(chip);
4467 	if (!ret)
4468 		chip->suspended = 1;
4469 	mutex_unlock(&chip->lock);
4470 
4471 	return ret;
4472 }
4473 
4474 /**
4475  * nand_resume - [MTD Interface] Resume the NAND flash
4476  * @mtd: MTD device structure
4477  */
4478 static void nand_resume(struct mtd_info *mtd)
4479 {
4480 	struct nand_chip *chip = mtd_to_nand(mtd);
4481 
4482 	mutex_lock(&chip->lock);
4483 	if (chip->suspended) {
4484 		if (chip->resume)
4485 			chip->resume(chip);
4486 		chip->suspended = 0;
4487 	} else {
4488 		pr_err("%s called for a chip which is not in suspended state\n",
4489 			__func__);
4490 	}
4491 	mutex_unlock(&chip->lock);
4492 }
4493 
4494 /**
4495  * nand_shutdown - [MTD Interface] Finish the current NAND operation and
4496  *                 prevent further operations
4497  * @mtd: MTD device structure
4498  */
4499 static void nand_shutdown(struct mtd_info *mtd)
4500 {
4501 	nand_suspend(mtd);
4502 }
4503 
4504 /**
4505  * nand_lock - [MTD Interface] Lock the NAND flash
4506  * @mtd: MTD device structure
4507  * @ofs: offset byte address
4508  * @len: number of bytes to lock (must be a multiple of block/page size)
4509  */
4510 static int nand_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
4511 {
4512 	struct nand_chip *chip = mtd_to_nand(mtd);
4513 
4514 	if (!chip->lock_area)
4515 		return -ENOTSUPP;
4516 
4517 	return chip->lock_area(chip, ofs, len);
4518 }
4519 
4520 /**
4521  * nand_unlock - [MTD Interface] Unlock the NAND flash
4522  * @mtd: MTD device structure
4523  * @ofs: offset byte address
4524  * @len: number of bytes to unlock (must be a multiple of block/page size)
4525  */
4526 static int nand_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
4527 {
4528 	struct nand_chip *chip = mtd_to_nand(mtd);
4529 
4530 	if (!chip->unlock_area)
4531 		return -ENOTSUPP;
4532 
4533 	return chip->unlock_area(chip, ofs, len);
4534 }
4535 
4536 /* Set default functions */
4537 static void nand_set_defaults(struct nand_chip *chip)
4538 {
4539 	/* If no controller is provided, use the dummy, legacy one. */
4540 	if (!chip->controller) {
4541 		chip->controller = &chip->legacy.dummy_controller;
4542 		nand_controller_init(chip->controller);
4543 	}
4544 
4545 	nand_legacy_set_defaults(chip);
4546 
4547 	if (!chip->buf_align)
4548 		chip->buf_align = 1;
4549 }
4550 
4551 /* Sanitize ONFI strings so we can safely print them */
4552 void sanitize_string(uint8_t *s, size_t len)
4553 {
4554 	ssize_t i;
4555 
4556 	/* Null terminate */
4557 	s[len - 1] = 0;
4558 
4559 	/* Remove non printable chars */
4560 	for (i = 0; i < len - 1; i++) {
4561 		if (s[i] < ' ' || s[i] > 127)
4562 			s[i] = '?';
4563 	}
4564 
4565 	/* Remove trailing spaces */
4566 	strim(s);
4567 }
4568 
4569 /*
4570  * nand_id_has_period - Check if an ID string has a given wraparound period
4571  * @id_data: the ID string
4572  * @arrlen: the length of the @id_data array
4573  * @period: the period of repitition
4574  *
4575  * Check if an ID string is repeated within a given sequence of bytes at
4576  * specific repetition interval period (e.g., {0x20,0x01,0x7F,0x20} has a
4577  * period of 3). This is a helper function for nand_id_len(). Returns non-zero
4578  * if the repetition has a period of @period; otherwise, returns zero.
4579  */
4580 static int nand_id_has_period(u8 *id_data, int arrlen, int period)
4581 {
4582 	int i, j;
4583 	for (i = 0; i < period; i++)
4584 		for (j = i + period; j < arrlen; j += period)
4585 			if (id_data[i] != id_data[j])
4586 				return 0;
4587 	return 1;
4588 }
4589 
4590 /*
4591  * nand_id_len - Get the length of an ID string returned by CMD_READID
4592  * @id_data: the ID string
4593  * @arrlen: the length of the @id_data array
4594 
4595  * Returns the length of the ID string, according to known wraparound/trailing
4596  * zero patterns. If no pattern exists, returns the length of the array.
4597  */
4598 static int nand_id_len(u8 *id_data, int arrlen)
4599 {
4600 	int last_nonzero, period;
4601 
4602 	/* Find last non-zero byte */
4603 	for (last_nonzero = arrlen - 1; last_nonzero >= 0; last_nonzero--)
4604 		if (id_data[last_nonzero])
4605 			break;
4606 
4607 	/* All zeros */
4608 	if (last_nonzero < 0)
4609 		return 0;
4610 
4611 	/* Calculate wraparound period */
4612 	for (period = 1; period < arrlen; period++)
4613 		if (nand_id_has_period(id_data, arrlen, period))
4614 			break;
4615 
4616 	/* There's a repeated pattern */
4617 	if (period < arrlen)
4618 		return period;
4619 
4620 	/* There are trailing zeros */
4621 	if (last_nonzero < arrlen - 1)
4622 		return last_nonzero + 1;
4623 
4624 	/* No pattern detected */
4625 	return arrlen;
4626 }
4627 
4628 /* Extract the bits of per cell from the 3rd byte of the extended ID */
4629 static int nand_get_bits_per_cell(u8 cellinfo)
4630 {
4631 	int bits;
4632 
4633 	bits = cellinfo & NAND_CI_CELLTYPE_MSK;
4634 	bits >>= NAND_CI_CELLTYPE_SHIFT;
4635 	return bits + 1;
4636 }
4637 
4638 /*
4639  * Many new NAND share similar device ID codes, which represent the size of the
4640  * chip. The rest of the parameters must be decoded according to generic or
4641  * manufacturer-specific "extended ID" decoding patterns.
4642  */
4643 void nand_decode_ext_id(struct nand_chip *chip)
4644 {
4645 	struct nand_memory_organization *memorg;
4646 	struct mtd_info *mtd = nand_to_mtd(chip);
4647 	int extid;
4648 	u8 *id_data = chip->id.data;
4649 
4650 	memorg = nanddev_get_memorg(&chip->base);
4651 
4652 	/* The 3rd id byte holds MLC / multichip data */
4653 	memorg->bits_per_cell = nand_get_bits_per_cell(id_data[2]);
4654 	/* The 4th id byte is the important one */
4655 	extid = id_data[3];
4656 
4657 	/* Calc pagesize */
4658 	memorg->pagesize = 1024 << (extid & 0x03);
4659 	mtd->writesize = memorg->pagesize;
4660 	extid >>= 2;
4661 	/* Calc oobsize */
4662 	memorg->oobsize = (8 << (extid & 0x01)) * (mtd->writesize >> 9);
4663 	mtd->oobsize = memorg->oobsize;
4664 	extid >>= 2;
4665 	/* Calc blocksize. Blocksize is multiples of 64KiB */
4666 	memorg->pages_per_eraseblock = ((64 * 1024) << (extid & 0x03)) /
4667 				       memorg->pagesize;
4668 	mtd->erasesize = (64 * 1024) << (extid & 0x03);
4669 	extid >>= 2;
4670 	/* Get buswidth information */
4671 	if (extid & 0x1)
4672 		chip->options |= NAND_BUSWIDTH_16;
4673 }
4674 EXPORT_SYMBOL_GPL(nand_decode_ext_id);
4675 
4676 /*
4677  * Old devices have chip data hardcoded in the device ID table. nand_decode_id
4678  * decodes a matching ID table entry and assigns the MTD size parameters for
4679  * the chip.
4680  */
4681 static void nand_decode_id(struct nand_chip *chip, struct nand_flash_dev *type)
4682 {
4683 	struct mtd_info *mtd = nand_to_mtd(chip);
4684 	struct nand_memory_organization *memorg;
4685 
4686 	memorg = nanddev_get_memorg(&chip->base);
4687 
4688 	memorg->pages_per_eraseblock = type->erasesize / type->pagesize;
4689 	mtd->erasesize = type->erasesize;
4690 	memorg->pagesize = type->pagesize;
4691 	mtd->writesize = memorg->pagesize;
4692 	memorg->oobsize = memorg->pagesize / 32;
4693 	mtd->oobsize = memorg->oobsize;
4694 
4695 	/* All legacy ID NAND are small-page, SLC */
4696 	memorg->bits_per_cell = 1;
4697 }
4698 
4699 /*
4700  * Set the bad block marker/indicator (BBM/BBI) patterns according to some
4701  * heuristic patterns using various detected parameters (e.g., manufacturer,
4702  * page size, cell-type information).
4703  */
4704 static void nand_decode_bbm_options(struct nand_chip *chip)
4705 {
4706 	struct mtd_info *mtd = nand_to_mtd(chip);
4707 
4708 	/* Set the bad block position */
4709 	if (mtd->writesize > 512 || (chip->options & NAND_BUSWIDTH_16))
4710 		chip->badblockpos = NAND_BBM_POS_LARGE;
4711 	else
4712 		chip->badblockpos = NAND_BBM_POS_SMALL;
4713 }
4714 
4715 static inline bool is_full_id_nand(struct nand_flash_dev *type)
4716 {
4717 	return type->id_len;
4718 }
4719 
4720 static bool find_full_id_nand(struct nand_chip *chip,
4721 			      struct nand_flash_dev *type)
4722 {
4723 	struct mtd_info *mtd = nand_to_mtd(chip);
4724 	struct nand_memory_organization *memorg;
4725 	u8 *id_data = chip->id.data;
4726 
4727 	memorg = nanddev_get_memorg(&chip->base);
4728 
4729 	if (!strncmp(type->id, id_data, type->id_len)) {
4730 		memorg->pagesize = type->pagesize;
4731 		mtd->writesize = memorg->pagesize;
4732 		memorg->pages_per_eraseblock = type->erasesize /
4733 					       type->pagesize;
4734 		mtd->erasesize = type->erasesize;
4735 		memorg->oobsize = type->oobsize;
4736 		mtd->oobsize = memorg->oobsize;
4737 
4738 		memorg->bits_per_cell = nand_get_bits_per_cell(id_data[2]);
4739 		memorg->eraseblocks_per_lun =
4740 			DIV_ROUND_DOWN_ULL((u64)type->chipsize << 20,
4741 					   memorg->pagesize *
4742 					   memorg->pages_per_eraseblock);
4743 		chip->options |= type->options;
4744 		chip->base.eccreq.strength = NAND_ECC_STRENGTH(type);
4745 		chip->base.eccreq.step_size = NAND_ECC_STEP(type);
4746 		chip->onfi_timing_mode_default =
4747 					type->onfi_timing_mode_default;
4748 
4749 		chip->parameters.model = kstrdup(type->name, GFP_KERNEL);
4750 		if (!chip->parameters.model)
4751 			return false;
4752 
4753 		return true;
4754 	}
4755 	return false;
4756 }
4757 
4758 /*
4759  * Manufacturer detection. Only used when the NAND is not ONFI or JEDEC
4760  * compliant and does not have a full-id or legacy-id entry in the nand_ids
4761  * table.
4762  */
4763 static void nand_manufacturer_detect(struct nand_chip *chip)
4764 {
4765 	/*
4766 	 * Try manufacturer detection if available and use
4767 	 * nand_decode_ext_id() otherwise.
4768 	 */
4769 	if (chip->manufacturer.desc && chip->manufacturer.desc->ops &&
4770 	    chip->manufacturer.desc->ops->detect) {
4771 		struct nand_memory_organization *memorg;
4772 
4773 		memorg = nanddev_get_memorg(&chip->base);
4774 
4775 		/* The 3rd id byte holds MLC / multichip data */
4776 		memorg->bits_per_cell = nand_get_bits_per_cell(chip->id.data[2]);
4777 		chip->manufacturer.desc->ops->detect(chip);
4778 	} else {
4779 		nand_decode_ext_id(chip);
4780 	}
4781 }
4782 
4783 /*
4784  * Manufacturer initialization. This function is called for all NANDs including
4785  * ONFI and JEDEC compliant ones.
4786  * Manufacturer drivers should put all their specific initialization code in
4787  * their ->init() hook.
4788  */
4789 static int nand_manufacturer_init(struct nand_chip *chip)
4790 {
4791 	if (!chip->manufacturer.desc || !chip->manufacturer.desc->ops ||
4792 	    !chip->manufacturer.desc->ops->init)
4793 		return 0;
4794 
4795 	return chip->manufacturer.desc->ops->init(chip);
4796 }
4797 
4798 /*
4799  * Manufacturer cleanup. This function is called for all NANDs including
4800  * ONFI and JEDEC compliant ones.
4801  * Manufacturer drivers should put all their specific cleanup code in their
4802  * ->cleanup() hook.
4803  */
4804 static void nand_manufacturer_cleanup(struct nand_chip *chip)
4805 {
4806 	/* Release manufacturer private data */
4807 	if (chip->manufacturer.desc && chip->manufacturer.desc->ops &&
4808 	    chip->manufacturer.desc->ops->cleanup)
4809 		chip->manufacturer.desc->ops->cleanup(chip);
4810 }
4811 
4812 static const char *
4813 nand_manufacturer_name(const struct nand_manufacturer *manufacturer)
4814 {
4815 	return manufacturer ? manufacturer->name : "Unknown";
4816 }
4817 
4818 /*
4819  * Get the flash and manufacturer id and lookup if the type is supported.
4820  */
4821 static int nand_detect(struct nand_chip *chip, struct nand_flash_dev *type)
4822 {
4823 	const struct nand_manufacturer *manufacturer;
4824 	struct mtd_info *mtd = nand_to_mtd(chip);
4825 	struct nand_memory_organization *memorg;
4826 	int busw, ret;
4827 	u8 *id_data = chip->id.data;
4828 	u8 maf_id, dev_id;
4829 	u64 targetsize;
4830 
4831 	/*
4832 	 * Let's start by initializing memorg fields that might be left
4833 	 * unassigned by the ID-based detection logic.
4834 	 */
4835 	memorg = nanddev_get_memorg(&chip->base);
4836 	memorg->planes_per_lun = 1;
4837 	memorg->luns_per_target = 1;
4838 
4839 	/*
4840 	 * Reset the chip, required by some chips (e.g. Micron MT29FxGxxxxx)
4841 	 * after power-up.
4842 	 */
4843 	ret = nand_reset(chip, 0);
4844 	if (ret)
4845 		return ret;
4846 
4847 	/* Select the device */
4848 	nand_select_target(chip, 0);
4849 
4850 	/* Send the command for reading device ID */
4851 	ret = nand_readid_op(chip, 0, id_data, 2);
4852 	if (ret)
4853 		return ret;
4854 
4855 	/* Read manufacturer and device IDs */
4856 	maf_id = id_data[0];
4857 	dev_id = id_data[1];
4858 
4859 	/*
4860 	 * Try again to make sure, as some systems the bus-hold or other
4861 	 * interface concerns can cause random data which looks like a
4862 	 * possibly credible NAND flash to appear. If the two results do
4863 	 * not match, ignore the device completely.
4864 	 */
4865 
4866 	/* Read entire ID string */
4867 	ret = nand_readid_op(chip, 0, id_data, sizeof(chip->id.data));
4868 	if (ret)
4869 		return ret;
4870 
4871 	if (id_data[0] != maf_id || id_data[1] != dev_id) {
4872 		pr_info("second ID read did not match %02x,%02x against %02x,%02x\n",
4873 			maf_id, dev_id, id_data[0], id_data[1]);
4874 		return -ENODEV;
4875 	}
4876 
4877 	chip->id.len = nand_id_len(id_data, ARRAY_SIZE(chip->id.data));
4878 
4879 	/* Try to identify manufacturer */
4880 	manufacturer = nand_get_manufacturer(maf_id);
4881 	chip->manufacturer.desc = manufacturer;
4882 
4883 	if (!type)
4884 		type = nand_flash_ids;
4885 
4886 	/*
4887 	 * Save the NAND_BUSWIDTH_16 flag before letting auto-detection logic
4888 	 * override it.
4889 	 * This is required to make sure initial NAND bus width set by the
4890 	 * NAND controller driver is coherent with the real NAND bus width
4891 	 * (extracted by auto-detection code).
4892 	 */
4893 	busw = chip->options & NAND_BUSWIDTH_16;
4894 
4895 	/*
4896 	 * The flag is only set (never cleared), reset it to its default value
4897 	 * before starting auto-detection.
4898 	 */
4899 	chip->options &= ~NAND_BUSWIDTH_16;
4900 
4901 	for (; type->name != NULL; type++) {
4902 		if (is_full_id_nand(type)) {
4903 			if (find_full_id_nand(chip, type))
4904 				goto ident_done;
4905 		} else if (dev_id == type->dev_id) {
4906 			break;
4907 		}
4908 	}
4909 
4910 	if (!type->name || !type->pagesize) {
4911 		/* Check if the chip is ONFI compliant */
4912 		ret = nand_onfi_detect(chip);
4913 		if (ret < 0)
4914 			return ret;
4915 		else if (ret)
4916 			goto ident_done;
4917 
4918 		/* Check if the chip is JEDEC compliant */
4919 		ret = nand_jedec_detect(chip);
4920 		if (ret < 0)
4921 			return ret;
4922 		else if (ret)
4923 			goto ident_done;
4924 	}
4925 
4926 	if (!type->name)
4927 		return -ENODEV;
4928 
4929 	chip->parameters.model = kstrdup(type->name, GFP_KERNEL);
4930 	if (!chip->parameters.model)
4931 		return -ENOMEM;
4932 
4933 	if (!type->pagesize)
4934 		nand_manufacturer_detect(chip);
4935 	else
4936 		nand_decode_id(chip, type);
4937 
4938 	/* Get chip options */
4939 	chip->options |= type->options;
4940 
4941 	memorg->eraseblocks_per_lun =
4942 			DIV_ROUND_DOWN_ULL((u64)type->chipsize << 20,
4943 					   memorg->pagesize *
4944 					   memorg->pages_per_eraseblock);
4945 
4946 ident_done:
4947 	if (!mtd->name)
4948 		mtd->name = chip->parameters.model;
4949 
4950 	if (chip->options & NAND_BUSWIDTH_AUTO) {
4951 		WARN_ON(busw & NAND_BUSWIDTH_16);
4952 		nand_set_defaults(chip);
4953 	} else if (busw != (chip->options & NAND_BUSWIDTH_16)) {
4954 		/*
4955 		 * Check, if buswidth is correct. Hardware drivers should set
4956 		 * chip correct!
4957 		 */
4958 		pr_info("device found, Manufacturer ID: 0x%02x, Chip ID: 0x%02x\n",
4959 			maf_id, dev_id);
4960 		pr_info("%s %s\n", nand_manufacturer_name(manufacturer),
4961 			mtd->name);
4962 		pr_warn("bus width %d instead of %d bits\n", busw ? 16 : 8,
4963 			(chip->options & NAND_BUSWIDTH_16) ? 16 : 8);
4964 		ret = -EINVAL;
4965 
4966 		goto free_detect_allocation;
4967 	}
4968 
4969 	nand_decode_bbm_options(chip);
4970 
4971 	/* Calculate the address shift from the page size */
4972 	chip->page_shift = ffs(mtd->writesize) - 1;
4973 	/* Convert chipsize to number of pages per chip -1 */
4974 	targetsize = nanddev_target_size(&chip->base);
4975 	chip->pagemask = (targetsize >> chip->page_shift) - 1;
4976 
4977 	chip->bbt_erase_shift = chip->phys_erase_shift =
4978 		ffs(mtd->erasesize) - 1;
4979 	if (targetsize & 0xffffffff)
4980 		chip->chip_shift = ffs((unsigned)targetsize) - 1;
4981 	else {
4982 		chip->chip_shift = ffs((unsigned)(targetsize >> 32));
4983 		chip->chip_shift += 32 - 1;
4984 	}
4985 
4986 	if (chip->chip_shift - chip->page_shift > 16)
4987 		chip->options |= NAND_ROW_ADDR_3;
4988 
4989 	chip->badblockbits = 8;
4990 
4991 	nand_legacy_adjust_cmdfunc(chip);
4992 
4993 	pr_info("device found, Manufacturer ID: 0x%02x, Chip ID: 0x%02x\n",
4994 		maf_id, dev_id);
4995 	pr_info("%s %s\n", nand_manufacturer_name(manufacturer),
4996 		chip->parameters.model);
4997 	pr_info("%d MiB, %s, erase size: %d KiB, page size: %d, OOB size: %d\n",
4998 		(int)(targetsize >> 20), nand_is_slc(chip) ? "SLC" : "MLC",
4999 		mtd->erasesize >> 10, mtd->writesize, mtd->oobsize);
5000 	return 0;
5001 
5002 free_detect_allocation:
5003 	kfree(chip->parameters.model);
5004 
5005 	return ret;
5006 }
5007 
5008 static const char * const nand_ecc_modes[] = {
5009 	[NAND_ECC_NONE]		= "none",
5010 	[NAND_ECC_SOFT]		= "soft",
5011 	[NAND_ECC_HW]		= "hw",
5012 	[NAND_ECC_HW_SYNDROME]	= "hw_syndrome",
5013 	[NAND_ECC_ON_DIE]	= "on-die",
5014 };
5015 
5016 static int of_get_nand_ecc_mode(struct device_node *np)
5017 {
5018 	const char *pm;
5019 	int err, i;
5020 
5021 	err = of_property_read_string(np, "nand-ecc-mode", &pm);
5022 	if (err < 0)
5023 		return err;
5024 
5025 	for (i = NAND_ECC_NONE; i < ARRAY_SIZE(nand_ecc_modes); i++)
5026 		if (!strcasecmp(pm, nand_ecc_modes[i]))
5027 			return i;
5028 
5029 	/*
5030 	 * For backward compatibility we support few obsoleted values that don't
5031 	 * have their mappings into the nand_ecc_mode enum anymore (they were
5032 	 * merged with other enums).
5033 	 */
5034 	if (!strcasecmp(pm, "soft_bch"))
5035 		return NAND_ECC_SOFT;
5036 
5037 	return -ENODEV;
5038 }
5039 
5040 static const char * const nand_ecc_algos[] = {
5041 	[NAND_ECC_HAMMING]	= "hamming",
5042 	[NAND_ECC_BCH]		= "bch",
5043 	[NAND_ECC_RS]		= "rs",
5044 };
5045 
5046 static enum nand_ecc_algo of_get_nand_ecc_algo(struct device_node *np)
5047 {
5048 	enum nand_ecc_algo ecc_algo;
5049 	const char *pm;
5050 	int err;
5051 
5052 	err = of_property_read_string(np, "nand-ecc-algo", &pm);
5053 	if (!err) {
5054 		for (ecc_algo = NAND_ECC_HAMMING;
5055 		     ecc_algo < ARRAY_SIZE(nand_ecc_algos);
5056 		     ecc_algo++) {
5057 			if (!strcasecmp(pm, nand_ecc_algos[ecc_algo]))
5058 				return ecc_algo;
5059 		}
5060 	}
5061 
5062 	/*
5063 	 * For backward compatibility we also read "nand-ecc-mode" checking
5064 	 * for some obsoleted values that were specifying ECC algorithm.
5065 	 */
5066 	err = of_property_read_string(np, "nand-ecc-mode", &pm);
5067 	if (!err) {
5068 		if (!strcasecmp(pm, "soft"))
5069 			return NAND_ECC_HAMMING;
5070 		else if (!strcasecmp(pm, "soft_bch"))
5071 			return NAND_ECC_BCH;
5072 	}
5073 
5074 	return NAND_ECC_UNKNOWN;
5075 }
5076 
5077 static int of_get_nand_ecc_step_size(struct device_node *np)
5078 {
5079 	int ret;
5080 	u32 val;
5081 
5082 	ret = of_property_read_u32(np, "nand-ecc-step-size", &val);
5083 	return ret ? ret : val;
5084 }
5085 
5086 static int of_get_nand_ecc_strength(struct device_node *np)
5087 {
5088 	int ret;
5089 	u32 val;
5090 
5091 	ret = of_property_read_u32(np, "nand-ecc-strength", &val);
5092 	return ret ? ret : val;
5093 }
5094 
5095 static int of_get_nand_bus_width(struct device_node *np)
5096 {
5097 	u32 val;
5098 
5099 	if (of_property_read_u32(np, "nand-bus-width", &val))
5100 		return 8;
5101 
5102 	switch (val) {
5103 	case 8:
5104 	case 16:
5105 		return val;
5106 	default:
5107 		return -EIO;
5108 	}
5109 }
5110 
5111 static bool of_get_nand_on_flash_bbt(struct device_node *np)
5112 {
5113 	return of_property_read_bool(np, "nand-on-flash-bbt");
5114 }
5115 
5116 static int nand_dt_init(struct nand_chip *chip)
5117 {
5118 	struct device_node *dn = nand_get_flash_node(chip);
5119 	enum nand_ecc_algo ecc_algo;
5120 	int ecc_mode, ecc_strength, ecc_step;
5121 
5122 	if (!dn)
5123 		return 0;
5124 
5125 	if (of_get_nand_bus_width(dn) == 16)
5126 		chip->options |= NAND_BUSWIDTH_16;
5127 
5128 	if (of_property_read_bool(dn, "nand-is-boot-medium"))
5129 		chip->options |= NAND_IS_BOOT_MEDIUM;
5130 
5131 	if (of_get_nand_on_flash_bbt(dn))
5132 		chip->bbt_options |= NAND_BBT_USE_FLASH;
5133 
5134 	ecc_mode = of_get_nand_ecc_mode(dn);
5135 	ecc_algo = of_get_nand_ecc_algo(dn);
5136 	ecc_strength = of_get_nand_ecc_strength(dn);
5137 	ecc_step = of_get_nand_ecc_step_size(dn);
5138 
5139 	if (ecc_mode >= 0)
5140 		chip->ecc.mode = ecc_mode;
5141 
5142 	if (ecc_algo != NAND_ECC_UNKNOWN)
5143 		chip->ecc.algo = ecc_algo;
5144 
5145 	if (ecc_strength >= 0)
5146 		chip->ecc.strength = ecc_strength;
5147 
5148 	if (ecc_step > 0)
5149 		chip->ecc.size = ecc_step;
5150 
5151 	if (of_property_read_bool(dn, "nand-ecc-maximize"))
5152 		chip->ecc.options |= NAND_ECC_MAXIMIZE;
5153 
5154 	return 0;
5155 }
5156 
5157 /**
5158  * nand_scan_ident - Scan for the NAND device
5159  * @chip: NAND chip object
5160  * @maxchips: number of chips to scan for
5161  * @table: alternative NAND ID table
5162  *
5163  * This is the first phase of the normal nand_scan() function. It reads the
5164  * flash ID and sets up MTD fields accordingly.
5165  *
5166  * This helper used to be called directly from controller drivers that needed
5167  * to tweak some ECC-related parameters before nand_scan_tail(). This separation
5168  * prevented dynamic allocations during this phase which was unconvenient and
5169  * as been banned for the benefit of the ->init_ecc()/cleanup_ecc() hooks.
5170  */
5171 static int nand_scan_ident(struct nand_chip *chip, unsigned int maxchips,
5172 			   struct nand_flash_dev *table)
5173 {
5174 	struct mtd_info *mtd = nand_to_mtd(chip);
5175 	struct nand_memory_organization *memorg;
5176 	int nand_maf_id, nand_dev_id;
5177 	unsigned int i;
5178 	int ret;
5179 
5180 	memorg = nanddev_get_memorg(&chip->base);
5181 
5182 	/* Assume all dies are deselected when we enter nand_scan_ident(). */
5183 	chip->cur_cs = -1;
5184 
5185 	mutex_init(&chip->lock);
5186 
5187 	/* Enforce the right timings for reset/detection */
5188 	onfi_fill_data_interface(chip, NAND_SDR_IFACE, 0);
5189 
5190 	ret = nand_dt_init(chip);
5191 	if (ret)
5192 		return ret;
5193 
5194 	if (!mtd->name && mtd->dev.parent)
5195 		mtd->name = dev_name(mtd->dev.parent);
5196 
5197 	/* Set the default functions */
5198 	nand_set_defaults(chip);
5199 
5200 	ret = nand_legacy_check_hooks(chip);
5201 	if (ret)
5202 		return ret;
5203 
5204 	memorg->ntargets = maxchips;
5205 
5206 	/* Read the flash type */
5207 	ret = nand_detect(chip, table);
5208 	if (ret) {
5209 		if (!(chip->options & NAND_SCAN_SILENT_NODEV))
5210 			pr_warn("No NAND device found\n");
5211 		nand_deselect_target(chip);
5212 		return ret;
5213 	}
5214 
5215 	nand_maf_id = chip->id.data[0];
5216 	nand_dev_id = chip->id.data[1];
5217 
5218 	nand_deselect_target(chip);
5219 
5220 	/* Check for a chip array */
5221 	for (i = 1; i < maxchips; i++) {
5222 		u8 id[2];
5223 
5224 		/* See comment in nand_get_flash_type for reset */
5225 		ret = nand_reset(chip, i);
5226 		if (ret)
5227 			break;
5228 
5229 		nand_select_target(chip, i);
5230 		/* Send the command for reading device ID */
5231 		ret = nand_readid_op(chip, 0, id, sizeof(id));
5232 		if (ret)
5233 			break;
5234 		/* Read manufacturer and device IDs */
5235 		if (nand_maf_id != id[0] || nand_dev_id != id[1]) {
5236 			nand_deselect_target(chip);
5237 			break;
5238 		}
5239 		nand_deselect_target(chip);
5240 	}
5241 	if (i > 1)
5242 		pr_info("%d chips detected\n", i);
5243 
5244 	/* Store the number of chips and calc total size for mtd */
5245 	memorg->ntargets = i;
5246 	mtd->size = i * nanddev_target_size(&chip->base);
5247 
5248 	return 0;
5249 }
5250 
5251 static void nand_scan_ident_cleanup(struct nand_chip *chip)
5252 {
5253 	kfree(chip->parameters.model);
5254 	kfree(chip->parameters.onfi);
5255 }
5256 
5257 static int nand_set_ecc_soft_ops(struct nand_chip *chip)
5258 {
5259 	struct mtd_info *mtd = nand_to_mtd(chip);
5260 	struct nand_ecc_ctrl *ecc = &chip->ecc;
5261 
5262 	if (WARN_ON(ecc->mode != NAND_ECC_SOFT))
5263 		return -EINVAL;
5264 
5265 	switch (ecc->algo) {
5266 	case NAND_ECC_HAMMING:
5267 		ecc->calculate = nand_calculate_ecc;
5268 		ecc->correct = nand_correct_data;
5269 		ecc->read_page = nand_read_page_swecc;
5270 		ecc->read_subpage = nand_read_subpage;
5271 		ecc->write_page = nand_write_page_swecc;
5272 		if (!ecc->read_page_raw)
5273 			ecc->read_page_raw = nand_read_page_raw;
5274 		if (!ecc->write_page_raw)
5275 			ecc->write_page_raw = nand_write_page_raw;
5276 		ecc->read_oob = nand_read_oob_std;
5277 		ecc->write_oob = nand_write_oob_std;
5278 		if (!ecc->size)
5279 			ecc->size = 256;
5280 		ecc->bytes = 3;
5281 		ecc->strength = 1;
5282 
5283 		if (IS_ENABLED(CONFIG_MTD_NAND_ECC_SW_HAMMING_SMC))
5284 			ecc->options |= NAND_ECC_SOFT_HAMMING_SM_ORDER;
5285 
5286 		return 0;
5287 	case NAND_ECC_BCH:
5288 		if (!mtd_nand_has_bch()) {
5289 			WARN(1, "CONFIG_MTD_NAND_ECC_SW_BCH not enabled\n");
5290 			return -EINVAL;
5291 		}
5292 		ecc->calculate = nand_bch_calculate_ecc;
5293 		ecc->correct = nand_bch_correct_data;
5294 		ecc->read_page = nand_read_page_swecc;
5295 		ecc->read_subpage = nand_read_subpage;
5296 		ecc->write_page = nand_write_page_swecc;
5297 		if (!ecc->read_page_raw)
5298 			ecc->read_page_raw = nand_read_page_raw;
5299 		if (!ecc->write_page_raw)
5300 			ecc->write_page_raw = nand_write_page_raw;
5301 		ecc->read_oob = nand_read_oob_std;
5302 		ecc->write_oob = nand_write_oob_std;
5303 
5304 		/*
5305 		* Board driver should supply ecc.size and ecc.strength
5306 		* values to select how many bits are correctable.
5307 		* Otherwise, default to 4 bits for large page devices.
5308 		*/
5309 		if (!ecc->size && (mtd->oobsize >= 64)) {
5310 			ecc->size = 512;
5311 			ecc->strength = 4;
5312 		}
5313 
5314 		/*
5315 		 * if no ecc placement scheme was provided pickup the default
5316 		 * large page one.
5317 		 */
5318 		if (!mtd->ooblayout) {
5319 			/* handle large page devices only */
5320 			if (mtd->oobsize < 64) {
5321 				WARN(1, "OOB layout is required when using software BCH on small pages\n");
5322 				return -EINVAL;
5323 			}
5324 
5325 			mtd_set_ooblayout(mtd, &nand_ooblayout_lp_ops);
5326 
5327 		}
5328 
5329 		/*
5330 		 * We can only maximize ECC config when the default layout is
5331 		 * used, otherwise we don't know how many bytes can really be
5332 		 * used.
5333 		 */
5334 		if (mtd->ooblayout == &nand_ooblayout_lp_ops &&
5335 		    ecc->options & NAND_ECC_MAXIMIZE) {
5336 			int steps, bytes;
5337 
5338 			/* Always prefer 1k blocks over 512bytes ones */
5339 			ecc->size = 1024;
5340 			steps = mtd->writesize / ecc->size;
5341 
5342 			/* Reserve 2 bytes for the BBM */
5343 			bytes = (mtd->oobsize - 2) / steps;
5344 			ecc->strength = bytes * 8 / fls(8 * ecc->size);
5345 		}
5346 
5347 		/* See nand_bch_init() for details. */
5348 		ecc->bytes = 0;
5349 		ecc->priv = nand_bch_init(mtd);
5350 		if (!ecc->priv) {
5351 			WARN(1, "BCH ECC initialization failed!\n");
5352 			return -EINVAL;
5353 		}
5354 		return 0;
5355 	default:
5356 		WARN(1, "Unsupported ECC algorithm!\n");
5357 		return -EINVAL;
5358 	}
5359 }
5360 
5361 /**
5362  * nand_check_ecc_caps - check the sanity of preset ECC settings
5363  * @chip: nand chip info structure
5364  * @caps: ECC caps info structure
5365  * @oobavail: OOB size that the ECC engine can use
5366  *
5367  * When ECC step size and strength are already set, check if they are supported
5368  * by the controller and the calculated ECC bytes fit within the chip's OOB.
5369  * On success, the calculated ECC bytes is set.
5370  */
5371 static int
5372 nand_check_ecc_caps(struct nand_chip *chip,
5373 		    const struct nand_ecc_caps *caps, int oobavail)
5374 {
5375 	struct mtd_info *mtd = nand_to_mtd(chip);
5376 	const struct nand_ecc_step_info *stepinfo;
5377 	int preset_step = chip->ecc.size;
5378 	int preset_strength = chip->ecc.strength;
5379 	int ecc_bytes, nsteps = mtd->writesize / preset_step;
5380 	int i, j;
5381 
5382 	for (i = 0; i < caps->nstepinfos; i++) {
5383 		stepinfo = &caps->stepinfos[i];
5384 
5385 		if (stepinfo->stepsize != preset_step)
5386 			continue;
5387 
5388 		for (j = 0; j < stepinfo->nstrengths; j++) {
5389 			if (stepinfo->strengths[j] != preset_strength)
5390 				continue;
5391 
5392 			ecc_bytes = caps->calc_ecc_bytes(preset_step,
5393 							 preset_strength);
5394 			if (WARN_ON_ONCE(ecc_bytes < 0))
5395 				return ecc_bytes;
5396 
5397 			if (ecc_bytes * nsteps > oobavail) {
5398 				pr_err("ECC (step, strength) = (%d, %d) does not fit in OOB",
5399 				       preset_step, preset_strength);
5400 				return -ENOSPC;
5401 			}
5402 
5403 			chip->ecc.bytes = ecc_bytes;
5404 
5405 			return 0;
5406 		}
5407 	}
5408 
5409 	pr_err("ECC (step, strength) = (%d, %d) not supported on this controller",
5410 	       preset_step, preset_strength);
5411 
5412 	return -ENOTSUPP;
5413 }
5414 
5415 /**
5416  * nand_match_ecc_req - meet the chip's requirement with least ECC bytes
5417  * @chip: nand chip info structure
5418  * @caps: ECC engine caps info structure
5419  * @oobavail: OOB size that the ECC engine can use
5420  *
5421  * If a chip's ECC requirement is provided, try to meet it with the least
5422  * number of ECC bytes (i.e. with the largest number of OOB-free bytes).
5423  * On success, the chosen ECC settings are set.
5424  */
5425 static int
5426 nand_match_ecc_req(struct nand_chip *chip,
5427 		   const struct nand_ecc_caps *caps, int oobavail)
5428 {
5429 	struct mtd_info *mtd = nand_to_mtd(chip);
5430 	const struct nand_ecc_step_info *stepinfo;
5431 	int req_step = chip->base.eccreq.step_size;
5432 	int req_strength = chip->base.eccreq.strength;
5433 	int req_corr, step_size, strength, nsteps, ecc_bytes, ecc_bytes_total;
5434 	int best_step, best_strength, best_ecc_bytes;
5435 	int best_ecc_bytes_total = INT_MAX;
5436 	int i, j;
5437 
5438 	/* No information provided by the NAND chip */
5439 	if (!req_step || !req_strength)
5440 		return -ENOTSUPP;
5441 
5442 	/* number of correctable bits the chip requires in a page */
5443 	req_corr = mtd->writesize / req_step * req_strength;
5444 
5445 	for (i = 0; i < caps->nstepinfos; i++) {
5446 		stepinfo = &caps->stepinfos[i];
5447 		step_size = stepinfo->stepsize;
5448 
5449 		for (j = 0; j < stepinfo->nstrengths; j++) {
5450 			strength = stepinfo->strengths[j];
5451 
5452 			/*
5453 			 * If both step size and strength are smaller than the
5454 			 * chip's requirement, it is not easy to compare the
5455 			 * resulted reliability.
5456 			 */
5457 			if (step_size < req_step && strength < req_strength)
5458 				continue;
5459 
5460 			if (mtd->writesize % step_size)
5461 				continue;
5462 
5463 			nsteps = mtd->writesize / step_size;
5464 
5465 			ecc_bytes = caps->calc_ecc_bytes(step_size, strength);
5466 			if (WARN_ON_ONCE(ecc_bytes < 0))
5467 				continue;
5468 			ecc_bytes_total = ecc_bytes * nsteps;
5469 
5470 			if (ecc_bytes_total > oobavail ||
5471 			    strength * nsteps < req_corr)
5472 				continue;
5473 
5474 			/*
5475 			 * We assume the best is to meet the chip's requrement
5476 			 * with the least number of ECC bytes.
5477 			 */
5478 			if (ecc_bytes_total < best_ecc_bytes_total) {
5479 				best_ecc_bytes_total = ecc_bytes_total;
5480 				best_step = step_size;
5481 				best_strength = strength;
5482 				best_ecc_bytes = ecc_bytes;
5483 			}
5484 		}
5485 	}
5486 
5487 	if (best_ecc_bytes_total == INT_MAX)
5488 		return -ENOTSUPP;
5489 
5490 	chip->ecc.size = best_step;
5491 	chip->ecc.strength = best_strength;
5492 	chip->ecc.bytes = best_ecc_bytes;
5493 
5494 	return 0;
5495 }
5496 
5497 /**
5498  * nand_maximize_ecc - choose the max ECC strength available
5499  * @chip: nand chip info structure
5500  * @caps: ECC engine caps info structure
5501  * @oobavail: OOB size that the ECC engine can use
5502  *
5503  * Choose the max ECC strength that is supported on the controller, and can fit
5504  * within the chip's OOB.  On success, the chosen ECC settings are set.
5505  */
5506 static int
5507 nand_maximize_ecc(struct nand_chip *chip,
5508 		  const struct nand_ecc_caps *caps, int oobavail)
5509 {
5510 	struct mtd_info *mtd = nand_to_mtd(chip);
5511 	const struct nand_ecc_step_info *stepinfo;
5512 	int step_size, strength, nsteps, ecc_bytes, corr;
5513 	int best_corr = 0;
5514 	int best_step = 0;
5515 	int best_strength, best_ecc_bytes;
5516 	int i, j;
5517 
5518 	for (i = 0; i < caps->nstepinfos; i++) {
5519 		stepinfo = &caps->stepinfos[i];
5520 		step_size = stepinfo->stepsize;
5521 
5522 		/* If chip->ecc.size is already set, respect it */
5523 		if (chip->ecc.size && step_size != chip->ecc.size)
5524 			continue;
5525 
5526 		for (j = 0; j < stepinfo->nstrengths; j++) {
5527 			strength = stepinfo->strengths[j];
5528 
5529 			if (mtd->writesize % step_size)
5530 				continue;
5531 
5532 			nsteps = mtd->writesize / step_size;
5533 
5534 			ecc_bytes = caps->calc_ecc_bytes(step_size, strength);
5535 			if (WARN_ON_ONCE(ecc_bytes < 0))
5536 				continue;
5537 
5538 			if (ecc_bytes * nsteps > oobavail)
5539 				continue;
5540 
5541 			corr = strength * nsteps;
5542 
5543 			/*
5544 			 * If the number of correctable bits is the same,
5545 			 * bigger step_size has more reliability.
5546 			 */
5547 			if (corr > best_corr ||
5548 			    (corr == best_corr && step_size > best_step)) {
5549 				best_corr = corr;
5550 				best_step = step_size;
5551 				best_strength = strength;
5552 				best_ecc_bytes = ecc_bytes;
5553 			}
5554 		}
5555 	}
5556 
5557 	if (!best_corr)
5558 		return -ENOTSUPP;
5559 
5560 	chip->ecc.size = best_step;
5561 	chip->ecc.strength = best_strength;
5562 	chip->ecc.bytes = best_ecc_bytes;
5563 
5564 	return 0;
5565 }
5566 
5567 /**
5568  * nand_ecc_choose_conf - Set the ECC strength and ECC step size
5569  * @chip: nand chip info structure
5570  * @caps: ECC engine caps info structure
5571  * @oobavail: OOB size that the ECC engine can use
5572  *
5573  * Choose the ECC configuration according to following logic
5574  *
5575  * 1. If both ECC step size and ECC strength are already set (usually by DT)
5576  *    then check if it is supported by this controller.
5577  * 2. If NAND_ECC_MAXIMIZE is set, then select maximum ECC strength.
5578  * 3. Otherwise, try to match the ECC step size and ECC strength closest
5579  *    to the chip's requirement. If available OOB size can't fit the chip
5580  *    requirement then fallback to the maximum ECC step size and ECC strength.
5581  *
5582  * On success, the chosen ECC settings are set.
5583  */
5584 int nand_ecc_choose_conf(struct nand_chip *chip,
5585 			 const struct nand_ecc_caps *caps, int oobavail)
5586 {
5587 	struct mtd_info *mtd = nand_to_mtd(chip);
5588 
5589 	if (WARN_ON(oobavail < 0 || oobavail > mtd->oobsize))
5590 		return -EINVAL;
5591 
5592 	if (chip->ecc.size && chip->ecc.strength)
5593 		return nand_check_ecc_caps(chip, caps, oobavail);
5594 
5595 	if (chip->ecc.options & NAND_ECC_MAXIMIZE)
5596 		return nand_maximize_ecc(chip, caps, oobavail);
5597 
5598 	if (!nand_match_ecc_req(chip, caps, oobavail))
5599 		return 0;
5600 
5601 	return nand_maximize_ecc(chip, caps, oobavail);
5602 }
5603 EXPORT_SYMBOL_GPL(nand_ecc_choose_conf);
5604 
5605 /*
5606  * Check if the chip configuration meet the datasheet requirements.
5607 
5608  * If our configuration corrects A bits per B bytes and the minimum
5609  * required correction level is X bits per Y bytes, then we must ensure
5610  * both of the following are true:
5611  *
5612  * (1) A / B >= X / Y
5613  * (2) A >= X
5614  *
5615  * Requirement (1) ensures we can correct for the required bitflip density.
5616  * Requirement (2) ensures we can correct even when all bitflips are clumped
5617  * in the same sector.
5618  */
5619 static bool nand_ecc_strength_good(struct nand_chip *chip)
5620 {
5621 	struct mtd_info *mtd = nand_to_mtd(chip);
5622 	struct nand_ecc_ctrl *ecc = &chip->ecc;
5623 	int corr, ds_corr;
5624 
5625 	if (ecc->size == 0 || chip->base.eccreq.step_size == 0)
5626 		/* Not enough information */
5627 		return true;
5628 
5629 	/*
5630 	 * We get the number of corrected bits per page to compare
5631 	 * the correction density.
5632 	 */
5633 	corr = (mtd->writesize * ecc->strength) / ecc->size;
5634 	ds_corr = (mtd->writesize * chip->base.eccreq.strength) /
5635 		  chip->base.eccreq.step_size;
5636 
5637 	return corr >= ds_corr && ecc->strength >= chip->base.eccreq.strength;
5638 }
5639 
5640 static int rawnand_erase(struct nand_device *nand, const struct nand_pos *pos)
5641 {
5642 	struct nand_chip *chip = container_of(nand, struct nand_chip,
5643 					      base);
5644 	unsigned int eb = nanddev_pos_to_row(nand, pos);
5645 	int ret;
5646 
5647 	eb >>= nand->rowconv.eraseblock_addr_shift;
5648 
5649 	nand_select_target(chip, pos->target);
5650 	ret = nand_erase_op(chip, eb);
5651 	nand_deselect_target(chip);
5652 
5653 	return ret;
5654 }
5655 
5656 static int rawnand_markbad(struct nand_device *nand,
5657 			   const struct nand_pos *pos)
5658 {
5659 	struct nand_chip *chip = container_of(nand, struct nand_chip,
5660 					      base);
5661 
5662 	return nand_markbad_bbm(chip, nanddev_pos_to_offs(nand, pos));
5663 }
5664 
5665 static bool rawnand_isbad(struct nand_device *nand, const struct nand_pos *pos)
5666 {
5667 	struct nand_chip *chip = container_of(nand, struct nand_chip,
5668 					      base);
5669 	int ret;
5670 
5671 	nand_select_target(chip, pos->target);
5672 	ret = nand_isbad_bbm(chip, nanddev_pos_to_offs(nand, pos));
5673 	nand_deselect_target(chip);
5674 
5675 	return ret;
5676 }
5677 
5678 static const struct nand_ops rawnand_ops = {
5679 	.erase = rawnand_erase,
5680 	.markbad = rawnand_markbad,
5681 	.isbad = rawnand_isbad,
5682 };
5683 
5684 /**
5685  * nand_scan_tail - Scan for the NAND device
5686  * @chip: NAND chip object
5687  *
5688  * This is the second phase of the normal nand_scan() function. It fills out
5689  * all the uninitialized function pointers with the defaults and scans for a
5690  * bad block table if appropriate.
5691  */
5692 static int nand_scan_tail(struct nand_chip *chip)
5693 {
5694 	struct mtd_info *mtd = nand_to_mtd(chip);
5695 	struct nand_ecc_ctrl *ecc = &chip->ecc;
5696 	int ret, i;
5697 
5698 	/* New bad blocks should be marked in OOB, flash-based BBT, or both */
5699 	if (WARN_ON((chip->bbt_options & NAND_BBT_NO_OOB_BBM) &&
5700 		   !(chip->bbt_options & NAND_BBT_USE_FLASH))) {
5701 		return -EINVAL;
5702 	}
5703 
5704 	chip->data_buf = kmalloc(mtd->writesize + mtd->oobsize, GFP_KERNEL);
5705 	if (!chip->data_buf)
5706 		return -ENOMEM;
5707 
5708 	/*
5709 	 * FIXME: some NAND manufacturer drivers expect the first die to be
5710 	 * selected when manufacturer->init() is called. They should be fixed
5711 	 * to explictly select the relevant die when interacting with the NAND
5712 	 * chip.
5713 	 */
5714 	nand_select_target(chip, 0);
5715 	ret = nand_manufacturer_init(chip);
5716 	nand_deselect_target(chip);
5717 	if (ret)
5718 		goto err_free_buf;
5719 
5720 	/* Set the internal oob buffer location, just after the page data */
5721 	chip->oob_poi = chip->data_buf + mtd->writesize;
5722 
5723 	/*
5724 	 * If no default placement scheme is given, select an appropriate one.
5725 	 */
5726 	if (!mtd->ooblayout &&
5727 	    !(ecc->mode == NAND_ECC_SOFT && ecc->algo == NAND_ECC_BCH)) {
5728 		switch (mtd->oobsize) {
5729 		case 8:
5730 		case 16:
5731 			mtd_set_ooblayout(mtd, &nand_ooblayout_sp_ops);
5732 			break;
5733 		case 64:
5734 		case 128:
5735 			mtd_set_ooblayout(mtd, &nand_ooblayout_lp_hamming_ops);
5736 			break;
5737 		default:
5738 			/*
5739 			 * Expose the whole OOB area to users if ECC_NONE
5740 			 * is passed. We could do that for all kind of
5741 			 * ->oobsize, but we must keep the old large/small
5742 			 * page with ECC layout when ->oobsize <= 128 for
5743 			 * compatibility reasons.
5744 			 */
5745 			if (ecc->mode == NAND_ECC_NONE) {
5746 				mtd_set_ooblayout(mtd,
5747 						&nand_ooblayout_lp_ops);
5748 				break;
5749 			}
5750 
5751 			WARN(1, "No oob scheme defined for oobsize %d\n",
5752 				mtd->oobsize);
5753 			ret = -EINVAL;
5754 			goto err_nand_manuf_cleanup;
5755 		}
5756 	}
5757 
5758 	/*
5759 	 * Check ECC mode, default to software if 3byte/512byte hardware ECC is
5760 	 * selected and we have 256 byte pagesize fallback to software ECC
5761 	 */
5762 
5763 	switch (ecc->mode) {
5764 	case NAND_ECC_HW:
5765 		/* Use standard hwecc read page function? */
5766 		if (!ecc->read_page)
5767 			ecc->read_page = nand_read_page_hwecc;
5768 		if (!ecc->write_page)
5769 			ecc->write_page = nand_write_page_hwecc;
5770 		if (!ecc->read_page_raw)
5771 			ecc->read_page_raw = nand_read_page_raw;
5772 		if (!ecc->write_page_raw)
5773 			ecc->write_page_raw = nand_write_page_raw;
5774 		if (!ecc->read_oob)
5775 			ecc->read_oob = nand_read_oob_std;
5776 		if (!ecc->write_oob)
5777 			ecc->write_oob = nand_write_oob_std;
5778 		if (!ecc->read_subpage)
5779 			ecc->read_subpage = nand_read_subpage;
5780 		if (!ecc->write_subpage && ecc->hwctl && ecc->calculate)
5781 			ecc->write_subpage = nand_write_subpage_hwecc;
5782 		fallthrough;
5783 	case NAND_ECC_HW_SYNDROME:
5784 		if ((!ecc->calculate || !ecc->correct || !ecc->hwctl) &&
5785 		    (!ecc->read_page ||
5786 		     ecc->read_page == nand_read_page_hwecc ||
5787 		     !ecc->write_page ||
5788 		     ecc->write_page == nand_write_page_hwecc)) {
5789 			WARN(1, "No ECC functions supplied; hardware ECC not possible\n");
5790 			ret = -EINVAL;
5791 			goto err_nand_manuf_cleanup;
5792 		}
5793 		/* Use standard syndrome read/write page function? */
5794 		if (!ecc->read_page)
5795 			ecc->read_page = nand_read_page_syndrome;
5796 		if (!ecc->write_page)
5797 			ecc->write_page = nand_write_page_syndrome;
5798 		if (!ecc->read_page_raw)
5799 			ecc->read_page_raw = nand_read_page_raw_syndrome;
5800 		if (!ecc->write_page_raw)
5801 			ecc->write_page_raw = nand_write_page_raw_syndrome;
5802 		if (!ecc->read_oob)
5803 			ecc->read_oob = nand_read_oob_syndrome;
5804 		if (!ecc->write_oob)
5805 			ecc->write_oob = nand_write_oob_syndrome;
5806 
5807 		if (mtd->writesize >= ecc->size) {
5808 			if (!ecc->strength) {
5809 				WARN(1, "Driver must set ecc.strength when using hardware ECC\n");
5810 				ret = -EINVAL;
5811 				goto err_nand_manuf_cleanup;
5812 			}
5813 			break;
5814 		}
5815 		pr_warn("%d byte HW ECC not possible on %d byte page size, fallback to SW ECC\n",
5816 			ecc->size, mtd->writesize);
5817 		ecc->mode = NAND_ECC_SOFT;
5818 		ecc->algo = NAND_ECC_HAMMING;
5819 		fallthrough;
5820 	case NAND_ECC_SOFT:
5821 		ret = nand_set_ecc_soft_ops(chip);
5822 		if (ret) {
5823 			ret = -EINVAL;
5824 			goto err_nand_manuf_cleanup;
5825 		}
5826 		break;
5827 
5828 	case NAND_ECC_ON_DIE:
5829 		if (!ecc->read_page || !ecc->write_page) {
5830 			WARN(1, "No ECC functions supplied; on-die ECC not possible\n");
5831 			ret = -EINVAL;
5832 			goto err_nand_manuf_cleanup;
5833 		}
5834 		if (!ecc->read_oob)
5835 			ecc->read_oob = nand_read_oob_std;
5836 		if (!ecc->write_oob)
5837 			ecc->write_oob = nand_write_oob_std;
5838 		break;
5839 
5840 	case NAND_ECC_NONE:
5841 		pr_warn("NAND_ECC_NONE selected by board driver. This is not recommended!\n");
5842 		ecc->read_page = nand_read_page_raw;
5843 		ecc->write_page = nand_write_page_raw;
5844 		ecc->read_oob = nand_read_oob_std;
5845 		ecc->read_page_raw = nand_read_page_raw;
5846 		ecc->write_page_raw = nand_write_page_raw;
5847 		ecc->write_oob = nand_write_oob_std;
5848 		ecc->size = mtd->writesize;
5849 		ecc->bytes = 0;
5850 		ecc->strength = 0;
5851 		break;
5852 
5853 	default:
5854 		WARN(1, "Invalid NAND_ECC_MODE %d\n", ecc->mode);
5855 		ret = -EINVAL;
5856 		goto err_nand_manuf_cleanup;
5857 	}
5858 
5859 	if (ecc->correct || ecc->calculate) {
5860 		ecc->calc_buf = kmalloc(mtd->oobsize, GFP_KERNEL);
5861 		ecc->code_buf = kmalloc(mtd->oobsize, GFP_KERNEL);
5862 		if (!ecc->calc_buf || !ecc->code_buf) {
5863 			ret = -ENOMEM;
5864 			goto err_nand_manuf_cleanup;
5865 		}
5866 	}
5867 
5868 	/* For many systems, the standard OOB write also works for raw */
5869 	if (!ecc->read_oob_raw)
5870 		ecc->read_oob_raw = ecc->read_oob;
5871 	if (!ecc->write_oob_raw)
5872 		ecc->write_oob_raw = ecc->write_oob;
5873 
5874 	/* propagate ecc info to mtd_info */
5875 	mtd->ecc_strength = ecc->strength;
5876 	mtd->ecc_step_size = ecc->size;
5877 
5878 	/*
5879 	 * Set the number of read / write steps for one page depending on ECC
5880 	 * mode.
5881 	 */
5882 	ecc->steps = mtd->writesize / ecc->size;
5883 	if (ecc->steps * ecc->size != mtd->writesize) {
5884 		WARN(1, "Invalid ECC parameters\n");
5885 		ret = -EINVAL;
5886 		goto err_nand_manuf_cleanup;
5887 	}
5888 	ecc->total = ecc->steps * ecc->bytes;
5889 	if (ecc->total > mtd->oobsize) {
5890 		WARN(1, "Total number of ECC bytes exceeded oobsize\n");
5891 		ret = -EINVAL;
5892 		goto err_nand_manuf_cleanup;
5893 	}
5894 
5895 	/*
5896 	 * The number of bytes available for a client to place data into
5897 	 * the out of band area.
5898 	 */
5899 	ret = mtd_ooblayout_count_freebytes(mtd);
5900 	if (ret < 0)
5901 		ret = 0;
5902 
5903 	mtd->oobavail = ret;
5904 
5905 	/* ECC sanity check: warn if it's too weak */
5906 	if (!nand_ecc_strength_good(chip))
5907 		pr_warn("WARNING: %s: the ECC used on your system (%db/%dB) is too weak compared to the one required by the NAND chip (%db/%dB)\n",
5908 			mtd->name, chip->ecc.strength, chip->ecc.size,
5909 			chip->base.eccreq.strength,
5910 			chip->base.eccreq.step_size);
5911 
5912 	/* Allow subpage writes up to ecc.steps. Not possible for MLC flash */
5913 	if (!(chip->options & NAND_NO_SUBPAGE_WRITE) && nand_is_slc(chip)) {
5914 		switch (ecc->steps) {
5915 		case 2:
5916 			mtd->subpage_sft = 1;
5917 			break;
5918 		case 4:
5919 		case 8:
5920 		case 16:
5921 			mtd->subpage_sft = 2;
5922 			break;
5923 		}
5924 	}
5925 	chip->subpagesize = mtd->writesize >> mtd->subpage_sft;
5926 
5927 	/* Invalidate the pagebuffer reference */
5928 	chip->pagecache.page = -1;
5929 
5930 	/* Large page NAND with SOFT_ECC should support subpage reads */
5931 	switch (ecc->mode) {
5932 	case NAND_ECC_SOFT:
5933 		if (chip->page_shift > 9)
5934 			chip->options |= NAND_SUBPAGE_READ;
5935 		break;
5936 
5937 	default:
5938 		break;
5939 	}
5940 
5941 	ret = nanddev_init(&chip->base, &rawnand_ops, mtd->owner);
5942 	if (ret)
5943 		goto err_nand_manuf_cleanup;
5944 
5945 	/* Adjust the MTD_CAP_ flags when NAND_ROM is set. */
5946 	if (chip->options & NAND_ROM)
5947 		mtd->flags = MTD_CAP_ROM;
5948 
5949 	/* Fill in remaining MTD driver data */
5950 	mtd->_erase = nand_erase;
5951 	mtd->_point = NULL;
5952 	mtd->_unpoint = NULL;
5953 	mtd->_panic_write = panic_nand_write;
5954 	mtd->_read_oob = nand_read_oob;
5955 	mtd->_write_oob = nand_write_oob;
5956 	mtd->_sync = nand_sync;
5957 	mtd->_lock = nand_lock;
5958 	mtd->_unlock = nand_unlock;
5959 	mtd->_suspend = nand_suspend;
5960 	mtd->_resume = nand_resume;
5961 	mtd->_reboot = nand_shutdown;
5962 	mtd->_block_isreserved = nand_block_isreserved;
5963 	mtd->_block_isbad = nand_block_isbad;
5964 	mtd->_block_markbad = nand_block_markbad;
5965 	mtd->_max_bad_blocks = nanddev_mtd_max_bad_blocks;
5966 
5967 	/*
5968 	 * Initialize bitflip_threshold to its default prior scan_bbt() call.
5969 	 * scan_bbt() might invoke mtd_read(), thus bitflip_threshold must be
5970 	 * properly set.
5971 	 */
5972 	if (!mtd->bitflip_threshold)
5973 		mtd->bitflip_threshold = DIV_ROUND_UP(mtd->ecc_strength * 3, 4);
5974 
5975 	/* Initialize the ->data_interface field. */
5976 	ret = nand_init_data_interface(chip);
5977 	if (ret)
5978 		goto err_nanddev_cleanup;
5979 
5980 	/* Enter fastest possible mode on all dies. */
5981 	for (i = 0; i < nanddev_ntargets(&chip->base); i++) {
5982 		ret = nand_setup_data_interface(chip, i);
5983 		if (ret)
5984 			goto err_nanddev_cleanup;
5985 	}
5986 
5987 	/* Check, if we should skip the bad block table scan */
5988 	if (chip->options & NAND_SKIP_BBTSCAN)
5989 		return 0;
5990 
5991 	/* Build bad block table */
5992 	ret = nand_create_bbt(chip);
5993 	if (ret)
5994 		goto err_nanddev_cleanup;
5995 
5996 	return 0;
5997 
5998 
5999 err_nanddev_cleanup:
6000 	nanddev_cleanup(&chip->base);
6001 
6002 err_nand_manuf_cleanup:
6003 	nand_manufacturer_cleanup(chip);
6004 
6005 err_free_buf:
6006 	kfree(chip->data_buf);
6007 	kfree(ecc->code_buf);
6008 	kfree(ecc->calc_buf);
6009 
6010 	return ret;
6011 }
6012 
6013 static int nand_attach(struct nand_chip *chip)
6014 {
6015 	if (chip->controller->ops && chip->controller->ops->attach_chip)
6016 		return chip->controller->ops->attach_chip(chip);
6017 
6018 	return 0;
6019 }
6020 
6021 static void nand_detach(struct nand_chip *chip)
6022 {
6023 	if (chip->controller->ops && chip->controller->ops->detach_chip)
6024 		chip->controller->ops->detach_chip(chip);
6025 }
6026 
6027 /**
6028  * nand_scan_with_ids - [NAND Interface] Scan for the NAND device
6029  * @chip: NAND chip object
6030  * @maxchips: number of chips to scan for.
6031  * @ids: optional flash IDs table
6032  *
6033  * This fills out all the uninitialized function pointers with the defaults.
6034  * The flash ID is read and the mtd/chip structures are filled with the
6035  * appropriate values.
6036  */
6037 int nand_scan_with_ids(struct nand_chip *chip, unsigned int maxchips,
6038 		       struct nand_flash_dev *ids)
6039 {
6040 	int ret;
6041 
6042 	if (!maxchips)
6043 		return -EINVAL;
6044 
6045 	ret = nand_scan_ident(chip, maxchips, ids);
6046 	if (ret)
6047 		return ret;
6048 
6049 	ret = nand_attach(chip);
6050 	if (ret)
6051 		goto cleanup_ident;
6052 
6053 	ret = nand_scan_tail(chip);
6054 	if (ret)
6055 		goto detach_chip;
6056 
6057 	return 0;
6058 
6059 detach_chip:
6060 	nand_detach(chip);
6061 cleanup_ident:
6062 	nand_scan_ident_cleanup(chip);
6063 
6064 	return ret;
6065 }
6066 EXPORT_SYMBOL(nand_scan_with_ids);
6067 
6068 /**
6069  * nand_cleanup - [NAND Interface] Free resources held by the NAND device
6070  * @chip: NAND chip object
6071  */
6072 void nand_cleanup(struct nand_chip *chip)
6073 {
6074 	if (chip->ecc.mode == NAND_ECC_SOFT &&
6075 	    chip->ecc.algo == NAND_ECC_BCH)
6076 		nand_bch_free((struct nand_bch_control *)chip->ecc.priv);
6077 
6078 	nanddev_cleanup(&chip->base);
6079 
6080 	/* Free bad block table memory */
6081 	kfree(chip->bbt);
6082 	kfree(chip->data_buf);
6083 	kfree(chip->ecc.code_buf);
6084 	kfree(chip->ecc.calc_buf);
6085 
6086 	/* Free bad block descriptor memory */
6087 	if (chip->badblock_pattern && chip->badblock_pattern->options
6088 			& NAND_BBT_DYNAMICSTRUCT)
6089 		kfree(chip->badblock_pattern);
6090 
6091 	/* Free manufacturer priv data. */
6092 	nand_manufacturer_cleanup(chip);
6093 
6094 	/* Free controller specific allocations after chip identification */
6095 	nand_detach(chip);
6096 
6097 	/* Free identification phase allocations */
6098 	nand_scan_ident_cleanup(chip);
6099 }
6100 
6101 EXPORT_SYMBOL_GPL(nand_cleanup);
6102 
6103 MODULE_LICENSE("GPL");
6104 MODULE_AUTHOR("Steven J. Hill <sjhill@realitydiluted.com>");
6105 MODULE_AUTHOR("Thomas Gleixner <tglx@linutronix.de>");
6106 MODULE_DESCRIPTION("Generic NAND flash driver code");
6107