1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * Freescale GPMI NAND Flash Driver
4  *
5  * Copyright (C) 2010-2015 Freescale Semiconductor, Inc.
6  * Copyright (C) 2008 Embedded Alley Solutions, Inc.
7  */
8 #include <linux/clk.h>
9 #include <linux/delay.h>
10 #include <linux/slab.h>
11 #include <linux/sched/task_stack.h>
12 #include <linux/interrupt.h>
13 #include <linux/module.h>
14 #include <linux/mtd/partitions.h>
15 #include <linux/of.h>
16 #include <linux/of_device.h>
17 #include <linux/pm_runtime.h>
18 #include <linux/dma/mxs-dma.h>
19 #include "gpmi-nand.h"
20 #include "gpmi-regs.h"
21 #include "bch-regs.h"
22 
23 /* Resource names for the GPMI NAND driver. */
24 #define GPMI_NAND_GPMI_REGS_ADDR_RES_NAME  "gpmi-nand"
25 #define GPMI_NAND_BCH_REGS_ADDR_RES_NAME   "bch"
26 #define GPMI_NAND_BCH_INTERRUPT_RES_NAME   "bch"
27 
28 /* Converts time to clock cycles */
29 #define TO_CYCLES(duration, period) DIV_ROUND_UP_ULL(duration, period)
30 
31 #define MXS_SET_ADDR		0x4
32 #define MXS_CLR_ADDR		0x8
33 /*
34  * Clear the bit and poll it cleared.  This is usually called with
35  * a reset address and mask being either SFTRST(bit 31) or CLKGATE
36  * (bit 30).
37  */
38 static int clear_poll_bit(void __iomem *addr, u32 mask)
39 {
40 	int timeout = 0x400;
41 
42 	/* clear the bit */
43 	writel(mask, addr + MXS_CLR_ADDR);
44 
45 	/*
46 	 * SFTRST needs 3 GPMI clocks to settle, the reference manual
47 	 * recommends to wait 1us.
48 	 */
49 	udelay(1);
50 
51 	/* poll the bit becoming clear */
52 	while ((readl(addr) & mask) && --timeout)
53 		/* nothing */;
54 
55 	return !timeout;
56 }
57 
58 #define MODULE_CLKGATE		(1 << 30)
59 #define MODULE_SFTRST		(1 << 31)
60 /*
61  * The current mxs_reset_block() will do two things:
62  *  [1] enable the module.
63  *  [2] reset the module.
64  *
65  * In most of the cases, it's ok.
66  * But in MX23, there is a hardware bug in the BCH block (see erratum #2847).
67  * If you try to soft reset the BCH block, it becomes unusable until
68  * the next hard reset. This case occurs in the NAND boot mode. When the board
69  * boots by NAND, the ROM of the chip will initialize the BCH blocks itself.
70  * So If the driver tries to reset the BCH again, the BCH will not work anymore.
71  * You will see a DMA timeout in this case. The bug has been fixed
72  * in the following chips, such as MX28.
73  *
74  * To avoid this bug, just add a new parameter `just_enable` for
75  * the mxs_reset_block(), and rewrite it here.
76  */
77 static int gpmi_reset_block(void __iomem *reset_addr, bool just_enable)
78 {
79 	int ret;
80 	int timeout = 0x400;
81 
82 	/* clear and poll SFTRST */
83 	ret = clear_poll_bit(reset_addr, MODULE_SFTRST);
84 	if (unlikely(ret))
85 		goto error;
86 
87 	/* clear CLKGATE */
88 	writel(MODULE_CLKGATE, reset_addr + MXS_CLR_ADDR);
89 
90 	if (!just_enable) {
91 		/* set SFTRST to reset the block */
92 		writel(MODULE_SFTRST, reset_addr + MXS_SET_ADDR);
93 		udelay(1);
94 
95 		/* poll CLKGATE becoming set */
96 		while ((!(readl(reset_addr) & MODULE_CLKGATE)) && --timeout)
97 			/* nothing */;
98 		if (unlikely(!timeout))
99 			goto error;
100 	}
101 
102 	/* clear and poll SFTRST */
103 	ret = clear_poll_bit(reset_addr, MODULE_SFTRST);
104 	if (unlikely(ret))
105 		goto error;
106 
107 	/* clear and poll CLKGATE */
108 	ret = clear_poll_bit(reset_addr, MODULE_CLKGATE);
109 	if (unlikely(ret))
110 		goto error;
111 
112 	return 0;
113 
114 error:
115 	pr_err("%s(%p): module reset timeout\n", __func__, reset_addr);
116 	return -ETIMEDOUT;
117 }
118 
119 static int __gpmi_enable_clk(struct gpmi_nand_data *this, bool v)
120 {
121 	struct clk *clk;
122 	int ret;
123 	int i;
124 
125 	for (i = 0; i < GPMI_CLK_MAX; i++) {
126 		clk = this->resources.clock[i];
127 		if (!clk)
128 			break;
129 
130 		if (v) {
131 			ret = clk_prepare_enable(clk);
132 			if (ret)
133 				goto err_clk;
134 		} else {
135 			clk_disable_unprepare(clk);
136 		}
137 	}
138 	return 0;
139 
140 err_clk:
141 	for (; i > 0; i--)
142 		clk_disable_unprepare(this->resources.clock[i - 1]);
143 	return ret;
144 }
145 
146 static int gpmi_init(struct gpmi_nand_data *this)
147 {
148 	struct resources *r = &this->resources;
149 	int ret;
150 
151 	ret = pm_runtime_get_sync(this->dev);
152 	if (ret < 0) {
153 		pm_runtime_put_noidle(this->dev);
154 		return ret;
155 	}
156 
157 	ret = gpmi_reset_block(r->gpmi_regs, false);
158 	if (ret)
159 		goto err_out;
160 
161 	/*
162 	 * Reset BCH here, too. We got failures otherwise :(
163 	 * See later BCH reset for explanation of MX23 and MX28 handling
164 	 */
165 	ret = gpmi_reset_block(r->bch_regs, GPMI_IS_MXS(this));
166 	if (ret)
167 		goto err_out;
168 
169 	/* Choose NAND mode. */
170 	writel(BM_GPMI_CTRL1_GPMI_MODE, r->gpmi_regs + HW_GPMI_CTRL1_CLR);
171 
172 	/* Set the IRQ polarity. */
173 	writel(BM_GPMI_CTRL1_ATA_IRQRDY_POLARITY,
174 				r->gpmi_regs + HW_GPMI_CTRL1_SET);
175 
176 	/* Disable Write-Protection. */
177 	writel(BM_GPMI_CTRL1_DEV_RESET, r->gpmi_regs + HW_GPMI_CTRL1_SET);
178 
179 	/* Select BCH ECC. */
180 	writel(BM_GPMI_CTRL1_BCH_MODE, r->gpmi_regs + HW_GPMI_CTRL1_SET);
181 
182 	/*
183 	 * Decouple the chip select from dma channel. We use dma0 for all
184 	 * the chips, force all NAND RDY_BUSY inputs to be sourced from
185 	 * RDY_BUSY0.
186 	 */
187 	writel(BM_GPMI_CTRL1_DECOUPLE_CS | BM_GPMI_CTRL1_GANGED_RDYBUSY,
188 	       r->gpmi_regs + HW_GPMI_CTRL1_SET);
189 
190 err_out:
191 	pm_runtime_mark_last_busy(this->dev);
192 	pm_runtime_put_autosuspend(this->dev);
193 	return ret;
194 }
195 
196 /* This function is very useful. It is called only when the bug occur. */
197 static void gpmi_dump_info(struct gpmi_nand_data *this)
198 {
199 	struct resources *r = &this->resources;
200 	struct bch_geometry *geo = &this->bch_geometry;
201 	u32 reg;
202 	int i;
203 
204 	dev_err(this->dev, "Show GPMI registers :\n");
205 	for (i = 0; i <= HW_GPMI_DEBUG / 0x10 + 1; i++) {
206 		reg = readl(r->gpmi_regs + i * 0x10);
207 		dev_err(this->dev, "offset 0x%.3x : 0x%.8x\n", i * 0x10, reg);
208 	}
209 
210 	/* start to print out the BCH info */
211 	dev_err(this->dev, "Show BCH registers :\n");
212 	for (i = 0; i <= HW_BCH_VERSION / 0x10 + 1; i++) {
213 		reg = readl(r->bch_regs + i * 0x10);
214 		dev_err(this->dev, "offset 0x%.3x : 0x%.8x\n", i * 0x10, reg);
215 	}
216 	dev_err(this->dev, "BCH Geometry :\n"
217 		"GF length              : %u\n"
218 		"ECC Strength           : %u\n"
219 		"Page Size in Bytes     : %u\n"
220 		"Metadata Size in Bytes : %u\n"
221 		"ECC Chunk Size in Bytes: %u\n"
222 		"ECC Chunk Count        : %u\n"
223 		"Payload Size in Bytes  : %u\n"
224 		"Auxiliary Size in Bytes: %u\n"
225 		"Auxiliary Status Offset: %u\n"
226 		"Block Mark Byte Offset : %u\n"
227 		"Block Mark Bit Offset  : %u\n",
228 		geo->gf_len,
229 		geo->ecc_strength,
230 		geo->page_size,
231 		geo->metadata_size,
232 		geo->ecc_chunk_size,
233 		geo->ecc_chunk_count,
234 		geo->payload_size,
235 		geo->auxiliary_size,
236 		geo->auxiliary_status_offset,
237 		geo->block_mark_byte_offset,
238 		geo->block_mark_bit_offset);
239 }
240 
241 static inline bool gpmi_check_ecc(struct gpmi_nand_data *this)
242 {
243 	struct bch_geometry *geo = &this->bch_geometry;
244 
245 	/* Do the sanity check. */
246 	if (GPMI_IS_MXS(this)) {
247 		/* The mx23/mx28 only support the GF13. */
248 		if (geo->gf_len == 14)
249 			return false;
250 	}
251 	return geo->ecc_strength <= this->devdata->bch_max_ecc_strength;
252 }
253 
254 /*
255  * If we can get the ECC information from the nand chip, we do not
256  * need to calculate them ourselves.
257  *
258  * We may have available oob space in this case.
259  */
260 static int set_geometry_by_ecc_info(struct gpmi_nand_data *this,
261 				    unsigned int ecc_strength,
262 				    unsigned int ecc_step)
263 {
264 	struct bch_geometry *geo = &this->bch_geometry;
265 	struct nand_chip *chip = &this->nand;
266 	struct mtd_info *mtd = nand_to_mtd(chip);
267 	unsigned int block_mark_bit_offset;
268 
269 	switch (ecc_step) {
270 	case SZ_512:
271 		geo->gf_len = 13;
272 		break;
273 	case SZ_1K:
274 		geo->gf_len = 14;
275 		break;
276 	default:
277 		dev_err(this->dev,
278 			"unsupported nand chip. ecc bits : %d, ecc size : %d\n",
279 			nanddev_get_ecc_requirements(&chip->base)->strength,
280 			nanddev_get_ecc_requirements(&chip->base)->step_size);
281 		return -EINVAL;
282 	}
283 	geo->ecc_chunk_size = ecc_step;
284 	geo->ecc_strength = round_up(ecc_strength, 2);
285 	if (!gpmi_check_ecc(this))
286 		return -EINVAL;
287 
288 	/* Keep the C >= O */
289 	if (geo->ecc_chunk_size < mtd->oobsize) {
290 		dev_err(this->dev,
291 			"unsupported nand chip. ecc size: %d, oob size : %d\n",
292 			ecc_step, mtd->oobsize);
293 		return -EINVAL;
294 	}
295 
296 	/* The default value, see comment in the legacy_set_geometry(). */
297 	geo->metadata_size = 10;
298 
299 	geo->ecc_chunk_count = mtd->writesize / geo->ecc_chunk_size;
300 
301 	/*
302 	 * Now, the NAND chip with 2K page(data chunk is 512byte) shows below:
303 	 *
304 	 *    |                          P                            |
305 	 *    |<----------------------------------------------------->|
306 	 *    |                                                       |
307 	 *    |                                        (Block Mark)   |
308 	 *    |                      P'                      |      | |     |
309 	 *    |<-------------------------------------------->|  D   | |  O' |
310 	 *    |                                              |<---->| |<--->|
311 	 *    V                                              V      V V     V
312 	 *    +---+----------+-+----------+-+----------+-+----------+-+-----+
313 	 *    | M |   data   |E|   data   |E|   data   |E|   data   |E|     |
314 	 *    +---+----------+-+----------+-+----------+-+----------+-+-----+
315 	 *                                                   ^              ^
316 	 *                                                   |      O       |
317 	 *                                                   |<------------>|
318 	 *                                                   |              |
319 	 *
320 	 *	P : the page size for BCH module.
321 	 *	E : The ECC strength.
322 	 *	G : the length of Galois Field.
323 	 *	N : The chunk count of per page.
324 	 *	M : the metasize of per page.
325 	 *	C : the ecc chunk size, aka the "data" above.
326 	 *	P': the nand chip's page size.
327 	 *	O : the nand chip's oob size.
328 	 *	O': the free oob.
329 	 *
330 	 *	The formula for P is :
331 	 *
332 	 *	            E * G * N
333 	 *	       P = ------------ + P' + M
334 	 *                      8
335 	 *
336 	 * The position of block mark moves forward in the ECC-based view
337 	 * of page, and the delta is:
338 	 *
339 	 *                   E * G * (N - 1)
340 	 *             D = (---------------- + M)
341 	 *                          8
342 	 *
343 	 * Please see the comment in legacy_set_geometry().
344 	 * With the condition C >= O , we still can get same result.
345 	 * So the bit position of the physical block mark within the ECC-based
346 	 * view of the page is :
347 	 *             (P' - D) * 8
348 	 */
349 	geo->page_size = mtd->writesize + geo->metadata_size +
350 		(geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8;
351 
352 	geo->payload_size = mtd->writesize;
353 
354 	geo->auxiliary_status_offset = ALIGN(geo->metadata_size, 4);
355 	geo->auxiliary_size = ALIGN(geo->metadata_size, 4)
356 				+ ALIGN(geo->ecc_chunk_count, 4);
357 
358 	if (!this->swap_block_mark)
359 		return 0;
360 
361 	/* For bit swap. */
362 	block_mark_bit_offset = mtd->writesize * 8 -
363 		(geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - 1)
364 				+ geo->metadata_size * 8);
365 
366 	geo->block_mark_byte_offset = block_mark_bit_offset / 8;
367 	geo->block_mark_bit_offset  = block_mark_bit_offset % 8;
368 	return 0;
369 }
370 
371 /*
372  *  Calculate the ECC strength by hand:
373  *	E : The ECC strength.
374  *	G : the length of Galois Field.
375  *	N : The chunk count of per page.
376  *	O : the oobsize of the NAND chip.
377  *	M : the metasize of per page.
378  *
379  *	The formula is :
380  *		E * G * N
381  *	      ------------ <= (O - M)
382  *                  8
383  *
384  *      So, we get E by:
385  *                    (O - M) * 8
386  *              E <= -------------
387  *                       G * N
388  */
389 static inline int get_ecc_strength(struct gpmi_nand_data *this)
390 {
391 	struct bch_geometry *geo = &this->bch_geometry;
392 	struct mtd_info	*mtd = nand_to_mtd(&this->nand);
393 	int ecc_strength;
394 
395 	ecc_strength = ((mtd->oobsize - geo->metadata_size) * 8)
396 			/ (geo->gf_len * geo->ecc_chunk_count);
397 
398 	/* We need the minor even number. */
399 	return round_down(ecc_strength, 2);
400 }
401 
402 static int legacy_set_geometry(struct gpmi_nand_data *this)
403 {
404 	struct bch_geometry *geo = &this->bch_geometry;
405 	struct mtd_info *mtd = nand_to_mtd(&this->nand);
406 	unsigned int metadata_size;
407 	unsigned int status_size;
408 	unsigned int block_mark_bit_offset;
409 
410 	/*
411 	 * The size of the metadata can be changed, though we set it to 10
412 	 * bytes now. But it can't be too large, because we have to save
413 	 * enough space for BCH.
414 	 */
415 	geo->metadata_size = 10;
416 
417 	/* The default for the length of Galois Field. */
418 	geo->gf_len = 13;
419 
420 	/* The default for chunk size. */
421 	geo->ecc_chunk_size = 512;
422 	while (geo->ecc_chunk_size < mtd->oobsize) {
423 		geo->ecc_chunk_size *= 2; /* keep C >= O */
424 		geo->gf_len = 14;
425 	}
426 
427 	geo->ecc_chunk_count = mtd->writesize / geo->ecc_chunk_size;
428 
429 	/* We use the same ECC strength for all chunks. */
430 	geo->ecc_strength = get_ecc_strength(this);
431 	if (!gpmi_check_ecc(this)) {
432 		dev_err(this->dev,
433 			"ecc strength: %d cannot be supported by the controller (%d)\n"
434 			"try to use minimum ecc strength that NAND chip required\n",
435 			geo->ecc_strength,
436 			this->devdata->bch_max_ecc_strength);
437 		return -EINVAL;
438 	}
439 
440 	geo->page_size = mtd->writesize + geo->metadata_size +
441 		(geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8;
442 	geo->payload_size = mtd->writesize;
443 
444 	/*
445 	 * The auxiliary buffer contains the metadata and the ECC status. The
446 	 * metadata is padded to the nearest 32-bit boundary. The ECC status
447 	 * contains one byte for every ECC chunk, and is also padded to the
448 	 * nearest 32-bit boundary.
449 	 */
450 	metadata_size = ALIGN(geo->metadata_size, 4);
451 	status_size   = ALIGN(geo->ecc_chunk_count, 4);
452 
453 	geo->auxiliary_size = metadata_size + status_size;
454 	geo->auxiliary_status_offset = metadata_size;
455 
456 	if (!this->swap_block_mark)
457 		return 0;
458 
459 	/*
460 	 * We need to compute the byte and bit offsets of
461 	 * the physical block mark within the ECC-based view of the page.
462 	 *
463 	 * NAND chip with 2K page shows below:
464 	 *                                             (Block Mark)
465 	 *                                                   |      |
466 	 *                                                   |  D   |
467 	 *                                                   |<---->|
468 	 *                                                   V      V
469 	 *    +---+----------+-+----------+-+----------+-+----------+-+
470 	 *    | M |   data   |E|   data   |E|   data   |E|   data   |E|
471 	 *    +---+----------+-+----------+-+----------+-+----------+-+
472 	 *
473 	 * The position of block mark moves forward in the ECC-based view
474 	 * of page, and the delta is:
475 	 *
476 	 *                   E * G * (N - 1)
477 	 *             D = (---------------- + M)
478 	 *                          8
479 	 *
480 	 * With the formula to compute the ECC strength, and the condition
481 	 *       : C >= O         (C is the ecc chunk size)
482 	 *
483 	 * It's easy to deduce to the following result:
484 	 *
485 	 *         E * G       (O - M)      C - M         C - M
486 	 *      ----------- <= ------- <=  --------  <  ---------
487 	 *           8            N           N          (N - 1)
488 	 *
489 	 *  So, we get:
490 	 *
491 	 *                   E * G * (N - 1)
492 	 *             D = (---------------- + M) < C
493 	 *                          8
494 	 *
495 	 *  The above inequality means the position of block mark
496 	 *  within the ECC-based view of the page is still in the data chunk,
497 	 *  and it's NOT in the ECC bits of the chunk.
498 	 *
499 	 *  Use the following to compute the bit position of the
500 	 *  physical block mark within the ECC-based view of the page:
501 	 *          (page_size - D) * 8
502 	 *
503 	 *  --Huang Shijie
504 	 */
505 	block_mark_bit_offset = mtd->writesize * 8 -
506 		(geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - 1)
507 				+ geo->metadata_size * 8);
508 
509 	geo->block_mark_byte_offset = block_mark_bit_offset / 8;
510 	geo->block_mark_bit_offset  = block_mark_bit_offset % 8;
511 	return 0;
512 }
513 
514 static int common_nfc_set_geometry(struct gpmi_nand_data *this)
515 {
516 	struct nand_chip *chip = &this->nand;
517 	const struct nand_ecc_props *requirements =
518 		nanddev_get_ecc_requirements(&chip->base);
519 
520 	if (chip->ecc.strength > 0 && chip->ecc.size > 0)
521 		return set_geometry_by_ecc_info(this, chip->ecc.strength,
522 						chip->ecc.size);
523 
524 	if ((of_property_read_bool(this->dev->of_node, "fsl,use-minimum-ecc"))
525 				|| legacy_set_geometry(this)) {
526 		if (!(requirements->strength > 0 && requirements->step_size > 0))
527 			return -EINVAL;
528 
529 		return set_geometry_by_ecc_info(this,
530 						requirements->strength,
531 						requirements->step_size);
532 	}
533 
534 	return 0;
535 }
536 
537 /* Configures the geometry for BCH.  */
538 static int bch_set_geometry(struct gpmi_nand_data *this)
539 {
540 	struct resources *r = &this->resources;
541 	int ret;
542 
543 	ret = common_nfc_set_geometry(this);
544 	if (ret)
545 		return ret;
546 
547 	ret = pm_runtime_get_sync(this->dev);
548 	if (ret < 0) {
549 		pm_runtime_put_autosuspend(this->dev);
550 		return ret;
551 	}
552 
553 	/*
554 	* Due to erratum #2847 of the MX23, the BCH cannot be soft reset on this
555 	* chip, otherwise it will lock up. So we skip resetting BCH on the MX23.
556 	* and MX28.
557 	*/
558 	ret = gpmi_reset_block(r->bch_regs, GPMI_IS_MXS(this));
559 	if (ret)
560 		goto err_out;
561 
562 	/* Set *all* chip selects to use layout 0. */
563 	writel(0, r->bch_regs + HW_BCH_LAYOUTSELECT);
564 
565 	ret = 0;
566 err_out:
567 	pm_runtime_mark_last_busy(this->dev);
568 	pm_runtime_put_autosuspend(this->dev);
569 
570 	return ret;
571 }
572 
573 /*
574  * <1> Firstly, we should know what's the GPMI-clock means.
575  *     The GPMI-clock is the internal clock in the gpmi nand controller.
576  *     If you set 100MHz to gpmi nand controller, the GPMI-clock's period
577  *     is 10ns. Mark the GPMI-clock's period as GPMI-clock-period.
578  *
579  * <2> Secondly, we should know what's the frequency on the nand chip pins.
580  *     The frequency on the nand chip pins is derived from the GPMI-clock.
581  *     We can get it from the following equation:
582  *
583  *         F = G / (DS + DH)
584  *
585  *         F  : the frequency on the nand chip pins.
586  *         G  : the GPMI clock, such as 100MHz.
587  *         DS : GPMI_HW_GPMI_TIMING0:DATA_SETUP
588  *         DH : GPMI_HW_GPMI_TIMING0:DATA_HOLD
589  *
590  * <3> Thirdly, when the frequency on the nand chip pins is above 33MHz,
591  *     the nand EDO(extended Data Out) timing could be applied.
592  *     The GPMI implements a feedback read strobe to sample the read data.
593  *     The feedback read strobe can be delayed to support the nand EDO timing
594  *     where the read strobe may deasserts before the read data is valid, and
595  *     read data is valid for some time after read strobe.
596  *
597  *     The following figure illustrates some aspects of a NAND Flash read:
598  *
599  *                   |<---tREA---->|
600  *                   |             |
601  *                   |         |   |
602  *                   |<--tRP-->|   |
603  *                   |         |   |
604  *                  __          ___|__________________________________
605  *     RDN            \________/   |
606  *                                 |
607  *                                 /---------\
608  *     Read Data    --------------<           >---------
609  *                                 \---------/
610  *                                |     |
611  *                                |<-D->|
612  *     FeedbackRDN  ________             ____________
613  *                          \___________/
614  *
615  *          D stands for delay, set in the HW_GPMI_CTRL1:RDN_DELAY.
616  *
617  *
618  * <4> Now, we begin to describe how to compute the right RDN_DELAY.
619  *
620  *  4.1) From the aspect of the nand chip pins:
621  *        Delay = (tREA + C - tRP)               {1}
622  *
623  *        tREA : the maximum read access time.
624  *        C    : a constant to adjust the delay. default is 4000ps.
625  *        tRP  : the read pulse width, which is exactly:
626  *                   tRP = (GPMI-clock-period) * DATA_SETUP
627  *
628  *  4.2) From the aspect of the GPMI nand controller:
629  *         Delay = RDN_DELAY * 0.125 * RP        {2}
630  *
631  *         RP   : the DLL reference period.
632  *            if (GPMI-clock-period > DLL_THRETHOLD)
633  *                   RP = GPMI-clock-period / 2;
634  *            else
635  *                   RP = GPMI-clock-period;
636  *
637  *            Set the HW_GPMI_CTRL1:HALF_PERIOD if GPMI-clock-period
638  *            is greater DLL_THRETHOLD. In other SOCs, the DLL_THRETHOLD
639  *            is 16000ps, but in mx6q, we use 12000ps.
640  *
641  *  4.3) since {1} equals {2}, we get:
642  *
643  *                     (tREA + 4000 - tRP) * 8
644  *         RDN_DELAY = -----------------------     {3}
645  *                           RP
646  */
647 static void gpmi_nfc_compute_timings(struct gpmi_nand_data *this,
648 				     const struct nand_sdr_timings *sdr)
649 {
650 	struct gpmi_nfc_hardware_timing *hw = &this->hw;
651 	unsigned int dll_threshold_ps = this->devdata->max_chain_delay;
652 	unsigned int period_ps, reference_period_ps;
653 	unsigned int data_setup_cycles, data_hold_cycles, addr_setup_cycles;
654 	unsigned int tRP_ps;
655 	bool use_half_period;
656 	int sample_delay_ps, sample_delay_factor;
657 	u16 busy_timeout_cycles;
658 	u8 wrn_dly_sel;
659 
660 	if (sdr->tRC_min >= 30000) {
661 		/* ONFI non-EDO modes [0-3] */
662 		hw->clk_rate = 22000000;
663 		wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_4_TO_8NS;
664 	} else if (sdr->tRC_min >= 25000) {
665 		/* ONFI EDO mode 4 */
666 		hw->clk_rate = 80000000;
667 		wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_NO_DELAY;
668 	} else {
669 		/* ONFI EDO mode 5 */
670 		hw->clk_rate = 100000000;
671 		wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_NO_DELAY;
672 	}
673 
674 	/* SDR core timings are given in picoseconds */
675 	period_ps = div_u64((u64)NSEC_PER_SEC * 1000, hw->clk_rate);
676 
677 	addr_setup_cycles = TO_CYCLES(sdr->tALS_min, period_ps);
678 	data_setup_cycles = TO_CYCLES(sdr->tDS_min, period_ps);
679 	data_hold_cycles = TO_CYCLES(sdr->tDH_min, period_ps);
680 	busy_timeout_cycles = TO_CYCLES(sdr->tWB_max + sdr->tR_max, period_ps);
681 
682 	hw->timing0 = BF_GPMI_TIMING0_ADDRESS_SETUP(addr_setup_cycles) |
683 		      BF_GPMI_TIMING0_DATA_HOLD(data_hold_cycles) |
684 		      BF_GPMI_TIMING0_DATA_SETUP(data_setup_cycles);
685 	hw->timing1 = BF_GPMI_TIMING1_BUSY_TIMEOUT(busy_timeout_cycles * 4096);
686 
687 	/*
688 	 * Derive NFC ideal delay from {3}:
689 	 *
690 	 *                     (tREA + 4000 - tRP) * 8
691 	 *         RDN_DELAY = -----------------------
692 	 *                                RP
693 	 */
694 	if (period_ps > dll_threshold_ps) {
695 		use_half_period = true;
696 		reference_period_ps = period_ps / 2;
697 	} else {
698 		use_half_period = false;
699 		reference_period_ps = period_ps;
700 	}
701 
702 	tRP_ps = data_setup_cycles * period_ps;
703 	sample_delay_ps = (sdr->tREA_max + 4000 - tRP_ps) * 8;
704 	if (sample_delay_ps > 0)
705 		sample_delay_factor = sample_delay_ps / reference_period_ps;
706 	else
707 		sample_delay_factor = 0;
708 
709 	hw->ctrl1n = BF_GPMI_CTRL1_WRN_DLY_SEL(wrn_dly_sel);
710 	if (sample_delay_factor)
711 		hw->ctrl1n |= BF_GPMI_CTRL1_RDN_DELAY(sample_delay_factor) |
712 			      BM_GPMI_CTRL1_DLL_ENABLE |
713 			      (use_half_period ? BM_GPMI_CTRL1_HALF_PERIOD : 0);
714 }
715 
716 static void gpmi_nfc_apply_timings(struct gpmi_nand_data *this)
717 {
718 	struct gpmi_nfc_hardware_timing *hw = &this->hw;
719 	struct resources *r = &this->resources;
720 	void __iomem *gpmi_regs = r->gpmi_regs;
721 	unsigned int dll_wait_time_us;
722 
723 	clk_set_rate(r->clock[0], hw->clk_rate);
724 
725 	writel(hw->timing0, gpmi_regs + HW_GPMI_TIMING0);
726 	writel(hw->timing1, gpmi_regs + HW_GPMI_TIMING1);
727 
728 	/*
729 	 * Clear several CTRL1 fields, DLL must be disabled when setting
730 	 * RDN_DELAY or HALF_PERIOD.
731 	 */
732 	writel(BM_GPMI_CTRL1_CLEAR_MASK, gpmi_regs + HW_GPMI_CTRL1_CLR);
733 	writel(hw->ctrl1n, gpmi_regs + HW_GPMI_CTRL1_SET);
734 
735 	/* Wait 64 clock cycles before using the GPMI after enabling the DLL */
736 	dll_wait_time_us = USEC_PER_SEC / hw->clk_rate * 64;
737 	if (!dll_wait_time_us)
738 		dll_wait_time_us = 1;
739 
740 	/* Wait for the DLL to settle. */
741 	udelay(dll_wait_time_us);
742 }
743 
744 static int gpmi_setup_interface(struct nand_chip *chip, int chipnr,
745 				const struct nand_interface_config *conf)
746 {
747 	struct gpmi_nand_data *this = nand_get_controller_data(chip);
748 	const struct nand_sdr_timings *sdr;
749 
750 	/* Retrieve required NAND timings */
751 	sdr = nand_get_sdr_timings(conf);
752 	if (IS_ERR(sdr))
753 		return PTR_ERR(sdr);
754 
755 	/* Only MX6 GPMI controller can reach EDO timings */
756 	if (sdr->tRC_min <= 25000 && !GPMI_IS_MX6(this))
757 		return -ENOTSUPP;
758 
759 	/* Stop here if this call was just a check */
760 	if (chipnr < 0)
761 		return 0;
762 
763 	/* Do the actual derivation of the controller timings */
764 	gpmi_nfc_compute_timings(this, sdr);
765 
766 	this->hw.must_apply_timings = true;
767 
768 	return 0;
769 }
770 
771 /* Clears a BCH interrupt. */
772 static void gpmi_clear_bch(struct gpmi_nand_data *this)
773 {
774 	struct resources *r = &this->resources;
775 	writel(BM_BCH_CTRL_COMPLETE_IRQ, r->bch_regs + HW_BCH_CTRL_CLR);
776 }
777 
778 static struct dma_chan *get_dma_chan(struct gpmi_nand_data *this)
779 {
780 	/* We use the DMA channel 0 to access all the nand chips. */
781 	return this->dma_chans[0];
782 }
783 
784 /* This will be called after the DMA operation is finished. */
785 static void dma_irq_callback(void *param)
786 {
787 	struct gpmi_nand_data *this = param;
788 	struct completion *dma_c = &this->dma_done;
789 
790 	complete(dma_c);
791 }
792 
793 static irqreturn_t bch_irq(int irq, void *cookie)
794 {
795 	struct gpmi_nand_data *this = cookie;
796 
797 	gpmi_clear_bch(this);
798 	complete(&this->bch_done);
799 	return IRQ_HANDLED;
800 }
801 
802 static int gpmi_raw_len_to_len(struct gpmi_nand_data *this, int raw_len)
803 {
804 	/*
805 	 * raw_len is the length to read/write including bch data which
806 	 * we are passed in exec_op. Calculate the data length from it.
807 	 */
808 	if (this->bch)
809 		return ALIGN_DOWN(raw_len, this->bch_geometry.ecc_chunk_size);
810 	else
811 		return raw_len;
812 }
813 
814 /* Can we use the upper's buffer directly for DMA? */
815 static bool prepare_data_dma(struct gpmi_nand_data *this, const void *buf,
816 			     int raw_len, struct scatterlist *sgl,
817 			     enum dma_data_direction dr)
818 {
819 	int ret;
820 	int len = gpmi_raw_len_to_len(this, raw_len);
821 
822 	/* first try to map the upper buffer directly */
823 	if (virt_addr_valid(buf) && !object_is_on_stack(buf)) {
824 		sg_init_one(sgl, buf, len);
825 		ret = dma_map_sg(this->dev, sgl, 1, dr);
826 		if (ret == 0)
827 			goto map_fail;
828 
829 		return true;
830 	}
831 
832 map_fail:
833 	/* We have to use our own DMA buffer. */
834 	sg_init_one(sgl, this->data_buffer_dma, len);
835 
836 	if (dr == DMA_TO_DEVICE && buf != this->data_buffer_dma)
837 		memcpy(this->data_buffer_dma, buf, len);
838 
839 	dma_map_sg(this->dev, sgl, 1, dr);
840 
841 	return false;
842 }
843 
844 /* add our owner bbt descriptor */
845 static uint8_t scan_ff_pattern[] = { 0xff };
846 static struct nand_bbt_descr gpmi_bbt_descr = {
847 	.options	= 0,
848 	.offs		= 0,
849 	.len		= 1,
850 	.pattern	= scan_ff_pattern
851 };
852 
853 /*
854  * We may change the layout if we can get the ECC info from the datasheet,
855  * else we will use all the (page + OOB).
856  */
857 static int gpmi_ooblayout_ecc(struct mtd_info *mtd, int section,
858 			      struct mtd_oob_region *oobregion)
859 {
860 	struct nand_chip *chip = mtd_to_nand(mtd);
861 	struct gpmi_nand_data *this = nand_get_controller_data(chip);
862 	struct bch_geometry *geo = &this->bch_geometry;
863 
864 	if (section)
865 		return -ERANGE;
866 
867 	oobregion->offset = 0;
868 	oobregion->length = geo->page_size - mtd->writesize;
869 
870 	return 0;
871 }
872 
873 static int gpmi_ooblayout_free(struct mtd_info *mtd, int section,
874 			       struct mtd_oob_region *oobregion)
875 {
876 	struct nand_chip *chip = mtd_to_nand(mtd);
877 	struct gpmi_nand_data *this = nand_get_controller_data(chip);
878 	struct bch_geometry *geo = &this->bch_geometry;
879 
880 	if (section)
881 		return -ERANGE;
882 
883 	/* The available oob size we have. */
884 	if (geo->page_size < mtd->writesize + mtd->oobsize) {
885 		oobregion->offset = geo->page_size - mtd->writesize;
886 		oobregion->length = mtd->oobsize - oobregion->offset;
887 	}
888 
889 	return 0;
890 }
891 
892 static const char * const gpmi_clks_for_mx2x[] = {
893 	"gpmi_io",
894 };
895 
896 static const struct mtd_ooblayout_ops gpmi_ooblayout_ops = {
897 	.ecc = gpmi_ooblayout_ecc,
898 	.free = gpmi_ooblayout_free,
899 };
900 
901 static const struct gpmi_devdata gpmi_devdata_imx23 = {
902 	.type = IS_MX23,
903 	.bch_max_ecc_strength = 20,
904 	.max_chain_delay = 16000,
905 	.clks = gpmi_clks_for_mx2x,
906 	.clks_count = ARRAY_SIZE(gpmi_clks_for_mx2x),
907 };
908 
909 static const struct gpmi_devdata gpmi_devdata_imx28 = {
910 	.type = IS_MX28,
911 	.bch_max_ecc_strength = 20,
912 	.max_chain_delay = 16000,
913 	.clks = gpmi_clks_for_mx2x,
914 	.clks_count = ARRAY_SIZE(gpmi_clks_for_mx2x),
915 };
916 
917 static const char * const gpmi_clks_for_mx6[] = {
918 	"gpmi_io", "gpmi_apb", "gpmi_bch", "gpmi_bch_apb", "per1_bch",
919 };
920 
921 static const struct gpmi_devdata gpmi_devdata_imx6q = {
922 	.type = IS_MX6Q,
923 	.bch_max_ecc_strength = 40,
924 	.max_chain_delay = 12000,
925 	.clks = gpmi_clks_for_mx6,
926 	.clks_count = ARRAY_SIZE(gpmi_clks_for_mx6),
927 };
928 
929 static const struct gpmi_devdata gpmi_devdata_imx6sx = {
930 	.type = IS_MX6SX,
931 	.bch_max_ecc_strength = 62,
932 	.max_chain_delay = 12000,
933 	.clks = gpmi_clks_for_mx6,
934 	.clks_count = ARRAY_SIZE(gpmi_clks_for_mx6),
935 };
936 
937 static const char * const gpmi_clks_for_mx7d[] = {
938 	"gpmi_io", "gpmi_bch_apb",
939 };
940 
941 static const struct gpmi_devdata gpmi_devdata_imx7d = {
942 	.type = IS_MX7D,
943 	.bch_max_ecc_strength = 62,
944 	.max_chain_delay = 12000,
945 	.clks = gpmi_clks_for_mx7d,
946 	.clks_count = ARRAY_SIZE(gpmi_clks_for_mx7d),
947 };
948 
949 static int acquire_register_block(struct gpmi_nand_data *this,
950 				  const char *res_name)
951 {
952 	struct platform_device *pdev = this->pdev;
953 	struct resources *res = &this->resources;
954 	struct resource *r;
955 	void __iomem *p;
956 
957 	r = platform_get_resource_byname(pdev, IORESOURCE_MEM, res_name);
958 	p = devm_ioremap_resource(&pdev->dev, r);
959 	if (IS_ERR(p))
960 		return PTR_ERR(p);
961 
962 	if (!strcmp(res_name, GPMI_NAND_GPMI_REGS_ADDR_RES_NAME))
963 		res->gpmi_regs = p;
964 	else if (!strcmp(res_name, GPMI_NAND_BCH_REGS_ADDR_RES_NAME))
965 		res->bch_regs = p;
966 	else
967 		dev_err(this->dev, "unknown resource name : %s\n", res_name);
968 
969 	return 0;
970 }
971 
972 static int acquire_bch_irq(struct gpmi_nand_data *this, irq_handler_t irq_h)
973 {
974 	struct platform_device *pdev = this->pdev;
975 	const char *res_name = GPMI_NAND_BCH_INTERRUPT_RES_NAME;
976 	struct resource *r;
977 	int err;
978 
979 	r = platform_get_resource_byname(pdev, IORESOURCE_IRQ, res_name);
980 	if (!r) {
981 		dev_err(this->dev, "Can't get resource for %s\n", res_name);
982 		return -ENODEV;
983 	}
984 
985 	err = devm_request_irq(this->dev, r->start, irq_h, 0, res_name, this);
986 	if (err)
987 		dev_err(this->dev, "error requesting BCH IRQ\n");
988 
989 	return err;
990 }
991 
992 static void release_dma_channels(struct gpmi_nand_data *this)
993 {
994 	unsigned int i;
995 	for (i = 0; i < DMA_CHANS; i++)
996 		if (this->dma_chans[i]) {
997 			dma_release_channel(this->dma_chans[i]);
998 			this->dma_chans[i] = NULL;
999 		}
1000 }
1001 
1002 static int acquire_dma_channels(struct gpmi_nand_data *this)
1003 {
1004 	struct platform_device *pdev = this->pdev;
1005 	struct dma_chan *dma_chan;
1006 	int ret = 0;
1007 
1008 	/* request dma channel */
1009 	dma_chan = dma_request_chan(&pdev->dev, "rx-tx");
1010 	if (IS_ERR(dma_chan)) {
1011 		ret = dev_err_probe(this->dev, PTR_ERR(dma_chan),
1012 				    "DMA channel request failed\n");
1013 		release_dma_channels(this);
1014 	} else {
1015 		this->dma_chans[0] = dma_chan;
1016 	}
1017 
1018 	return ret;
1019 }
1020 
1021 static int gpmi_get_clks(struct gpmi_nand_data *this)
1022 {
1023 	struct resources *r = &this->resources;
1024 	struct clk *clk;
1025 	int err, i;
1026 
1027 	for (i = 0; i < this->devdata->clks_count; i++) {
1028 		clk = devm_clk_get(this->dev, this->devdata->clks[i]);
1029 		if (IS_ERR(clk)) {
1030 			err = PTR_ERR(clk);
1031 			goto err_clock;
1032 		}
1033 
1034 		r->clock[i] = clk;
1035 	}
1036 
1037 	if (GPMI_IS_MX6(this))
1038 		/*
1039 		 * Set the default value for the gpmi clock.
1040 		 *
1041 		 * If you want to use the ONFI nand which is in the
1042 		 * Synchronous Mode, you should change the clock as you need.
1043 		 */
1044 		clk_set_rate(r->clock[0], 22000000);
1045 
1046 	return 0;
1047 
1048 err_clock:
1049 	dev_dbg(this->dev, "failed in finding the clocks.\n");
1050 	return err;
1051 }
1052 
1053 static int acquire_resources(struct gpmi_nand_data *this)
1054 {
1055 	int ret;
1056 
1057 	ret = acquire_register_block(this, GPMI_NAND_GPMI_REGS_ADDR_RES_NAME);
1058 	if (ret)
1059 		goto exit_regs;
1060 
1061 	ret = acquire_register_block(this, GPMI_NAND_BCH_REGS_ADDR_RES_NAME);
1062 	if (ret)
1063 		goto exit_regs;
1064 
1065 	ret = acquire_bch_irq(this, bch_irq);
1066 	if (ret)
1067 		goto exit_regs;
1068 
1069 	ret = acquire_dma_channels(this);
1070 	if (ret)
1071 		goto exit_regs;
1072 
1073 	ret = gpmi_get_clks(this);
1074 	if (ret)
1075 		goto exit_clock;
1076 	return 0;
1077 
1078 exit_clock:
1079 	release_dma_channels(this);
1080 exit_regs:
1081 	return ret;
1082 }
1083 
1084 static void release_resources(struct gpmi_nand_data *this)
1085 {
1086 	release_dma_channels(this);
1087 }
1088 
1089 static void gpmi_free_dma_buffer(struct gpmi_nand_data *this)
1090 {
1091 	struct device *dev = this->dev;
1092 	struct bch_geometry *geo = &this->bch_geometry;
1093 
1094 	if (this->auxiliary_virt && virt_addr_valid(this->auxiliary_virt))
1095 		dma_free_coherent(dev, geo->auxiliary_size,
1096 					this->auxiliary_virt,
1097 					this->auxiliary_phys);
1098 	kfree(this->data_buffer_dma);
1099 	kfree(this->raw_buffer);
1100 
1101 	this->data_buffer_dma	= NULL;
1102 	this->raw_buffer	= NULL;
1103 }
1104 
1105 /* Allocate the DMA buffers */
1106 static int gpmi_alloc_dma_buffer(struct gpmi_nand_data *this)
1107 {
1108 	struct bch_geometry *geo = &this->bch_geometry;
1109 	struct device *dev = this->dev;
1110 	struct mtd_info *mtd = nand_to_mtd(&this->nand);
1111 
1112 	/*
1113 	 * [2] Allocate a read/write data buffer.
1114 	 *     The gpmi_alloc_dma_buffer can be called twice.
1115 	 *     We allocate a PAGE_SIZE length buffer if gpmi_alloc_dma_buffer
1116 	 *     is called before the NAND identification; and we allocate a
1117 	 *     buffer of the real NAND page size when the gpmi_alloc_dma_buffer
1118 	 *     is called after.
1119 	 */
1120 	this->data_buffer_dma = kzalloc(mtd->writesize ?: PAGE_SIZE,
1121 					GFP_DMA | GFP_KERNEL);
1122 	if (this->data_buffer_dma == NULL)
1123 		goto error_alloc;
1124 
1125 	this->auxiliary_virt = dma_alloc_coherent(dev, geo->auxiliary_size,
1126 					&this->auxiliary_phys, GFP_DMA);
1127 	if (!this->auxiliary_virt)
1128 		goto error_alloc;
1129 
1130 	this->raw_buffer = kzalloc((mtd->writesize ?: PAGE_SIZE) + mtd->oobsize, GFP_KERNEL);
1131 	if (!this->raw_buffer)
1132 		goto error_alloc;
1133 
1134 	return 0;
1135 
1136 error_alloc:
1137 	gpmi_free_dma_buffer(this);
1138 	return -ENOMEM;
1139 }
1140 
1141 /*
1142  * Handles block mark swapping.
1143  * It can be called in swapping the block mark, or swapping it back,
1144  * because the the operations are the same.
1145  */
1146 static void block_mark_swapping(struct gpmi_nand_data *this,
1147 				void *payload, void *auxiliary)
1148 {
1149 	struct bch_geometry *nfc_geo = &this->bch_geometry;
1150 	unsigned char *p;
1151 	unsigned char *a;
1152 	unsigned int  bit;
1153 	unsigned char mask;
1154 	unsigned char from_data;
1155 	unsigned char from_oob;
1156 
1157 	if (!this->swap_block_mark)
1158 		return;
1159 
1160 	/*
1161 	 * If control arrives here, we're swapping. Make some convenience
1162 	 * variables.
1163 	 */
1164 	bit = nfc_geo->block_mark_bit_offset;
1165 	p   = payload + nfc_geo->block_mark_byte_offset;
1166 	a   = auxiliary;
1167 
1168 	/*
1169 	 * Get the byte from the data area that overlays the block mark. Since
1170 	 * the ECC engine applies its own view to the bits in the page, the
1171 	 * physical block mark won't (in general) appear on a byte boundary in
1172 	 * the data.
1173 	 */
1174 	from_data = (p[0] >> bit) | (p[1] << (8 - bit));
1175 
1176 	/* Get the byte from the OOB. */
1177 	from_oob = a[0];
1178 
1179 	/* Swap them. */
1180 	a[0] = from_data;
1181 
1182 	mask = (0x1 << bit) - 1;
1183 	p[0] = (p[0] & mask) | (from_oob << bit);
1184 
1185 	mask = ~0 << bit;
1186 	p[1] = (p[1] & mask) | (from_oob >> (8 - bit));
1187 }
1188 
1189 static int gpmi_count_bitflips(struct nand_chip *chip, void *buf, int first,
1190 			       int last, int meta)
1191 {
1192 	struct gpmi_nand_data *this = nand_get_controller_data(chip);
1193 	struct bch_geometry *nfc_geo = &this->bch_geometry;
1194 	struct mtd_info *mtd = nand_to_mtd(chip);
1195 	int i;
1196 	unsigned char *status;
1197 	unsigned int max_bitflips = 0;
1198 
1199 	/* Loop over status bytes, accumulating ECC status. */
1200 	status = this->auxiliary_virt + ALIGN(meta, 4);
1201 
1202 	for (i = first; i < last; i++, status++) {
1203 		if ((*status == STATUS_GOOD) || (*status == STATUS_ERASED))
1204 			continue;
1205 
1206 		if (*status == STATUS_UNCORRECTABLE) {
1207 			int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len;
1208 			u8 *eccbuf = this->raw_buffer;
1209 			int offset, bitoffset;
1210 			int eccbytes;
1211 			int flips;
1212 
1213 			/* Read ECC bytes into our internal raw_buffer */
1214 			offset = nfc_geo->metadata_size * 8;
1215 			offset += ((8 * nfc_geo->ecc_chunk_size) + eccbits) * (i + 1);
1216 			offset -= eccbits;
1217 			bitoffset = offset % 8;
1218 			eccbytes = DIV_ROUND_UP(offset + eccbits, 8);
1219 			offset /= 8;
1220 			eccbytes -= offset;
1221 			nand_change_read_column_op(chip, offset, eccbuf,
1222 						   eccbytes, false);
1223 
1224 			/*
1225 			 * ECC data are not byte aligned and we may have
1226 			 * in-band data in the first and last byte of
1227 			 * eccbuf. Set non-eccbits to one so that
1228 			 * nand_check_erased_ecc_chunk() does not count them
1229 			 * as bitflips.
1230 			 */
1231 			if (bitoffset)
1232 				eccbuf[0] |= GENMASK(bitoffset - 1, 0);
1233 
1234 			bitoffset = (bitoffset + eccbits) % 8;
1235 			if (bitoffset)
1236 				eccbuf[eccbytes - 1] |= GENMASK(7, bitoffset);
1237 
1238 			/*
1239 			 * The ECC hardware has an uncorrectable ECC status
1240 			 * code in case we have bitflips in an erased page. As
1241 			 * nothing was written into this subpage the ECC is
1242 			 * obviously wrong and we can not trust it. We assume
1243 			 * at this point that we are reading an erased page and
1244 			 * try to correct the bitflips in buffer up to
1245 			 * ecc_strength bitflips. If this is a page with random
1246 			 * data, we exceed this number of bitflips and have a
1247 			 * ECC failure. Otherwise we use the corrected buffer.
1248 			 */
1249 			if (i == 0) {
1250 				/* The first block includes metadata */
1251 				flips = nand_check_erased_ecc_chunk(
1252 						buf + i * nfc_geo->ecc_chunk_size,
1253 						nfc_geo->ecc_chunk_size,
1254 						eccbuf, eccbytes,
1255 						this->auxiliary_virt,
1256 						nfc_geo->metadata_size,
1257 						nfc_geo->ecc_strength);
1258 			} else {
1259 				flips = nand_check_erased_ecc_chunk(
1260 						buf + i * nfc_geo->ecc_chunk_size,
1261 						nfc_geo->ecc_chunk_size,
1262 						eccbuf, eccbytes,
1263 						NULL, 0,
1264 						nfc_geo->ecc_strength);
1265 			}
1266 
1267 			if (flips > 0) {
1268 				max_bitflips = max_t(unsigned int, max_bitflips,
1269 						     flips);
1270 				mtd->ecc_stats.corrected += flips;
1271 				continue;
1272 			}
1273 
1274 			mtd->ecc_stats.failed++;
1275 			continue;
1276 		}
1277 
1278 		mtd->ecc_stats.corrected += *status;
1279 		max_bitflips = max_t(unsigned int, max_bitflips, *status);
1280 	}
1281 
1282 	return max_bitflips;
1283 }
1284 
1285 static void gpmi_bch_layout_std(struct gpmi_nand_data *this)
1286 {
1287 	struct bch_geometry *geo = &this->bch_geometry;
1288 	unsigned int ecc_strength = geo->ecc_strength >> 1;
1289 	unsigned int gf_len = geo->gf_len;
1290 	unsigned int block_size = geo->ecc_chunk_size;
1291 
1292 	this->bch_flashlayout0 =
1293 		BF_BCH_FLASH0LAYOUT0_NBLOCKS(geo->ecc_chunk_count - 1) |
1294 		BF_BCH_FLASH0LAYOUT0_META_SIZE(geo->metadata_size) |
1295 		BF_BCH_FLASH0LAYOUT0_ECC0(ecc_strength, this) |
1296 		BF_BCH_FLASH0LAYOUT0_GF(gf_len, this) |
1297 		BF_BCH_FLASH0LAYOUT0_DATA0_SIZE(block_size, this);
1298 
1299 	this->bch_flashlayout1 =
1300 		BF_BCH_FLASH0LAYOUT1_PAGE_SIZE(geo->page_size) |
1301 		BF_BCH_FLASH0LAYOUT1_ECCN(ecc_strength, this) |
1302 		BF_BCH_FLASH0LAYOUT1_GF(gf_len, this) |
1303 		BF_BCH_FLASH0LAYOUT1_DATAN_SIZE(block_size, this);
1304 }
1305 
1306 static int gpmi_ecc_read_page(struct nand_chip *chip, uint8_t *buf,
1307 			      int oob_required, int page)
1308 {
1309 	struct gpmi_nand_data *this = nand_get_controller_data(chip);
1310 	struct mtd_info *mtd = nand_to_mtd(chip);
1311 	struct bch_geometry *geo = &this->bch_geometry;
1312 	unsigned int max_bitflips;
1313 	int ret;
1314 
1315 	gpmi_bch_layout_std(this);
1316 	this->bch = true;
1317 
1318 	ret = nand_read_page_op(chip, page, 0, buf, geo->page_size);
1319 	if (ret)
1320 		return ret;
1321 
1322 	max_bitflips = gpmi_count_bitflips(chip, buf, 0,
1323 					   geo->ecc_chunk_count,
1324 					   geo->auxiliary_status_offset);
1325 
1326 	/* handle the block mark swapping */
1327 	block_mark_swapping(this, buf, this->auxiliary_virt);
1328 
1329 	if (oob_required) {
1330 		/*
1331 		 * It's time to deliver the OOB bytes. See gpmi_ecc_read_oob()
1332 		 * for details about our policy for delivering the OOB.
1333 		 *
1334 		 * We fill the caller's buffer with set bits, and then copy the
1335 		 * block mark to th caller's buffer. Note that, if block mark
1336 		 * swapping was necessary, it has already been done, so we can
1337 		 * rely on the first byte of the auxiliary buffer to contain
1338 		 * the block mark.
1339 		 */
1340 		memset(chip->oob_poi, ~0, mtd->oobsize);
1341 		chip->oob_poi[0] = ((uint8_t *)this->auxiliary_virt)[0];
1342 	}
1343 
1344 	return max_bitflips;
1345 }
1346 
1347 /* Fake a virtual small page for the subpage read */
1348 static int gpmi_ecc_read_subpage(struct nand_chip *chip, uint32_t offs,
1349 				 uint32_t len, uint8_t *buf, int page)
1350 {
1351 	struct gpmi_nand_data *this = nand_get_controller_data(chip);
1352 	struct bch_geometry *geo = &this->bch_geometry;
1353 	int size = chip->ecc.size; /* ECC chunk size */
1354 	int meta, n, page_size;
1355 	unsigned int max_bitflips;
1356 	unsigned int ecc_strength;
1357 	int first, last, marker_pos;
1358 	int ecc_parity_size;
1359 	int col = 0;
1360 	int ret;
1361 
1362 	/* The size of ECC parity */
1363 	ecc_parity_size = geo->gf_len * geo->ecc_strength / 8;
1364 
1365 	/* Align it with the chunk size */
1366 	first = offs / size;
1367 	last = (offs + len - 1) / size;
1368 
1369 	if (this->swap_block_mark) {
1370 		/*
1371 		 * Find the chunk which contains the Block Marker.
1372 		 * If this chunk is in the range of [first, last],
1373 		 * we have to read out the whole page.
1374 		 * Why? since we had swapped the data at the position of Block
1375 		 * Marker to the metadata which is bound with the chunk 0.
1376 		 */
1377 		marker_pos = geo->block_mark_byte_offset / size;
1378 		if (last >= marker_pos && first <= marker_pos) {
1379 			dev_dbg(this->dev,
1380 				"page:%d, first:%d, last:%d, marker at:%d\n",
1381 				page, first, last, marker_pos);
1382 			return gpmi_ecc_read_page(chip, buf, 0, page);
1383 		}
1384 	}
1385 
1386 	meta = geo->metadata_size;
1387 	if (first) {
1388 		col = meta + (size + ecc_parity_size) * first;
1389 		meta = 0;
1390 		buf = buf + first * size;
1391 	}
1392 
1393 	ecc_parity_size = geo->gf_len * geo->ecc_strength / 8;
1394 
1395 	n = last - first + 1;
1396 	page_size = meta + (size + ecc_parity_size) * n;
1397 	ecc_strength = geo->ecc_strength >> 1;
1398 
1399 	this->bch_flashlayout0 = BF_BCH_FLASH0LAYOUT0_NBLOCKS(n - 1) |
1400 		BF_BCH_FLASH0LAYOUT0_META_SIZE(meta) |
1401 		BF_BCH_FLASH0LAYOUT0_ECC0(ecc_strength, this) |
1402 		BF_BCH_FLASH0LAYOUT0_GF(geo->gf_len, this) |
1403 		BF_BCH_FLASH0LAYOUT0_DATA0_SIZE(geo->ecc_chunk_size, this);
1404 
1405 	this->bch_flashlayout1 = BF_BCH_FLASH0LAYOUT1_PAGE_SIZE(page_size) |
1406 		BF_BCH_FLASH0LAYOUT1_ECCN(ecc_strength, this) |
1407 		BF_BCH_FLASH0LAYOUT1_GF(geo->gf_len, this) |
1408 		BF_BCH_FLASH0LAYOUT1_DATAN_SIZE(geo->ecc_chunk_size, this);
1409 
1410 	this->bch = true;
1411 
1412 	ret = nand_read_page_op(chip, page, col, buf, page_size);
1413 	if (ret)
1414 		return ret;
1415 
1416 	dev_dbg(this->dev, "page:%d(%d:%d)%d, chunk:(%d:%d), BCH PG size:%d\n",
1417 		page, offs, len, col, first, n, page_size);
1418 
1419 	max_bitflips = gpmi_count_bitflips(chip, buf, first, last, meta);
1420 
1421 	return max_bitflips;
1422 }
1423 
1424 static int gpmi_ecc_write_page(struct nand_chip *chip, const uint8_t *buf,
1425 			       int oob_required, int page)
1426 {
1427 	struct mtd_info *mtd = nand_to_mtd(chip);
1428 	struct gpmi_nand_data *this = nand_get_controller_data(chip);
1429 	struct bch_geometry *nfc_geo = &this->bch_geometry;
1430 	int ret;
1431 
1432 	dev_dbg(this->dev, "ecc write page.\n");
1433 
1434 	gpmi_bch_layout_std(this);
1435 	this->bch = true;
1436 
1437 	memcpy(this->auxiliary_virt, chip->oob_poi, nfc_geo->auxiliary_size);
1438 
1439 	if (this->swap_block_mark) {
1440 		/*
1441 		 * When doing bad block marker swapping we must always copy the
1442 		 * input buffer as we can't modify the const buffer.
1443 		 */
1444 		memcpy(this->data_buffer_dma, buf, mtd->writesize);
1445 		buf = this->data_buffer_dma;
1446 		block_mark_swapping(this, this->data_buffer_dma,
1447 				    this->auxiliary_virt);
1448 	}
1449 
1450 	ret = nand_prog_page_op(chip, page, 0, buf, nfc_geo->page_size);
1451 
1452 	return ret;
1453 }
1454 
1455 /*
1456  * There are several places in this driver where we have to handle the OOB and
1457  * block marks. This is the function where things are the most complicated, so
1458  * this is where we try to explain it all. All the other places refer back to
1459  * here.
1460  *
1461  * These are the rules, in order of decreasing importance:
1462  *
1463  * 1) Nothing the caller does can be allowed to imperil the block mark.
1464  *
1465  * 2) In read operations, the first byte of the OOB we return must reflect the
1466  *    true state of the block mark, no matter where that block mark appears in
1467  *    the physical page.
1468  *
1469  * 3) ECC-based read operations return an OOB full of set bits (since we never
1470  *    allow ECC-based writes to the OOB, it doesn't matter what ECC-based reads
1471  *    return).
1472  *
1473  * 4) "Raw" read operations return a direct view of the physical bytes in the
1474  *    page, using the conventional definition of which bytes are data and which
1475  *    are OOB. This gives the caller a way to see the actual, physical bytes
1476  *    in the page, without the distortions applied by our ECC engine.
1477  *
1478  *
1479  * What we do for this specific read operation depends on two questions:
1480  *
1481  * 1) Are we doing a "raw" read, or an ECC-based read?
1482  *
1483  * 2) Are we using block mark swapping or transcription?
1484  *
1485  * There are four cases, illustrated by the following Karnaugh map:
1486  *
1487  *                    |           Raw           |         ECC-based       |
1488  *       -------------+-------------------------+-------------------------+
1489  *                    | Read the conventional   |                         |
1490  *                    | OOB at the end of the   |                         |
1491  *       Swapping     | page and return it. It  |                         |
1492  *                    | contains exactly what   |                         |
1493  *                    | we want.                | Read the block mark and |
1494  *       -------------+-------------------------+ return it in a buffer   |
1495  *                    | Read the conventional   | full of set bits.       |
1496  *                    | OOB at the end of the   |                         |
1497  *                    | page and also the block |                         |
1498  *       Transcribing | mark in the metadata.   |                         |
1499  *                    | Copy the block mark     |                         |
1500  *                    | into the first byte of  |                         |
1501  *                    | the OOB.                |                         |
1502  *       -------------+-------------------------+-------------------------+
1503  *
1504  * Note that we break rule #4 in the Transcribing/Raw case because we're not
1505  * giving an accurate view of the actual, physical bytes in the page (we're
1506  * overwriting the block mark). That's OK because it's more important to follow
1507  * rule #2.
1508  *
1509  * It turns out that knowing whether we want an "ECC-based" or "raw" read is not
1510  * easy. When reading a page, for example, the NAND Flash MTD code calls our
1511  * ecc.read_page or ecc.read_page_raw function. Thus, the fact that MTD wants an
1512  * ECC-based or raw view of the page is implicit in which function it calls
1513  * (there is a similar pair of ECC-based/raw functions for writing).
1514  */
1515 static int gpmi_ecc_read_oob(struct nand_chip *chip, int page)
1516 {
1517 	struct mtd_info *mtd = nand_to_mtd(chip);
1518 	struct gpmi_nand_data *this = nand_get_controller_data(chip);
1519 	int ret;
1520 
1521 	/* clear the OOB buffer */
1522 	memset(chip->oob_poi, ~0, mtd->oobsize);
1523 
1524 	/* Read out the conventional OOB. */
1525 	ret = nand_read_page_op(chip, page, mtd->writesize, chip->oob_poi,
1526 				mtd->oobsize);
1527 	if (ret)
1528 		return ret;
1529 
1530 	/*
1531 	 * Now, we want to make sure the block mark is correct. In the
1532 	 * non-transcribing case (!GPMI_IS_MX23()), we already have it.
1533 	 * Otherwise, we need to explicitly read it.
1534 	 */
1535 	if (GPMI_IS_MX23(this)) {
1536 		/* Read the block mark into the first byte of the OOB buffer. */
1537 		ret = nand_read_page_op(chip, page, 0, chip->oob_poi, 1);
1538 		if (ret)
1539 			return ret;
1540 	}
1541 
1542 	return 0;
1543 }
1544 
1545 static int gpmi_ecc_write_oob(struct nand_chip *chip, int page)
1546 {
1547 	struct mtd_info *mtd = nand_to_mtd(chip);
1548 	struct mtd_oob_region of = { };
1549 
1550 	/* Do we have available oob area? */
1551 	mtd_ooblayout_free(mtd, 0, &of);
1552 	if (!of.length)
1553 		return -EPERM;
1554 
1555 	if (!nand_is_slc(chip))
1556 		return -EPERM;
1557 
1558 	return nand_prog_page_op(chip, page, mtd->writesize + of.offset,
1559 				 chip->oob_poi + of.offset, of.length);
1560 }
1561 
1562 /*
1563  * This function reads a NAND page without involving the ECC engine (no HW
1564  * ECC correction).
1565  * The tricky part in the GPMI/BCH controller is that it stores ECC bits
1566  * inline (interleaved with payload DATA), and do not align data chunk on
1567  * byte boundaries.
1568  * We thus need to take care moving the payload data and ECC bits stored in the
1569  * page into the provided buffers, which is why we're using nand_extract_bits().
1570  *
1571  * See set_geometry_by_ecc_info inline comments to have a full description
1572  * of the layout used by the GPMI controller.
1573  */
1574 static int gpmi_ecc_read_page_raw(struct nand_chip *chip, uint8_t *buf,
1575 				  int oob_required, int page)
1576 {
1577 	struct mtd_info *mtd = nand_to_mtd(chip);
1578 	struct gpmi_nand_data *this = nand_get_controller_data(chip);
1579 	struct bch_geometry *nfc_geo = &this->bch_geometry;
1580 	int eccsize = nfc_geo->ecc_chunk_size;
1581 	int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len;
1582 	u8 *tmp_buf = this->raw_buffer;
1583 	size_t src_bit_off;
1584 	size_t oob_bit_off;
1585 	size_t oob_byte_off;
1586 	uint8_t *oob = chip->oob_poi;
1587 	int step;
1588 	int ret;
1589 
1590 	ret = nand_read_page_op(chip, page, 0, tmp_buf,
1591 				mtd->writesize + mtd->oobsize);
1592 	if (ret)
1593 		return ret;
1594 
1595 	/*
1596 	 * If required, swap the bad block marker and the data stored in the
1597 	 * metadata section, so that we don't wrongly consider a block as bad.
1598 	 *
1599 	 * See the layout description for a detailed explanation on why this
1600 	 * is needed.
1601 	 */
1602 	if (this->swap_block_mark)
1603 		swap(tmp_buf[0], tmp_buf[mtd->writesize]);
1604 
1605 	/*
1606 	 * Copy the metadata section into the oob buffer (this section is
1607 	 * guaranteed to be aligned on a byte boundary).
1608 	 */
1609 	if (oob_required)
1610 		memcpy(oob, tmp_buf, nfc_geo->metadata_size);
1611 
1612 	oob_bit_off = nfc_geo->metadata_size * 8;
1613 	src_bit_off = oob_bit_off;
1614 
1615 	/* Extract interleaved payload data and ECC bits */
1616 	for (step = 0; step < nfc_geo->ecc_chunk_count; step++) {
1617 		if (buf)
1618 			nand_extract_bits(buf, step * eccsize, tmp_buf,
1619 					  src_bit_off, eccsize * 8);
1620 		src_bit_off += eccsize * 8;
1621 
1622 		/* Align last ECC block to align a byte boundary */
1623 		if (step == nfc_geo->ecc_chunk_count - 1 &&
1624 		    (oob_bit_off + eccbits) % 8)
1625 			eccbits += 8 - ((oob_bit_off + eccbits) % 8);
1626 
1627 		if (oob_required)
1628 			nand_extract_bits(oob, oob_bit_off, tmp_buf,
1629 					  src_bit_off, eccbits);
1630 
1631 		src_bit_off += eccbits;
1632 		oob_bit_off += eccbits;
1633 	}
1634 
1635 	if (oob_required) {
1636 		oob_byte_off = oob_bit_off / 8;
1637 
1638 		if (oob_byte_off < mtd->oobsize)
1639 			memcpy(oob + oob_byte_off,
1640 			       tmp_buf + mtd->writesize + oob_byte_off,
1641 			       mtd->oobsize - oob_byte_off);
1642 	}
1643 
1644 	return 0;
1645 }
1646 
1647 /*
1648  * This function writes a NAND page without involving the ECC engine (no HW
1649  * ECC generation).
1650  * The tricky part in the GPMI/BCH controller is that it stores ECC bits
1651  * inline (interleaved with payload DATA), and do not align data chunk on
1652  * byte boundaries.
1653  * We thus need to take care moving the OOB area at the right place in the
1654  * final page, which is why we're using nand_extract_bits().
1655  *
1656  * See set_geometry_by_ecc_info inline comments to have a full description
1657  * of the layout used by the GPMI controller.
1658  */
1659 static int gpmi_ecc_write_page_raw(struct nand_chip *chip, const uint8_t *buf,
1660 				   int oob_required, int page)
1661 {
1662 	struct mtd_info *mtd = nand_to_mtd(chip);
1663 	struct gpmi_nand_data *this = nand_get_controller_data(chip);
1664 	struct bch_geometry *nfc_geo = &this->bch_geometry;
1665 	int eccsize = nfc_geo->ecc_chunk_size;
1666 	int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len;
1667 	u8 *tmp_buf = this->raw_buffer;
1668 	uint8_t *oob = chip->oob_poi;
1669 	size_t dst_bit_off;
1670 	size_t oob_bit_off;
1671 	size_t oob_byte_off;
1672 	int step;
1673 
1674 	/*
1675 	 * Initialize all bits to 1 in case we don't have a buffer for the
1676 	 * payload or oob data in order to leave unspecified bits of data
1677 	 * to their initial state.
1678 	 */
1679 	if (!buf || !oob_required)
1680 		memset(tmp_buf, 0xff, mtd->writesize + mtd->oobsize);
1681 
1682 	/*
1683 	 * First copy the metadata section (stored in oob buffer) at the
1684 	 * beginning of the page, as imposed by the GPMI layout.
1685 	 */
1686 	memcpy(tmp_buf, oob, nfc_geo->metadata_size);
1687 	oob_bit_off = nfc_geo->metadata_size * 8;
1688 	dst_bit_off = oob_bit_off;
1689 
1690 	/* Interleave payload data and ECC bits */
1691 	for (step = 0; step < nfc_geo->ecc_chunk_count; step++) {
1692 		if (buf)
1693 			nand_extract_bits(tmp_buf, dst_bit_off, buf,
1694 					  step * eccsize * 8, eccsize * 8);
1695 		dst_bit_off += eccsize * 8;
1696 
1697 		/* Align last ECC block to align a byte boundary */
1698 		if (step == nfc_geo->ecc_chunk_count - 1 &&
1699 		    (oob_bit_off + eccbits) % 8)
1700 			eccbits += 8 - ((oob_bit_off + eccbits) % 8);
1701 
1702 		if (oob_required)
1703 			nand_extract_bits(tmp_buf, dst_bit_off, oob,
1704 					  oob_bit_off, eccbits);
1705 
1706 		dst_bit_off += eccbits;
1707 		oob_bit_off += eccbits;
1708 	}
1709 
1710 	oob_byte_off = oob_bit_off / 8;
1711 
1712 	if (oob_required && oob_byte_off < mtd->oobsize)
1713 		memcpy(tmp_buf + mtd->writesize + oob_byte_off,
1714 		       oob + oob_byte_off, mtd->oobsize - oob_byte_off);
1715 
1716 	/*
1717 	 * If required, swap the bad block marker and the first byte of the
1718 	 * metadata section, so that we don't modify the bad block marker.
1719 	 *
1720 	 * See the layout description for a detailed explanation on why this
1721 	 * is needed.
1722 	 */
1723 	if (this->swap_block_mark)
1724 		swap(tmp_buf[0], tmp_buf[mtd->writesize]);
1725 
1726 	return nand_prog_page_op(chip, page, 0, tmp_buf,
1727 				 mtd->writesize + mtd->oobsize);
1728 }
1729 
1730 static int gpmi_ecc_read_oob_raw(struct nand_chip *chip, int page)
1731 {
1732 	return gpmi_ecc_read_page_raw(chip, NULL, 1, page);
1733 }
1734 
1735 static int gpmi_ecc_write_oob_raw(struct nand_chip *chip, int page)
1736 {
1737 	return gpmi_ecc_write_page_raw(chip, NULL, 1, page);
1738 }
1739 
1740 static int gpmi_block_markbad(struct nand_chip *chip, loff_t ofs)
1741 {
1742 	struct mtd_info *mtd = nand_to_mtd(chip);
1743 	struct gpmi_nand_data *this = nand_get_controller_data(chip);
1744 	int ret = 0;
1745 	uint8_t *block_mark;
1746 	int column, page, chipnr;
1747 
1748 	chipnr = (int)(ofs >> chip->chip_shift);
1749 	nand_select_target(chip, chipnr);
1750 
1751 	column = !GPMI_IS_MX23(this) ? mtd->writesize : 0;
1752 
1753 	/* Write the block mark. */
1754 	block_mark = this->data_buffer_dma;
1755 	block_mark[0] = 0; /* bad block marker */
1756 
1757 	/* Shift to get page */
1758 	page = (int)(ofs >> chip->page_shift);
1759 
1760 	ret = nand_prog_page_op(chip, page, column, block_mark, 1);
1761 
1762 	nand_deselect_target(chip);
1763 
1764 	return ret;
1765 }
1766 
1767 static int nand_boot_set_geometry(struct gpmi_nand_data *this)
1768 {
1769 	struct boot_rom_geometry *geometry = &this->rom_geometry;
1770 
1771 	/*
1772 	 * Set the boot block stride size.
1773 	 *
1774 	 * In principle, we should be reading this from the OTP bits, since
1775 	 * that's where the ROM is going to get it. In fact, we don't have any
1776 	 * way to read the OTP bits, so we go with the default and hope for the
1777 	 * best.
1778 	 */
1779 	geometry->stride_size_in_pages = 64;
1780 
1781 	/*
1782 	 * Set the search area stride exponent.
1783 	 *
1784 	 * In principle, we should be reading this from the OTP bits, since
1785 	 * that's where the ROM is going to get it. In fact, we don't have any
1786 	 * way to read the OTP bits, so we go with the default and hope for the
1787 	 * best.
1788 	 */
1789 	geometry->search_area_stride_exponent = 2;
1790 	return 0;
1791 }
1792 
1793 static const char  *fingerprint = "STMP";
1794 static int mx23_check_transcription_stamp(struct gpmi_nand_data *this)
1795 {
1796 	struct boot_rom_geometry *rom_geo = &this->rom_geometry;
1797 	struct device *dev = this->dev;
1798 	struct nand_chip *chip = &this->nand;
1799 	unsigned int search_area_size_in_strides;
1800 	unsigned int stride;
1801 	unsigned int page;
1802 	u8 *buffer = nand_get_data_buf(chip);
1803 	int found_an_ncb_fingerprint = false;
1804 	int ret;
1805 
1806 	/* Compute the number of strides in a search area. */
1807 	search_area_size_in_strides = 1 << rom_geo->search_area_stride_exponent;
1808 
1809 	nand_select_target(chip, 0);
1810 
1811 	/*
1812 	 * Loop through the first search area, looking for the NCB fingerprint.
1813 	 */
1814 	dev_dbg(dev, "Scanning for an NCB fingerprint...\n");
1815 
1816 	for (stride = 0; stride < search_area_size_in_strides; stride++) {
1817 		/* Compute the page addresses. */
1818 		page = stride * rom_geo->stride_size_in_pages;
1819 
1820 		dev_dbg(dev, "Looking for a fingerprint in page 0x%x\n", page);
1821 
1822 		/*
1823 		 * Read the NCB fingerprint. The fingerprint is four bytes long
1824 		 * and starts in the 12th byte of the page.
1825 		 */
1826 		ret = nand_read_page_op(chip, page, 12, buffer,
1827 					strlen(fingerprint));
1828 		if (ret)
1829 			continue;
1830 
1831 		/* Look for the fingerprint. */
1832 		if (!memcmp(buffer, fingerprint, strlen(fingerprint))) {
1833 			found_an_ncb_fingerprint = true;
1834 			break;
1835 		}
1836 
1837 	}
1838 
1839 	nand_deselect_target(chip);
1840 
1841 	if (found_an_ncb_fingerprint)
1842 		dev_dbg(dev, "\tFound a fingerprint\n");
1843 	else
1844 		dev_dbg(dev, "\tNo fingerprint found\n");
1845 	return found_an_ncb_fingerprint;
1846 }
1847 
1848 /* Writes a transcription stamp. */
1849 static int mx23_write_transcription_stamp(struct gpmi_nand_data *this)
1850 {
1851 	struct device *dev = this->dev;
1852 	struct boot_rom_geometry *rom_geo = &this->rom_geometry;
1853 	struct nand_chip *chip = &this->nand;
1854 	struct mtd_info *mtd = nand_to_mtd(chip);
1855 	unsigned int block_size_in_pages;
1856 	unsigned int search_area_size_in_strides;
1857 	unsigned int search_area_size_in_pages;
1858 	unsigned int search_area_size_in_blocks;
1859 	unsigned int block;
1860 	unsigned int stride;
1861 	unsigned int page;
1862 	u8 *buffer = nand_get_data_buf(chip);
1863 	int status;
1864 
1865 	/* Compute the search area geometry. */
1866 	block_size_in_pages = mtd->erasesize / mtd->writesize;
1867 	search_area_size_in_strides = 1 << rom_geo->search_area_stride_exponent;
1868 	search_area_size_in_pages = search_area_size_in_strides *
1869 					rom_geo->stride_size_in_pages;
1870 	search_area_size_in_blocks =
1871 		  (search_area_size_in_pages + (block_size_in_pages - 1)) /
1872 				    block_size_in_pages;
1873 
1874 	dev_dbg(dev, "Search Area Geometry :\n");
1875 	dev_dbg(dev, "\tin Blocks : %u\n", search_area_size_in_blocks);
1876 	dev_dbg(dev, "\tin Strides: %u\n", search_area_size_in_strides);
1877 	dev_dbg(dev, "\tin Pages  : %u\n", search_area_size_in_pages);
1878 
1879 	nand_select_target(chip, 0);
1880 
1881 	/* Loop over blocks in the first search area, erasing them. */
1882 	dev_dbg(dev, "Erasing the search area...\n");
1883 
1884 	for (block = 0; block < search_area_size_in_blocks; block++) {
1885 		/* Erase this block. */
1886 		dev_dbg(dev, "\tErasing block 0x%x\n", block);
1887 		status = nand_erase_op(chip, block);
1888 		if (status)
1889 			dev_err(dev, "[%s] Erase failed.\n", __func__);
1890 	}
1891 
1892 	/* Write the NCB fingerprint into the page buffer. */
1893 	memset(buffer, ~0, mtd->writesize);
1894 	memcpy(buffer + 12, fingerprint, strlen(fingerprint));
1895 
1896 	/* Loop through the first search area, writing NCB fingerprints. */
1897 	dev_dbg(dev, "Writing NCB fingerprints...\n");
1898 	for (stride = 0; stride < search_area_size_in_strides; stride++) {
1899 		/* Compute the page addresses. */
1900 		page = stride * rom_geo->stride_size_in_pages;
1901 
1902 		/* Write the first page of the current stride. */
1903 		dev_dbg(dev, "Writing an NCB fingerprint in page 0x%x\n", page);
1904 
1905 		status = chip->ecc.write_page_raw(chip, buffer, 0, page);
1906 		if (status)
1907 			dev_err(dev, "[%s] Write failed.\n", __func__);
1908 	}
1909 
1910 	nand_deselect_target(chip);
1911 
1912 	return 0;
1913 }
1914 
1915 static int mx23_boot_init(struct gpmi_nand_data  *this)
1916 {
1917 	struct device *dev = this->dev;
1918 	struct nand_chip *chip = &this->nand;
1919 	struct mtd_info *mtd = nand_to_mtd(chip);
1920 	unsigned int block_count;
1921 	unsigned int block;
1922 	int     chipnr;
1923 	int     page;
1924 	loff_t  byte;
1925 	uint8_t block_mark;
1926 	int     ret = 0;
1927 
1928 	/*
1929 	 * If control arrives here, we can't use block mark swapping, which
1930 	 * means we're forced to use transcription. First, scan for the
1931 	 * transcription stamp. If we find it, then we don't have to do
1932 	 * anything -- the block marks are already transcribed.
1933 	 */
1934 	if (mx23_check_transcription_stamp(this))
1935 		return 0;
1936 
1937 	/*
1938 	 * If control arrives here, we couldn't find a transcription stamp, so
1939 	 * so we presume the block marks are in the conventional location.
1940 	 */
1941 	dev_dbg(dev, "Transcribing bad block marks...\n");
1942 
1943 	/* Compute the number of blocks in the entire medium. */
1944 	block_count = nanddev_eraseblocks_per_target(&chip->base);
1945 
1946 	/*
1947 	 * Loop over all the blocks in the medium, transcribing block marks as
1948 	 * we go.
1949 	 */
1950 	for (block = 0; block < block_count; block++) {
1951 		/*
1952 		 * Compute the chip, page and byte addresses for this block's
1953 		 * conventional mark.
1954 		 */
1955 		chipnr = block >> (chip->chip_shift - chip->phys_erase_shift);
1956 		page = block << (chip->phys_erase_shift - chip->page_shift);
1957 		byte = block <<  chip->phys_erase_shift;
1958 
1959 		/* Send the command to read the conventional block mark. */
1960 		nand_select_target(chip, chipnr);
1961 		ret = nand_read_page_op(chip, page, mtd->writesize, &block_mark,
1962 					1);
1963 		nand_deselect_target(chip);
1964 
1965 		if (ret)
1966 			continue;
1967 
1968 		/*
1969 		 * Check if the block is marked bad. If so, we need to mark it
1970 		 * again, but this time the result will be a mark in the
1971 		 * location where we transcribe block marks.
1972 		 */
1973 		if (block_mark != 0xff) {
1974 			dev_dbg(dev, "Transcribing mark in block %u\n", block);
1975 			ret = chip->legacy.block_markbad(chip, byte);
1976 			if (ret)
1977 				dev_err(dev,
1978 					"Failed to mark block bad with ret %d\n",
1979 					ret);
1980 		}
1981 	}
1982 
1983 	/* Write the stamp that indicates we've transcribed the block marks. */
1984 	mx23_write_transcription_stamp(this);
1985 	return 0;
1986 }
1987 
1988 static int nand_boot_init(struct gpmi_nand_data  *this)
1989 {
1990 	nand_boot_set_geometry(this);
1991 
1992 	/* This is ROM arch-specific initilization before the BBT scanning. */
1993 	if (GPMI_IS_MX23(this))
1994 		return mx23_boot_init(this);
1995 	return 0;
1996 }
1997 
1998 static int gpmi_set_geometry(struct gpmi_nand_data *this)
1999 {
2000 	int ret;
2001 
2002 	/* Free the temporary DMA memory for reading ID. */
2003 	gpmi_free_dma_buffer(this);
2004 
2005 	/* Set up the NFC geometry which is used by BCH. */
2006 	ret = bch_set_geometry(this);
2007 	if (ret) {
2008 		dev_err(this->dev, "Error setting BCH geometry : %d\n", ret);
2009 		return ret;
2010 	}
2011 
2012 	/* Alloc the new DMA buffers according to the pagesize and oobsize */
2013 	return gpmi_alloc_dma_buffer(this);
2014 }
2015 
2016 static int gpmi_init_last(struct gpmi_nand_data *this)
2017 {
2018 	struct nand_chip *chip = &this->nand;
2019 	struct mtd_info *mtd = nand_to_mtd(chip);
2020 	struct nand_ecc_ctrl *ecc = &chip->ecc;
2021 	struct bch_geometry *bch_geo = &this->bch_geometry;
2022 	int ret;
2023 
2024 	/* Set up the medium geometry */
2025 	ret = gpmi_set_geometry(this);
2026 	if (ret)
2027 		return ret;
2028 
2029 	/* Init the nand_ecc_ctrl{} */
2030 	ecc->read_page	= gpmi_ecc_read_page;
2031 	ecc->write_page	= gpmi_ecc_write_page;
2032 	ecc->read_oob	= gpmi_ecc_read_oob;
2033 	ecc->write_oob	= gpmi_ecc_write_oob;
2034 	ecc->read_page_raw = gpmi_ecc_read_page_raw;
2035 	ecc->write_page_raw = gpmi_ecc_write_page_raw;
2036 	ecc->read_oob_raw = gpmi_ecc_read_oob_raw;
2037 	ecc->write_oob_raw = gpmi_ecc_write_oob_raw;
2038 	ecc->engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2039 	ecc->size	= bch_geo->ecc_chunk_size;
2040 	ecc->strength	= bch_geo->ecc_strength;
2041 	mtd_set_ooblayout(mtd, &gpmi_ooblayout_ops);
2042 
2043 	/*
2044 	 * We only enable the subpage read when:
2045 	 *  (1) the chip is imx6, and
2046 	 *  (2) the size of the ECC parity is byte aligned.
2047 	 */
2048 	if (GPMI_IS_MX6(this) &&
2049 		((bch_geo->gf_len * bch_geo->ecc_strength) % 8) == 0) {
2050 		ecc->read_subpage = gpmi_ecc_read_subpage;
2051 		chip->options |= NAND_SUBPAGE_READ;
2052 	}
2053 
2054 	return 0;
2055 }
2056 
2057 static int gpmi_nand_attach_chip(struct nand_chip *chip)
2058 {
2059 	struct gpmi_nand_data *this = nand_get_controller_data(chip);
2060 	int ret;
2061 
2062 	if (chip->bbt_options & NAND_BBT_USE_FLASH) {
2063 		chip->bbt_options |= NAND_BBT_NO_OOB;
2064 
2065 		if (of_property_read_bool(this->dev->of_node,
2066 					  "fsl,no-blockmark-swap"))
2067 			this->swap_block_mark = false;
2068 	}
2069 	dev_dbg(this->dev, "Blockmark swapping %sabled\n",
2070 		this->swap_block_mark ? "en" : "dis");
2071 
2072 	ret = gpmi_init_last(this);
2073 	if (ret)
2074 		return ret;
2075 
2076 	chip->options |= NAND_SKIP_BBTSCAN;
2077 
2078 	return 0;
2079 }
2080 
2081 static struct gpmi_transfer *get_next_transfer(struct gpmi_nand_data *this)
2082 {
2083 	struct gpmi_transfer *transfer = &this->transfers[this->ntransfers];
2084 
2085 	this->ntransfers++;
2086 
2087 	if (this->ntransfers == GPMI_MAX_TRANSFERS)
2088 		return NULL;
2089 
2090 	return transfer;
2091 }
2092 
2093 static struct dma_async_tx_descriptor *gpmi_chain_command(
2094 	struct gpmi_nand_data *this, u8 cmd, const u8 *addr, int naddr)
2095 {
2096 	struct dma_chan *channel = get_dma_chan(this);
2097 	struct dma_async_tx_descriptor *desc;
2098 	struct gpmi_transfer *transfer;
2099 	int chip = this->nand.cur_cs;
2100 	u32 pio[3];
2101 
2102 	/* [1] send out the PIO words */
2103 	pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WRITE)
2104 		| BM_GPMI_CTRL0_WORD_LENGTH
2105 		| BF_GPMI_CTRL0_CS(chip, this)
2106 		| BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
2107 		| BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_CLE)
2108 		| BM_GPMI_CTRL0_ADDRESS_INCREMENT
2109 		| BF_GPMI_CTRL0_XFER_COUNT(naddr + 1);
2110 	pio[1] = 0;
2111 	pio[2] = 0;
2112 	desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio),
2113 				      DMA_TRANS_NONE, 0);
2114 	if (!desc)
2115 		return NULL;
2116 
2117 	transfer = get_next_transfer(this);
2118 	if (!transfer)
2119 		return NULL;
2120 
2121 	transfer->cmdbuf[0] = cmd;
2122 	if (naddr)
2123 		memcpy(&transfer->cmdbuf[1], addr, naddr);
2124 
2125 	sg_init_one(&transfer->sgl, transfer->cmdbuf, naddr + 1);
2126 	dma_map_sg(this->dev, &transfer->sgl, 1, DMA_TO_DEVICE);
2127 
2128 	transfer->direction = DMA_TO_DEVICE;
2129 
2130 	desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1, DMA_MEM_TO_DEV,
2131 				       MXS_DMA_CTRL_WAIT4END);
2132 	return desc;
2133 }
2134 
2135 static struct dma_async_tx_descriptor *gpmi_chain_wait_ready(
2136 	struct gpmi_nand_data *this)
2137 {
2138 	struct dma_chan *channel = get_dma_chan(this);
2139 	u32 pio[2];
2140 
2141 	pio[0] =  BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WAIT_FOR_READY)
2142 		| BM_GPMI_CTRL0_WORD_LENGTH
2143 		| BF_GPMI_CTRL0_CS(this->nand.cur_cs, this)
2144 		| BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
2145 		| BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA)
2146 		| BF_GPMI_CTRL0_XFER_COUNT(0);
2147 	pio[1] = 0;
2148 
2149 	return mxs_dmaengine_prep_pio(channel, pio, 2, DMA_TRANS_NONE,
2150 				MXS_DMA_CTRL_WAIT4END | MXS_DMA_CTRL_WAIT4RDY);
2151 }
2152 
2153 static struct dma_async_tx_descriptor *gpmi_chain_data_read(
2154 	struct gpmi_nand_data *this, void *buf, int raw_len, bool *direct)
2155 {
2156 	struct dma_async_tx_descriptor *desc;
2157 	struct dma_chan *channel = get_dma_chan(this);
2158 	struct gpmi_transfer *transfer;
2159 	u32 pio[6] = {};
2160 
2161 	transfer = get_next_transfer(this);
2162 	if (!transfer)
2163 		return NULL;
2164 
2165 	transfer->direction = DMA_FROM_DEVICE;
2166 
2167 	*direct = prepare_data_dma(this, buf, raw_len, &transfer->sgl,
2168 				   DMA_FROM_DEVICE);
2169 
2170 	pio[0] =  BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__READ)
2171 		| BM_GPMI_CTRL0_WORD_LENGTH
2172 		| BF_GPMI_CTRL0_CS(this->nand.cur_cs, this)
2173 		| BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
2174 		| BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA)
2175 		| BF_GPMI_CTRL0_XFER_COUNT(raw_len);
2176 
2177 	if (this->bch) {
2178 		pio[2] =  BM_GPMI_ECCCTRL_ENABLE_ECC
2179 			| BF_GPMI_ECCCTRL_ECC_CMD(BV_GPMI_ECCCTRL_ECC_CMD__BCH_DECODE)
2180 			| BF_GPMI_ECCCTRL_BUFFER_MASK(BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_PAGE
2181 				| BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_AUXONLY);
2182 		pio[3] = raw_len;
2183 		pio[4] = transfer->sgl.dma_address;
2184 		pio[5] = this->auxiliary_phys;
2185 	}
2186 
2187 	desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio),
2188 				      DMA_TRANS_NONE, 0);
2189 	if (!desc)
2190 		return NULL;
2191 
2192 	if (!this->bch)
2193 		desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1,
2194 					     DMA_DEV_TO_MEM,
2195 					     MXS_DMA_CTRL_WAIT4END);
2196 
2197 	return desc;
2198 }
2199 
2200 static struct dma_async_tx_descriptor *gpmi_chain_data_write(
2201 	struct gpmi_nand_data *this, const void *buf, int raw_len)
2202 {
2203 	struct dma_chan *channel = get_dma_chan(this);
2204 	struct dma_async_tx_descriptor *desc;
2205 	struct gpmi_transfer *transfer;
2206 	u32 pio[6] = {};
2207 
2208 	transfer = get_next_transfer(this);
2209 	if (!transfer)
2210 		return NULL;
2211 
2212 	transfer->direction = DMA_TO_DEVICE;
2213 
2214 	prepare_data_dma(this, buf, raw_len, &transfer->sgl, DMA_TO_DEVICE);
2215 
2216 	pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WRITE)
2217 		| BM_GPMI_CTRL0_WORD_LENGTH
2218 		| BF_GPMI_CTRL0_CS(this->nand.cur_cs, this)
2219 		| BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
2220 		| BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA)
2221 		| BF_GPMI_CTRL0_XFER_COUNT(raw_len);
2222 
2223 	if (this->bch) {
2224 		pio[2] = BM_GPMI_ECCCTRL_ENABLE_ECC
2225 			| BF_GPMI_ECCCTRL_ECC_CMD(BV_GPMI_ECCCTRL_ECC_CMD__BCH_ENCODE)
2226 			| BF_GPMI_ECCCTRL_BUFFER_MASK(BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_PAGE |
2227 					BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_AUXONLY);
2228 		pio[3] = raw_len;
2229 		pio[4] = transfer->sgl.dma_address;
2230 		pio[5] = this->auxiliary_phys;
2231 	}
2232 
2233 	desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio),
2234 				      DMA_TRANS_NONE,
2235 				      (this->bch ? MXS_DMA_CTRL_WAIT4END : 0));
2236 	if (!desc)
2237 		return NULL;
2238 
2239 	if (!this->bch)
2240 		desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1,
2241 					       DMA_MEM_TO_DEV,
2242 					       MXS_DMA_CTRL_WAIT4END);
2243 
2244 	return desc;
2245 }
2246 
2247 static int gpmi_nfc_exec_op(struct nand_chip *chip,
2248 			     const struct nand_operation *op,
2249 			     bool check_only)
2250 {
2251 	const struct nand_op_instr *instr;
2252 	struct gpmi_nand_data *this = nand_get_controller_data(chip);
2253 	struct dma_async_tx_descriptor *desc = NULL;
2254 	int i, ret, buf_len = 0, nbufs = 0;
2255 	u8 cmd = 0;
2256 	void *buf_read = NULL;
2257 	const void *buf_write = NULL;
2258 	bool direct = false;
2259 	struct completion *dma_completion, *bch_completion;
2260 	unsigned long to;
2261 
2262 	if (check_only)
2263 		return 0;
2264 
2265 	this->ntransfers = 0;
2266 	for (i = 0; i < GPMI_MAX_TRANSFERS; i++)
2267 		this->transfers[i].direction = DMA_NONE;
2268 
2269 	ret = pm_runtime_get_sync(this->dev);
2270 	if (ret < 0) {
2271 		pm_runtime_put_noidle(this->dev);
2272 		return ret;
2273 	}
2274 
2275 	/*
2276 	 * This driver currently supports only one NAND chip. Plus, dies share
2277 	 * the same configuration. So once timings have been applied on the
2278 	 * controller side, they will not change anymore. When the time will
2279 	 * come, the check on must_apply_timings will have to be dropped.
2280 	 */
2281 	if (this->hw.must_apply_timings) {
2282 		this->hw.must_apply_timings = false;
2283 		gpmi_nfc_apply_timings(this);
2284 	}
2285 
2286 	dev_dbg(this->dev, "%s: %d instructions\n", __func__, op->ninstrs);
2287 
2288 	for (i = 0; i < op->ninstrs; i++) {
2289 		instr = &op->instrs[i];
2290 
2291 		nand_op_trace("  ", instr);
2292 
2293 		switch (instr->type) {
2294 		case NAND_OP_WAITRDY_INSTR:
2295 			desc = gpmi_chain_wait_ready(this);
2296 			break;
2297 		case NAND_OP_CMD_INSTR:
2298 			cmd = instr->ctx.cmd.opcode;
2299 
2300 			/*
2301 			 * When this command has an address cycle chain it
2302 			 * together with the address cycle
2303 			 */
2304 			if (i + 1 != op->ninstrs &&
2305 			    op->instrs[i + 1].type == NAND_OP_ADDR_INSTR)
2306 				continue;
2307 
2308 			desc = gpmi_chain_command(this, cmd, NULL, 0);
2309 
2310 			break;
2311 		case NAND_OP_ADDR_INSTR:
2312 			desc = gpmi_chain_command(this, cmd, instr->ctx.addr.addrs,
2313 						  instr->ctx.addr.naddrs);
2314 			break;
2315 		case NAND_OP_DATA_OUT_INSTR:
2316 			buf_write = instr->ctx.data.buf.out;
2317 			buf_len = instr->ctx.data.len;
2318 			nbufs++;
2319 
2320 			desc = gpmi_chain_data_write(this, buf_write, buf_len);
2321 
2322 			break;
2323 		case NAND_OP_DATA_IN_INSTR:
2324 			if (!instr->ctx.data.len)
2325 				break;
2326 			buf_read = instr->ctx.data.buf.in;
2327 			buf_len = instr->ctx.data.len;
2328 			nbufs++;
2329 
2330 			desc = gpmi_chain_data_read(this, buf_read, buf_len,
2331 						   &direct);
2332 			break;
2333 		}
2334 
2335 		if (!desc) {
2336 			ret = -ENXIO;
2337 			goto unmap;
2338 		}
2339 	}
2340 
2341 	dev_dbg(this->dev, "%s setup done\n", __func__);
2342 
2343 	if (nbufs > 1) {
2344 		dev_err(this->dev, "Multiple data instructions not supported\n");
2345 		ret = -EINVAL;
2346 		goto unmap;
2347 	}
2348 
2349 	if (this->bch) {
2350 		writel(this->bch_flashlayout0,
2351 		       this->resources.bch_regs + HW_BCH_FLASH0LAYOUT0);
2352 		writel(this->bch_flashlayout1,
2353 		       this->resources.bch_regs + HW_BCH_FLASH0LAYOUT1);
2354 	}
2355 
2356 	desc->callback = dma_irq_callback;
2357 	desc->callback_param = this;
2358 	dma_completion = &this->dma_done;
2359 	bch_completion = NULL;
2360 
2361 	init_completion(dma_completion);
2362 
2363 	if (this->bch && buf_read) {
2364 		writel(BM_BCH_CTRL_COMPLETE_IRQ_EN,
2365 		       this->resources.bch_regs + HW_BCH_CTRL_SET);
2366 		bch_completion = &this->bch_done;
2367 		init_completion(bch_completion);
2368 	}
2369 
2370 	dmaengine_submit(desc);
2371 	dma_async_issue_pending(get_dma_chan(this));
2372 
2373 	to = wait_for_completion_timeout(dma_completion, msecs_to_jiffies(1000));
2374 	if (!to) {
2375 		dev_err(this->dev, "DMA timeout, last DMA\n");
2376 		gpmi_dump_info(this);
2377 		ret = -ETIMEDOUT;
2378 		goto unmap;
2379 	}
2380 
2381 	if (this->bch && buf_read) {
2382 		to = wait_for_completion_timeout(bch_completion, msecs_to_jiffies(1000));
2383 		if (!to) {
2384 			dev_err(this->dev, "BCH timeout, last DMA\n");
2385 			gpmi_dump_info(this);
2386 			ret = -ETIMEDOUT;
2387 			goto unmap;
2388 		}
2389 	}
2390 
2391 	writel(BM_BCH_CTRL_COMPLETE_IRQ_EN,
2392 	       this->resources.bch_regs + HW_BCH_CTRL_CLR);
2393 	gpmi_clear_bch(this);
2394 
2395 	ret = 0;
2396 
2397 unmap:
2398 	for (i = 0; i < this->ntransfers; i++) {
2399 		struct gpmi_transfer *transfer = &this->transfers[i];
2400 
2401 		if (transfer->direction != DMA_NONE)
2402 			dma_unmap_sg(this->dev, &transfer->sgl, 1,
2403 				     transfer->direction);
2404 	}
2405 
2406 	if (!ret && buf_read && !direct)
2407 		memcpy(buf_read, this->data_buffer_dma,
2408 		       gpmi_raw_len_to_len(this, buf_len));
2409 
2410 	this->bch = false;
2411 
2412 	pm_runtime_mark_last_busy(this->dev);
2413 	pm_runtime_put_autosuspend(this->dev);
2414 
2415 	return ret;
2416 }
2417 
2418 static const struct nand_controller_ops gpmi_nand_controller_ops = {
2419 	.attach_chip = gpmi_nand_attach_chip,
2420 	.setup_interface = gpmi_setup_interface,
2421 	.exec_op = gpmi_nfc_exec_op,
2422 };
2423 
2424 static int gpmi_nand_init(struct gpmi_nand_data *this)
2425 {
2426 	struct nand_chip *chip = &this->nand;
2427 	struct mtd_info  *mtd = nand_to_mtd(chip);
2428 	int ret;
2429 
2430 	/* init the MTD data structures */
2431 	mtd->name		= "gpmi-nand";
2432 	mtd->dev.parent		= this->dev;
2433 
2434 	/* init the nand_chip{}, we don't support a 16-bit NAND Flash bus. */
2435 	nand_set_controller_data(chip, this);
2436 	nand_set_flash_node(chip, this->pdev->dev.of_node);
2437 	chip->legacy.block_markbad = gpmi_block_markbad;
2438 	chip->badblock_pattern	= &gpmi_bbt_descr;
2439 	chip->options		|= NAND_NO_SUBPAGE_WRITE;
2440 
2441 	/* Set up swap_block_mark, must be set before the gpmi_set_geometry() */
2442 	this->swap_block_mark = !GPMI_IS_MX23(this);
2443 
2444 	/*
2445 	 * Allocate a temporary DMA buffer for reading ID in the
2446 	 * nand_scan_ident().
2447 	 */
2448 	this->bch_geometry.payload_size = 1024;
2449 	this->bch_geometry.auxiliary_size = 128;
2450 	ret = gpmi_alloc_dma_buffer(this);
2451 	if (ret)
2452 		goto err_out;
2453 
2454 	nand_controller_init(&this->base);
2455 	this->base.ops = &gpmi_nand_controller_ops;
2456 	chip->controller = &this->base;
2457 
2458 	ret = nand_scan(chip, GPMI_IS_MX6(this) ? 2 : 1);
2459 	if (ret)
2460 		goto err_out;
2461 
2462 	ret = nand_boot_init(this);
2463 	if (ret)
2464 		goto err_nand_cleanup;
2465 	ret = nand_create_bbt(chip);
2466 	if (ret)
2467 		goto err_nand_cleanup;
2468 
2469 	ret = mtd_device_register(mtd, NULL, 0);
2470 	if (ret)
2471 		goto err_nand_cleanup;
2472 	return 0;
2473 
2474 err_nand_cleanup:
2475 	nand_cleanup(chip);
2476 err_out:
2477 	gpmi_free_dma_buffer(this);
2478 	return ret;
2479 }
2480 
2481 static const struct of_device_id gpmi_nand_id_table[] = {
2482 	{ .compatible = "fsl,imx23-gpmi-nand", .data = &gpmi_devdata_imx23, },
2483 	{ .compatible = "fsl,imx28-gpmi-nand", .data = &gpmi_devdata_imx28, },
2484 	{ .compatible = "fsl,imx6q-gpmi-nand", .data = &gpmi_devdata_imx6q, },
2485 	{ .compatible = "fsl,imx6sx-gpmi-nand", .data = &gpmi_devdata_imx6sx, },
2486 	{ .compatible = "fsl,imx7d-gpmi-nand", .data = &gpmi_devdata_imx7d,},
2487 	{}
2488 };
2489 MODULE_DEVICE_TABLE(of, gpmi_nand_id_table);
2490 
2491 static int gpmi_nand_probe(struct platform_device *pdev)
2492 {
2493 	struct gpmi_nand_data *this;
2494 	int ret;
2495 
2496 	this = devm_kzalloc(&pdev->dev, sizeof(*this), GFP_KERNEL);
2497 	if (!this)
2498 		return -ENOMEM;
2499 
2500 	this->devdata = of_device_get_match_data(&pdev->dev);
2501 	platform_set_drvdata(pdev, this);
2502 	this->pdev  = pdev;
2503 	this->dev   = &pdev->dev;
2504 
2505 	ret = acquire_resources(this);
2506 	if (ret)
2507 		goto exit_acquire_resources;
2508 
2509 	ret = __gpmi_enable_clk(this, true);
2510 	if (ret)
2511 		goto exit_acquire_resources;
2512 
2513 	pm_runtime_set_autosuspend_delay(&pdev->dev, 500);
2514 	pm_runtime_use_autosuspend(&pdev->dev);
2515 	pm_runtime_set_active(&pdev->dev);
2516 	pm_runtime_enable(&pdev->dev);
2517 	pm_runtime_get_sync(&pdev->dev);
2518 
2519 	ret = gpmi_init(this);
2520 	if (ret)
2521 		goto exit_nfc_init;
2522 
2523 	ret = gpmi_nand_init(this);
2524 	if (ret)
2525 		goto exit_nfc_init;
2526 
2527 	pm_runtime_mark_last_busy(&pdev->dev);
2528 	pm_runtime_put_autosuspend(&pdev->dev);
2529 
2530 	dev_info(this->dev, "driver registered.\n");
2531 
2532 	return 0;
2533 
2534 exit_nfc_init:
2535 	pm_runtime_put(&pdev->dev);
2536 	pm_runtime_disable(&pdev->dev);
2537 	release_resources(this);
2538 exit_acquire_resources:
2539 
2540 	return ret;
2541 }
2542 
2543 static int gpmi_nand_remove(struct platform_device *pdev)
2544 {
2545 	struct gpmi_nand_data *this = platform_get_drvdata(pdev);
2546 	struct nand_chip *chip = &this->nand;
2547 	int ret;
2548 
2549 	pm_runtime_put_sync(&pdev->dev);
2550 	pm_runtime_disable(&pdev->dev);
2551 
2552 	ret = mtd_device_unregister(nand_to_mtd(chip));
2553 	WARN_ON(ret);
2554 	nand_cleanup(chip);
2555 	gpmi_free_dma_buffer(this);
2556 	release_resources(this);
2557 	return 0;
2558 }
2559 
2560 #ifdef CONFIG_PM_SLEEP
2561 static int gpmi_pm_suspend(struct device *dev)
2562 {
2563 	struct gpmi_nand_data *this = dev_get_drvdata(dev);
2564 
2565 	release_dma_channels(this);
2566 	return 0;
2567 }
2568 
2569 static int gpmi_pm_resume(struct device *dev)
2570 {
2571 	struct gpmi_nand_data *this = dev_get_drvdata(dev);
2572 	int ret;
2573 
2574 	ret = acquire_dma_channels(this);
2575 	if (ret < 0)
2576 		return ret;
2577 
2578 	/* re-init the GPMI registers */
2579 	ret = gpmi_init(this);
2580 	if (ret) {
2581 		dev_err(this->dev, "Error setting GPMI : %d\n", ret);
2582 		return ret;
2583 	}
2584 
2585 	/* Set flag to get timing setup restored for next exec_op */
2586 	if (this->hw.clk_rate)
2587 		this->hw.must_apply_timings = true;
2588 
2589 	/* re-init the BCH registers */
2590 	ret = bch_set_geometry(this);
2591 	if (ret) {
2592 		dev_err(this->dev, "Error setting BCH : %d\n", ret);
2593 		return ret;
2594 	}
2595 
2596 	return 0;
2597 }
2598 #endif /* CONFIG_PM_SLEEP */
2599 
2600 static int __maybe_unused gpmi_runtime_suspend(struct device *dev)
2601 {
2602 	struct gpmi_nand_data *this = dev_get_drvdata(dev);
2603 
2604 	return __gpmi_enable_clk(this, false);
2605 }
2606 
2607 static int __maybe_unused gpmi_runtime_resume(struct device *dev)
2608 {
2609 	struct gpmi_nand_data *this = dev_get_drvdata(dev);
2610 
2611 	return __gpmi_enable_clk(this, true);
2612 }
2613 
2614 static const struct dev_pm_ops gpmi_pm_ops = {
2615 	SET_SYSTEM_SLEEP_PM_OPS(gpmi_pm_suspend, gpmi_pm_resume)
2616 	SET_RUNTIME_PM_OPS(gpmi_runtime_suspend, gpmi_runtime_resume, NULL)
2617 };
2618 
2619 static struct platform_driver gpmi_nand_driver = {
2620 	.driver = {
2621 		.name = "gpmi-nand",
2622 		.pm = &gpmi_pm_ops,
2623 		.of_match_table = gpmi_nand_id_table,
2624 	},
2625 	.probe   = gpmi_nand_probe,
2626 	.remove  = gpmi_nand_remove,
2627 };
2628 module_platform_driver(gpmi_nand_driver);
2629 
2630 MODULE_AUTHOR("Freescale Semiconductor, Inc.");
2631 MODULE_DESCRIPTION("i.MX GPMI NAND Flash Controller Driver");
2632 MODULE_LICENSE("GPL");
2633