1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Marvell NAND flash controller driver
4  *
5  * Copyright (C) 2017 Marvell
6  * Author: Miquel RAYNAL <miquel.raynal@free-electrons.com>
7  *
8  *
9  * This NAND controller driver handles two versions of the hardware,
10  * one is called NFCv1 and is available on PXA SoCs and the other is
11  * called NFCv2 and is available on Armada SoCs.
12  *
13  * The main visible difference is that NFCv1 only has Hamming ECC
14  * capabilities, while NFCv2 also embeds a BCH ECC engine. Also, DMA
15  * is not used with NFCv2.
16  *
17  * The ECC layouts are depicted in details in Marvell AN-379, but here
18  * is a brief description.
19  *
20  * When using Hamming, the data is split in 512B chunks (either 1, 2
21  * or 4) and each chunk will have its own ECC "digest" of 6B at the
22  * beginning of the OOB area and eventually the remaining free OOB
23  * bytes (also called "spare" bytes in the driver). This engine
24  * corrects up to 1 bit per chunk and detects reliably an error if
25  * there are at most 2 bitflips. Here is the page layout used by the
26  * controller when Hamming is chosen:
27  *
28  * +-------------------------------------------------------------+
29  * | Data 1 | ... | Data N | ECC 1 | ... | ECCN | Free OOB bytes |
30  * +-------------------------------------------------------------+
31  *
32  * When using the BCH engine, there are N identical (data + free OOB +
33  * ECC) sections and potentially an extra one to deal with
34  * configurations where the chosen (data + free OOB + ECC) sizes do
35  * not align with the page (data + OOB) size. ECC bytes are always
36  * 30B per ECC chunk. Here is the page layout used by the controller
37  * when BCH is chosen:
38  *
39  * +-----------------------------------------
40  * | Data 1 | Free OOB bytes 1 | ECC 1 | ...
41  * +-----------------------------------------
42  *
43  *      -------------------------------------------
44  *       ... | Data N | Free OOB bytes N | ECC N |
45  *      -------------------------------------------
46  *
47  *           --------------------------------------------+
48  *            Last Data | Last Free OOB bytes | Last ECC |
49  *           --------------------------------------------+
50  *
51  * In both cases, the layout seen by the user is always: all data
52  * first, then all free OOB bytes and finally all ECC bytes. With BCH,
53  * ECC bytes are 30B long and are padded with 0xFF to align on 32
54  * bytes.
55  *
56  * The controller has certain limitations that are handled by the
57  * driver:
58  *   - It can only read 2k at a time. To overcome this limitation, the
59  *     driver issues data cycles on the bus, without issuing new
60  *     CMD + ADDR cycles. The Marvell term is "naked" operations.
61  *   - The ECC strength in BCH mode cannot be tuned. It is fixed 16
62  *     bits. What can be tuned is the ECC block size as long as it
63  *     stays between 512B and 2kiB. It's usually chosen based on the
64  *     chip ECC requirements. For instance, using 2kiB ECC chunks
65  *     provides 4b/512B correctability.
66  *   - The controller will always treat data bytes, free OOB bytes
67  *     and ECC bytes in that order, no matter what the real layout is
68  *     (which is usually all data then all OOB bytes). The
69  *     marvell_nfc_layouts array below contains the currently
70  *     supported layouts.
71  *   - Because of these weird layouts, the Bad Block Markers can be
72  *     located in data section. In this case, the NAND_BBT_NO_OOB_BBM
73  *     option must be set to prevent scanning/writing bad block
74  *     markers.
75  */
76 
77 #include <linux/module.h>
78 #include <linux/clk.h>
79 #include <linux/mtd/rawnand.h>
80 #include <linux/of_platform.h>
81 #include <linux/iopoll.h>
82 #include <linux/interrupt.h>
83 #include <linux/slab.h>
84 #include <linux/mfd/syscon.h>
85 #include <linux/regmap.h>
86 #include <asm/unaligned.h>
87 
88 #include <linux/dmaengine.h>
89 #include <linux/dma-mapping.h>
90 #include <linux/dma/pxa-dma.h>
91 #include <linux/platform_data/mtd-nand-pxa3xx.h>
92 
93 /* Data FIFO granularity, FIFO reads/writes must be a multiple of this length */
94 #define FIFO_DEPTH		8
95 #define FIFO_REP(x)		(x / sizeof(u32))
96 #define BCH_SEQ_READS		(32 / FIFO_DEPTH)
97 /* NFC does not support transfers of larger chunks at a time */
98 #define MAX_CHUNK_SIZE		2112
99 /* NFCv1 cannot read more that 7 bytes of ID */
100 #define NFCV1_READID_LEN	7
101 /* Polling is done at a pace of POLL_PERIOD us until POLL_TIMEOUT is reached */
102 #define POLL_PERIOD		0
103 #define POLL_TIMEOUT		100000
104 /* Interrupt maximum wait period in ms */
105 #define IRQ_TIMEOUT		1000
106 /* Latency in clock cycles between SoC pins and NFC logic */
107 #define MIN_RD_DEL_CNT		3
108 /* Maximum number of contiguous address cycles */
109 #define MAX_ADDRESS_CYC_NFCV1	5
110 #define MAX_ADDRESS_CYC_NFCV2	7
111 /* System control registers/bits to enable the NAND controller on some SoCs */
112 #define GENCONF_SOC_DEVICE_MUX	0x208
113 #define GENCONF_SOC_DEVICE_MUX_NFC_EN BIT(0)
114 #define GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST BIT(20)
115 #define GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST BIT(21)
116 #define GENCONF_SOC_DEVICE_MUX_NFC_INT_EN BIT(25)
117 #define GENCONF_CLK_GATING_CTRL	0x220
118 #define GENCONF_CLK_GATING_CTRL_ND_GATE BIT(2)
119 #define GENCONF_ND_CLK_CTRL	0x700
120 #define GENCONF_ND_CLK_CTRL_EN	BIT(0)
121 
122 /* NAND controller data flash control register */
123 #define NDCR			0x00
124 #define NDCR_ALL_INT		GENMASK(11, 0)
125 #define NDCR_CS1_CMDDM		BIT(7)
126 #define NDCR_CS0_CMDDM		BIT(8)
127 #define NDCR_RDYM		BIT(11)
128 #define NDCR_ND_ARB_EN		BIT(12)
129 #define NDCR_RA_START		BIT(15)
130 #define NDCR_RD_ID_CNT(x)	(min_t(unsigned int, x, 0x7) << 16)
131 #define NDCR_PAGE_SZ(x)		(x >= 2048 ? BIT(24) : 0)
132 #define NDCR_DWIDTH_M		BIT(26)
133 #define NDCR_DWIDTH_C		BIT(27)
134 #define NDCR_ND_RUN		BIT(28)
135 #define NDCR_DMA_EN		BIT(29)
136 #define NDCR_ECC_EN		BIT(30)
137 #define NDCR_SPARE_EN		BIT(31)
138 #define NDCR_GENERIC_FIELDS_MASK (~(NDCR_RA_START | NDCR_PAGE_SZ(2048) | \
139 				    NDCR_DWIDTH_M | NDCR_DWIDTH_C))
140 
141 /* NAND interface timing parameter 0 register */
142 #define NDTR0			0x04
143 #define NDTR0_TRP(x)		((min_t(unsigned int, x, 0xF) & 0x7) << 0)
144 #define NDTR0_TRH(x)		(min_t(unsigned int, x, 0x7) << 3)
145 #define NDTR0_ETRP(x)		((min_t(unsigned int, x, 0xF) & 0x8) << 3)
146 #define NDTR0_SEL_NRE_EDGE	BIT(7)
147 #define NDTR0_TWP(x)		(min_t(unsigned int, x, 0x7) << 8)
148 #define NDTR0_TWH(x)		(min_t(unsigned int, x, 0x7) << 11)
149 #define NDTR0_TCS(x)		(min_t(unsigned int, x, 0x7) << 16)
150 #define NDTR0_TCH(x)		(min_t(unsigned int, x, 0x7) << 19)
151 #define NDTR0_RD_CNT_DEL(x)	(min_t(unsigned int, x, 0xF) << 22)
152 #define NDTR0_SELCNTR		BIT(26)
153 #define NDTR0_TADL(x)		(min_t(unsigned int, x, 0x1F) << 27)
154 
155 /* NAND interface timing parameter 1 register */
156 #define NDTR1			0x0C
157 #define NDTR1_TAR(x)		(min_t(unsigned int, x, 0xF) << 0)
158 #define NDTR1_TWHR(x)		(min_t(unsigned int, x, 0xF) << 4)
159 #define NDTR1_TRHW(x)		(min_t(unsigned int, x / 16, 0x3) << 8)
160 #define NDTR1_PRESCALE		BIT(14)
161 #define NDTR1_WAIT_MODE		BIT(15)
162 #define NDTR1_TR(x)		(min_t(unsigned int, x, 0xFFFF) << 16)
163 
164 /* NAND controller status register */
165 #define NDSR			0x14
166 #define NDSR_WRCMDREQ		BIT(0)
167 #define NDSR_RDDREQ		BIT(1)
168 #define NDSR_WRDREQ		BIT(2)
169 #define NDSR_CORERR		BIT(3)
170 #define NDSR_UNCERR		BIT(4)
171 #define NDSR_CMDD(cs)		BIT(8 - cs)
172 #define NDSR_RDY(rb)		BIT(11 + rb)
173 #define NDSR_ERRCNT(x)		((x >> 16) & 0x1F)
174 
175 /* NAND ECC control register */
176 #define NDECCCTRL		0x28
177 #define NDECCCTRL_BCH_EN	BIT(0)
178 
179 /* NAND controller data buffer register */
180 #define NDDB			0x40
181 
182 /* NAND controller command buffer 0 register */
183 #define NDCB0			0x48
184 #define NDCB0_CMD1(x)		((x & 0xFF) << 0)
185 #define NDCB0_CMD2(x)		((x & 0xFF) << 8)
186 #define NDCB0_ADDR_CYC(x)	((x & 0x7) << 16)
187 #define NDCB0_ADDR_GET_NUM_CYC(x) (((x) >> 16) & 0x7)
188 #define NDCB0_DBC		BIT(19)
189 #define NDCB0_CMD_TYPE(x)	((x & 0x7) << 21)
190 #define NDCB0_CSEL		BIT(24)
191 #define NDCB0_RDY_BYP		BIT(27)
192 #define NDCB0_LEN_OVRD		BIT(28)
193 #define NDCB0_CMD_XTYPE(x)	((x & 0x7) << 29)
194 
195 /* NAND controller command buffer 1 register */
196 #define NDCB1			0x4C
197 #define NDCB1_COLS(x)		((x & 0xFFFF) << 0)
198 #define NDCB1_ADDRS_PAGE(x)	(x << 16)
199 
200 /* NAND controller command buffer 2 register */
201 #define NDCB2			0x50
202 #define NDCB2_ADDR5_PAGE(x)	(((x >> 16) & 0xFF) << 0)
203 #define NDCB2_ADDR5_CYC(x)	((x & 0xFF) << 0)
204 
205 /* NAND controller command buffer 3 register */
206 #define NDCB3			0x54
207 #define NDCB3_ADDR6_CYC(x)	((x & 0xFF) << 16)
208 #define NDCB3_ADDR7_CYC(x)	((x & 0xFF) << 24)
209 
210 /* NAND controller command buffer 0 register 'type' and 'xtype' fields */
211 #define TYPE_READ		0
212 #define TYPE_WRITE		1
213 #define TYPE_ERASE		2
214 #define TYPE_READ_ID		3
215 #define TYPE_STATUS		4
216 #define TYPE_RESET		5
217 #define TYPE_NAKED_CMD		6
218 #define TYPE_NAKED_ADDR		7
219 #define TYPE_MASK		7
220 #define XTYPE_MONOLITHIC_RW	0
221 #define XTYPE_LAST_NAKED_RW	1
222 #define XTYPE_FINAL_COMMAND	3
223 #define XTYPE_READ		4
224 #define XTYPE_WRITE_DISPATCH	4
225 #define XTYPE_NAKED_RW		5
226 #define XTYPE_COMMAND_DISPATCH	6
227 #define XTYPE_MASK		7
228 
229 /**
230  * Marvell ECC engine works differently than the others, in order to limit the
231  * size of the IP, hardware engineers chose to set a fixed strength at 16 bits
232  * per subpage, and depending on a the desired strength needed by the NAND chip,
233  * a particular layout mixing data/spare/ecc is defined, with a possible last
234  * chunk smaller that the others.
235  *
236  * @writesize:		Full page size on which the layout applies
237  * @chunk:		Desired ECC chunk size on which the layout applies
238  * @strength:		Desired ECC strength (per chunk size bytes) on which the
239  *			layout applies
240  * @nchunks:		Total number of chunks
241  * @full_chunk_cnt:	Number of full-sized chunks, which is the number of
242  *			repetitions of the pattern:
243  *			(data_bytes + spare_bytes + ecc_bytes).
244  * @data_bytes:		Number of data bytes per chunk
245  * @spare_bytes:	Number of spare bytes per chunk
246  * @ecc_bytes:		Number of ecc bytes per chunk
247  * @last_data_bytes:	Number of data bytes in the last chunk
248  * @last_spare_bytes:	Number of spare bytes in the last chunk
249  * @last_ecc_bytes:	Number of ecc bytes in the last chunk
250  */
251 struct marvell_hw_ecc_layout {
252 	/* Constraints */
253 	int writesize;
254 	int chunk;
255 	int strength;
256 	/* Corresponding layout */
257 	int nchunks;
258 	int full_chunk_cnt;
259 	int data_bytes;
260 	int spare_bytes;
261 	int ecc_bytes;
262 	int last_data_bytes;
263 	int last_spare_bytes;
264 	int last_ecc_bytes;
265 };
266 
267 #define MARVELL_LAYOUT(ws, dc, ds, nc, fcc, db, sb, eb, ldb, lsb, leb)	\
268 	{								\
269 		.writesize = ws,					\
270 		.chunk = dc,						\
271 		.strength = ds,						\
272 		.nchunks = nc,						\
273 		.full_chunk_cnt = fcc,					\
274 		.data_bytes = db,					\
275 		.spare_bytes = sb,					\
276 		.ecc_bytes = eb,					\
277 		.last_data_bytes = ldb,					\
278 		.last_spare_bytes = lsb,				\
279 		.last_ecc_bytes = leb,					\
280 	}
281 
282 /* Layouts explained in AN-379_Marvell_SoC_NFC_ECC */
283 static const struct marvell_hw_ecc_layout marvell_nfc_layouts[] = {
284 	MARVELL_LAYOUT(  512,   512,  1,  1,  1,  512,  8,  8,  0,  0,  0),
285 	MARVELL_LAYOUT( 2048,   512,  1,  1,  1, 2048, 40, 24,  0,  0,  0),
286 	MARVELL_LAYOUT( 2048,   512,  4,  1,  1, 2048, 32, 30,  0,  0,  0),
287 	MARVELL_LAYOUT( 2048,   512,  8,  2,  1, 1024,  0, 30,1024,32, 30),
288 	MARVELL_LAYOUT( 4096,   512,  4,  2,  2, 2048, 32, 30,  0,  0,  0),
289 	MARVELL_LAYOUT( 4096,   512,  8,  5,  4, 1024,  0, 30,  0, 64, 30),
290 	MARVELL_LAYOUT( 8192,   512,  4,  4,  4, 2048,  0, 30,  0,  0,  0),
291 	MARVELL_LAYOUT( 8192,   512,  8,  9,  8, 1024,  0, 30,  0, 160, 30),
292 };
293 
294 /**
295  * The Nand Flash Controller has up to 4 CE and 2 RB pins. The CE selection
296  * is made by a field in NDCB0 register, and in another field in NDCB2 register.
297  * The datasheet describes the logic with an error: ADDR5 field is once
298  * declared at the beginning of NDCB2, and another time at its end. Because the
299  * ADDR5 field of NDCB2 may be used by other bytes, it would be more logical
300  * to use the last bit of this field instead of the first ones.
301  *
302  * @cs:			Wanted CE lane.
303  * @ndcb0_csel:		Value of the NDCB0 register with or without the flag
304  *			selecting the wanted CE lane. This is set once when
305  *			the Device Tree is probed.
306  * @rb:			Ready/Busy pin for the flash chip
307  */
308 struct marvell_nand_chip_sel {
309 	unsigned int cs;
310 	u32 ndcb0_csel;
311 	unsigned int rb;
312 };
313 
314 /**
315  * NAND chip structure: stores NAND chip device related information
316  *
317  * @chip:		Base NAND chip structure
318  * @node:		Used to store NAND chips into a list
319  * @layout		NAND layout when using hardware ECC
320  * @ndcr:		Controller register value for this NAND chip
321  * @ndtr0:		Timing registers 0 value for this NAND chip
322  * @ndtr1:		Timing registers 1 value for this NAND chip
323  * @selected_die:	Current active CS
324  * @nsels:		Number of CS lines required by the NAND chip
325  * @sels:		Array of CS lines descriptions
326  */
327 struct marvell_nand_chip {
328 	struct nand_chip chip;
329 	struct list_head node;
330 	const struct marvell_hw_ecc_layout *layout;
331 	u32 ndcr;
332 	u32 ndtr0;
333 	u32 ndtr1;
334 	int addr_cyc;
335 	int selected_die;
336 	unsigned int nsels;
337 	struct marvell_nand_chip_sel sels[0];
338 };
339 
340 static inline struct marvell_nand_chip *to_marvell_nand(struct nand_chip *chip)
341 {
342 	return container_of(chip, struct marvell_nand_chip, chip);
343 }
344 
345 static inline struct marvell_nand_chip_sel *to_nand_sel(struct marvell_nand_chip
346 							*nand)
347 {
348 	return &nand->sels[nand->selected_die];
349 }
350 
351 /**
352  * NAND controller capabilities for distinction between compatible strings
353  *
354  * @max_cs_nb:		Number of Chip Select lines available
355  * @max_rb_nb:		Number of Ready/Busy lines available
356  * @need_system_controller: Indicates if the SoC needs to have access to the
357  *                      system controller (ie. to enable the NAND controller)
358  * @legacy_of_bindings:	Indicates if DT parsing must be done using the old
359  *			fashion way
360  * @is_nfcv2:		NFCv2 has numerous enhancements compared to NFCv1, ie.
361  *			BCH error detection and correction algorithm,
362  *			NDCB3 register has been added
363  * @use_dma:		Use dma for data transfers
364  */
365 struct marvell_nfc_caps {
366 	unsigned int max_cs_nb;
367 	unsigned int max_rb_nb;
368 	bool need_system_controller;
369 	bool legacy_of_bindings;
370 	bool is_nfcv2;
371 	bool use_dma;
372 };
373 
374 /**
375  * NAND controller structure: stores Marvell NAND controller information
376  *
377  * @controller:		Base controller structure
378  * @dev:		Parent device (used to print error messages)
379  * @regs:		NAND controller registers
380  * @core_clk:		Core clock
381  * @reg_clk:		Registers clock
382  * @complete:		Completion object to wait for NAND controller events
383  * @assigned_cs:	Bitmask describing already assigned CS lines
384  * @chips:		List containing all the NAND chips attached to
385  *			this NAND controller
386  * @caps:		NAND controller capabilities for each compatible string
387  * @dma_chan:		DMA channel (NFCv1 only)
388  * @dma_buf:		32-bit aligned buffer for DMA transfers (NFCv1 only)
389  */
390 struct marvell_nfc {
391 	struct nand_controller controller;
392 	struct device *dev;
393 	void __iomem *regs;
394 	struct clk *core_clk;
395 	struct clk *reg_clk;
396 	struct completion complete;
397 	unsigned long assigned_cs;
398 	struct list_head chips;
399 	struct nand_chip *selected_chip;
400 	const struct marvell_nfc_caps *caps;
401 
402 	/* DMA (NFCv1 only) */
403 	bool use_dma;
404 	struct dma_chan *dma_chan;
405 	u8 *dma_buf;
406 };
407 
408 static inline struct marvell_nfc *to_marvell_nfc(struct nand_controller *ctrl)
409 {
410 	return container_of(ctrl, struct marvell_nfc, controller);
411 }
412 
413 /**
414  * NAND controller timings expressed in NAND Controller clock cycles
415  *
416  * @tRP:		ND_nRE pulse width
417  * @tRH:		ND_nRE high duration
418  * @tWP:		ND_nWE pulse time
419  * @tWH:		ND_nWE high duration
420  * @tCS:		Enable signal setup time
421  * @tCH:		Enable signal hold time
422  * @tADL:		Address to write data delay
423  * @tAR:		ND_ALE low to ND_nRE low delay
424  * @tWHR:		ND_nWE high to ND_nRE low for status read
425  * @tRHW:		ND_nRE high duration, read to write delay
426  * @tR:			ND_nWE high to ND_nRE low for read
427  */
428 struct marvell_nfc_timings {
429 	/* NDTR0 fields */
430 	unsigned int tRP;
431 	unsigned int tRH;
432 	unsigned int tWP;
433 	unsigned int tWH;
434 	unsigned int tCS;
435 	unsigned int tCH;
436 	unsigned int tADL;
437 	/* NDTR1 fields */
438 	unsigned int tAR;
439 	unsigned int tWHR;
440 	unsigned int tRHW;
441 	unsigned int tR;
442 };
443 
444 /**
445  * Derives a duration in numbers of clock cycles.
446  *
447  * @ps: Duration in pico-seconds
448  * @period_ns:  Clock period in nano-seconds
449  *
450  * Convert the duration in nano-seconds, then divide by the period and
451  * return the number of clock periods.
452  */
453 #define TO_CYCLES(ps, period_ns) (DIV_ROUND_UP(ps / 1000, period_ns))
454 #define TO_CYCLES64(ps, period_ns) (DIV_ROUND_UP_ULL(div_u64(ps, 1000), \
455 						     period_ns))
456 
457 /**
458  * NAND driver structure filled during the parsing of the ->exec_op() subop
459  * subset of instructions.
460  *
461  * @ndcb:		Array of values written to NDCBx registers
462  * @cle_ale_delay_ns:	Optional delay after the last CMD or ADDR cycle
463  * @rdy_timeout_ms:	Timeout for waits on Ready/Busy pin
464  * @rdy_delay_ns:	Optional delay after waiting for the RB pin
465  * @data_delay_ns:	Optional delay after the data xfer
466  * @data_instr_idx:	Index of the data instruction in the subop
467  * @data_instr:		Pointer to the data instruction in the subop
468  */
469 struct marvell_nfc_op {
470 	u32 ndcb[4];
471 	unsigned int cle_ale_delay_ns;
472 	unsigned int rdy_timeout_ms;
473 	unsigned int rdy_delay_ns;
474 	unsigned int data_delay_ns;
475 	unsigned int data_instr_idx;
476 	const struct nand_op_instr *data_instr;
477 };
478 
479 /*
480  * Internal helper to conditionnally apply a delay (from the above structure,
481  * most of the time).
482  */
483 static void cond_delay(unsigned int ns)
484 {
485 	if (!ns)
486 		return;
487 
488 	if (ns < 10000)
489 		ndelay(ns);
490 	else
491 		udelay(DIV_ROUND_UP(ns, 1000));
492 }
493 
494 /*
495  * The controller has many flags that could generate interrupts, most of them
496  * are disabled and polling is used. For the very slow signals, using interrupts
497  * may relax the CPU charge.
498  */
499 static void marvell_nfc_disable_int(struct marvell_nfc *nfc, u32 int_mask)
500 {
501 	u32 reg;
502 
503 	/* Writing 1 disables the interrupt */
504 	reg = readl_relaxed(nfc->regs + NDCR);
505 	writel_relaxed(reg | int_mask, nfc->regs + NDCR);
506 }
507 
508 static void marvell_nfc_enable_int(struct marvell_nfc *nfc, u32 int_mask)
509 {
510 	u32 reg;
511 
512 	/* Writing 0 enables the interrupt */
513 	reg = readl_relaxed(nfc->regs + NDCR);
514 	writel_relaxed(reg & ~int_mask, nfc->regs + NDCR);
515 }
516 
517 static u32 marvell_nfc_clear_int(struct marvell_nfc *nfc, u32 int_mask)
518 {
519 	u32 reg;
520 
521 	reg = readl_relaxed(nfc->regs + NDSR);
522 	writel_relaxed(int_mask, nfc->regs + NDSR);
523 
524 	return reg & int_mask;
525 }
526 
527 static void marvell_nfc_force_byte_access(struct nand_chip *chip,
528 					  bool force_8bit)
529 {
530 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
531 	u32 ndcr;
532 
533 	/*
534 	 * Callers of this function do not verify if the NAND is using a 16-bit
535 	 * an 8-bit bus for normal operations, so we need to take care of that
536 	 * here by leaving the configuration unchanged if the NAND does not have
537 	 * the NAND_BUSWIDTH_16 flag set.
538 	 */
539 	if (!(chip->options & NAND_BUSWIDTH_16))
540 		return;
541 
542 	ndcr = readl_relaxed(nfc->regs + NDCR);
543 
544 	if (force_8bit)
545 		ndcr &= ~(NDCR_DWIDTH_M | NDCR_DWIDTH_C);
546 	else
547 		ndcr |= NDCR_DWIDTH_M | NDCR_DWIDTH_C;
548 
549 	writel_relaxed(ndcr, nfc->regs + NDCR);
550 }
551 
552 static int marvell_nfc_wait_ndrun(struct nand_chip *chip)
553 {
554 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
555 	u32 val;
556 	int ret;
557 
558 	/*
559 	 * The command is being processed, wait for the ND_RUN bit to be
560 	 * cleared by the NFC. If not, we must clear it by hand.
561 	 */
562 	ret = readl_relaxed_poll_timeout(nfc->regs + NDCR, val,
563 					 (val & NDCR_ND_RUN) == 0,
564 					 POLL_PERIOD, POLL_TIMEOUT);
565 	if (ret) {
566 		dev_err(nfc->dev, "Timeout on NAND controller run mode\n");
567 		writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
568 			       nfc->regs + NDCR);
569 		return ret;
570 	}
571 
572 	return 0;
573 }
574 
575 /*
576  * Any time a command has to be sent to the controller, the following sequence
577  * has to be followed:
578  * - call marvell_nfc_prepare_cmd()
579  *      -> activate the ND_RUN bit that will kind of 'start a job'
580  *      -> wait the signal indicating the NFC is waiting for a command
581  * - send the command (cmd and address cycles)
582  * - enventually send or receive the data
583  * - call marvell_nfc_end_cmd() with the corresponding flag
584  *      -> wait the flag to be triggered or cancel the job with a timeout
585  *
586  * The following helpers are here to factorize the code a bit so that
587  * specialized functions responsible for executing the actual NAND
588  * operations do not have to replicate the same code blocks.
589  */
590 static int marvell_nfc_prepare_cmd(struct nand_chip *chip)
591 {
592 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
593 	u32 ndcr, val;
594 	int ret;
595 
596 	/* Poll ND_RUN and clear NDSR before issuing any command */
597 	ret = marvell_nfc_wait_ndrun(chip);
598 	if (ret) {
599 		dev_err(nfc->dev, "Last operation did not succeed\n");
600 		return ret;
601 	}
602 
603 	ndcr = readl_relaxed(nfc->regs + NDCR);
604 	writel_relaxed(readl(nfc->regs + NDSR), nfc->regs + NDSR);
605 
606 	/* Assert ND_RUN bit and wait the NFC to be ready */
607 	writel_relaxed(ndcr | NDCR_ND_RUN, nfc->regs + NDCR);
608 	ret = readl_relaxed_poll_timeout(nfc->regs + NDSR, val,
609 					 val & NDSR_WRCMDREQ,
610 					 POLL_PERIOD, POLL_TIMEOUT);
611 	if (ret) {
612 		dev_err(nfc->dev, "Timeout on WRCMDRE\n");
613 		return -ETIMEDOUT;
614 	}
615 
616 	/* Command may be written, clear WRCMDREQ status bit */
617 	writel_relaxed(NDSR_WRCMDREQ, nfc->regs + NDSR);
618 
619 	return 0;
620 }
621 
622 static void marvell_nfc_send_cmd(struct nand_chip *chip,
623 				 struct marvell_nfc_op *nfc_op)
624 {
625 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
626 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
627 
628 	dev_dbg(nfc->dev, "\nNDCR:  0x%08x\n"
629 		"NDCB0: 0x%08x\nNDCB1: 0x%08x\nNDCB2: 0x%08x\nNDCB3: 0x%08x\n",
630 		(u32)readl_relaxed(nfc->regs + NDCR), nfc_op->ndcb[0],
631 		nfc_op->ndcb[1], nfc_op->ndcb[2], nfc_op->ndcb[3]);
632 
633 	writel_relaxed(to_nand_sel(marvell_nand)->ndcb0_csel | nfc_op->ndcb[0],
634 		       nfc->regs + NDCB0);
635 	writel_relaxed(nfc_op->ndcb[1], nfc->regs + NDCB0);
636 	writel(nfc_op->ndcb[2], nfc->regs + NDCB0);
637 
638 	/*
639 	 * Write NDCB0 four times only if LEN_OVRD is set or if ADDR6 or ADDR7
640 	 * fields are used (only available on NFCv2).
641 	 */
642 	if (nfc_op->ndcb[0] & NDCB0_LEN_OVRD ||
643 	    NDCB0_ADDR_GET_NUM_CYC(nfc_op->ndcb[0]) >= 6) {
644 		if (!WARN_ON_ONCE(!nfc->caps->is_nfcv2))
645 			writel(nfc_op->ndcb[3], nfc->regs + NDCB0);
646 	}
647 }
648 
649 static int marvell_nfc_end_cmd(struct nand_chip *chip, int flag,
650 			       const char *label)
651 {
652 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
653 	u32 val;
654 	int ret;
655 
656 	ret = readl_relaxed_poll_timeout(nfc->regs + NDSR, val,
657 					 val & flag,
658 					 POLL_PERIOD, POLL_TIMEOUT);
659 
660 	if (ret) {
661 		dev_err(nfc->dev, "Timeout on %s (NDSR: 0x%08x)\n",
662 			label, val);
663 		if (nfc->dma_chan)
664 			dmaengine_terminate_all(nfc->dma_chan);
665 		return ret;
666 	}
667 
668 	/*
669 	 * DMA function uses this helper to poll on CMDD bits without wanting
670 	 * them to be cleared.
671 	 */
672 	if (nfc->use_dma && (readl_relaxed(nfc->regs + NDCR) & NDCR_DMA_EN))
673 		return 0;
674 
675 	writel_relaxed(flag, nfc->regs + NDSR);
676 
677 	return 0;
678 }
679 
680 static int marvell_nfc_wait_cmdd(struct nand_chip *chip)
681 {
682 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
683 	int cs_flag = NDSR_CMDD(to_nand_sel(marvell_nand)->ndcb0_csel);
684 
685 	return marvell_nfc_end_cmd(chip, cs_flag, "CMDD");
686 }
687 
688 static int marvell_nfc_wait_op(struct nand_chip *chip, unsigned int timeout_ms)
689 {
690 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
691 	u32 pending;
692 	int ret;
693 
694 	/* Timeout is expressed in ms */
695 	if (!timeout_ms)
696 		timeout_ms = IRQ_TIMEOUT;
697 
698 	init_completion(&nfc->complete);
699 
700 	marvell_nfc_enable_int(nfc, NDCR_RDYM);
701 	ret = wait_for_completion_timeout(&nfc->complete,
702 					  msecs_to_jiffies(timeout_ms));
703 	marvell_nfc_disable_int(nfc, NDCR_RDYM);
704 	pending = marvell_nfc_clear_int(nfc, NDSR_RDY(0) | NDSR_RDY(1));
705 
706 	/*
707 	 * In case the interrupt was not served in the required time frame,
708 	 * check if the ISR was not served or if something went actually wrong.
709 	 */
710 	if (ret && !pending) {
711 		dev_err(nfc->dev, "Timeout waiting for RB signal\n");
712 		return -ETIMEDOUT;
713 	}
714 
715 	return 0;
716 }
717 
718 static void marvell_nfc_select_target(struct nand_chip *chip,
719 				      unsigned int die_nr)
720 {
721 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
722 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
723 	u32 ndcr_generic;
724 
725 	if (chip == nfc->selected_chip && die_nr == marvell_nand->selected_die)
726 		return;
727 
728 	writel_relaxed(marvell_nand->ndtr0, nfc->regs + NDTR0);
729 	writel_relaxed(marvell_nand->ndtr1, nfc->regs + NDTR1);
730 
731 	/*
732 	 * Reset the NDCR register to a clean state for this particular chip,
733 	 * also clear ND_RUN bit.
734 	 */
735 	ndcr_generic = readl_relaxed(nfc->regs + NDCR) &
736 		       NDCR_GENERIC_FIELDS_MASK & ~NDCR_ND_RUN;
737 	writel_relaxed(ndcr_generic | marvell_nand->ndcr, nfc->regs + NDCR);
738 
739 	/* Also reset the interrupt status register */
740 	marvell_nfc_clear_int(nfc, NDCR_ALL_INT);
741 
742 	nfc->selected_chip = chip;
743 	marvell_nand->selected_die = die_nr;
744 }
745 
746 static irqreturn_t marvell_nfc_isr(int irq, void *dev_id)
747 {
748 	struct marvell_nfc *nfc = dev_id;
749 	u32 st = readl_relaxed(nfc->regs + NDSR);
750 	u32 ien = (~readl_relaxed(nfc->regs + NDCR)) & NDCR_ALL_INT;
751 
752 	/*
753 	 * RDY interrupt mask is one bit in NDCR while there are two status
754 	 * bit in NDSR (RDY[cs0/cs2] and RDY[cs1/cs3]).
755 	 */
756 	if (st & NDSR_RDY(1))
757 		st |= NDSR_RDY(0);
758 
759 	if (!(st & ien))
760 		return IRQ_NONE;
761 
762 	marvell_nfc_disable_int(nfc, st & NDCR_ALL_INT);
763 
764 	if (st & (NDSR_RDY(0) | NDSR_RDY(1)))
765 		complete(&nfc->complete);
766 
767 	return IRQ_HANDLED;
768 }
769 
770 /* HW ECC related functions */
771 static void marvell_nfc_enable_hw_ecc(struct nand_chip *chip)
772 {
773 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
774 	u32 ndcr = readl_relaxed(nfc->regs + NDCR);
775 
776 	if (!(ndcr & NDCR_ECC_EN)) {
777 		writel_relaxed(ndcr | NDCR_ECC_EN, nfc->regs + NDCR);
778 
779 		/*
780 		 * When enabling BCH, set threshold to 0 to always know the
781 		 * number of corrected bitflips.
782 		 */
783 		if (chip->ecc.algo == NAND_ECC_BCH)
784 			writel_relaxed(NDECCCTRL_BCH_EN, nfc->regs + NDECCCTRL);
785 	}
786 }
787 
788 static void marvell_nfc_disable_hw_ecc(struct nand_chip *chip)
789 {
790 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
791 	u32 ndcr = readl_relaxed(nfc->regs + NDCR);
792 
793 	if (ndcr & NDCR_ECC_EN) {
794 		writel_relaxed(ndcr & ~NDCR_ECC_EN, nfc->regs + NDCR);
795 		if (chip->ecc.algo == NAND_ECC_BCH)
796 			writel_relaxed(0, nfc->regs + NDECCCTRL);
797 	}
798 }
799 
800 /* DMA related helpers */
801 static void marvell_nfc_enable_dma(struct marvell_nfc *nfc)
802 {
803 	u32 reg;
804 
805 	reg = readl_relaxed(nfc->regs + NDCR);
806 	writel_relaxed(reg | NDCR_DMA_EN, nfc->regs + NDCR);
807 }
808 
809 static void marvell_nfc_disable_dma(struct marvell_nfc *nfc)
810 {
811 	u32 reg;
812 
813 	reg = readl_relaxed(nfc->regs + NDCR);
814 	writel_relaxed(reg & ~NDCR_DMA_EN, nfc->regs + NDCR);
815 }
816 
817 /* Read/write PIO/DMA accessors */
818 static int marvell_nfc_xfer_data_dma(struct marvell_nfc *nfc,
819 				     enum dma_data_direction direction,
820 				     unsigned int len)
821 {
822 	unsigned int dma_len = min_t(int, ALIGN(len, 32), MAX_CHUNK_SIZE);
823 	struct dma_async_tx_descriptor *tx;
824 	struct scatterlist sg;
825 	dma_cookie_t cookie;
826 	int ret;
827 
828 	marvell_nfc_enable_dma(nfc);
829 	/* Prepare the DMA transfer */
830 	sg_init_one(&sg, nfc->dma_buf, dma_len);
831 	dma_map_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
832 	tx = dmaengine_prep_slave_sg(nfc->dma_chan, &sg, 1,
833 				     direction == DMA_FROM_DEVICE ?
834 				     DMA_DEV_TO_MEM : DMA_MEM_TO_DEV,
835 				     DMA_PREP_INTERRUPT);
836 	if (!tx) {
837 		dev_err(nfc->dev, "Could not prepare DMA S/G list\n");
838 		return -ENXIO;
839 	}
840 
841 	/* Do the task and wait for it to finish */
842 	cookie = dmaengine_submit(tx);
843 	ret = dma_submit_error(cookie);
844 	if (ret)
845 		return -EIO;
846 
847 	dma_async_issue_pending(nfc->dma_chan);
848 	ret = marvell_nfc_wait_cmdd(nfc->selected_chip);
849 	dma_unmap_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
850 	marvell_nfc_disable_dma(nfc);
851 	if (ret) {
852 		dev_err(nfc->dev, "Timeout waiting for DMA (status: %d)\n",
853 			dmaengine_tx_status(nfc->dma_chan, cookie, NULL));
854 		dmaengine_terminate_all(nfc->dma_chan);
855 		return -ETIMEDOUT;
856 	}
857 
858 	return 0;
859 }
860 
861 static int marvell_nfc_xfer_data_in_pio(struct marvell_nfc *nfc, u8 *in,
862 					unsigned int len)
863 {
864 	unsigned int last_len = len % FIFO_DEPTH;
865 	unsigned int last_full_offset = round_down(len, FIFO_DEPTH);
866 	int i;
867 
868 	for (i = 0; i < last_full_offset; i += FIFO_DEPTH)
869 		ioread32_rep(nfc->regs + NDDB, in + i, FIFO_REP(FIFO_DEPTH));
870 
871 	if (last_len) {
872 		u8 tmp_buf[FIFO_DEPTH];
873 
874 		ioread32_rep(nfc->regs + NDDB, tmp_buf, FIFO_REP(FIFO_DEPTH));
875 		memcpy(in + last_full_offset, tmp_buf, last_len);
876 	}
877 
878 	return 0;
879 }
880 
881 static int marvell_nfc_xfer_data_out_pio(struct marvell_nfc *nfc, const u8 *out,
882 					 unsigned int len)
883 {
884 	unsigned int last_len = len % FIFO_DEPTH;
885 	unsigned int last_full_offset = round_down(len, FIFO_DEPTH);
886 	int i;
887 
888 	for (i = 0; i < last_full_offset; i += FIFO_DEPTH)
889 		iowrite32_rep(nfc->regs + NDDB, out + i, FIFO_REP(FIFO_DEPTH));
890 
891 	if (last_len) {
892 		u8 tmp_buf[FIFO_DEPTH];
893 
894 		memcpy(tmp_buf, out + last_full_offset, last_len);
895 		iowrite32_rep(nfc->regs + NDDB, tmp_buf, FIFO_REP(FIFO_DEPTH));
896 	}
897 
898 	return 0;
899 }
900 
901 static void marvell_nfc_check_empty_chunk(struct nand_chip *chip,
902 					  u8 *data, int data_len,
903 					  u8 *spare, int spare_len,
904 					  u8 *ecc, int ecc_len,
905 					  unsigned int *max_bitflips)
906 {
907 	struct mtd_info *mtd = nand_to_mtd(chip);
908 	int bf;
909 
910 	/*
911 	 * Blank pages (all 0xFF) that have not been written may be recognized
912 	 * as bad if bitflips occur, so whenever an uncorrectable error occurs,
913 	 * check if the entire page (with ECC bytes) is actually blank or not.
914 	 */
915 	if (!data)
916 		data_len = 0;
917 	if (!spare)
918 		spare_len = 0;
919 	if (!ecc)
920 		ecc_len = 0;
921 
922 	bf = nand_check_erased_ecc_chunk(data, data_len, ecc, ecc_len,
923 					 spare, spare_len, chip->ecc.strength);
924 	if (bf < 0) {
925 		mtd->ecc_stats.failed++;
926 		return;
927 	}
928 
929 	/* Update the stats and max_bitflips */
930 	mtd->ecc_stats.corrected += bf;
931 	*max_bitflips = max_t(unsigned int, *max_bitflips, bf);
932 }
933 
934 /*
935  * Check a chunk is correct or not according to hardware ECC engine.
936  * mtd->ecc_stats.corrected is updated, as well as max_bitflips, however
937  * mtd->ecc_stats.failure is not, the function will instead return a non-zero
938  * value indicating that a check on the emptyness of the subpage must be
939  * performed before declaring the subpage corrupted.
940  */
941 static int marvell_nfc_hw_ecc_correct(struct nand_chip *chip,
942 				      unsigned int *max_bitflips)
943 {
944 	struct mtd_info *mtd = nand_to_mtd(chip);
945 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
946 	int bf = 0;
947 	u32 ndsr;
948 
949 	ndsr = readl_relaxed(nfc->regs + NDSR);
950 
951 	/* Check uncorrectable error flag */
952 	if (ndsr & NDSR_UNCERR) {
953 		writel_relaxed(ndsr, nfc->regs + NDSR);
954 
955 		/*
956 		 * Do not increment ->ecc_stats.failed now, instead, return a
957 		 * non-zero value to indicate that this chunk was apparently
958 		 * bad, and it should be check to see if it empty or not. If
959 		 * the chunk (with ECC bytes) is not declared empty, the calling
960 		 * function must increment the failure count.
961 		 */
962 		return -EBADMSG;
963 	}
964 
965 	/* Check correctable error flag */
966 	if (ndsr & NDSR_CORERR) {
967 		writel_relaxed(ndsr, nfc->regs + NDSR);
968 
969 		if (chip->ecc.algo == NAND_ECC_BCH)
970 			bf = NDSR_ERRCNT(ndsr);
971 		else
972 			bf = 1;
973 	}
974 
975 	/* Update the stats and max_bitflips */
976 	mtd->ecc_stats.corrected += bf;
977 	*max_bitflips = max_t(unsigned int, *max_bitflips, bf);
978 
979 	return 0;
980 }
981 
982 /* Hamming read helpers */
983 static int marvell_nfc_hw_ecc_hmg_do_read_page(struct nand_chip *chip,
984 					       u8 *data_buf, u8 *oob_buf,
985 					       bool raw, int page)
986 {
987 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
988 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
989 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
990 	struct marvell_nfc_op nfc_op = {
991 		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_READ) |
992 			   NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
993 			   NDCB0_DBC |
994 			   NDCB0_CMD1(NAND_CMD_READ0) |
995 			   NDCB0_CMD2(NAND_CMD_READSTART),
996 		.ndcb[1] = NDCB1_ADDRS_PAGE(page),
997 		.ndcb[2] = NDCB2_ADDR5_PAGE(page),
998 	};
999 	unsigned int oob_bytes = lt->spare_bytes + (raw ? lt->ecc_bytes : 0);
1000 	int ret;
1001 
1002 	/* NFCv2 needs more information about the operation being executed */
1003 	if (nfc->caps->is_nfcv2)
1004 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1005 
1006 	ret = marvell_nfc_prepare_cmd(chip);
1007 	if (ret)
1008 		return ret;
1009 
1010 	marvell_nfc_send_cmd(chip, &nfc_op);
1011 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1012 				  "RDDREQ while draining FIFO (data/oob)");
1013 	if (ret)
1014 		return ret;
1015 
1016 	/*
1017 	 * Read the page then the OOB area. Unlike what is shown in current
1018 	 * documentation, spare bytes are protected by the ECC engine, and must
1019 	 * be at the beginning of the OOB area or running this driver on legacy
1020 	 * systems will prevent the discovery of the BBM/BBT.
1021 	 */
1022 	if (nfc->use_dma) {
1023 		marvell_nfc_xfer_data_dma(nfc, DMA_FROM_DEVICE,
1024 					  lt->data_bytes + oob_bytes);
1025 		memcpy(data_buf, nfc->dma_buf, lt->data_bytes);
1026 		memcpy(oob_buf, nfc->dma_buf + lt->data_bytes, oob_bytes);
1027 	} else {
1028 		marvell_nfc_xfer_data_in_pio(nfc, data_buf, lt->data_bytes);
1029 		marvell_nfc_xfer_data_in_pio(nfc, oob_buf, oob_bytes);
1030 	}
1031 
1032 	ret = marvell_nfc_wait_cmdd(chip);
1033 	return ret;
1034 }
1035 
1036 static int marvell_nfc_hw_ecc_hmg_read_page_raw(struct nand_chip *chip, u8 *buf,
1037 						int oob_required, int page)
1038 {
1039 	marvell_nfc_select_target(chip, chip->cur_cs);
1040 	return marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi,
1041 						   true, page);
1042 }
1043 
1044 static int marvell_nfc_hw_ecc_hmg_read_page(struct nand_chip *chip, u8 *buf,
1045 					    int oob_required, int page)
1046 {
1047 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1048 	unsigned int full_sz = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1049 	int max_bitflips = 0, ret;
1050 	u8 *raw_buf;
1051 
1052 	marvell_nfc_select_target(chip, chip->cur_cs);
1053 	marvell_nfc_enable_hw_ecc(chip);
1054 	marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi, false,
1055 					    page);
1056 	ret = marvell_nfc_hw_ecc_correct(chip, &max_bitflips);
1057 	marvell_nfc_disable_hw_ecc(chip);
1058 
1059 	if (!ret)
1060 		return max_bitflips;
1061 
1062 	/*
1063 	 * When ECC failures are detected, check if the full page has been
1064 	 * written or not. Ignore the failure if it is actually empty.
1065 	 */
1066 	raw_buf = kmalloc(full_sz, GFP_KERNEL);
1067 	if (!raw_buf)
1068 		return -ENOMEM;
1069 
1070 	marvell_nfc_hw_ecc_hmg_do_read_page(chip, raw_buf, raw_buf +
1071 					    lt->data_bytes, true, page);
1072 	marvell_nfc_check_empty_chunk(chip, raw_buf, full_sz, NULL, 0, NULL, 0,
1073 				      &max_bitflips);
1074 	kfree(raw_buf);
1075 
1076 	return max_bitflips;
1077 }
1078 
1079 /*
1080  * Spare area in Hamming layouts is not protected by the ECC engine (even if
1081  * it appears before the ECC bytes when reading), the ->read_oob_raw() function
1082  * also stands for ->read_oob().
1083  */
1084 static int marvell_nfc_hw_ecc_hmg_read_oob_raw(struct nand_chip *chip, int page)
1085 {
1086 	/* Invalidate page cache */
1087 	chip->pagebuf = -1;
1088 
1089 	marvell_nfc_select_target(chip, chip->cur_cs);
1090 	return marvell_nfc_hw_ecc_hmg_do_read_page(chip, chip->data_buf,
1091 						   chip->oob_poi, true, page);
1092 }
1093 
1094 /* Hamming write helpers */
1095 static int marvell_nfc_hw_ecc_hmg_do_write_page(struct nand_chip *chip,
1096 						const u8 *data_buf,
1097 						const u8 *oob_buf, bool raw,
1098 						int page)
1099 {
1100 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1101 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1102 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1103 	struct marvell_nfc_op nfc_op = {
1104 		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_WRITE) |
1105 			   NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1106 			   NDCB0_CMD1(NAND_CMD_SEQIN) |
1107 			   NDCB0_CMD2(NAND_CMD_PAGEPROG) |
1108 			   NDCB0_DBC,
1109 		.ndcb[1] = NDCB1_ADDRS_PAGE(page),
1110 		.ndcb[2] = NDCB2_ADDR5_PAGE(page),
1111 	};
1112 	unsigned int oob_bytes = lt->spare_bytes + (raw ? lt->ecc_bytes : 0);
1113 	int ret;
1114 
1115 	/* NFCv2 needs more information about the operation being executed */
1116 	if (nfc->caps->is_nfcv2)
1117 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1118 
1119 	ret = marvell_nfc_prepare_cmd(chip);
1120 	if (ret)
1121 		return ret;
1122 
1123 	marvell_nfc_send_cmd(chip, &nfc_op);
1124 	ret = marvell_nfc_end_cmd(chip, NDSR_WRDREQ,
1125 				  "WRDREQ while loading FIFO (data)");
1126 	if (ret)
1127 		return ret;
1128 
1129 	/* Write the page then the OOB area */
1130 	if (nfc->use_dma) {
1131 		memcpy(nfc->dma_buf, data_buf, lt->data_bytes);
1132 		memcpy(nfc->dma_buf + lt->data_bytes, oob_buf, oob_bytes);
1133 		marvell_nfc_xfer_data_dma(nfc, DMA_TO_DEVICE, lt->data_bytes +
1134 					  lt->ecc_bytes + lt->spare_bytes);
1135 	} else {
1136 		marvell_nfc_xfer_data_out_pio(nfc, data_buf, lt->data_bytes);
1137 		marvell_nfc_xfer_data_out_pio(nfc, oob_buf, oob_bytes);
1138 	}
1139 
1140 	ret = marvell_nfc_wait_cmdd(chip);
1141 	if (ret)
1142 		return ret;
1143 
1144 	ret = marvell_nfc_wait_op(chip,
1145 				  PSEC_TO_MSEC(chip->data_interface.timings.sdr.tPROG_max));
1146 	return ret;
1147 }
1148 
1149 static int marvell_nfc_hw_ecc_hmg_write_page_raw(struct nand_chip *chip,
1150 						 const u8 *buf,
1151 						 int oob_required, int page)
1152 {
1153 	marvell_nfc_select_target(chip, chip->cur_cs);
1154 	return marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1155 						    true, page);
1156 }
1157 
1158 static int marvell_nfc_hw_ecc_hmg_write_page(struct nand_chip *chip,
1159 					     const u8 *buf,
1160 					     int oob_required, int page)
1161 {
1162 	int ret;
1163 
1164 	marvell_nfc_select_target(chip, chip->cur_cs);
1165 	marvell_nfc_enable_hw_ecc(chip);
1166 	ret = marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1167 						   false, page);
1168 	marvell_nfc_disable_hw_ecc(chip);
1169 
1170 	return ret;
1171 }
1172 
1173 /*
1174  * Spare area in Hamming layouts is not protected by the ECC engine (even if
1175  * it appears before the ECC bytes when reading), the ->write_oob_raw() function
1176  * also stands for ->write_oob().
1177  */
1178 static int marvell_nfc_hw_ecc_hmg_write_oob_raw(struct nand_chip *chip,
1179 						int page)
1180 {
1181 	struct mtd_info *mtd = nand_to_mtd(chip);
1182 
1183 	/* Invalidate page cache */
1184 	chip->pagebuf = -1;
1185 
1186 	memset(chip->data_buf, 0xFF, mtd->writesize);
1187 
1188 	marvell_nfc_select_target(chip, chip->cur_cs);
1189 	return marvell_nfc_hw_ecc_hmg_do_write_page(chip, chip->data_buf,
1190 						    chip->oob_poi, true, page);
1191 }
1192 
1193 /* BCH read helpers */
1194 static int marvell_nfc_hw_ecc_bch_read_page_raw(struct nand_chip *chip, u8 *buf,
1195 						int oob_required, int page)
1196 {
1197 	struct mtd_info *mtd = nand_to_mtd(chip);
1198 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1199 	u8 *oob = chip->oob_poi;
1200 	int chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1201 	int ecc_offset = (lt->full_chunk_cnt * lt->spare_bytes) +
1202 		lt->last_spare_bytes;
1203 	int data_len = lt->data_bytes;
1204 	int spare_len = lt->spare_bytes;
1205 	int ecc_len = lt->ecc_bytes;
1206 	int chunk;
1207 
1208 	marvell_nfc_select_target(chip, chip->cur_cs);
1209 
1210 	if (oob_required)
1211 		memset(chip->oob_poi, 0xFF, mtd->oobsize);
1212 
1213 	nand_read_page_op(chip, page, 0, NULL, 0);
1214 
1215 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1216 		/* Update last chunk length */
1217 		if (chunk >= lt->full_chunk_cnt) {
1218 			data_len = lt->last_data_bytes;
1219 			spare_len = lt->last_spare_bytes;
1220 			ecc_len = lt->last_ecc_bytes;
1221 		}
1222 
1223 		/* Read data bytes*/
1224 		nand_change_read_column_op(chip, chunk * chunk_size,
1225 					   buf + (lt->data_bytes * chunk),
1226 					   data_len, false);
1227 
1228 		/* Read spare bytes */
1229 		nand_read_data_op(chip, oob + (lt->spare_bytes * chunk),
1230 				  spare_len, false);
1231 
1232 		/* Read ECC bytes */
1233 		nand_read_data_op(chip, oob + ecc_offset +
1234 				  (ALIGN(lt->ecc_bytes, 32) * chunk),
1235 				  ecc_len, false);
1236 	}
1237 
1238 	return 0;
1239 }
1240 
1241 static void marvell_nfc_hw_ecc_bch_read_chunk(struct nand_chip *chip, int chunk,
1242 					      u8 *data, unsigned int data_len,
1243 					      u8 *spare, unsigned int spare_len,
1244 					      int page)
1245 {
1246 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1247 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1248 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1249 	int i, ret;
1250 	struct marvell_nfc_op nfc_op = {
1251 		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_READ) |
1252 			   NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1253 			   NDCB0_LEN_OVRD,
1254 		.ndcb[1] = NDCB1_ADDRS_PAGE(page),
1255 		.ndcb[2] = NDCB2_ADDR5_PAGE(page),
1256 		.ndcb[3] = data_len + spare_len,
1257 	};
1258 
1259 	ret = marvell_nfc_prepare_cmd(chip);
1260 	if (ret)
1261 		return;
1262 
1263 	if (chunk == 0)
1264 		nfc_op.ndcb[0] |= NDCB0_DBC |
1265 				  NDCB0_CMD1(NAND_CMD_READ0) |
1266 				  NDCB0_CMD2(NAND_CMD_READSTART);
1267 
1268 	/*
1269 	 * Trigger the monolithic read on the first chunk, then naked read on
1270 	 * intermediate chunks and finally a last naked read on the last chunk.
1271 	 */
1272 	if (chunk == 0)
1273 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1274 	else if (chunk < lt->nchunks - 1)
1275 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_NAKED_RW);
1276 	else
1277 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1278 
1279 	marvell_nfc_send_cmd(chip, &nfc_op);
1280 
1281 	/*
1282 	 * According to the datasheet, when reading from NDDB
1283 	 * with BCH enabled, after each 32 bytes reads, we
1284 	 * have to make sure that the NDSR.RDDREQ bit is set.
1285 	 *
1286 	 * Drain the FIFO, 8 32-bit reads at a time, and skip
1287 	 * the polling on the last read.
1288 	 *
1289 	 * Length is a multiple of 32 bytes, hence it is a multiple of 8 too.
1290 	 */
1291 	for (i = 0; i < data_len; i += FIFO_DEPTH * BCH_SEQ_READS) {
1292 		marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1293 				    "RDDREQ while draining FIFO (data)");
1294 		marvell_nfc_xfer_data_in_pio(nfc, data,
1295 					     FIFO_DEPTH * BCH_SEQ_READS);
1296 		data += FIFO_DEPTH * BCH_SEQ_READS;
1297 	}
1298 
1299 	for (i = 0; i < spare_len; i += FIFO_DEPTH * BCH_SEQ_READS) {
1300 		marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1301 				    "RDDREQ while draining FIFO (OOB)");
1302 		marvell_nfc_xfer_data_in_pio(nfc, spare,
1303 					     FIFO_DEPTH * BCH_SEQ_READS);
1304 		spare += FIFO_DEPTH * BCH_SEQ_READS;
1305 	}
1306 }
1307 
1308 static int marvell_nfc_hw_ecc_bch_read_page(struct nand_chip *chip,
1309 					    u8 *buf, int oob_required,
1310 					    int page)
1311 {
1312 	struct mtd_info *mtd = nand_to_mtd(chip);
1313 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1314 	int data_len = lt->data_bytes, spare_len = lt->spare_bytes;
1315 	u8 *data = buf, *spare = chip->oob_poi;
1316 	int max_bitflips = 0;
1317 	u32 failure_mask = 0;
1318 	int chunk, ret;
1319 
1320 	marvell_nfc_select_target(chip, chip->cur_cs);
1321 
1322 	/*
1323 	 * With BCH, OOB is not fully used (and thus not read entirely), not
1324 	 * expected bytes could show up at the end of the OOB buffer if not
1325 	 * explicitly erased.
1326 	 */
1327 	if (oob_required)
1328 		memset(chip->oob_poi, 0xFF, mtd->oobsize);
1329 
1330 	marvell_nfc_enable_hw_ecc(chip);
1331 
1332 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1333 		/* Update length for the last chunk */
1334 		if (chunk >= lt->full_chunk_cnt) {
1335 			data_len = lt->last_data_bytes;
1336 			spare_len = lt->last_spare_bytes;
1337 		}
1338 
1339 		/* Read the chunk and detect number of bitflips */
1340 		marvell_nfc_hw_ecc_bch_read_chunk(chip, chunk, data, data_len,
1341 						  spare, spare_len, page);
1342 		ret = marvell_nfc_hw_ecc_correct(chip, &max_bitflips);
1343 		if (ret)
1344 			failure_mask |= BIT(chunk);
1345 
1346 		data += data_len;
1347 		spare += spare_len;
1348 	}
1349 
1350 	marvell_nfc_disable_hw_ecc(chip);
1351 
1352 	if (!failure_mask)
1353 		return max_bitflips;
1354 
1355 	/*
1356 	 * Please note that dumping the ECC bytes during a normal read with OOB
1357 	 * area would add a significant overhead as ECC bytes are "consumed" by
1358 	 * the controller in normal mode and must be re-read in raw mode. To
1359 	 * avoid dropping the performances, we prefer not to include them. The
1360 	 * user should re-read the page in raw mode if ECC bytes are required.
1361 	 */
1362 
1363 	/*
1364 	 * In case there is any subpage read error reported by ->correct(), we
1365 	 * usually re-read only ECC bytes in raw mode and check if the whole
1366 	 * page is empty. In this case, it is normal that the ECC check failed
1367 	 * and we just ignore the error.
1368 	 *
1369 	 * However, it has been empirically observed that for some layouts (e.g
1370 	 * 2k page, 8b strength per 512B chunk), the controller tries to correct
1371 	 * bits and may create itself bitflips in the erased area. To overcome
1372 	 * this strange behavior, the whole page is re-read in raw mode, not
1373 	 * only the ECC bytes.
1374 	 */
1375 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1376 		int data_off_in_page, spare_off_in_page, ecc_off_in_page;
1377 		int data_off, spare_off, ecc_off;
1378 		int data_len, spare_len, ecc_len;
1379 
1380 		/* No failure reported for this chunk, move to the next one */
1381 		if (!(failure_mask & BIT(chunk)))
1382 			continue;
1383 
1384 		data_off_in_page = chunk * (lt->data_bytes + lt->spare_bytes +
1385 					    lt->ecc_bytes);
1386 		spare_off_in_page = data_off_in_page +
1387 			(chunk < lt->full_chunk_cnt ? lt->data_bytes :
1388 						      lt->last_data_bytes);
1389 		ecc_off_in_page = spare_off_in_page +
1390 			(chunk < lt->full_chunk_cnt ? lt->spare_bytes :
1391 						      lt->last_spare_bytes);
1392 
1393 		data_off = chunk * lt->data_bytes;
1394 		spare_off = chunk * lt->spare_bytes;
1395 		ecc_off = (lt->full_chunk_cnt * lt->spare_bytes) +
1396 			  lt->last_spare_bytes +
1397 			  (chunk * (lt->ecc_bytes + 2));
1398 
1399 		data_len = chunk < lt->full_chunk_cnt ? lt->data_bytes :
1400 							lt->last_data_bytes;
1401 		spare_len = chunk < lt->full_chunk_cnt ? lt->spare_bytes :
1402 							 lt->last_spare_bytes;
1403 		ecc_len = chunk < lt->full_chunk_cnt ? lt->ecc_bytes :
1404 						       lt->last_ecc_bytes;
1405 
1406 		/*
1407 		 * Only re-read the ECC bytes, unless we are using the 2k/8b
1408 		 * layout which is buggy in the sense that the ECC engine will
1409 		 * try to correct data bytes anyway, creating bitflips. In this
1410 		 * case, re-read the entire page.
1411 		 */
1412 		if (lt->writesize == 2048 && lt->strength == 8) {
1413 			nand_change_read_column_op(chip, data_off_in_page,
1414 						   buf + data_off, data_len,
1415 						   false);
1416 			nand_change_read_column_op(chip, spare_off_in_page,
1417 						   chip->oob_poi + spare_off, spare_len,
1418 						   false);
1419 		}
1420 
1421 		nand_change_read_column_op(chip, ecc_off_in_page,
1422 					   chip->oob_poi + ecc_off, ecc_len,
1423 					   false);
1424 
1425 		/* Check the entire chunk (data + spare + ecc) for emptyness */
1426 		marvell_nfc_check_empty_chunk(chip, buf + data_off, data_len,
1427 					      chip->oob_poi + spare_off, spare_len,
1428 					      chip->oob_poi + ecc_off, ecc_len,
1429 					      &max_bitflips);
1430 	}
1431 
1432 	return max_bitflips;
1433 }
1434 
1435 static int marvell_nfc_hw_ecc_bch_read_oob_raw(struct nand_chip *chip, int page)
1436 {
1437 	/* Invalidate page cache */
1438 	chip->pagebuf = -1;
1439 
1440 	return chip->ecc.read_page_raw(chip, chip->data_buf, true, page);
1441 }
1442 
1443 static int marvell_nfc_hw_ecc_bch_read_oob(struct nand_chip *chip, int page)
1444 {
1445 	/* Invalidate page cache */
1446 	chip->pagebuf = -1;
1447 
1448 	return chip->ecc.read_page(chip, chip->data_buf, true, page);
1449 }
1450 
1451 /* BCH write helpers */
1452 static int marvell_nfc_hw_ecc_bch_write_page_raw(struct nand_chip *chip,
1453 						 const u8 *buf,
1454 						 int oob_required, int page)
1455 {
1456 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1457 	int full_chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1458 	int data_len = lt->data_bytes;
1459 	int spare_len = lt->spare_bytes;
1460 	int ecc_len = lt->ecc_bytes;
1461 	int spare_offset = 0;
1462 	int ecc_offset = (lt->full_chunk_cnt * lt->spare_bytes) +
1463 		lt->last_spare_bytes;
1464 	int chunk;
1465 
1466 	marvell_nfc_select_target(chip, chip->cur_cs);
1467 
1468 	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1469 
1470 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1471 		if (chunk >= lt->full_chunk_cnt) {
1472 			data_len = lt->last_data_bytes;
1473 			spare_len = lt->last_spare_bytes;
1474 			ecc_len = lt->last_ecc_bytes;
1475 		}
1476 
1477 		/* Point to the column of the next chunk */
1478 		nand_change_write_column_op(chip, chunk * full_chunk_size,
1479 					    NULL, 0, false);
1480 
1481 		/* Write the data */
1482 		nand_write_data_op(chip, buf + (chunk * lt->data_bytes),
1483 				   data_len, false);
1484 
1485 		if (!oob_required)
1486 			continue;
1487 
1488 		/* Write the spare bytes */
1489 		if (spare_len)
1490 			nand_write_data_op(chip, chip->oob_poi + spare_offset,
1491 					   spare_len, false);
1492 
1493 		/* Write the ECC bytes */
1494 		if (ecc_len)
1495 			nand_write_data_op(chip, chip->oob_poi + ecc_offset,
1496 					   ecc_len, false);
1497 
1498 		spare_offset += spare_len;
1499 		ecc_offset += ALIGN(ecc_len, 32);
1500 	}
1501 
1502 	return nand_prog_page_end_op(chip);
1503 }
1504 
1505 static int
1506 marvell_nfc_hw_ecc_bch_write_chunk(struct nand_chip *chip, int chunk,
1507 				   const u8 *data, unsigned int data_len,
1508 				   const u8 *spare, unsigned int spare_len,
1509 				   int page)
1510 {
1511 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1512 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1513 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1514 	u32 xtype;
1515 	int ret;
1516 	struct marvell_nfc_op nfc_op = {
1517 		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_WRITE) | NDCB0_LEN_OVRD,
1518 		.ndcb[3] = data_len + spare_len,
1519 	};
1520 
1521 	/*
1522 	 * First operation dispatches the CMD_SEQIN command, issue the address
1523 	 * cycles and asks for the first chunk of data.
1524 	 * All operations in the middle (if any) will issue a naked write and
1525 	 * also ask for data.
1526 	 * Last operation (if any) asks for the last chunk of data through a
1527 	 * last naked write.
1528 	 */
1529 	if (chunk == 0) {
1530 		if (lt->nchunks == 1)
1531 			xtype = XTYPE_MONOLITHIC_RW;
1532 		else
1533 			xtype = XTYPE_WRITE_DISPATCH;
1534 
1535 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(xtype) |
1536 				  NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1537 				  NDCB0_CMD1(NAND_CMD_SEQIN);
1538 		nfc_op.ndcb[1] |= NDCB1_ADDRS_PAGE(page);
1539 		nfc_op.ndcb[2] |= NDCB2_ADDR5_PAGE(page);
1540 	} else if (chunk < lt->nchunks - 1) {
1541 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_NAKED_RW);
1542 	} else {
1543 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1544 	}
1545 
1546 	/* Always dispatch the PAGEPROG command on the last chunk */
1547 	if (chunk == lt->nchunks - 1)
1548 		nfc_op.ndcb[0] |= NDCB0_CMD2(NAND_CMD_PAGEPROG) | NDCB0_DBC;
1549 
1550 	ret = marvell_nfc_prepare_cmd(chip);
1551 	if (ret)
1552 		return ret;
1553 
1554 	marvell_nfc_send_cmd(chip, &nfc_op);
1555 	ret = marvell_nfc_end_cmd(chip, NDSR_WRDREQ,
1556 				  "WRDREQ while loading FIFO (data)");
1557 	if (ret)
1558 		return ret;
1559 
1560 	/* Transfer the contents */
1561 	iowrite32_rep(nfc->regs + NDDB, data, FIFO_REP(data_len));
1562 	iowrite32_rep(nfc->regs + NDDB, spare, FIFO_REP(spare_len));
1563 
1564 	return 0;
1565 }
1566 
1567 static int marvell_nfc_hw_ecc_bch_write_page(struct nand_chip *chip,
1568 					     const u8 *buf,
1569 					     int oob_required, int page)
1570 {
1571 	struct mtd_info *mtd = nand_to_mtd(chip);
1572 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1573 	const u8 *data = buf;
1574 	const u8 *spare = chip->oob_poi;
1575 	int data_len = lt->data_bytes;
1576 	int spare_len = lt->spare_bytes;
1577 	int chunk, ret;
1578 
1579 	marvell_nfc_select_target(chip, chip->cur_cs);
1580 
1581 	/* Spare data will be written anyway, so clear it to avoid garbage */
1582 	if (!oob_required)
1583 		memset(chip->oob_poi, 0xFF, mtd->oobsize);
1584 
1585 	marvell_nfc_enable_hw_ecc(chip);
1586 
1587 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1588 		if (chunk >= lt->full_chunk_cnt) {
1589 			data_len = lt->last_data_bytes;
1590 			spare_len = lt->last_spare_bytes;
1591 		}
1592 
1593 		marvell_nfc_hw_ecc_bch_write_chunk(chip, chunk, data, data_len,
1594 						   spare, spare_len, page);
1595 		data += data_len;
1596 		spare += spare_len;
1597 
1598 		/*
1599 		 * Waiting only for CMDD or PAGED is not enough, ECC are
1600 		 * partially written. No flag is set once the operation is
1601 		 * really finished but the ND_RUN bit is cleared, so wait for it
1602 		 * before stepping into the next command.
1603 		 */
1604 		marvell_nfc_wait_ndrun(chip);
1605 	}
1606 
1607 	ret = marvell_nfc_wait_op(chip,
1608 				  PSEC_TO_MSEC(chip->data_interface.timings.sdr.tPROG_max));
1609 
1610 	marvell_nfc_disable_hw_ecc(chip);
1611 
1612 	if (ret)
1613 		return ret;
1614 
1615 	return 0;
1616 }
1617 
1618 static int marvell_nfc_hw_ecc_bch_write_oob_raw(struct nand_chip *chip,
1619 						int page)
1620 {
1621 	struct mtd_info *mtd = nand_to_mtd(chip);
1622 
1623 	/* Invalidate page cache */
1624 	chip->pagebuf = -1;
1625 
1626 	memset(chip->data_buf, 0xFF, mtd->writesize);
1627 
1628 	return chip->ecc.write_page_raw(chip, chip->data_buf, true, page);
1629 }
1630 
1631 static int marvell_nfc_hw_ecc_bch_write_oob(struct nand_chip *chip, int page)
1632 {
1633 	struct mtd_info *mtd = nand_to_mtd(chip);
1634 
1635 	/* Invalidate page cache */
1636 	chip->pagebuf = -1;
1637 
1638 	memset(chip->data_buf, 0xFF, mtd->writesize);
1639 
1640 	return chip->ecc.write_page(chip, chip->data_buf, true, page);
1641 }
1642 
1643 /* NAND framework ->exec_op() hooks and related helpers */
1644 static void marvell_nfc_parse_instructions(struct nand_chip *chip,
1645 					   const struct nand_subop *subop,
1646 					   struct marvell_nfc_op *nfc_op)
1647 {
1648 	const struct nand_op_instr *instr = NULL;
1649 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1650 	bool first_cmd = true;
1651 	unsigned int op_id;
1652 	int i;
1653 
1654 	/* Reset the input structure as most of its fields will be OR'ed */
1655 	memset(nfc_op, 0, sizeof(struct marvell_nfc_op));
1656 
1657 	for (op_id = 0; op_id < subop->ninstrs; op_id++) {
1658 		unsigned int offset, naddrs;
1659 		const u8 *addrs;
1660 		int len;
1661 
1662 		instr = &subop->instrs[op_id];
1663 
1664 		switch (instr->type) {
1665 		case NAND_OP_CMD_INSTR:
1666 			if (first_cmd)
1667 				nfc_op->ndcb[0] |=
1668 					NDCB0_CMD1(instr->ctx.cmd.opcode);
1669 			else
1670 				nfc_op->ndcb[0] |=
1671 					NDCB0_CMD2(instr->ctx.cmd.opcode) |
1672 					NDCB0_DBC;
1673 
1674 			nfc_op->cle_ale_delay_ns = instr->delay_ns;
1675 			first_cmd = false;
1676 			break;
1677 
1678 		case NAND_OP_ADDR_INSTR:
1679 			offset = nand_subop_get_addr_start_off(subop, op_id);
1680 			naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
1681 			addrs = &instr->ctx.addr.addrs[offset];
1682 
1683 			nfc_op->ndcb[0] |= NDCB0_ADDR_CYC(naddrs);
1684 
1685 			for (i = 0; i < min_t(unsigned int, 4, naddrs); i++)
1686 				nfc_op->ndcb[1] |= addrs[i] << (8 * i);
1687 
1688 			if (naddrs >= 5)
1689 				nfc_op->ndcb[2] |= NDCB2_ADDR5_CYC(addrs[4]);
1690 			if (naddrs >= 6)
1691 				nfc_op->ndcb[3] |= NDCB3_ADDR6_CYC(addrs[5]);
1692 			if (naddrs == 7)
1693 				nfc_op->ndcb[3] |= NDCB3_ADDR7_CYC(addrs[6]);
1694 
1695 			nfc_op->cle_ale_delay_ns = instr->delay_ns;
1696 			break;
1697 
1698 		case NAND_OP_DATA_IN_INSTR:
1699 			nfc_op->data_instr = instr;
1700 			nfc_op->data_instr_idx = op_id;
1701 			nfc_op->ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ);
1702 			if (nfc->caps->is_nfcv2) {
1703 				nfc_op->ndcb[0] |=
1704 					NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW) |
1705 					NDCB0_LEN_OVRD;
1706 				len = nand_subop_get_data_len(subop, op_id);
1707 				nfc_op->ndcb[3] |= round_up(len, FIFO_DEPTH);
1708 			}
1709 			nfc_op->data_delay_ns = instr->delay_ns;
1710 			break;
1711 
1712 		case NAND_OP_DATA_OUT_INSTR:
1713 			nfc_op->data_instr = instr;
1714 			nfc_op->data_instr_idx = op_id;
1715 			nfc_op->ndcb[0] |= NDCB0_CMD_TYPE(TYPE_WRITE);
1716 			if (nfc->caps->is_nfcv2) {
1717 				nfc_op->ndcb[0] |=
1718 					NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW) |
1719 					NDCB0_LEN_OVRD;
1720 				len = nand_subop_get_data_len(subop, op_id);
1721 				nfc_op->ndcb[3] |= round_up(len, FIFO_DEPTH);
1722 			}
1723 			nfc_op->data_delay_ns = instr->delay_ns;
1724 			break;
1725 
1726 		case NAND_OP_WAITRDY_INSTR:
1727 			nfc_op->rdy_timeout_ms = instr->ctx.waitrdy.timeout_ms;
1728 			nfc_op->rdy_delay_ns = instr->delay_ns;
1729 			break;
1730 		}
1731 	}
1732 }
1733 
1734 static int marvell_nfc_xfer_data_pio(struct nand_chip *chip,
1735 				     const struct nand_subop *subop,
1736 				     struct marvell_nfc_op *nfc_op)
1737 {
1738 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1739 	const struct nand_op_instr *instr = nfc_op->data_instr;
1740 	unsigned int op_id = nfc_op->data_instr_idx;
1741 	unsigned int len = nand_subop_get_data_len(subop, op_id);
1742 	unsigned int offset = nand_subop_get_data_start_off(subop, op_id);
1743 	bool reading = (instr->type == NAND_OP_DATA_IN_INSTR);
1744 	int ret;
1745 
1746 	if (instr->ctx.data.force_8bit)
1747 		marvell_nfc_force_byte_access(chip, true);
1748 
1749 	if (reading) {
1750 		u8 *in = instr->ctx.data.buf.in + offset;
1751 
1752 		ret = marvell_nfc_xfer_data_in_pio(nfc, in, len);
1753 	} else {
1754 		const u8 *out = instr->ctx.data.buf.out + offset;
1755 
1756 		ret = marvell_nfc_xfer_data_out_pio(nfc, out, len);
1757 	}
1758 
1759 	if (instr->ctx.data.force_8bit)
1760 		marvell_nfc_force_byte_access(chip, false);
1761 
1762 	return ret;
1763 }
1764 
1765 static int marvell_nfc_monolithic_access_exec(struct nand_chip *chip,
1766 					      const struct nand_subop *subop)
1767 {
1768 	struct marvell_nfc_op nfc_op;
1769 	bool reading;
1770 	int ret;
1771 
1772 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1773 	reading = (nfc_op.data_instr->type == NAND_OP_DATA_IN_INSTR);
1774 
1775 	ret = marvell_nfc_prepare_cmd(chip);
1776 	if (ret)
1777 		return ret;
1778 
1779 	marvell_nfc_send_cmd(chip, &nfc_op);
1780 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ | NDSR_WRDREQ,
1781 				  "RDDREQ/WRDREQ while draining raw data");
1782 	if (ret)
1783 		return ret;
1784 
1785 	cond_delay(nfc_op.cle_ale_delay_ns);
1786 
1787 	if (reading) {
1788 		if (nfc_op.rdy_timeout_ms) {
1789 			ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1790 			if (ret)
1791 				return ret;
1792 		}
1793 
1794 		cond_delay(nfc_op.rdy_delay_ns);
1795 	}
1796 
1797 	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1798 	ret = marvell_nfc_wait_cmdd(chip);
1799 	if (ret)
1800 		return ret;
1801 
1802 	cond_delay(nfc_op.data_delay_ns);
1803 
1804 	if (!reading) {
1805 		if (nfc_op.rdy_timeout_ms) {
1806 			ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1807 			if (ret)
1808 				return ret;
1809 		}
1810 
1811 		cond_delay(nfc_op.rdy_delay_ns);
1812 	}
1813 
1814 	/*
1815 	 * NDCR ND_RUN bit should be cleared automatically at the end of each
1816 	 * operation but experience shows that the behavior is buggy when it
1817 	 * comes to writes (with LEN_OVRD). Clear it by hand in this case.
1818 	 */
1819 	if (!reading) {
1820 		struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1821 
1822 		writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
1823 			       nfc->regs + NDCR);
1824 	}
1825 
1826 	return 0;
1827 }
1828 
1829 static int marvell_nfc_naked_access_exec(struct nand_chip *chip,
1830 					 const struct nand_subop *subop)
1831 {
1832 	struct marvell_nfc_op nfc_op;
1833 	int ret;
1834 
1835 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1836 
1837 	/*
1838 	 * Naked access are different in that they need to be flagged as naked
1839 	 * by the controller. Reset the controller registers fields that inform
1840 	 * on the type and refill them according to the ongoing operation.
1841 	 */
1842 	nfc_op.ndcb[0] &= ~(NDCB0_CMD_TYPE(TYPE_MASK) |
1843 			    NDCB0_CMD_XTYPE(XTYPE_MASK));
1844 	switch (subop->instrs[0].type) {
1845 	case NAND_OP_CMD_INSTR:
1846 		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_NAKED_CMD);
1847 		break;
1848 	case NAND_OP_ADDR_INSTR:
1849 		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_NAKED_ADDR);
1850 		break;
1851 	case NAND_OP_DATA_IN_INSTR:
1852 		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ) |
1853 				  NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1854 		break;
1855 	case NAND_OP_DATA_OUT_INSTR:
1856 		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_WRITE) |
1857 				  NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1858 		break;
1859 	default:
1860 		/* This should never happen */
1861 		break;
1862 	}
1863 
1864 	ret = marvell_nfc_prepare_cmd(chip);
1865 	if (ret)
1866 		return ret;
1867 
1868 	marvell_nfc_send_cmd(chip, &nfc_op);
1869 
1870 	if (!nfc_op.data_instr) {
1871 		ret = marvell_nfc_wait_cmdd(chip);
1872 		cond_delay(nfc_op.cle_ale_delay_ns);
1873 		return ret;
1874 	}
1875 
1876 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ | NDSR_WRDREQ,
1877 				  "RDDREQ/WRDREQ while draining raw data");
1878 	if (ret)
1879 		return ret;
1880 
1881 	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1882 	ret = marvell_nfc_wait_cmdd(chip);
1883 	if (ret)
1884 		return ret;
1885 
1886 	/*
1887 	 * NDCR ND_RUN bit should be cleared automatically at the end of each
1888 	 * operation but experience shows that the behavior is buggy when it
1889 	 * comes to writes (with LEN_OVRD). Clear it by hand in this case.
1890 	 */
1891 	if (subop->instrs[0].type == NAND_OP_DATA_OUT_INSTR) {
1892 		struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1893 
1894 		writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
1895 			       nfc->regs + NDCR);
1896 	}
1897 
1898 	return 0;
1899 }
1900 
1901 static int marvell_nfc_naked_waitrdy_exec(struct nand_chip *chip,
1902 					  const struct nand_subop *subop)
1903 {
1904 	struct marvell_nfc_op nfc_op;
1905 	int ret;
1906 
1907 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1908 
1909 	ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1910 	cond_delay(nfc_op.rdy_delay_ns);
1911 
1912 	return ret;
1913 }
1914 
1915 static int marvell_nfc_read_id_type_exec(struct nand_chip *chip,
1916 					 const struct nand_subop *subop)
1917 {
1918 	struct marvell_nfc_op nfc_op;
1919 	int ret;
1920 
1921 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1922 	nfc_op.ndcb[0] &= ~NDCB0_CMD_TYPE(TYPE_READ);
1923 	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ_ID);
1924 
1925 	ret = marvell_nfc_prepare_cmd(chip);
1926 	if (ret)
1927 		return ret;
1928 
1929 	marvell_nfc_send_cmd(chip, &nfc_op);
1930 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1931 				  "RDDREQ while reading ID");
1932 	if (ret)
1933 		return ret;
1934 
1935 	cond_delay(nfc_op.cle_ale_delay_ns);
1936 
1937 	if (nfc_op.rdy_timeout_ms) {
1938 		ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1939 		if (ret)
1940 			return ret;
1941 	}
1942 
1943 	cond_delay(nfc_op.rdy_delay_ns);
1944 
1945 	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1946 	ret = marvell_nfc_wait_cmdd(chip);
1947 	if (ret)
1948 		return ret;
1949 
1950 	cond_delay(nfc_op.data_delay_ns);
1951 
1952 	return 0;
1953 }
1954 
1955 static int marvell_nfc_read_status_exec(struct nand_chip *chip,
1956 					const struct nand_subop *subop)
1957 {
1958 	struct marvell_nfc_op nfc_op;
1959 	int ret;
1960 
1961 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1962 	nfc_op.ndcb[0] &= ~NDCB0_CMD_TYPE(TYPE_READ);
1963 	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_STATUS);
1964 
1965 	ret = marvell_nfc_prepare_cmd(chip);
1966 	if (ret)
1967 		return ret;
1968 
1969 	marvell_nfc_send_cmd(chip, &nfc_op);
1970 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1971 				  "RDDREQ while reading status");
1972 	if (ret)
1973 		return ret;
1974 
1975 	cond_delay(nfc_op.cle_ale_delay_ns);
1976 
1977 	if (nfc_op.rdy_timeout_ms) {
1978 		ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1979 		if (ret)
1980 			return ret;
1981 	}
1982 
1983 	cond_delay(nfc_op.rdy_delay_ns);
1984 
1985 	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1986 	ret = marvell_nfc_wait_cmdd(chip);
1987 	if (ret)
1988 		return ret;
1989 
1990 	cond_delay(nfc_op.data_delay_ns);
1991 
1992 	return 0;
1993 }
1994 
1995 static int marvell_nfc_reset_cmd_type_exec(struct nand_chip *chip,
1996 					   const struct nand_subop *subop)
1997 {
1998 	struct marvell_nfc_op nfc_op;
1999 	int ret;
2000 
2001 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
2002 	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_RESET);
2003 
2004 	ret = marvell_nfc_prepare_cmd(chip);
2005 	if (ret)
2006 		return ret;
2007 
2008 	marvell_nfc_send_cmd(chip, &nfc_op);
2009 	ret = marvell_nfc_wait_cmdd(chip);
2010 	if (ret)
2011 		return ret;
2012 
2013 	cond_delay(nfc_op.cle_ale_delay_ns);
2014 
2015 	ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
2016 	if (ret)
2017 		return ret;
2018 
2019 	cond_delay(nfc_op.rdy_delay_ns);
2020 
2021 	return 0;
2022 }
2023 
2024 static int marvell_nfc_erase_cmd_type_exec(struct nand_chip *chip,
2025 					   const struct nand_subop *subop)
2026 {
2027 	struct marvell_nfc_op nfc_op;
2028 	int ret;
2029 
2030 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
2031 	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_ERASE);
2032 
2033 	ret = marvell_nfc_prepare_cmd(chip);
2034 	if (ret)
2035 		return ret;
2036 
2037 	marvell_nfc_send_cmd(chip, &nfc_op);
2038 	ret = marvell_nfc_wait_cmdd(chip);
2039 	if (ret)
2040 		return ret;
2041 
2042 	cond_delay(nfc_op.cle_ale_delay_ns);
2043 
2044 	ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
2045 	if (ret)
2046 		return ret;
2047 
2048 	cond_delay(nfc_op.rdy_delay_ns);
2049 
2050 	return 0;
2051 }
2052 
2053 static const struct nand_op_parser marvell_nfcv2_op_parser = NAND_OP_PARSER(
2054 	/* Monolithic reads/writes */
2055 	NAND_OP_PARSER_PATTERN(
2056 		marvell_nfc_monolithic_access_exec,
2057 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2058 		NAND_OP_PARSER_PAT_ADDR_ELEM(true, MAX_ADDRESS_CYC_NFCV2),
2059 		NAND_OP_PARSER_PAT_CMD_ELEM(true),
2060 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
2061 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_CHUNK_SIZE)),
2062 	NAND_OP_PARSER_PATTERN(
2063 		marvell_nfc_monolithic_access_exec,
2064 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2065 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV2),
2066 		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_CHUNK_SIZE),
2067 		NAND_OP_PARSER_PAT_CMD_ELEM(true),
2068 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
2069 	/* Naked commands */
2070 	NAND_OP_PARSER_PATTERN(
2071 		marvell_nfc_naked_access_exec,
2072 		NAND_OP_PARSER_PAT_CMD_ELEM(false)),
2073 	NAND_OP_PARSER_PATTERN(
2074 		marvell_nfc_naked_access_exec,
2075 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV2)),
2076 	NAND_OP_PARSER_PATTERN(
2077 		marvell_nfc_naked_access_exec,
2078 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_CHUNK_SIZE)),
2079 	NAND_OP_PARSER_PATTERN(
2080 		marvell_nfc_naked_access_exec,
2081 		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_CHUNK_SIZE)),
2082 	NAND_OP_PARSER_PATTERN(
2083 		marvell_nfc_naked_waitrdy_exec,
2084 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2085 	);
2086 
2087 static const struct nand_op_parser marvell_nfcv1_op_parser = NAND_OP_PARSER(
2088 	/* Naked commands not supported, use a function for each pattern */
2089 	NAND_OP_PARSER_PATTERN(
2090 		marvell_nfc_read_id_type_exec,
2091 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2092 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV1),
2093 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 8)),
2094 	NAND_OP_PARSER_PATTERN(
2095 		marvell_nfc_erase_cmd_type_exec,
2096 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2097 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV1),
2098 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2099 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2100 	NAND_OP_PARSER_PATTERN(
2101 		marvell_nfc_read_status_exec,
2102 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2103 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 1)),
2104 	NAND_OP_PARSER_PATTERN(
2105 		marvell_nfc_reset_cmd_type_exec,
2106 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2107 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2108 	NAND_OP_PARSER_PATTERN(
2109 		marvell_nfc_naked_waitrdy_exec,
2110 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2111 	);
2112 
2113 static int marvell_nfc_exec_op(struct nand_chip *chip,
2114 			       const struct nand_operation *op,
2115 			       bool check_only)
2116 {
2117 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2118 
2119 	marvell_nfc_select_target(chip, op->cs);
2120 
2121 	if (nfc->caps->is_nfcv2)
2122 		return nand_op_parser_exec_op(chip, &marvell_nfcv2_op_parser,
2123 					      op, check_only);
2124 	else
2125 		return nand_op_parser_exec_op(chip, &marvell_nfcv1_op_parser,
2126 					      op, check_only);
2127 }
2128 
2129 /*
2130  * Layouts were broken in old pxa3xx_nand driver, these are supposed to be
2131  * usable.
2132  */
2133 static int marvell_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2134 				      struct mtd_oob_region *oobregion)
2135 {
2136 	struct nand_chip *chip = mtd_to_nand(mtd);
2137 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
2138 
2139 	if (section)
2140 		return -ERANGE;
2141 
2142 	oobregion->length = (lt->full_chunk_cnt * lt->ecc_bytes) +
2143 			    lt->last_ecc_bytes;
2144 	oobregion->offset = mtd->oobsize - oobregion->length;
2145 
2146 	return 0;
2147 }
2148 
2149 static int marvell_nand_ooblayout_free(struct mtd_info *mtd, int section,
2150 				       struct mtd_oob_region *oobregion)
2151 {
2152 	struct nand_chip *chip = mtd_to_nand(mtd);
2153 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
2154 
2155 	if (section)
2156 		return -ERANGE;
2157 
2158 	/*
2159 	 * Bootrom looks in bytes 0 & 5 for bad blocks for the
2160 	 * 4KB page / 4bit BCH combination.
2161 	 */
2162 	if (mtd->writesize == SZ_4K && lt->data_bytes == SZ_2K)
2163 		oobregion->offset = 6;
2164 	else
2165 		oobregion->offset = 2;
2166 
2167 	oobregion->length = (lt->full_chunk_cnt * lt->spare_bytes) +
2168 			    lt->last_spare_bytes - oobregion->offset;
2169 
2170 	return 0;
2171 }
2172 
2173 static const struct mtd_ooblayout_ops marvell_nand_ooblayout_ops = {
2174 	.ecc = marvell_nand_ooblayout_ecc,
2175 	.free = marvell_nand_ooblayout_free,
2176 };
2177 
2178 static int marvell_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
2179 					 struct nand_ecc_ctrl *ecc)
2180 {
2181 	struct nand_chip *chip = mtd_to_nand(mtd);
2182 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2183 	const struct marvell_hw_ecc_layout *l;
2184 	int i;
2185 
2186 	if (!nfc->caps->is_nfcv2 &&
2187 	    (mtd->writesize + mtd->oobsize > MAX_CHUNK_SIZE)) {
2188 		dev_err(nfc->dev,
2189 			"NFCv1: writesize (%d) cannot be bigger than a chunk (%d)\n",
2190 			mtd->writesize, MAX_CHUNK_SIZE - mtd->oobsize);
2191 		return -ENOTSUPP;
2192 	}
2193 
2194 	to_marvell_nand(chip)->layout = NULL;
2195 	for (i = 0; i < ARRAY_SIZE(marvell_nfc_layouts); i++) {
2196 		l = &marvell_nfc_layouts[i];
2197 		if (mtd->writesize == l->writesize &&
2198 		    ecc->size == l->chunk && ecc->strength == l->strength) {
2199 			to_marvell_nand(chip)->layout = l;
2200 			break;
2201 		}
2202 	}
2203 
2204 	if (!to_marvell_nand(chip)->layout ||
2205 	    (!nfc->caps->is_nfcv2 && ecc->strength > 1)) {
2206 		dev_err(nfc->dev,
2207 			"ECC strength %d at page size %d is not supported\n",
2208 			ecc->strength, mtd->writesize);
2209 		return -ENOTSUPP;
2210 	}
2211 
2212 	/* Special care for the layout 2k/8-bit/512B  */
2213 	if (l->writesize == 2048 && l->strength == 8) {
2214 		if (mtd->oobsize < 128) {
2215 			dev_err(nfc->dev, "Requested layout needs at least 128 OOB bytes\n");
2216 			return -ENOTSUPP;
2217 		} else {
2218 			chip->bbt_options |= NAND_BBT_NO_OOB_BBM;
2219 		}
2220 	}
2221 
2222 	mtd_set_ooblayout(mtd, &marvell_nand_ooblayout_ops);
2223 	ecc->steps = l->nchunks;
2224 	ecc->size = l->data_bytes;
2225 
2226 	if (ecc->strength == 1) {
2227 		chip->ecc.algo = NAND_ECC_HAMMING;
2228 		ecc->read_page_raw = marvell_nfc_hw_ecc_hmg_read_page_raw;
2229 		ecc->read_page = marvell_nfc_hw_ecc_hmg_read_page;
2230 		ecc->read_oob_raw = marvell_nfc_hw_ecc_hmg_read_oob_raw;
2231 		ecc->read_oob = ecc->read_oob_raw;
2232 		ecc->write_page_raw = marvell_nfc_hw_ecc_hmg_write_page_raw;
2233 		ecc->write_page = marvell_nfc_hw_ecc_hmg_write_page;
2234 		ecc->write_oob_raw = marvell_nfc_hw_ecc_hmg_write_oob_raw;
2235 		ecc->write_oob = ecc->write_oob_raw;
2236 	} else {
2237 		chip->ecc.algo = NAND_ECC_BCH;
2238 		ecc->strength = 16;
2239 		ecc->read_page_raw = marvell_nfc_hw_ecc_bch_read_page_raw;
2240 		ecc->read_page = marvell_nfc_hw_ecc_bch_read_page;
2241 		ecc->read_oob_raw = marvell_nfc_hw_ecc_bch_read_oob_raw;
2242 		ecc->read_oob = marvell_nfc_hw_ecc_bch_read_oob;
2243 		ecc->write_page_raw = marvell_nfc_hw_ecc_bch_write_page_raw;
2244 		ecc->write_page = marvell_nfc_hw_ecc_bch_write_page;
2245 		ecc->write_oob_raw = marvell_nfc_hw_ecc_bch_write_oob_raw;
2246 		ecc->write_oob = marvell_nfc_hw_ecc_bch_write_oob;
2247 	}
2248 
2249 	return 0;
2250 }
2251 
2252 static int marvell_nand_ecc_init(struct mtd_info *mtd,
2253 				 struct nand_ecc_ctrl *ecc)
2254 {
2255 	struct nand_chip *chip = mtd_to_nand(mtd);
2256 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2257 	int ret;
2258 
2259 	if (ecc->mode != NAND_ECC_NONE && (!ecc->size || !ecc->strength)) {
2260 		if (chip->ecc_step_ds && chip->ecc_strength_ds) {
2261 			ecc->size = chip->ecc_step_ds;
2262 			ecc->strength = chip->ecc_strength_ds;
2263 		} else {
2264 			dev_info(nfc->dev,
2265 				 "No minimum ECC strength, using 1b/512B\n");
2266 			ecc->size = 512;
2267 			ecc->strength = 1;
2268 		}
2269 	}
2270 
2271 	switch (ecc->mode) {
2272 	case NAND_ECC_HW:
2273 		ret = marvell_nand_hw_ecc_ctrl_init(mtd, ecc);
2274 		if (ret)
2275 			return ret;
2276 		break;
2277 	case NAND_ECC_NONE:
2278 	case NAND_ECC_SOFT:
2279 	case NAND_ECC_ON_DIE:
2280 		if (!nfc->caps->is_nfcv2 && mtd->writesize != SZ_512 &&
2281 		    mtd->writesize != SZ_2K) {
2282 			dev_err(nfc->dev, "NFCv1 cannot write %d bytes pages\n",
2283 				mtd->writesize);
2284 			return -EINVAL;
2285 		}
2286 		break;
2287 	default:
2288 		return -EINVAL;
2289 	}
2290 
2291 	return 0;
2292 }
2293 
2294 static u8 bbt_pattern[] = {'M', 'V', 'B', 'b', 't', '0' };
2295 static u8 bbt_mirror_pattern[] = {'1', 't', 'b', 'B', 'V', 'M' };
2296 
2297 static struct nand_bbt_descr bbt_main_descr = {
2298 	.options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
2299 		   NAND_BBT_2BIT | NAND_BBT_VERSION,
2300 	.offs =	8,
2301 	.len = 6,
2302 	.veroffs = 14,
2303 	.maxblocks = 8,	/* Last 8 blocks in each chip */
2304 	.pattern = bbt_pattern
2305 };
2306 
2307 static struct nand_bbt_descr bbt_mirror_descr = {
2308 	.options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
2309 		   NAND_BBT_2BIT | NAND_BBT_VERSION,
2310 	.offs =	8,
2311 	.len = 6,
2312 	.veroffs = 14,
2313 	.maxblocks = 8,	/* Last 8 blocks in each chip */
2314 	.pattern = bbt_mirror_pattern
2315 };
2316 
2317 static int marvell_nfc_setup_data_interface(struct nand_chip *chip, int chipnr,
2318 					    const struct nand_data_interface
2319 					    *conf)
2320 {
2321 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
2322 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2323 	unsigned int period_ns = 1000000000 / clk_get_rate(nfc->core_clk) * 2;
2324 	const struct nand_sdr_timings *sdr;
2325 	struct marvell_nfc_timings nfc_tmg;
2326 	int read_delay;
2327 
2328 	sdr = nand_get_sdr_timings(conf);
2329 	if (IS_ERR(sdr))
2330 		return PTR_ERR(sdr);
2331 
2332 	/*
2333 	 * SDR timings are given in pico-seconds while NFC timings must be
2334 	 * expressed in NAND controller clock cycles, which is half of the
2335 	 * frequency of the accessible ECC clock retrieved by clk_get_rate().
2336 	 * This is not written anywhere in the datasheet but was observed
2337 	 * with an oscilloscope.
2338 	 *
2339 	 * NFC datasheet gives equations from which thoses calculations
2340 	 * are derived, they tend to be slightly more restrictives than the
2341 	 * given core timings and may improve the overall speed.
2342 	 */
2343 	nfc_tmg.tRP = TO_CYCLES(DIV_ROUND_UP(sdr->tRC_min, 2), period_ns) - 1;
2344 	nfc_tmg.tRH = nfc_tmg.tRP;
2345 	nfc_tmg.tWP = TO_CYCLES(DIV_ROUND_UP(sdr->tWC_min, 2), period_ns) - 1;
2346 	nfc_tmg.tWH = nfc_tmg.tWP;
2347 	nfc_tmg.tCS = TO_CYCLES(sdr->tCS_min, period_ns);
2348 	nfc_tmg.tCH = TO_CYCLES(sdr->tCH_min, period_ns) - 1;
2349 	nfc_tmg.tADL = TO_CYCLES(sdr->tADL_min, period_ns);
2350 	/*
2351 	 * Read delay is the time of propagation from SoC pins to NFC internal
2352 	 * logic. With non-EDO timings, this is MIN_RD_DEL_CNT clock cycles. In
2353 	 * EDO mode, an additional delay of tRH must be taken into account so
2354 	 * the data is sampled on the falling edge instead of the rising edge.
2355 	 */
2356 	read_delay = sdr->tRC_min >= 30000 ?
2357 		MIN_RD_DEL_CNT : MIN_RD_DEL_CNT + nfc_tmg.tRH;
2358 
2359 	nfc_tmg.tAR = TO_CYCLES(sdr->tAR_min, period_ns);
2360 	/*
2361 	 * tWHR and tRHW are supposed to be read to write delays (and vice
2362 	 * versa) but in some cases, ie. when doing a change column, they must
2363 	 * be greater than that to be sure tCCS delay is respected.
2364 	 */
2365 	nfc_tmg.tWHR = TO_CYCLES(max_t(int, sdr->tWHR_min, sdr->tCCS_min),
2366 				 period_ns) - 2,
2367 	nfc_tmg.tRHW = TO_CYCLES(max_t(int, sdr->tRHW_min, sdr->tCCS_min),
2368 				 period_ns);
2369 
2370 	/*
2371 	 * NFCv2: Use WAIT_MODE (wait for RB line), do not rely only on delays.
2372 	 * NFCv1: No WAIT_MODE, tR must be maximal.
2373 	 */
2374 	if (nfc->caps->is_nfcv2) {
2375 		nfc_tmg.tR = TO_CYCLES(sdr->tWB_max, period_ns);
2376 	} else {
2377 		nfc_tmg.tR = TO_CYCLES64(sdr->tWB_max + sdr->tR_max,
2378 					 period_ns);
2379 		if (nfc_tmg.tR + 3 > nfc_tmg.tCH)
2380 			nfc_tmg.tR = nfc_tmg.tCH - 3;
2381 		else
2382 			nfc_tmg.tR = 0;
2383 	}
2384 
2385 	if (chipnr < 0)
2386 		return 0;
2387 
2388 	marvell_nand->ndtr0 =
2389 		NDTR0_TRP(nfc_tmg.tRP) |
2390 		NDTR0_TRH(nfc_tmg.tRH) |
2391 		NDTR0_ETRP(nfc_tmg.tRP) |
2392 		NDTR0_TWP(nfc_tmg.tWP) |
2393 		NDTR0_TWH(nfc_tmg.tWH) |
2394 		NDTR0_TCS(nfc_tmg.tCS) |
2395 		NDTR0_TCH(nfc_tmg.tCH);
2396 
2397 	marvell_nand->ndtr1 =
2398 		NDTR1_TAR(nfc_tmg.tAR) |
2399 		NDTR1_TWHR(nfc_tmg.tWHR) |
2400 		NDTR1_TR(nfc_tmg.tR);
2401 
2402 	if (nfc->caps->is_nfcv2) {
2403 		marvell_nand->ndtr0 |=
2404 			NDTR0_RD_CNT_DEL(read_delay) |
2405 			NDTR0_SELCNTR |
2406 			NDTR0_TADL(nfc_tmg.tADL);
2407 
2408 		marvell_nand->ndtr1 |=
2409 			NDTR1_TRHW(nfc_tmg.tRHW) |
2410 			NDTR1_WAIT_MODE;
2411 	}
2412 
2413 	return 0;
2414 }
2415 
2416 static int marvell_nand_attach_chip(struct nand_chip *chip)
2417 {
2418 	struct mtd_info *mtd = nand_to_mtd(chip);
2419 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
2420 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2421 	struct pxa3xx_nand_platform_data *pdata = dev_get_platdata(nfc->dev);
2422 	int ret;
2423 
2424 	if (pdata && pdata->flash_bbt)
2425 		chip->bbt_options |= NAND_BBT_USE_FLASH;
2426 
2427 	if (chip->bbt_options & NAND_BBT_USE_FLASH) {
2428 		/*
2429 		 * We'll use a bad block table stored in-flash and don't
2430 		 * allow writing the bad block marker to the flash.
2431 		 */
2432 		chip->bbt_options |= NAND_BBT_NO_OOB_BBM;
2433 		chip->bbt_td = &bbt_main_descr;
2434 		chip->bbt_md = &bbt_mirror_descr;
2435 	}
2436 
2437 	/* Save the chip-specific fields of NDCR */
2438 	marvell_nand->ndcr = NDCR_PAGE_SZ(mtd->writesize);
2439 	if (chip->options & NAND_BUSWIDTH_16)
2440 		marvell_nand->ndcr |= NDCR_DWIDTH_M | NDCR_DWIDTH_C;
2441 
2442 	/*
2443 	 * On small page NANDs, only one cycle is needed to pass the
2444 	 * column address.
2445 	 */
2446 	if (mtd->writesize <= 512) {
2447 		marvell_nand->addr_cyc = 1;
2448 	} else {
2449 		marvell_nand->addr_cyc = 2;
2450 		marvell_nand->ndcr |= NDCR_RA_START;
2451 	}
2452 
2453 	/*
2454 	 * Now add the number of cycles needed to pass the row
2455 	 * address.
2456 	 *
2457 	 * Addressing a chip using CS 2 or 3 should also need the third row
2458 	 * cycle but due to inconsistance in the documentation and lack of
2459 	 * hardware to test this situation, this case is not supported.
2460 	 */
2461 	if (chip->options & NAND_ROW_ADDR_3)
2462 		marvell_nand->addr_cyc += 3;
2463 	else
2464 		marvell_nand->addr_cyc += 2;
2465 
2466 	if (pdata) {
2467 		chip->ecc.size = pdata->ecc_step_size;
2468 		chip->ecc.strength = pdata->ecc_strength;
2469 	}
2470 
2471 	ret = marvell_nand_ecc_init(mtd, &chip->ecc);
2472 	if (ret) {
2473 		dev_err(nfc->dev, "ECC init failed: %d\n", ret);
2474 		return ret;
2475 	}
2476 
2477 	if (chip->ecc.mode == NAND_ECC_HW) {
2478 		/*
2479 		 * Subpage write not available with hardware ECC, prohibit also
2480 		 * subpage read as in userspace subpage access would still be
2481 		 * allowed and subpage write, if used, would lead to numerous
2482 		 * uncorrectable ECC errors.
2483 		 */
2484 		chip->options |= NAND_NO_SUBPAGE_WRITE;
2485 	}
2486 
2487 	if (pdata || nfc->caps->legacy_of_bindings) {
2488 		/*
2489 		 * We keep the MTD name unchanged to avoid breaking platforms
2490 		 * where the MTD cmdline parser is used and the bootloader
2491 		 * has not been updated to use the new naming scheme.
2492 		 */
2493 		mtd->name = "pxa3xx_nand-0";
2494 	} else if (!mtd->name) {
2495 		/*
2496 		 * If the new bindings are used and the bootloader has not been
2497 		 * updated to pass a new mtdparts parameter on the cmdline, you
2498 		 * should define the following property in your NAND node, ie:
2499 		 *
2500 		 *	label = "main-storage";
2501 		 *
2502 		 * This way, mtd->name will be set by the core when
2503 		 * nand_set_flash_node() is called.
2504 		 */
2505 		mtd->name = devm_kasprintf(nfc->dev, GFP_KERNEL,
2506 					   "%s:nand.%d", dev_name(nfc->dev),
2507 					   marvell_nand->sels[0].cs);
2508 		if (!mtd->name) {
2509 			dev_err(nfc->dev, "Failed to allocate mtd->name\n");
2510 			return -ENOMEM;
2511 		}
2512 	}
2513 
2514 	return 0;
2515 }
2516 
2517 static const struct nand_controller_ops marvell_nand_controller_ops = {
2518 	.attach_chip = marvell_nand_attach_chip,
2519 	.exec_op = marvell_nfc_exec_op,
2520 	.setup_data_interface = marvell_nfc_setup_data_interface,
2521 };
2522 
2523 static int marvell_nand_chip_init(struct device *dev, struct marvell_nfc *nfc,
2524 				  struct device_node *np)
2525 {
2526 	struct pxa3xx_nand_platform_data *pdata = dev_get_platdata(dev);
2527 	struct marvell_nand_chip *marvell_nand;
2528 	struct mtd_info *mtd;
2529 	struct nand_chip *chip;
2530 	int nsels, ret, i;
2531 	u32 cs, rb;
2532 
2533 	/*
2534 	 * The legacy "num-cs" property indicates the number of CS on the only
2535 	 * chip connected to the controller (legacy bindings does not support
2536 	 * more than one chip). The CS and RB pins are always the #0.
2537 	 *
2538 	 * When not using legacy bindings, a couple of "reg" and "nand-rb"
2539 	 * properties must be filled. For each chip, expressed as a subnode,
2540 	 * "reg" points to the CS lines and "nand-rb" to the RB line.
2541 	 */
2542 	if (pdata || nfc->caps->legacy_of_bindings) {
2543 		nsels = 1;
2544 	} else {
2545 		nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
2546 		if (nsels <= 0) {
2547 			dev_err(dev, "missing/invalid reg property\n");
2548 			return -EINVAL;
2549 		}
2550 	}
2551 
2552 	/* Alloc the nand chip structure */
2553 	marvell_nand = devm_kzalloc(dev,
2554 				    struct_size(marvell_nand, sels, nsels),
2555 				    GFP_KERNEL);
2556 	if (!marvell_nand) {
2557 		dev_err(dev, "could not allocate chip structure\n");
2558 		return -ENOMEM;
2559 	}
2560 
2561 	marvell_nand->nsels = nsels;
2562 	marvell_nand->selected_die = -1;
2563 
2564 	for (i = 0; i < nsels; i++) {
2565 		if (pdata || nfc->caps->legacy_of_bindings) {
2566 			/*
2567 			 * Legacy bindings use the CS lines in natural
2568 			 * order (0, 1, ...)
2569 			 */
2570 			cs = i;
2571 		} else {
2572 			/* Retrieve CS id */
2573 			ret = of_property_read_u32_index(np, "reg", i, &cs);
2574 			if (ret) {
2575 				dev_err(dev, "could not retrieve reg property: %d\n",
2576 					ret);
2577 				return ret;
2578 			}
2579 		}
2580 
2581 		if (cs >= nfc->caps->max_cs_nb) {
2582 			dev_err(dev, "invalid reg value: %u (max CS = %d)\n",
2583 				cs, nfc->caps->max_cs_nb);
2584 			return -EINVAL;
2585 		}
2586 
2587 		if (test_and_set_bit(cs, &nfc->assigned_cs)) {
2588 			dev_err(dev, "CS %d already assigned\n", cs);
2589 			return -EINVAL;
2590 		}
2591 
2592 		/*
2593 		 * The cs variable represents the chip select id, which must be
2594 		 * converted in bit fields for NDCB0 and NDCB2 to select the
2595 		 * right chip. Unfortunately, due to a lack of information on
2596 		 * the subject and incoherent documentation, the user should not
2597 		 * use CS1 and CS3 at all as asserting them is not supported in
2598 		 * a reliable way (due to multiplexing inside ADDR5 field).
2599 		 */
2600 		marvell_nand->sels[i].cs = cs;
2601 		switch (cs) {
2602 		case 0:
2603 		case 2:
2604 			marvell_nand->sels[i].ndcb0_csel = 0;
2605 			break;
2606 		case 1:
2607 		case 3:
2608 			marvell_nand->sels[i].ndcb0_csel = NDCB0_CSEL;
2609 			break;
2610 		default:
2611 			return -EINVAL;
2612 		}
2613 
2614 		/* Retrieve RB id */
2615 		if (pdata || nfc->caps->legacy_of_bindings) {
2616 			/* Legacy bindings always use RB #0 */
2617 			rb = 0;
2618 		} else {
2619 			ret = of_property_read_u32_index(np, "nand-rb", i,
2620 							 &rb);
2621 			if (ret) {
2622 				dev_err(dev,
2623 					"could not retrieve RB property: %d\n",
2624 					ret);
2625 				return ret;
2626 			}
2627 		}
2628 
2629 		if (rb >= nfc->caps->max_rb_nb) {
2630 			dev_err(dev, "invalid reg value: %u (max RB = %d)\n",
2631 				rb, nfc->caps->max_rb_nb);
2632 			return -EINVAL;
2633 		}
2634 
2635 		marvell_nand->sels[i].rb = rb;
2636 	}
2637 
2638 	chip = &marvell_nand->chip;
2639 	chip->controller = &nfc->controller;
2640 	nand_set_flash_node(chip, np);
2641 
2642 	if (!of_property_read_bool(np, "marvell,nand-keep-config"))
2643 		chip->options |= NAND_KEEP_TIMINGS;
2644 
2645 	mtd = nand_to_mtd(chip);
2646 	mtd->dev.parent = dev;
2647 
2648 	/*
2649 	 * Default to HW ECC engine mode. If the nand-ecc-mode property is given
2650 	 * in the DT node, this entry will be overwritten in nand_scan_ident().
2651 	 */
2652 	chip->ecc.mode = NAND_ECC_HW;
2653 
2654 	/*
2655 	 * Save a reference value for timing registers before
2656 	 * ->setup_data_interface() is called.
2657 	 */
2658 	marvell_nand->ndtr0 = readl_relaxed(nfc->regs + NDTR0);
2659 	marvell_nand->ndtr1 = readl_relaxed(nfc->regs + NDTR1);
2660 
2661 	chip->options |= NAND_BUSWIDTH_AUTO;
2662 
2663 	ret = nand_scan(chip, marvell_nand->nsels);
2664 	if (ret) {
2665 		dev_err(dev, "could not scan the nand chip\n");
2666 		return ret;
2667 	}
2668 
2669 	if (pdata)
2670 		/* Legacy bindings support only one chip */
2671 		ret = mtd_device_register(mtd, pdata->parts, pdata->nr_parts);
2672 	else
2673 		ret = mtd_device_register(mtd, NULL, 0);
2674 	if (ret) {
2675 		dev_err(dev, "failed to register mtd device: %d\n", ret);
2676 		nand_release(chip);
2677 		return ret;
2678 	}
2679 
2680 	list_add_tail(&marvell_nand->node, &nfc->chips);
2681 
2682 	return 0;
2683 }
2684 
2685 static int marvell_nand_chips_init(struct device *dev, struct marvell_nfc *nfc)
2686 {
2687 	struct device_node *np = dev->of_node;
2688 	struct device_node *nand_np;
2689 	int max_cs = nfc->caps->max_cs_nb;
2690 	int nchips;
2691 	int ret;
2692 
2693 	if (!np)
2694 		nchips = 1;
2695 	else
2696 		nchips = of_get_child_count(np);
2697 
2698 	if (nchips > max_cs) {
2699 		dev_err(dev, "too many NAND chips: %d (max = %d CS)\n", nchips,
2700 			max_cs);
2701 		return -EINVAL;
2702 	}
2703 
2704 	/*
2705 	 * Legacy bindings do not use child nodes to exhibit NAND chip
2706 	 * properties and layout. Instead, NAND properties are mixed with the
2707 	 * controller ones, and partitions are defined as direct subnodes of the
2708 	 * NAND controller node.
2709 	 */
2710 	if (nfc->caps->legacy_of_bindings) {
2711 		ret = marvell_nand_chip_init(dev, nfc, np);
2712 		return ret;
2713 	}
2714 
2715 	for_each_child_of_node(np, nand_np) {
2716 		ret = marvell_nand_chip_init(dev, nfc, nand_np);
2717 		if (ret) {
2718 			of_node_put(nand_np);
2719 			return ret;
2720 		}
2721 	}
2722 
2723 	return 0;
2724 }
2725 
2726 static void marvell_nand_chips_cleanup(struct marvell_nfc *nfc)
2727 {
2728 	struct marvell_nand_chip *entry, *temp;
2729 
2730 	list_for_each_entry_safe(entry, temp, &nfc->chips, node) {
2731 		nand_release(&entry->chip);
2732 		list_del(&entry->node);
2733 	}
2734 }
2735 
2736 static int marvell_nfc_init_dma(struct marvell_nfc *nfc)
2737 {
2738 	struct platform_device *pdev = container_of(nfc->dev,
2739 						    struct platform_device,
2740 						    dev);
2741 	struct dma_slave_config config = {};
2742 	struct resource *r;
2743 	int ret;
2744 
2745 	if (!IS_ENABLED(CONFIG_PXA_DMA)) {
2746 		dev_warn(nfc->dev,
2747 			 "DMA not enabled in configuration\n");
2748 		return -ENOTSUPP;
2749 	}
2750 
2751 	ret = dma_set_mask_and_coherent(nfc->dev, DMA_BIT_MASK(32));
2752 	if (ret)
2753 		return ret;
2754 
2755 	nfc->dma_chan =	dma_request_slave_channel(nfc->dev, "data");
2756 	if (!nfc->dma_chan) {
2757 		dev_err(nfc->dev,
2758 			"Unable to request data DMA channel\n");
2759 		return -ENODEV;
2760 	}
2761 
2762 	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2763 	if (!r)
2764 		return -ENXIO;
2765 
2766 	config.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2767 	config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2768 	config.src_addr = r->start + NDDB;
2769 	config.dst_addr = r->start + NDDB;
2770 	config.src_maxburst = 32;
2771 	config.dst_maxburst = 32;
2772 	ret = dmaengine_slave_config(nfc->dma_chan, &config);
2773 	if (ret < 0) {
2774 		dev_err(nfc->dev, "Failed to configure DMA channel\n");
2775 		return ret;
2776 	}
2777 
2778 	/*
2779 	 * DMA must act on length multiple of 32 and this length may be
2780 	 * bigger than the destination buffer. Use this buffer instead
2781 	 * for DMA transfers and then copy the desired amount of data to
2782 	 * the provided buffer.
2783 	 */
2784 	nfc->dma_buf = kmalloc(MAX_CHUNK_SIZE, GFP_KERNEL | GFP_DMA);
2785 	if (!nfc->dma_buf)
2786 		return -ENOMEM;
2787 
2788 	nfc->use_dma = true;
2789 
2790 	return 0;
2791 }
2792 
2793 static void marvell_nfc_reset(struct marvell_nfc *nfc)
2794 {
2795 	/*
2796 	 * ECC operations and interruptions are only enabled when specifically
2797 	 * needed. ECC shall not be activated in the early stages (fails probe).
2798 	 * Arbiter flag, even if marked as "reserved", must be set (empirical).
2799 	 * SPARE_EN bit must always be set or ECC bytes will not be at the same
2800 	 * offset in the read page and this will fail the protection.
2801 	 */
2802 	writel_relaxed(NDCR_ALL_INT | NDCR_ND_ARB_EN | NDCR_SPARE_EN |
2803 		       NDCR_RD_ID_CNT(NFCV1_READID_LEN), nfc->regs + NDCR);
2804 	writel_relaxed(0xFFFFFFFF, nfc->regs + NDSR);
2805 	writel_relaxed(0, nfc->regs + NDECCCTRL);
2806 }
2807 
2808 static int marvell_nfc_init(struct marvell_nfc *nfc)
2809 {
2810 	struct device_node *np = nfc->dev->of_node;
2811 
2812 	/*
2813 	 * Some SoCs like A7k/A8k need to enable manually the NAND
2814 	 * controller, gated clocks and reset bits to avoid being bootloader
2815 	 * dependent. This is done through the use of the System Functions
2816 	 * registers.
2817 	 */
2818 	if (nfc->caps->need_system_controller) {
2819 		struct regmap *sysctrl_base =
2820 			syscon_regmap_lookup_by_phandle(np,
2821 							"marvell,system-controller");
2822 
2823 		if (IS_ERR(sysctrl_base))
2824 			return PTR_ERR(sysctrl_base);
2825 
2826 		regmap_write(sysctrl_base, GENCONF_SOC_DEVICE_MUX,
2827 			     GENCONF_SOC_DEVICE_MUX_NFC_EN |
2828 			     GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST |
2829 			     GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST |
2830 			     GENCONF_SOC_DEVICE_MUX_NFC_INT_EN);
2831 
2832 		regmap_update_bits(sysctrl_base, GENCONF_CLK_GATING_CTRL,
2833 				   GENCONF_CLK_GATING_CTRL_ND_GATE,
2834 				   GENCONF_CLK_GATING_CTRL_ND_GATE);
2835 
2836 		regmap_update_bits(sysctrl_base, GENCONF_ND_CLK_CTRL,
2837 				   GENCONF_ND_CLK_CTRL_EN,
2838 				   GENCONF_ND_CLK_CTRL_EN);
2839 	}
2840 
2841 	/* Configure the DMA if appropriate */
2842 	if (!nfc->caps->is_nfcv2)
2843 		marvell_nfc_init_dma(nfc);
2844 
2845 	marvell_nfc_reset(nfc);
2846 
2847 	return 0;
2848 }
2849 
2850 static int marvell_nfc_probe(struct platform_device *pdev)
2851 {
2852 	struct device *dev = &pdev->dev;
2853 	struct resource *r;
2854 	struct marvell_nfc *nfc;
2855 	int ret;
2856 	int irq;
2857 
2858 	nfc = devm_kzalloc(&pdev->dev, sizeof(struct marvell_nfc),
2859 			   GFP_KERNEL);
2860 	if (!nfc)
2861 		return -ENOMEM;
2862 
2863 	nfc->dev = dev;
2864 	nand_controller_init(&nfc->controller);
2865 	nfc->controller.ops = &marvell_nand_controller_ops;
2866 	INIT_LIST_HEAD(&nfc->chips);
2867 
2868 	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2869 	nfc->regs = devm_ioremap_resource(dev, r);
2870 	if (IS_ERR(nfc->regs))
2871 		return PTR_ERR(nfc->regs);
2872 
2873 	irq = platform_get_irq(pdev, 0);
2874 	if (irq < 0) {
2875 		dev_err(dev, "failed to retrieve irq\n");
2876 		return irq;
2877 	}
2878 
2879 	nfc->core_clk = devm_clk_get(&pdev->dev, "core");
2880 
2881 	/* Managed the legacy case (when the first clock was not named) */
2882 	if (nfc->core_clk == ERR_PTR(-ENOENT))
2883 		nfc->core_clk = devm_clk_get(&pdev->dev, NULL);
2884 
2885 	if (IS_ERR(nfc->core_clk))
2886 		return PTR_ERR(nfc->core_clk);
2887 
2888 	ret = clk_prepare_enable(nfc->core_clk);
2889 	if (ret)
2890 		return ret;
2891 
2892 	nfc->reg_clk = devm_clk_get(&pdev->dev, "reg");
2893 	if (IS_ERR(nfc->reg_clk)) {
2894 		if (PTR_ERR(nfc->reg_clk) != -ENOENT) {
2895 			ret = PTR_ERR(nfc->reg_clk);
2896 			goto unprepare_core_clk;
2897 		}
2898 
2899 		nfc->reg_clk = NULL;
2900 	}
2901 
2902 	ret = clk_prepare_enable(nfc->reg_clk);
2903 	if (ret)
2904 		goto unprepare_core_clk;
2905 
2906 	marvell_nfc_disable_int(nfc, NDCR_ALL_INT);
2907 	marvell_nfc_clear_int(nfc, NDCR_ALL_INT);
2908 	ret = devm_request_irq(dev, irq, marvell_nfc_isr,
2909 			       0, "marvell-nfc", nfc);
2910 	if (ret)
2911 		goto unprepare_reg_clk;
2912 
2913 	/* Get NAND controller capabilities */
2914 	if (pdev->id_entry)
2915 		nfc->caps = (void *)pdev->id_entry->driver_data;
2916 	else
2917 		nfc->caps = of_device_get_match_data(&pdev->dev);
2918 
2919 	if (!nfc->caps) {
2920 		dev_err(dev, "Could not retrieve NFC caps\n");
2921 		ret = -EINVAL;
2922 		goto unprepare_reg_clk;
2923 	}
2924 
2925 	/* Init the controller and then probe the chips */
2926 	ret = marvell_nfc_init(nfc);
2927 	if (ret)
2928 		goto unprepare_reg_clk;
2929 
2930 	platform_set_drvdata(pdev, nfc);
2931 
2932 	ret = marvell_nand_chips_init(dev, nfc);
2933 	if (ret)
2934 		goto unprepare_reg_clk;
2935 
2936 	return 0;
2937 
2938 unprepare_reg_clk:
2939 	clk_disable_unprepare(nfc->reg_clk);
2940 unprepare_core_clk:
2941 	clk_disable_unprepare(nfc->core_clk);
2942 
2943 	return ret;
2944 }
2945 
2946 static int marvell_nfc_remove(struct platform_device *pdev)
2947 {
2948 	struct marvell_nfc *nfc = platform_get_drvdata(pdev);
2949 
2950 	marvell_nand_chips_cleanup(nfc);
2951 
2952 	if (nfc->use_dma) {
2953 		dmaengine_terminate_all(nfc->dma_chan);
2954 		dma_release_channel(nfc->dma_chan);
2955 	}
2956 
2957 	clk_disable_unprepare(nfc->reg_clk);
2958 	clk_disable_unprepare(nfc->core_clk);
2959 
2960 	return 0;
2961 }
2962 
2963 static int __maybe_unused marvell_nfc_suspend(struct device *dev)
2964 {
2965 	struct marvell_nfc *nfc = dev_get_drvdata(dev);
2966 	struct marvell_nand_chip *chip;
2967 
2968 	list_for_each_entry(chip, &nfc->chips, node)
2969 		marvell_nfc_wait_ndrun(&chip->chip);
2970 
2971 	clk_disable_unprepare(nfc->reg_clk);
2972 	clk_disable_unprepare(nfc->core_clk);
2973 
2974 	return 0;
2975 }
2976 
2977 static int __maybe_unused marvell_nfc_resume(struct device *dev)
2978 {
2979 	struct marvell_nfc *nfc = dev_get_drvdata(dev);
2980 	int ret;
2981 
2982 	ret = clk_prepare_enable(nfc->core_clk);
2983 	if (ret < 0)
2984 		return ret;
2985 
2986 	ret = clk_prepare_enable(nfc->reg_clk);
2987 	if (ret < 0)
2988 		return ret;
2989 
2990 	/*
2991 	 * Reset nfc->selected_chip so the next command will cause the timing
2992 	 * registers to be restored in marvell_nfc_select_chip().
2993 	 */
2994 	nfc->selected_chip = NULL;
2995 
2996 	/* Reset registers that have lost their contents */
2997 	marvell_nfc_reset(nfc);
2998 
2999 	return 0;
3000 }
3001 
3002 static const struct dev_pm_ops marvell_nfc_pm_ops = {
3003 	SET_SYSTEM_SLEEP_PM_OPS(marvell_nfc_suspend, marvell_nfc_resume)
3004 };
3005 
3006 static const struct marvell_nfc_caps marvell_armada_8k_nfc_caps = {
3007 	.max_cs_nb = 4,
3008 	.max_rb_nb = 2,
3009 	.need_system_controller = true,
3010 	.is_nfcv2 = true,
3011 };
3012 
3013 static const struct marvell_nfc_caps marvell_armada370_nfc_caps = {
3014 	.max_cs_nb = 4,
3015 	.max_rb_nb = 2,
3016 	.is_nfcv2 = true,
3017 };
3018 
3019 static const struct marvell_nfc_caps marvell_pxa3xx_nfc_caps = {
3020 	.max_cs_nb = 2,
3021 	.max_rb_nb = 1,
3022 	.use_dma = true,
3023 };
3024 
3025 static const struct marvell_nfc_caps marvell_armada_8k_nfc_legacy_caps = {
3026 	.max_cs_nb = 4,
3027 	.max_rb_nb = 2,
3028 	.need_system_controller = true,
3029 	.legacy_of_bindings = true,
3030 	.is_nfcv2 = true,
3031 };
3032 
3033 static const struct marvell_nfc_caps marvell_armada370_nfc_legacy_caps = {
3034 	.max_cs_nb = 4,
3035 	.max_rb_nb = 2,
3036 	.legacy_of_bindings = true,
3037 	.is_nfcv2 = true,
3038 };
3039 
3040 static const struct marvell_nfc_caps marvell_pxa3xx_nfc_legacy_caps = {
3041 	.max_cs_nb = 2,
3042 	.max_rb_nb = 1,
3043 	.legacy_of_bindings = true,
3044 	.use_dma = true,
3045 };
3046 
3047 static const struct platform_device_id marvell_nfc_platform_ids[] = {
3048 	{
3049 		.name = "pxa3xx-nand",
3050 		.driver_data = (kernel_ulong_t)&marvell_pxa3xx_nfc_legacy_caps,
3051 	},
3052 	{ /* sentinel */ },
3053 };
3054 MODULE_DEVICE_TABLE(platform, marvell_nfc_platform_ids);
3055 
3056 static const struct of_device_id marvell_nfc_of_ids[] = {
3057 	{
3058 		.compatible = "marvell,armada-8k-nand-controller",
3059 		.data = &marvell_armada_8k_nfc_caps,
3060 	},
3061 	{
3062 		.compatible = "marvell,armada370-nand-controller",
3063 		.data = &marvell_armada370_nfc_caps,
3064 	},
3065 	{
3066 		.compatible = "marvell,pxa3xx-nand-controller",
3067 		.data = &marvell_pxa3xx_nfc_caps,
3068 	},
3069 	/* Support for old/deprecated bindings: */
3070 	{
3071 		.compatible = "marvell,armada-8k-nand",
3072 		.data = &marvell_armada_8k_nfc_legacy_caps,
3073 	},
3074 	{
3075 		.compatible = "marvell,armada370-nand",
3076 		.data = &marvell_armada370_nfc_legacy_caps,
3077 	},
3078 	{
3079 		.compatible = "marvell,pxa3xx-nand",
3080 		.data = &marvell_pxa3xx_nfc_legacy_caps,
3081 	},
3082 	{ /* sentinel */ },
3083 };
3084 MODULE_DEVICE_TABLE(of, marvell_nfc_of_ids);
3085 
3086 static struct platform_driver marvell_nfc_driver = {
3087 	.driver	= {
3088 		.name		= "marvell-nfc",
3089 		.of_match_table = marvell_nfc_of_ids,
3090 		.pm		= &marvell_nfc_pm_ops,
3091 	},
3092 	.id_table = marvell_nfc_platform_ids,
3093 	.probe = marvell_nfc_probe,
3094 	.remove	= marvell_nfc_remove,
3095 };
3096 module_platform_driver(marvell_nfc_driver);
3097 
3098 MODULE_LICENSE("GPL");
3099 MODULE_DESCRIPTION("Marvell NAND controller driver");
3100