1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Marvell NAND flash controller driver
4  *
5  * Copyright (C) 2017 Marvell
6  * Author: Miquel RAYNAL <miquel.raynal@free-electrons.com>
7  *
8  *
9  * This NAND controller driver handles two versions of the hardware,
10  * one is called NFCv1 and is available on PXA SoCs and the other is
11  * called NFCv2 and is available on Armada SoCs.
12  *
13  * The main visible difference is that NFCv1 only has Hamming ECC
14  * capabilities, while NFCv2 also embeds a BCH ECC engine. Also, DMA
15  * is not used with NFCv2.
16  *
17  * The ECC layouts are depicted in details in Marvell AN-379, but here
18  * is a brief description.
19  *
20  * When using Hamming, the data is split in 512B chunks (either 1, 2
21  * or 4) and each chunk will have its own ECC "digest" of 6B at the
22  * beginning of the OOB area and eventually the remaining free OOB
23  * bytes (also called "spare" bytes in the driver). This engine
24  * corrects up to 1 bit per chunk and detects reliably an error if
25  * there are at most 2 bitflips. Here is the page layout used by the
26  * controller when Hamming is chosen:
27  *
28  * +-------------------------------------------------------------+
29  * | Data 1 | ... | Data N | ECC 1 | ... | ECCN | Free OOB bytes |
30  * +-------------------------------------------------------------+
31  *
32  * When using the BCH engine, there are N identical (data + free OOB +
33  * ECC) sections and potentially an extra one to deal with
34  * configurations where the chosen (data + free OOB + ECC) sizes do
35  * not align with the page (data + OOB) size. ECC bytes are always
36  * 30B per ECC chunk. Here is the page layout used by the controller
37  * when BCH is chosen:
38  *
39  * +-----------------------------------------
40  * | Data 1 | Free OOB bytes 1 | ECC 1 | ...
41  * +-----------------------------------------
42  *
43  *      -------------------------------------------
44  *       ... | Data N | Free OOB bytes N | ECC N |
45  *      -------------------------------------------
46  *
47  *           --------------------------------------------+
48  *            Last Data | Last Free OOB bytes | Last ECC |
49  *           --------------------------------------------+
50  *
51  * In both cases, the layout seen by the user is always: all data
52  * first, then all free OOB bytes and finally all ECC bytes. With BCH,
53  * ECC bytes are 30B long and are padded with 0xFF to align on 32
54  * bytes.
55  *
56  * The controller has certain limitations that are handled by the
57  * driver:
58  *   - It can only read 2k at a time. To overcome this limitation, the
59  *     driver issues data cycles on the bus, without issuing new
60  *     CMD + ADDR cycles. The Marvell term is "naked" operations.
61  *   - The ECC strength in BCH mode cannot be tuned. It is fixed 16
62  *     bits. What can be tuned is the ECC block size as long as it
63  *     stays between 512B and 2kiB. It's usually chosen based on the
64  *     chip ECC requirements. For instance, using 2kiB ECC chunks
65  *     provides 4b/512B correctability.
66  *   - The controller will always treat data bytes, free OOB bytes
67  *     and ECC bytes in that order, no matter what the real layout is
68  *     (which is usually all data then all OOB bytes). The
69  *     marvell_nfc_layouts array below contains the currently
70  *     supported layouts.
71  *   - Because of these weird layouts, the Bad Block Markers can be
72  *     located in data section. In this case, the NAND_BBT_NO_OOB_BBM
73  *     option must be set to prevent scanning/writing bad block
74  *     markers.
75  */
76 
77 #include <linux/module.h>
78 #include <linux/clk.h>
79 #include <linux/mtd/rawnand.h>
80 #include <linux/of.h>
81 #include <linux/iopoll.h>
82 #include <linux/interrupt.h>
83 #include <linux/platform_device.h>
84 #include <linux/slab.h>
85 #include <linux/mfd/syscon.h>
86 #include <linux/regmap.h>
87 #include <asm/unaligned.h>
88 
89 #include <linux/dmaengine.h>
90 #include <linux/dma-mapping.h>
91 #include <linux/dma/pxa-dma.h>
92 #include <linux/platform_data/mtd-nand-pxa3xx.h>
93 
94 /* Data FIFO granularity, FIFO reads/writes must be a multiple of this length */
95 #define FIFO_DEPTH		8
96 #define FIFO_REP(x)		(x / sizeof(u32))
97 #define BCH_SEQ_READS		(32 / FIFO_DEPTH)
98 /* NFC does not support transfers of larger chunks at a time */
99 #define MAX_CHUNK_SIZE		2112
100 /* NFCv1 cannot read more that 7 bytes of ID */
101 #define NFCV1_READID_LEN	7
102 /* Polling is done at a pace of POLL_PERIOD us until POLL_TIMEOUT is reached */
103 #define POLL_PERIOD		0
104 #define POLL_TIMEOUT		100000
105 /* Interrupt maximum wait period in ms */
106 #define IRQ_TIMEOUT		1000
107 /* Latency in clock cycles between SoC pins and NFC logic */
108 #define MIN_RD_DEL_CNT		3
109 /* Maximum number of contiguous address cycles */
110 #define MAX_ADDRESS_CYC_NFCV1	5
111 #define MAX_ADDRESS_CYC_NFCV2	7
112 /* System control registers/bits to enable the NAND controller on some SoCs */
113 #define GENCONF_SOC_DEVICE_MUX	0x208
114 #define GENCONF_SOC_DEVICE_MUX_NFC_EN BIT(0)
115 #define GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST BIT(20)
116 #define GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST BIT(21)
117 #define GENCONF_SOC_DEVICE_MUX_NFC_INT_EN BIT(25)
118 #define GENCONF_SOC_DEVICE_MUX_NFC_DEVBUS_ARB_EN BIT(27)
119 #define GENCONF_CLK_GATING_CTRL	0x220
120 #define GENCONF_CLK_GATING_CTRL_ND_GATE BIT(2)
121 #define GENCONF_ND_CLK_CTRL	0x700
122 #define GENCONF_ND_CLK_CTRL_EN	BIT(0)
123 
124 /* NAND controller data flash control register */
125 #define NDCR			0x00
126 #define NDCR_ALL_INT		GENMASK(11, 0)
127 #define NDCR_CS1_CMDDM		BIT(7)
128 #define NDCR_CS0_CMDDM		BIT(8)
129 #define NDCR_RDYM		BIT(11)
130 #define NDCR_ND_ARB_EN		BIT(12)
131 #define NDCR_RA_START		BIT(15)
132 #define NDCR_RD_ID_CNT(x)	(min_t(unsigned int, x, 0x7) << 16)
133 #define NDCR_PAGE_SZ(x)		(x >= 2048 ? BIT(24) : 0)
134 #define NDCR_DWIDTH_M		BIT(26)
135 #define NDCR_DWIDTH_C		BIT(27)
136 #define NDCR_ND_RUN		BIT(28)
137 #define NDCR_DMA_EN		BIT(29)
138 #define NDCR_ECC_EN		BIT(30)
139 #define NDCR_SPARE_EN		BIT(31)
140 #define NDCR_GENERIC_FIELDS_MASK (~(NDCR_RA_START | NDCR_PAGE_SZ(2048) | \
141 				    NDCR_DWIDTH_M | NDCR_DWIDTH_C))
142 
143 /* NAND interface timing parameter 0 register */
144 #define NDTR0			0x04
145 #define NDTR0_TRP(x)		((min_t(unsigned int, x, 0xF) & 0x7) << 0)
146 #define NDTR0_TRH(x)		(min_t(unsigned int, x, 0x7) << 3)
147 #define NDTR0_ETRP(x)		((min_t(unsigned int, x, 0xF) & 0x8) << 3)
148 #define NDTR0_SEL_NRE_EDGE	BIT(7)
149 #define NDTR0_TWP(x)		(min_t(unsigned int, x, 0x7) << 8)
150 #define NDTR0_TWH(x)		(min_t(unsigned int, x, 0x7) << 11)
151 #define NDTR0_TCS(x)		(min_t(unsigned int, x, 0x7) << 16)
152 #define NDTR0_TCH(x)		(min_t(unsigned int, x, 0x7) << 19)
153 #define NDTR0_RD_CNT_DEL(x)	(min_t(unsigned int, x, 0xF) << 22)
154 #define NDTR0_SELCNTR		BIT(26)
155 #define NDTR0_TADL(x)		(min_t(unsigned int, x, 0x1F) << 27)
156 
157 /* NAND interface timing parameter 1 register */
158 #define NDTR1			0x0C
159 #define NDTR1_TAR(x)		(min_t(unsigned int, x, 0xF) << 0)
160 #define NDTR1_TWHR(x)		(min_t(unsigned int, x, 0xF) << 4)
161 #define NDTR1_TRHW(x)		(min_t(unsigned int, x / 16, 0x3) << 8)
162 #define NDTR1_PRESCALE		BIT(14)
163 #define NDTR1_WAIT_MODE		BIT(15)
164 #define NDTR1_TR(x)		(min_t(unsigned int, x, 0xFFFF) << 16)
165 
166 /* NAND controller status register */
167 #define NDSR			0x14
168 #define NDSR_WRCMDREQ		BIT(0)
169 #define NDSR_RDDREQ		BIT(1)
170 #define NDSR_WRDREQ		BIT(2)
171 #define NDSR_CORERR		BIT(3)
172 #define NDSR_UNCERR		BIT(4)
173 #define NDSR_CMDD(cs)		BIT(8 - cs)
174 #define NDSR_RDY(rb)		BIT(11 + rb)
175 #define NDSR_ERRCNT(x)		((x >> 16) & 0x1F)
176 
177 /* NAND ECC control register */
178 #define NDECCCTRL		0x28
179 #define NDECCCTRL_BCH_EN	BIT(0)
180 
181 /* NAND controller data buffer register */
182 #define NDDB			0x40
183 
184 /* NAND controller command buffer 0 register */
185 #define NDCB0			0x48
186 #define NDCB0_CMD1(x)		((x & 0xFF) << 0)
187 #define NDCB0_CMD2(x)		((x & 0xFF) << 8)
188 #define NDCB0_ADDR_CYC(x)	((x & 0x7) << 16)
189 #define NDCB0_ADDR_GET_NUM_CYC(x) (((x) >> 16) & 0x7)
190 #define NDCB0_DBC		BIT(19)
191 #define NDCB0_CMD_TYPE(x)	((x & 0x7) << 21)
192 #define NDCB0_CSEL		BIT(24)
193 #define NDCB0_RDY_BYP		BIT(27)
194 #define NDCB0_LEN_OVRD		BIT(28)
195 #define NDCB0_CMD_XTYPE(x)	((x & 0x7) << 29)
196 
197 /* NAND controller command buffer 1 register */
198 #define NDCB1			0x4C
199 #define NDCB1_COLS(x)		((x & 0xFFFF) << 0)
200 #define NDCB1_ADDRS_PAGE(x)	(x << 16)
201 
202 /* NAND controller command buffer 2 register */
203 #define NDCB2			0x50
204 #define NDCB2_ADDR5_PAGE(x)	(((x >> 16) & 0xFF) << 0)
205 #define NDCB2_ADDR5_CYC(x)	((x & 0xFF) << 0)
206 
207 /* NAND controller command buffer 3 register */
208 #define NDCB3			0x54
209 #define NDCB3_ADDR6_CYC(x)	((x & 0xFF) << 16)
210 #define NDCB3_ADDR7_CYC(x)	((x & 0xFF) << 24)
211 
212 /* NAND controller command buffer 0 register 'type' and 'xtype' fields */
213 #define TYPE_READ		0
214 #define TYPE_WRITE		1
215 #define TYPE_ERASE		2
216 #define TYPE_READ_ID		3
217 #define TYPE_STATUS		4
218 #define TYPE_RESET		5
219 #define TYPE_NAKED_CMD		6
220 #define TYPE_NAKED_ADDR		7
221 #define TYPE_MASK		7
222 #define XTYPE_MONOLITHIC_RW	0
223 #define XTYPE_LAST_NAKED_RW	1
224 #define XTYPE_FINAL_COMMAND	3
225 #define XTYPE_READ		4
226 #define XTYPE_WRITE_DISPATCH	4
227 #define XTYPE_NAKED_RW		5
228 #define XTYPE_COMMAND_DISPATCH	6
229 #define XTYPE_MASK		7
230 
231 /**
232  * struct marvell_hw_ecc_layout - layout of Marvell ECC
233  *
234  * Marvell ECC engine works differently than the others, in order to limit the
235  * size of the IP, hardware engineers chose to set a fixed strength at 16 bits
236  * per subpage, and depending on a the desired strength needed by the NAND chip,
237  * a particular layout mixing data/spare/ecc is defined, with a possible last
238  * chunk smaller that the others.
239  *
240  * @writesize:		Full page size on which the layout applies
241  * @chunk:		Desired ECC chunk size on which the layout applies
242  * @strength:		Desired ECC strength (per chunk size bytes) on which the
243  *			layout applies
244  * @nchunks:		Total number of chunks
245  * @full_chunk_cnt:	Number of full-sized chunks, which is the number of
246  *			repetitions of the pattern:
247  *			(data_bytes + spare_bytes + ecc_bytes).
248  * @data_bytes:		Number of data bytes per chunk
249  * @spare_bytes:	Number of spare bytes per chunk
250  * @ecc_bytes:		Number of ecc bytes per chunk
251  * @last_data_bytes:	Number of data bytes in the last chunk
252  * @last_spare_bytes:	Number of spare bytes in the last chunk
253  * @last_ecc_bytes:	Number of ecc bytes in the last chunk
254  */
255 struct marvell_hw_ecc_layout {
256 	/* Constraints */
257 	int writesize;
258 	int chunk;
259 	int strength;
260 	/* Corresponding layout */
261 	int nchunks;
262 	int full_chunk_cnt;
263 	int data_bytes;
264 	int spare_bytes;
265 	int ecc_bytes;
266 	int last_data_bytes;
267 	int last_spare_bytes;
268 	int last_ecc_bytes;
269 };
270 
271 #define MARVELL_LAYOUT(ws, dc, ds, nc, fcc, db, sb, eb, ldb, lsb, leb)	\
272 	{								\
273 		.writesize = ws,					\
274 		.chunk = dc,						\
275 		.strength = ds,						\
276 		.nchunks = nc,						\
277 		.full_chunk_cnt = fcc,					\
278 		.data_bytes = db,					\
279 		.spare_bytes = sb,					\
280 		.ecc_bytes = eb,					\
281 		.last_data_bytes = ldb,					\
282 		.last_spare_bytes = lsb,				\
283 		.last_ecc_bytes = leb,					\
284 	}
285 
286 /* Layouts explained in AN-379_Marvell_SoC_NFC_ECC */
287 static const struct marvell_hw_ecc_layout marvell_nfc_layouts[] = {
288 	MARVELL_LAYOUT(  512,   512,  1,  1,  1,  512,  8,  8,  0,  0,  0),
289 	MARVELL_LAYOUT( 2048,   512,  1,  1,  1, 2048, 40, 24,  0,  0,  0),
290 	MARVELL_LAYOUT( 2048,   512,  4,  1,  1, 2048, 32, 30,  0,  0,  0),
291 	MARVELL_LAYOUT( 2048,   512,  8,  2,  1, 1024,  0, 30,1024,32, 30),
292 	MARVELL_LAYOUT( 2048,   512,  8,  2,  1, 1024,  0, 30,1024,64, 30),
293 	MARVELL_LAYOUT( 2048,   512,  16, 4,  4, 512,   0, 30,  0, 32, 30),
294 	MARVELL_LAYOUT( 4096,   512,  4,  2,  2, 2048, 32, 30,  0,  0,  0),
295 	MARVELL_LAYOUT( 4096,   512,  8,  4,  4, 1024,  0, 30,  0, 64, 30),
296 	MARVELL_LAYOUT( 4096,   512,  16, 8,  8, 512,   0, 30,  0, 32, 30),
297 	MARVELL_LAYOUT( 8192,   512,  4,  4,  4, 2048,  0, 30,  0,  0,  0),
298 	MARVELL_LAYOUT( 8192,   512,  8,  8,  8, 1024,  0, 30,  0, 160, 30),
299 	MARVELL_LAYOUT( 8192,   512,  16, 16, 16, 512,  0, 30,  0,  32, 30),
300 };
301 
302 /**
303  * struct marvell_nand_chip_sel - CS line description
304  *
305  * The Nand Flash Controller has up to 4 CE and 2 RB pins. The CE selection
306  * is made by a field in NDCB0 register, and in another field in NDCB2 register.
307  * The datasheet describes the logic with an error: ADDR5 field is once
308  * declared at the beginning of NDCB2, and another time at its end. Because the
309  * ADDR5 field of NDCB2 may be used by other bytes, it would be more logical
310  * to use the last bit of this field instead of the first ones.
311  *
312  * @cs:			Wanted CE lane.
313  * @ndcb0_csel:		Value of the NDCB0 register with or without the flag
314  *			selecting the wanted CE lane. This is set once when
315  *			the Device Tree is probed.
316  * @rb:			Ready/Busy pin for the flash chip
317  */
318 struct marvell_nand_chip_sel {
319 	unsigned int cs;
320 	u32 ndcb0_csel;
321 	unsigned int rb;
322 };
323 
324 /**
325  * struct marvell_nand_chip - stores NAND chip device related information
326  *
327  * @chip:		Base NAND chip structure
328  * @node:		Used to store NAND chips into a list
329  * @layout:		NAND layout when using hardware ECC
330  * @ndcr:		Controller register value for this NAND chip
331  * @ndtr0:		Timing registers 0 value for this NAND chip
332  * @ndtr1:		Timing registers 1 value for this NAND chip
333  * @addr_cyc:		Amount of cycles needed to pass column address
334  * @selected_die:	Current active CS
335  * @nsels:		Number of CS lines required by the NAND chip
336  * @sels:		Array of CS lines descriptions
337  */
338 struct marvell_nand_chip {
339 	struct nand_chip chip;
340 	struct list_head node;
341 	const struct marvell_hw_ecc_layout *layout;
342 	u32 ndcr;
343 	u32 ndtr0;
344 	u32 ndtr1;
345 	int addr_cyc;
346 	int selected_die;
347 	unsigned int nsels;
348 	struct marvell_nand_chip_sel sels[];
349 };
350 
351 static inline struct marvell_nand_chip *to_marvell_nand(struct nand_chip *chip)
352 {
353 	return container_of(chip, struct marvell_nand_chip, chip);
354 }
355 
356 static inline struct marvell_nand_chip_sel *to_nand_sel(struct marvell_nand_chip
357 							*nand)
358 {
359 	return &nand->sels[nand->selected_die];
360 }
361 
362 /**
363  * struct marvell_nfc_caps - NAND controller capabilities for distinction
364  *                           between compatible strings
365  *
366  * @max_cs_nb:		Number of Chip Select lines available
367  * @max_rb_nb:		Number of Ready/Busy lines available
368  * @need_system_controller: Indicates if the SoC needs to have access to the
369  *                      system controller (ie. to enable the NAND controller)
370  * @legacy_of_bindings:	Indicates if DT parsing must be done using the old
371  *			fashion way
372  * @is_nfcv2:		NFCv2 has numerous enhancements compared to NFCv1, ie.
373  *			BCH error detection and correction algorithm,
374  *			NDCB3 register has been added
375  * @use_dma:		Use dma for data transfers
376  * @max_mode_number:	Maximum timing mode supported by the controller
377  */
378 struct marvell_nfc_caps {
379 	unsigned int max_cs_nb;
380 	unsigned int max_rb_nb;
381 	bool need_system_controller;
382 	bool legacy_of_bindings;
383 	bool is_nfcv2;
384 	bool use_dma;
385 	unsigned int max_mode_number;
386 };
387 
388 /**
389  * struct marvell_nfc - stores Marvell NAND controller information
390  *
391  * @controller:		Base controller structure
392  * @dev:		Parent device (used to print error messages)
393  * @regs:		NAND controller registers
394  * @core_clk:		Core clock
395  * @reg_clk:		Registers clock
396  * @complete:		Completion object to wait for NAND controller events
397  * @assigned_cs:	Bitmask describing already assigned CS lines
398  * @chips:		List containing all the NAND chips attached to
399  *			this NAND controller
400  * @selected_chip:	Currently selected target chip
401  * @caps:		NAND controller capabilities for each compatible string
402  * @use_dma:		Whetner DMA is used
403  * @dma_chan:		DMA channel (NFCv1 only)
404  * @dma_buf:		32-bit aligned buffer for DMA transfers (NFCv1 only)
405  */
406 struct marvell_nfc {
407 	struct nand_controller controller;
408 	struct device *dev;
409 	void __iomem *regs;
410 	struct clk *core_clk;
411 	struct clk *reg_clk;
412 	struct completion complete;
413 	unsigned long assigned_cs;
414 	struct list_head chips;
415 	struct nand_chip *selected_chip;
416 	const struct marvell_nfc_caps *caps;
417 
418 	/* DMA (NFCv1 only) */
419 	bool use_dma;
420 	struct dma_chan *dma_chan;
421 	u8 *dma_buf;
422 };
423 
424 static inline struct marvell_nfc *to_marvell_nfc(struct nand_controller *ctrl)
425 {
426 	return container_of(ctrl, struct marvell_nfc, controller);
427 }
428 
429 /**
430  * struct marvell_nfc_timings - NAND controller timings expressed in NAND
431  *                              Controller clock cycles
432  *
433  * @tRP:		ND_nRE pulse width
434  * @tRH:		ND_nRE high duration
435  * @tWP:		ND_nWE pulse time
436  * @tWH:		ND_nWE high duration
437  * @tCS:		Enable signal setup time
438  * @tCH:		Enable signal hold time
439  * @tADL:		Address to write data delay
440  * @tAR:		ND_ALE low to ND_nRE low delay
441  * @tWHR:		ND_nWE high to ND_nRE low for status read
442  * @tRHW:		ND_nRE high duration, read to write delay
443  * @tR:			ND_nWE high to ND_nRE low for read
444  */
445 struct marvell_nfc_timings {
446 	/* NDTR0 fields */
447 	unsigned int tRP;
448 	unsigned int tRH;
449 	unsigned int tWP;
450 	unsigned int tWH;
451 	unsigned int tCS;
452 	unsigned int tCH;
453 	unsigned int tADL;
454 	/* NDTR1 fields */
455 	unsigned int tAR;
456 	unsigned int tWHR;
457 	unsigned int tRHW;
458 	unsigned int tR;
459 };
460 
461 /**
462  * TO_CYCLES() - Derives a duration in numbers of clock cycles.
463  *
464  * @ps: Duration in pico-seconds
465  * @period_ns:  Clock period in nano-seconds
466  *
467  * Convert the duration in nano-seconds, then divide by the period and
468  * return the number of clock periods.
469  */
470 #define TO_CYCLES(ps, period_ns) (DIV_ROUND_UP(ps / 1000, period_ns))
471 #define TO_CYCLES64(ps, period_ns) (DIV_ROUND_UP_ULL(div_u64(ps, 1000), \
472 						     period_ns))
473 
474 /**
475  * struct marvell_nfc_op - filled during the parsing of the ->exec_op()
476  *                         subop subset of instructions.
477  *
478  * @ndcb:		Array of values written to NDCBx registers
479  * @cle_ale_delay_ns:	Optional delay after the last CMD or ADDR cycle
480  * @rdy_timeout_ms:	Timeout for waits on Ready/Busy pin
481  * @rdy_delay_ns:	Optional delay after waiting for the RB pin
482  * @data_delay_ns:	Optional delay after the data xfer
483  * @data_instr_idx:	Index of the data instruction in the subop
484  * @data_instr:		Pointer to the data instruction in the subop
485  */
486 struct marvell_nfc_op {
487 	u32 ndcb[4];
488 	unsigned int cle_ale_delay_ns;
489 	unsigned int rdy_timeout_ms;
490 	unsigned int rdy_delay_ns;
491 	unsigned int data_delay_ns;
492 	unsigned int data_instr_idx;
493 	const struct nand_op_instr *data_instr;
494 };
495 
496 /*
497  * Internal helper to conditionnally apply a delay (from the above structure,
498  * most of the time).
499  */
500 static void cond_delay(unsigned int ns)
501 {
502 	if (!ns)
503 		return;
504 
505 	if (ns < 10000)
506 		ndelay(ns);
507 	else
508 		udelay(DIV_ROUND_UP(ns, 1000));
509 }
510 
511 /*
512  * The controller has many flags that could generate interrupts, most of them
513  * are disabled and polling is used. For the very slow signals, using interrupts
514  * may relax the CPU charge.
515  */
516 static void marvell_nfc_disable_int(struct marvell_nfc *nfc, u32 int_mask)
517 {
518 	u32 reg;
519 
520 	/* Writing 1 disables the interrupt */
521 	reg = readl_relaxed(nfc->regs + NDCR);
522 	writel_relaxed(reg | int_mask, nfc->regs + NDCR);
523 }
524 
525 static void marvell_nfc_enable_int(struct marvell_nfc *nfc, u32 int_mask)
526 {
527 	u32 reg;
528 
529 	/* Writing 0 enables the interrupt */
530 	reg = readl_relaxed(nfc->regs + NDCR);
531 	writel_relaxed(reg & ~int_mask, nfc->regs + NDCR);
532 }
533 
534 static u32 marvell_nfc_clear_int(struct marvell_nfc *nfc, u32 int_mask)
535 {
536 	u32 reg;
537 
538 	reg = readl_relaxed(nfc->regs + NDSR);
539 	writel_relaxed(int_mask, nfc->regs + NDSR);
540 
541 	return reg & int_mask;
542 }
543 
544 static void marvell_nfc_force_byte_access(struct nand_chip *chip,
545 					  bool force_8bit)
546 {
547 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
548 	u32 ndcr;
549 
550 	/*
551 	 * Callers of this function do not verify if the NAND is using a 16-bit
552 	 * an 8-bit bus for normal operations, so we need to take care of that
553 	 * here by leaving the configuration unchanged if the NAND does not have
554 	 * the NAND_BUSWIDTH_16 flag set.
555 	 */
556 	if (!(chip->options & NAND_BUSWIDTH_16))
557 		return;
558 
559 	ndcr = readl_relaxed(nfc->regs + NDCR);
560 
561 	if (force_8bit)
562 		ndcr &= ~(NDCR_DWIDTH_M | NDCR_DWIDTH_C);
563 	else
564 		ndcr |= NDCR_DWIDTH_M | NDCR_DWIDTH_C;
565 
566 	writel_relaxed(ndcr, nfc->regs + NDCR);
567 }
568 
569 static int marvell_nfc_wait_ndrun(struct nand_chip *chip)
570 {
571 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
572 	u32 val;
573 	int ret;
574 
575 	/*
576 	 * The command is being processed, wait for the ND_RUN bit to be
577 	 * cleared by the NFC. If not, we must clear it by hand.
578 	 */
579 	ret = readl_relaxed_poll_timeout(nfc->regs + NDCR, val,
580 					 (val & NDCR_ND_RUN) == 0,
581 					 POLL_PERIOD, POLL_TIMEOUT);
582 	if (ret) {
583 		dev_err(nfc->dev, "Timeout on NAND controller run mode\n");
584 		writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
585 			       nfc->regs + NDCR);
586 		return ret;
587 	}
588 
589 	return 0;
590 }
591 
592 /*
593  * Any time a command has to be sent to the controller, the following sequence
594  * has to be followed:
595  * - call marvell_nfc_prepare_cmd()
596  *      -> activate the ND_RUN bit that will kind of 'start a job'
597  *      -> wait the signal indicating the NFC is waiting for a command
598  * - send the command (cmd and address cycles)
599  * - enventually send or receive the data
600  * - call marvell_nfc_end_cmd() with the corresponding flag
601  *      -> wait the flag to be triggered or cancel the job with a timeout
602  *
603  * The following helpers are here to factorize the code a bit so that
604  * specialized functions responsible for executing the actual NAND
605  * operations do not have to replicate the same code blocks.
606  */
607 static int marvell_nfc_prepare_cmd(struct nand_chip *chip)
608 {
609 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
610 	u32 ndcr, val;
611 	int ret;
612 
613 	/* Poll ND_RUN and clear NDSR before issuing any command */
614 	ret = marvell_nfc_wait_ndrun(chip);
615 	if (ret) {
616 		dev_err(nfc->dev, "Last operation did not succeed\n");
617 		return ret;
618 	}
619 
620 	ndcr = readl_relaxed(nfc->regs + NDCR);
621 	writel_relaxed(readl(nfc->regs + NDSR), nfc->regs + NDSR);
622 
623 	/* Assert ND_RUN bit and wait the NFC to be ready */
624 	writel_relaxed(ndcr | NDCR_ND_RUN, nfc->regs + NDCR);
625 	ret = readl_relaxed_poll_timeout(nfc->regs + NDSR, val,
626 					 val & NDSR_WRCMDREQ,
627 					 POLL_PERIOD, POLL_TIMEOUT);
628 	if (ret) {
629 		dev_err(nfc->dev, "Timeout on WRCMDRE\n");
630 		return -ETIMEDOUT;
631 	}
632 
633 	/* Command may be written, clear WRCMDREQ status bit */
634 	writel_relaxed(NDSR_WRCMDREQ, nfc->regs + NDSR);
635 
636 	return 0;
637 }
638 
639 static void marvell_nfc_send_cmd(struct nand_chip *chip,
640 				 struct marvell_nfc_op *nfc_op)
641 {
642 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
643 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
644 
645 	dev_dbg(nfc->dev, "\nNDCR:  0x%08x\n"
646 		"NDCB0: 0x%08x\nNDCB1: 0x%08x\nNDCB2: 0x%08x\nNDCB3: 0x%08x\n",
647 		(u32)readl_relaxed(nfc->regs + NDCR), nfc_op->ndcb[0],
648 		nfc_op->ndcb[1], nfc_op->ndcb[2], nfc_op->ndcb[3]);
649 
650 	writel_relaxed(to_nand_sel(marvell_nand)->ndcb0_csel | nfc_op->ndcb[0],
651 		       nfc->regs + NDCB0);
652 	writel_relaxed(nfc_op->ndcb[1], nfc->regs + NDCB0);
653 	writel(nfc_op->ndcb[2], nfc->regs + NDCB0);
654 
655 	/*
656 	 * Write NDCB0 four times only if LEN_OVRD is set or if ADDR6 or ADDR7
657 	 * fields are used (only available on NFCv2).
658 	 */
659 	if (nfc_op->ndcb[0] & NDCB0_LEN_OVRD ||
660 	    NDCB0_ADDR_GET_NUM_CYC(nfc_op->ndcb[0]) >= 6) {
661 		if (!WARN_ON_ONCE(!nfc->caps->is_nfcv2))
662 			writel(nfc_op->ndcb[3], nfc->regs + NDCB0);
663 	}
664 }
665 
666 static int marvell_nfc_end_cmd(struct nand_chip *chip, int flag,
667 			       const char *label)
668 {
669 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
670 	u32 val;
671 	int ret;
672 
673 	ret = readl_relaxed_poll_timeout(nfc->regs + NDSR, val,
674 					 val & flag,
675 					 POLL_PERIOD, POLL_TIMEOUT);
676 
677 	if (ret) {
678 		dev_err(nfc->dev, "Timeout on %s (NDSR: 0x%08x)\n",
679 			label, val);
680 		if (nfc->dma_chan)
681 			dmaengine_terminate_all(nfc->dma_chan);
682 		return ret;
683 	}
684 
685 	/*
686 	 * DMA function uses this helper to poll on CMDD bits without wanting
687 	 * them to be cleared.
688 	 */
689 	if (nfc->use_dma && (readl_relaxed(nfc->regs + NDCR) & NDCR_DMA_EN))
690 		return 0;
691 
692 	writel_relaxed(flag, nfc->regs + NDSR);
693 
694 	return 0;
695 }
696 
697 static int marvell_nfc_wait_cmdd(struct nand_chip *chip)
698 {
699 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
700 	int cs_flag = NDSR_CMDD(to_nand_sel(marvell_nand)->ndcb0_csel);
701 
702 	return marvell_nfc_end_cmd(chip, cs_flag, "CMDD");
703 }
704 
705 static int marvell_nfc_poll_status(struct marvell_nfc *nfc, u32 mask,
706 				   u32 expected_val, unsigned long timeout_ms)
707 {
708 	unsigned long limit;
709 	u32 st;
710 
711 	limit = jiffies + msecs_to_jiffies(timeout_ms);
712 	do {
713 		st = readl_relaxed(nfc->regs + NDSR);
714 		if (st & NDSR_RDY(1))
715 			st |= NDSR_RDY(0);
716 
717 		if ((st & mask) == expected_val)
718 			return 0;
719 
720 		cpu_relax();
721 	} while (time_after(limit, jiffies));
722 
723 	return -ETIMEDOUT;
724 }
725 
726 static int marvell_nfc_wait_op(struct nand_chip *chip, unsigned int timeout_ms)
727 {
728 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
729 	struct mtd_info *mtd = nand_to_mtd(chip);
730 	u32 pending;
731 	int ret;
732 
733 	/* Timeout is expressed in ms */
734 	if (!timeout_ms)
735 		timeout_ms = IRQ_TIMEOUT;
736 
737 	if (mtd->oops_panic_write) {
738 		ret = marvell_nfc_poll_status(nfc, NDSR_RDY(0),
739 					      NDSR_RDY(0),
740 					      timeout_ms);
741 	} else {
742 		init_completion(&nfc->complete);
743 
744 		marvell_nfc_enable_int(nfc, NDCR_RDYM);
745 		ret = wait_for_completion_timeout(&nfc->complete,
746 						  msecs_to_jiffies(timeout_ms));
747 		marvell_nfc_disable_int(nfc, NDCR_RDYM);
748 	}
749 	pending = marvell_nfc_clear_int(nfc, NDSR_RDY(0) | NDSR_RDY(1));
750 
751 	/*
752 	 * In case the interrupt was not served in the required time frame,
753 	 * check if the ISR was not served or if something went actually wrong.
754 	 */
755 	if (!ret && !pending) {
756 		dev_err(nfc->dev, "Timeout waiting for RB signal\n");
757 		return -ETIMEDOUT;
758 	}
759 
760 	return 0;
761 }
762 
763 static void marvell_nfc_select_target(struct nand_chip *chip,
764 				      unsigned int die_nr)
765 {
766 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
767 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
768 	u32 ndcr_generic;
769 
770 	/*
771 	 * Reset the NDCR register to a clean state for this particular chip,
772 	 * also clear ND_RUN bit.
773 	 */
774 	ndcr_generic = readl_relaxed(nfc->regs + NDCR) &
775 		       NDCR_GENERIC_FIELDS_MASK & ~NDCR_ND_RUN;
776 	writel_relaxed(ndcr_generic | marvell_nand->ndcr, nfc->regs + NDCR);
777 
778 	/* Also reset the interrupt status register */
779 	marvell_nfc_clear_int(nfc, NDCR_ALL_INT);
780 
781 	if (chip == nfc->selected_chip && die_nr == marvell_nand->selected_die)
782 		return;
783 
784 	writel_relaxed(marvell_nand->ndtr0, nfc->regs + NDTR0);
785 	writel_relaxed(marvell_nand->ndtr1, nfc->regs + NDTR1);
786 
787 	nfc->selected_chip = chip;
788 	marvell_nand->selected_die = die_nr;
789 }
790 
791 static irqreturn_t marvell_nfc_isr(int irq, void *dev_id)
792 {
793 	struct marvell_nfc *nfc = dev_id;
794 	u32 st = readl_relaxed(nfc->regs + NDSR);
795 	u32 ien = (~readl_relaxed(nfc->regs + NDCR)) & NDCR_ALL_INT;
796 
797 	/*
798 	 * RDY interrupt mask is one bit in NDCR while there are two status
799 	 * bit in NDSR (RDY[cs0/cs2] and RDY[cs1/cs3]).
800 	 */
801 	if (st & NDSR_RDY(1))
802 		st |= NDSR_RDY(0);
803 
804 	if (!(st & ien))
805 		return IRQ_NONE;
806 
807 	marvell_nfc_disable_int(nfc, st & NDCR_ALL_INT);
808 
809 	if (st & (NDSR_RDY(0) | NDSR_RDY(1)))
810 		complete(&nfc->complete);
811 
812 	return IRQ_HANDLED;
813 }
814 
815 /* HW ECC related functions */
816 static void marvell_nfc_enable_hw_ecc(struct nand_chip *chip)
817 {
818 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
819 	u32 ndcr = readl_relaxed(nfc->regs + NDCR);
820 
821 	if (!(ndcr & NDCR_ECC_EN)) {
822 		writel_relaxed(ndcr | NDCR_ECC_EN, nfc->regs + NDCR);
823 
824 		/*
825 		 * When enabling BCH, set threshold to 0 to always know the
826 		 * number of corrected bitflips.
827 		 */
828 		if (chip->ecc.algo == NAND_ECC_ALGO_BCH)
829 			writel_relaxed(NDECCCTRL_BCH_EN, nfc->regs + NDECCCTRL);
830 	}
831 }
832 
833 static void marvell_nfc_disable_hw_ecc(struct nand_chip *chip)
834 {
835 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
836 	u32 ndcr = readl_relaxed(nfc->regs + NDCR);
837 
838 	if (ndcr & NDCR_ECC_EN) {
839 		writel_relaxed(ndcr & ~NDCR_ECC_EN, nfc->regs + NDCR);
840 		if (chip->ecc.algo == NAND_ECC_ALGO_BCH)
841 			writel_relaxed(0, nfc->regs + NDECCCTRL);
842 	}
843 }
844 
845 /* DMA related helpers */
846 static void marvell_nfc_enable_dma(struct marvell_nfc *nfc)
847 {
848 	u32 reg;
849 
850 	reg = readl_relaxed(nfc->regs + NDCR);
851 	writel_relaxed(reg | NDCR_DMA_EN, nfc->regs + NDCR);
852 }
853 
854 static void marvell_nfc_disable_dma(struct marvell_nfc *nfc)
855 {
856 	u32 reg;
857 
858 	reg = readl_relaxed(nfc->regs + NDCR);
859 	writel_relaxed(reg & ~NDCR_DMA_EN, nfc->regs + NDCR);
860 }
861 
862 /* Read/write PIO/DMA accessors */
863 static int marvell_nfc_xfer_data_dma(struct marvell_nfc *nfc,
864 				     enum dma_data_direction direction,
865 				     unsigned int len)
866 {
867 	unsigned int dma_len = min_t(int, ALIGN(len, 32), MAX_CHUNK_SIZE);
868 	struct dma_async_tx_descriptor *tx;
869 	struct scatterlist sg;
870 	dma_cookie_t cookie;
871 	int ret;
872 
873 	marvell_nfc_enable_dma(nfc);
874 	/* Prepare the DMA transfer */
875 	sg_init_one(&sg, nfc->dma_buf, dma_len);
876 	ret = dma_map_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
877 	if (!ret) {
878 		dev_err(nfc->dev, "Could not map DMA S/G list\n");
879 		return -ENXIO;
880 	}
881 
882 	tx = dmaengine_prep_slave_sg(nfc->dma_chan, &sg, 1,
883 				     direction == DMA_FROM_DEVICE ?
884 				     DMA_DEV_TO_MEM : DMA_MEM_TO_DEV,
885 				     DMA_PREP_INTERRUPT);
886 	if (!tx) {
887 		dev_err(nfc->dev, "Could not prepare DMA S/G list\n");
888 		dma_unmap_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
889 		return -ENXIO;
890 	}
891 
892 	/* Do the task and wait for it to finish */
893 	cookie = dmaengine_submit(tx);
894 	ret = dma_submit_error(cookie);
895 	if (ret)
896 		return -EIO;
897 
898 	dma_async_issue_pending(nfc->dma_chan);
899 	ret = marvell_nfc_wait_cmdd(nfc->selected_chip);
900 	dma_unmap_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
901 	marvell_nfc_disable_dma(nfc);
902 	if (ret) {
903 		dev_err(nfc->dev, "Timeout waiting for DMA (status: %d)\n",
904 			dmaengine_tx_status(nfc->dma_chan, cookie, NULL));
905 		dmaengine_terminate_all(nfc->dma_chan);
906 		return -ETIMEDOUT;
907 	}
908 
909 	return 0;
910 }
911 
912 static int marvell_nfc_xfer_data_in_pio(struct marvell_nfc *nfc, u8 *in,
913 					unsigned int len)
914 {
915 	unsigned int last_len = len % FIFO_DEPTH;
916 	unsigned int last_full_offset = round_down(len, FIFO_DEPTH);
917 	int i;
918 
919 	for (i = 0; i < last_full_offset; i += FIFO_DEPTH)
920 		ioread32_rep(nfc->regs + NDDB, in + i, FIFO_REP(FIFO_DEPTH));
921 
922 	if (last_len) {
923 		u8 tmp_buf[FIFO_DEPTH];
924 
925 		ioread32_rep(nfc->regs + NDDB, tmp_buf, FIFO_REP(FIFO_DEPTH));
926 		memcpy(in + last_full_offset, tmp_buf, last_len);
927 	}
928 
929 	return 0;
930 }
931 
932 static int marvell_nfc_xfer_data_out_pio(struct marvell_nfc *nfc, const u8 *out,
933 					 unsigned int len)
934 {
935 	unsigned int last_len = len % FIFO_DEPTH;
936 	unsigned int last_full_offset = round_down(len, FIFO_DEPTH);
937 	int i;
938 
939 	for (i = 0; i < last_full_offset; i += FIFO_DEPTH)
940 		iowrite32_rep(nfc->regs + NDDB, out + i, FIFO_REP(FIFO_DEPTH));
941 
942 	if (last_len) {
943 		u8 tmp_buf[FIFO_DEPTH];
944 
945 		memcpy(tmp_buf, out + last_full_offset, last_len);
946 		iowrite32_rep(nfc->regs + NDDB, tmp_buf, FIFO_REP(FIFO_DEPTH));
947 	}
948 
949 	return 0;
950 }
951 
952 static void marvell_nfc_check_empty_chunk(struct nand_chip *chip,
953 					  u8 *data, int data_len,
954 					  u8 *spare, int spare_len,
955 					  u8 *ecc, int ecc_len,
956 					  unsigned int *max_bitflips)
957 {
958 	struct mtd_info *mtd = nand_to_mtd(chip);
959 	int bf;
960 
961 	/*
962 	 * Blank pages (all 0xFF) that have not been written may be recognized
963 	 * as bad if bitflips occur, so whenever an uncorrectable error occurs,
964 	 * check if the entire page (with ECC bytes) is actually blank or not.
965 	 */
966 	if (!data)
967 		data_len = 0;
968 	if (!spare)
969 		spare_len = 0;
970 	if (!ecc)
971 		ecc_len = 0;
972 
973 	bf = nand_check_erased_ecc_chunk(data, data_len, ecc, ecc_len,
974 					 spare, spare_len, chip->ecc.strength);
975 	if (bf < 0) {
976 		mtd->ecc_stats.failed++;
977 		return;
978 	}
979 
980 	/* Update the stats and max_bitflips */
981 	mtd->ecc_stats.corrected += bf;
982 	*max_bitflips = max_t(unsigned int, *max_bitflips, bf);
983 }
984 
985 /*
986  * Check if a chunk is correct or not according to the hardware ECC engine.
987  * mtd->ecc_stats.corrected is updated, as well as max_bitflips, however
988  * mtd->ecc_stats.failure is not, the function will instead return a non-zero
989  * value indicating that a check on the emptyness of the subpage must be
990  * performed before actually declaring the subpage as "corrupted".
991  */
992 static int marvell_nfc_hw_ecc_check_bitflips(struct nand_chip *chip,
993 					     unsigned int *max_bitflips)
994 {
995 	struct mtd_info *mtd = nand_to_mtd(chip);
996 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
997 	int bf = 0;
998 	u32 ndsr;
999 
1000 	ndsr = readl_relaxed(nfc->regs + NDSR);
1001 
1002 	/* Check uncorrectable error flag */
1003 	if (ndsr & NDSR_UNCERR) {
1004 		writel_relaxed(ndsr, nfc->regs + NDSR);
1005 
1006 		/*
1007 		 * Do not increment ->ecc_stats.failed now, instead, return a
1008 		 * non-zero value to indicate that this chunk was apparently
1009 		 * bad, and it should be check to see if it empty or not. If
1010 		 * the chunk (with ECC bytes) is not declared empty, the calling
1011 		 * function must increment the failure count.
1012 		 */
1013 		return -EBADMSG;
1014 	}
1015 
1016 	/* Check correctable error flag */
1017 	if (ndsr & NDSR_CORERR) {
1018 		writel_relaxed(ndsr, nfc->regs + NDSR);
1019 
1020 		if (chip->ecc.algo == NAND_ECC_ALGO_BCH)
1021 			bf = NDSR_ERRCNT(ndsr);
1022 		else
1023 			bf = 1;
1024 	}
1025 
1026 	/* Update the stats and max_bitflips */
1027 	mtd->ecc_stats.corrected += bf;
1028 	*max_bitflips = max_t(unsigned int, *max_bitflips, bf);
1029 
1030 	return 0;
1031 }
1032 
1033 /* Hamming read helpers */
1034 static int marvell_nfc_hw_ecc_hmg_do_read_page(struct nand_chip *chip,
1035 					       u8 *data_buf, u8 *oob_buf,
1036 					       bool raw, int page)
1037 {
1038 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1039 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1040 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1041 	struct marvell_nfc_op nfc_op = {
1042 		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_READ) |
1043 			   NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1044 			   NDCB0_DBC |
1045 			   NDCB0_CMD1(NAND_CMD_READ0) |
1046 			   NDCB0_CMD2(NAND_CMD_READSTART),
1047 		.ndcb[1] = NDCB1_ADDRS_PAGE(page),
1048 		.ndcb[2] = NDCB2_ADDR5_PAGE(page),
1049 	};
1050 	unsigned int oob_bytes = lt->spare_bytes + (raw ? lt->ecc_bytes : 0);
1051 	int ret;
1052 
1053 	/* NFCv2 needs more information about the operation being executed */
1054 	if (nfc->caps->is_nfcv2)
1055 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1056 
1057 	ret = marvell_nfc_prepare_cmd(chip);
1058 	if (ret)
1059 		return ret;
1060 
1061 	marvell_nfc_send_cmd(chip, &nfc_op);
1062 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1063 				  "RDDREQ while draining FIFO (data/oob)");
1064 	if (ret)
1065 		return ret;
1066 
1067 	/*
1068 	 * Read the page then the OOB area. Unlike what is shown in current
1069 	 * documentation, spare bytes are protected by the ECC engine, and must
1070 	 * be at the beginning of the OOB area or running this driver on legacy
1071 	 * systems will prevent the discovery of the BBM/BBT.
1072 	 */
1073 	if (nfc->use_dma) {
1074 		marvell_nfc_xfer_data_dma(nfc, DMA_FROM_DEVICE,
1075 					  lt->data_bytes + oob_bytes);
1076 		memcpy(data_buf, nfc->dma_buf, lt->data_bytes);
1077 		memcpy(oob_buf, nfc->dma_buf + lt->data_bytes, oob_bytes);
1078 	} else {
1079 		marvell_nfc_xfer_data_in_pio(nfc, data_buf, lt->data_bytes);
1080 		marvell_nfc_xfer_data_in_pio(nfc, oob_buf, oob_bytes);
1081 	}
1082 
1083 	ret = marvell_nfc_wait_cmdd(chip);
1084 	return ret;
1085 }
1086 
1087 static int marvell_nfc_hw_ecc_hmg_read_page_raw(struct nand_chip *chip, u8 *buf,
1088 						int oob_required, int page)
1089 {
1090 	marvell_nfc_select_target(chip, chip->cur_cs);
1091 	return marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi,
1092 						   true, page);
1093 }
1094 
1095 static int marvell_nfc_hw_ecc_hmg_read_page(struct nand_chip *chip, u8 *buf,
1096 					    int oob_required, int page)
1097 {
1098 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1099 	unsigned int full_sz = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1100 	int max_bitflips = 0, ret;
1101 	u8 *raw_buf;
1102 
1103 	marvell_nfc_select_target(chip, chip->cur_cs);
1104 	marvell_nfc_enable_hw_ecc(chip);
1105 	marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi, false,
1106 					    page);
1107 	ret = marvell_nfc_hw_ecc_check_bitflips(chip, &max_bitflips);
1108 	marvell_nfc_disable_hw_ecc(chip);
1109 
1110 	if (!ret)
1111 		return max_bitflips;
1112 
1113 	/*
1114 	 * When ECC failures are detected, check if the full page has been
1115 	 * written or not. Ignore the failure if it is actually empty.
1116 	 */
1117 	raw_buf = kmalloc(full_sz, GFP_KERNEL);
1118 	if (!raw_buf)
1119 		return -ENOMEM;
1120 
1121 	marvell_nfc_hw_ecc_hmg_do_read_page(chip, raw_buf, raw_buf +
1122 					    lt->data_bytes, true, page);
1123 	marvell_nfc_check_empty_chunk(chip, raw_buf, full_sz, NULL, 0, NULL, 0,
1124 				      &max_bitflips);
1125 	kfree(raw_buf);
1126 
1127 	return max_bitflips;
1128 }
1129 
1130 /*
1131  * Spare area in Hamming layouts is not protected by the ECC engine (even if
1132  * it appears before the ECC bytes when reading), the ->read_oob_raw() function
1133  * also stands for ->read_oob().
1134  */
1135 static int marvell_nfc_hw_ecc_hmg_read_oob_raw(struct nand_chip *chip, int page)
1136 {
1137 	u8 *buf = nand_get_data_buf(chip);
1138 
1139 	marvell_nfc_select_target(chip, chip->cur_cs);
1140 	return marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi,
1141 						   true, page);
1142 }
1143 
1144 /* Hamming write helpers */
1145 static int marvell_nfc_hw_ecc_hmg_do_write_page(struct nand_chip *chip,
1146 						const u8 *data_buf,
1147 						const u8 *oob_buf, bool raw,
1148 						int page)
1149 {
1150 	const struct nand_sdr_timings *sdr =
1151 		nand_get_sdr_timings(nand_get_interface_config(chip));
1152 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1153 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1154 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1155 	struct marvell_nfc_op nfc_op = {
1156 		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_WRITE) |
1157 			   NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1158 			   NDCB0_CMD1(NAND_CMD_SEQIN) |
1159 			   NDCB0_CMD2(NAND_CMD_PAGEPROG) |
1160 			   NDCB0_DBC,
1161 		.ndcb[1] = NDCB1_ADDRS_PAGE(page),
1162 		.ndcb[2] = NDCB2_ADDR5_PAGE(page),
1163 	};
1164 	unsigned int oob_bytes = lt->spare_bytes + (raw ? lt->ecc_bytes : 0);
1165 	u8 status;
1166 	int ret;
1167 
1168 	/* NFCv2 needs more information about the operation being executed */
1169 	if (nfc->caps->is_nfcv2)
1170 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1171 
1172 	ret = marvell_nfc_prepare_cmd(chip);
1173 	if (ret)
1174 		return ret;
1175 
1176 	marvell_nfc_send_cmd(chip, &nfc_op);
1177 	ret = marvell_nfc_end_cmd(chip, NDSR_WRDREQ,
1178 				  "WRDREQ while loading FIFO (data)");
1179 	if (ret)
1180 		return ret;
1181 
1182 	/* Write the page then the OOB area */
1183 	if (nfc->use_dma) {
1184 		memcpy(nfc->dma_buf, data_buf, lt->data_bytes);
1185 		memcpy(nfc->dma_buf + lt->data_bytes, oob_buf, oob_bytes);
1186 		marvell_nfc_xfer_data_dma(nfc, DMA_TO_DEVICE, lt->data_bytes +
1187 					  lt->ecc_bytes + lt->spare_bytes);
1188 	} else {
1189 		marvell_nfc_xfer_data_out_pio(nfc, data_buf, lt->data_bytes);
1190 		marvell_nfc_xfer_data_out_pio(nfc, oob_buf, oob_bytes);
1191 	}
1192 
1193 	ret = marvell_nfc_wait_cmdd(chip);
1194 	if (ret)
1195 		return ret;
1196 
1197 	ret = marvell_nfc_wait_op(chip,
1198 				  PSEC_TO_MSEC(sdr->tPROG_max));
1199 	if (ret)
1200 		return ret;
1201 
1202 	/* Check write status on the chip side */
1203 	ret = nand_status_op(chip, &status);
1204 	if (ret)
1205 		return ret;
1206 
1207 	if (status & NAND_STATUS_FAIL)
1208 		return -EIO;
1209 
1210 	return 0;
1211 }
1212 
1213 static int marvell_nfc_hw_ecc_hmg_write_page_raw(struct nand_chip *chip,
1214 						 const u8 *buf,
1215 						 int oob_required, int page)
1216 {
1217 	marvell_nfc_select_target(chip, chip->cur_cs);
1218 	return marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1219 						    true, page);
1220 }
1221 
1222 static int marvell_nfc_hw_ecc_hmg_write_page(struct nand_chip *chip,
1223 					     const u8 *buf,
1224 					     int oob_required, int page)
1225 {
1226 	int ret;
1227 
1228 	marvell_nfc_select_target(chip, chip->cur_cs);
1229 	marvell_nfc_enable_hw_ecc(chip);
1230 	ret = marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1231 						   false, page);
1232 	marvell_nfc_disable_hw_ecc(chip);
1233 
1234 	return ret;
1235 }
1236 
1237 /*
1238  * Spare area in Hamming layouts is not protected by the ECC engine (even if
1239  * it appears before the ECC bytes when reading), the ->write_oob_raw() function
1240  * also stands for ->write_oob().
1241  */
1242 static int marvell_nfc_hw_ecc_hmg_write_oob_raw(struct nand_chip *chip,
1243 						int page)
1244 {
1245 	struct mtd_info *mtd = nand_to_mtd(chip);
1246 	u8 *buf = nand_get_data_buf(chip);
1247 
1248 	memset(buf, 0xFF, mtd->writesize);
1249 
1250 	marvell_nfc_select_target(chip, chip->cur_cs);
1251 	return marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1252 						    true, page);
1253 }
1254 
1255 /* BCH read helpers */
1256 static int marvell_nfc_hw_ecc_bch_read_page_raw(struct nand_chip *chip, u8 *buf,
1257 						int oob_required, int page)
1258 {
1259 	struct mtd_info *mtd = nand_to_mtd(chip);
1260 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1261 	u8 *oob = chip->oob_poi;
1262 	int chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1263 	int ecc_offset = (lt->full_chunk_cnt * lt->spare_bytes) +
1264 		lt->last_spare_bytes;
1265 	int data_len = lt->data_bytes;
1266 	int spare_len = lt->spare_bytes;
1267 	int ecc_len = lt->ecc_bytes;
1268 	int chunk;
1269 
1270 	marvell_nfc_select_target(chip, chip->cur_cs);
1271 
1272 	if (oob_required)
1273 		memset(chip->oob_poi, 0xFF, mtd->oobsize);
1274 
1275 	nand_read_page_op(chip, page, 0, NULL, 0);
1276 
1277 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1278 		/* Update last chunk length */
1279 		if (chunk >= lt->full_chunk_cnt) {
1280 			data_len = lt->last_data_bytes;
1281 			spare_len = lt->last_spare_bytes;
1282 			ecc_len = lt->last_ecc_bytes;
1283 		}
1284 
1285 		/* Read data bytes*/
1286 		nand_change_read_column_op(chip, chunk * chunk_size,
1287 					   buf + (lt->data_bytes * chunk),
1288 					   data_len, false);
1289 
1290 		/* Read spare bytes */
1291 		nand_read_data_op(chip, oob + (lt->spare_bytes * chunk),
1292 				  spare_len, false, false);
1293 
1294 		/* Read ECC bytes */
1295 		nand_read_data_op(chip, oob + ecc_offset +
1296 				  (ALIGN(lt->ecc_bytes, 32) * chunk),
1297 				  ecc_len, false, false);
1298 	}
1299 
1300 	return 0;
1301 }
1302 
1303 static void marvell_nfc_hw_ecc_bch_read_chunk(struct nand_chip *chip, int chunk,
1304 					      u8 *data, unsigned int data_len,
1305 					      u8 *spare, unsigned int spare_len,
1306 					      int page)
1307 {
1308 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1309 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1310 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1311 	int i, ret;
1312 	struct marvell_nfc_op nfc_op = {
1313 		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_READ) |
1314 			   NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1315 			   NDCB0_LEN_OVRD,
1316 		.ndcb[1] = NDCB1_ADDRS_PAGE(page),
1317 		.ndcb[2] = NDCB2_ADDR5_PAGE(page),
1318 		.ndcb[3] = data_len + spare_len,
1319 	};
1320 
1321 	ret = marvell_nfc_prepare_cmd(chip);
1322 	if (ret)
1323 		return;
1324 
1325 	if (chunk == 0)
1326 		nfc_op.ndcb[0] |= NDCB0_DBC |
1327 				  NDCB0_CMD1(NAND_CMD_READ0) |
1328 				  NDCB0_CMD2(NAND_CMD_READSTART);
1329 
1330 	/*
1331 	 * Trigger the monolithic read on the first chunk, then naked read on
1332 	 * intermediate chunks and finally a last naked read on the last chunk.
1333 	 */
1334 	if (chunk == 0)
1335 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1336 	else if (chunk < lt->nchunks - 1)
1337 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_NAKED_RW);
1338 	else
1339 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1340 
1341 	marvell_nfc_send_cmd(chip, &nfc_op);
1342 
1343 	/*
1344 	 * According to the datasheet, when reading from NDDB
1345 	 * with BCH enabled, after each 32 bytes reads, we
1346 	 * have to make sure that the NDSR.RDDREQ bit is set.
1347 	 *
1348 	 * Drain the FIFO, 8 32-bit reads at a time, and skip
1349 	 * the polling on the last read.
1350 	 *
1351 	 * Length is a multiple of 32 bytes, hence it is a multiple of 8 too.
1352 	 */
1353 	for (i = 0; i < data_len; i += FIFO_DEPTH * BCH_SEQ_READS) {
1354 		marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1355 				    "RDDREQ while draining FIFO (data)");
1356 		marvell_nfc_xfer_data_in_pio(nfc, data,
1357 					     FIFO_DEPTH * BCH_SEQ_READS);
1358 		data += FIFO_DEPTH * BCH_SEQ_READS;
1359 	}
1360 
1361 	for (i = 0; i < spare_len; i += FIFO_DEPTH * BCH_SEQ_READS) {
1362 		marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1363 				    "RDDREQ while draining FIFO (OOB)");
1364 		marvell_nfc_xfer_data_in_pio(nfc, spare,
1365 					     FIFO_DEPTH * BCH_SEQ_READS);
1366 		spare += FIFO_DEPTH * BCH_SEQ_READS;
1367 	}
1368 }
1369 
1370 static int marvell_nfc_hw_ecc_bch_read_page(struct nand_chip *chip,
1371 					    u8 *buf, int oob_required,
1372 					    int page)
1373 {
1374 	struct mtd_info *mtd = nand_to_mtd(chip);
1375 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1376 	int data_len = lt->data_bytes, spare_len = lt->spare_bytes;
1377 	u8 *data = buf, *spare = chip->oob_poi;
1378 	int max_bitflips = 0;
1379 	u32 failure_mask = 0;
1380 	int chunk, ret;
1381 
1382 	marvell_nfc_select_target(chip, chip->cur_cs);
1383 
1384 	/*
1385 	 * With BCH, OOB is not fully used (and thus not read entirely), not
1386 	 * expected bytes could show up at the end of the OOB buffer if not
1387 	 * explicitly erased.
1388 	 */
1389 	if (oob_required)
1390 		memset(chip->oob_poi, 0xFF, mtd->oobsize);
1391 
1392 	marvell_nfc_enable_hw_ecc(chip);
1393 
1394 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1395 		/* Update length for the last chunk */
1396 		if (chunk >= lt->full_chunk_cnt) {
1397 			data_len = lt->last_data_bytes;
1398 			spare_len = lt->last_spare_bytes;
1399 		}
1400 
1401 		/* Read the chunk and detect number of bitflips */
1402 		marvell_nfc_hw_ecc_bch_read_chunk(chip, chunk, data, data_len,
1403 						  spare, spare_len, page);
1404 		ret = marvell_nfc_hw_ecc_check_bitflips(chip, &max_bitflips);
1405 		if (ret)
1406 			failure_mask |= BIT(chunk);
1407 
1408 		data += data_len;
1409 		spare += spare_len;
1410 	}
1411 
1412 	marvell_nfc_disable_hw_ecc(chip);
1413 
1414 	if (!failure_mask)
1415 		return max_bitflips;
1416 
1417 	/*
1418 	 * Please note that dumping the ECC bytes during a normal read with OOB
1419 	 * area would add a significant overhead as ECC bytes are "consumed" by
1420 	 * the controller in normal mode and must be re-read in raw mode. To
1421 	 * avoid dropping the performances, we prefer not to include them. The
1422 	 * user should re-read the page in raw mode if ECC bytes are required.
1423 	 */
1424 
1425 	/*
1426 	 * In case there is any subpage read error, we usually re-read only ECC
1427 	 * bytes in raw mode and check if the whole page is empty. In this case,
1428 	 * it is normal that the ECC check failed and we just ignore the error.
1429 	 *
1430 	 * However, it has been empirically observed that for some layouts (e.g
1431 	 * 2k page, 8b strength per 512B chunk), the controller tries to correct
1432 	 * bits and may create itself bitflips in the erased area. To overcome
1433 	 * this strange behavior, the whole page is re-read in raw mode, not
1434 	 * only the ECC bytes.
1435 	 */
1436 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1437 		int data_off_in_page, spare_off_in_page, ecc_off_in_page;
1438 		int data_off, spare_off, ecc_off;
1439 		int data_len, spare_len, ecc_len;
1440 
1441 		/* No failure reported for this chunk, move to the next one */
1442 		if (!(failure_mask & BIT(chunk)))
1443 			continue;
1444 
1445 		data_off_in_page = chunk * (lt->data_bytes + lt->spare_bytes +
1446 					    lt->ecc_bytes);
1447 		spare_off_in_page = data_off_in_page +
1448 			(chunk < lt->full_chunk_cnt ? lt->data_bytes :
1449 						      lt->last_data_bytes);
1450 		ecc_off_in_page = spare_off_in_page +
1451 			(chunk < lt->full_chunk_cnt ? lt->spare_bytes :
1452 						      lt->last_spare_bytes);
1453 
1454 		data_off = chunk * lt->data_bytes;
1455 		spare_off = chunk * lt->spare_bytes;
1456 		ecc_off = (lt->full_chunk_cnt * lt->spare_bytes) +
1457 			  lt->last_spare_bytes +
1458 			  (chunk * (lt->ecc_bytes + 2));
1459 
1460 		data_len = chunk < lt->full_chunk_cnt ? lt->data_bytes :
1461 							lt->last_data_bytes;
1462 		spare_len = chunk < lt->full_chunk_cnt ? lt->spare_bytes :
1463 							 lt->last_spare_bytes;
1464 		ecc_len = chunk < lt->full_chunk_cnt ? lt->ecc_bytes :
1465 						       lt->last_ecc_bytes;
1466 
1467 		/*
1468 		 * Only re-read the ECC bytes, unless we are using the 2k/8b
1469 		 * layout which is buggy in the sense that the ECC engine will
1470 		 * try to correct data bytes anyway, creating bitflips. In this
1471 		 * case, re-read the entire page.
1472 		 */
1473 		if (lt->writesize == 2048 && lt->strength == 8) {
1474 			nand_change_read_column_op(chip, data_off_in_page,
1475 						   buf + data_off, data_len,
1476 						   false);
1477 			nand_change_read_column_op(chip, spare_off_in_page,
1478 						   chip->oob_poi + spare_off, spare_len,
1479 						   false);
1480 		}
1481 
1482 		nand_change_read_column_op(chip, ecc_off_in_page,
1483 					   chip->oob_poi + ecc_off, ecc_len,
1484 					   false);
1485 
1486 		/* Check the entire chunk (data + spare + ecc) for emptyness */
1487 		marvell_nfc_check_empty_chunk(chip, buf + data_off, data_len,
1488 					      chip->oob_poi + spare_off, spare_len,
1489 					      chip->oob_poi + ecc_off, ecc_len,
1490 					      &max_bitflips);
1491 	}
1492 
1493 	return max_bitflips;
1494 }
1495 
1496 static int marvell_nfc_hw_ecc_bch_read_oob_raw(struct nand_chip *chip, int page)
1497 {
1498 	u8 *buf = nand_get_data_buf(chip);
1499 
1500 	return chip->ecc.read_page_raw(chip, buf, true, page);
1501 }
1502 
1503 static int marvell_nfc_hw_ecc_bch_read_oob(struct nand_chip *chip, int page)
1504 {
1505 	u8 *buf = nand_get_data_buf(chip);
1506 
1507 	return chip->ecc.read_page(chip, buf, true, page);
1508 }
1509 
1510 /* BCH write helpers */
1511 static int marvell_nfc_hw_ecc_bch_write_page_raw(struct nand_chip *chip,
1512 						 const u8 *buf,
1513 						 int oob_required, int page)
1514 {
1515 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1516 	int full_chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1517 	int data_len = lt->data_bytes;
1518 	int spare_len = lt->spare_bytes;
1519 	int ecc_len = lt->ecc_bytes;
1520 	int spare_offset = 0;
1521 	int ecc_offset = (lt->full_chunk_cnt * lt->spare_bytes) +
1522 		lt->last_spare_bytes;
1523 	int chunk;
1524 
1525 	marvell_nfc_select_target(chip, chip->cur_cs);
1526 
1527 	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1528 
1529 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1530 		if (chunk >= lt->full_chunk_cnt) {
1531 			data_len = lt->last_data_bytes;
1532 			spare_len = lt->last_spare_bytes;
1533 			ecc_len = lt->last_ecc_bytes;
1534 		}
1535 
1536 		/* Point to the column of the next chunk */
1537 		nand_change_write_column_op(chip, chunk * full_chunk_size,
1538 					    NULL, 0, false);
1539 
1540 		/* Write the data */
1541 		nand_write_data_op(chip, buf + (chunk * lt->data_bytes),
1542 				   data_len, false);
1543 
1544 		if (!oob_required)
1545 			continue;
1546 
1547 		/* Write the spare bytes */
1548 		if (spare_len)
1549 			nand_write_data_op(chip, chip->oob_poi + spare_offset,
1550 					   spare_len, false);
1551 
1552 		/* Write the ECC bytes */
1553 		if (ecc_len)
1554 			nand_write_data_op(chip, chip->oob_poi + ecc_offset,
1555 					   ecc_len, false);
1556 
1557 		spare_offset += spare_len;
1558 		ecc_offset += ALIGN(ecc_len, 32);
1559 	}
1560 
1561 	return nand_prog_page_end_op(chip);
1562 }
1563 
1564 static int
1565 marvell_nfc_hw_ecc_bch_write_chunk(struct nand_chip *chip, int chunk,
1566 				   const u8 *data, unsigned int data_len,
1567 				   const u8 *spare, unsigned int spare_len,
1568 				   int page)
1569 {
1570 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1571 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1572 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1573 	u32 xtype;
1574 	int ret;
1575 	struct marvell_nfc_op nfc_op = {
1576 		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_WRITE) | NDCB0_LEN_OVRD,
1577 		.ndcb[3] = data_len + spare_len,
1578 	};
1579 
1580 	/*
1581 	 * First operation dispatches the CMD_SEQIN command, issue the address
1582 	 * cycles and asks for the first chunk of data.
1583 	 * All operations in the middle (if any) will issue a naked write and
1584 	 * also ask for data.
1585 	 * Last operation (if any) asks for the last chunk of data through a
1586 	 * last naked write.
1587 	 */
1588 	if (chunk == 0) {
1589 		if (lt->nchunks == 1)
1590 			xtype = XTYPE_MONOLITHIC_RW;
1591 		else
1592 			xtype = XTYPE_WRITE_DISPATCH;
1593 
1594 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(xtype) |
1595 				  NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1596 				  NDCB0_CMD1(NAND_CMD_SEQIN);
1597 		nfc_op.ndcb[1] |= NDCB1_ADDRS_PAGE(page);
1598 		nfc_op.ndcb[2] |= NDCB2_ADDR5_PAGE(page);
1599 	} else if (chunk < lt->nchunks - 1) {
1600 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_NAKED_RW);
1601 	} else {
1602 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1603 	}
1604 
1605 	/* Always dispatch the PAGEPROG command on the last chunk */
1606 	if (chunk == lt->nchunks - 1)
1607 		nfc_op.ndcb[0] |= NDCB0_CMD2(NAND_CMD_PAGEPROG) | NDCB0_DBC;
1608 
1609 	ret = marvell_nfc_prepare_cmd(chip);
1610 	if (ret)
1611 		return ret;
1612 
1613 	marvell_nfc_send_cmd(chip, &nfc_op);
1614 	ret = marvell_nfc_end_cmd(chip, NDSR_WRDREQ,
1615 				  "WRDREQ while loading FIFO (data)");
1616 	if (ret)
1617 		return ret;
1618 
1619 	/* Transfer the contents */
1620 	iowrite32_rep(nfc->regs + NDDB, data, FIFO_REP(data_len));
1621 	iowrite32_rep(nfc->regs + NDDB, spare, FIFO_REP(spare_len));
1622 
1623 	return 0;
1624 }
1625 
1626 static int marvell_nfc_hw_ecc_bch_write_page(struct nand_chip *chip,
1627 					     const u8 *buf,
1628 					     int oob_required, int page)
1629 {
1630 	const struct nand_sdr_timings *sdr =
1631 		nand_get_sdr_timings(nand_get_interface_config(chip));
1632 	struct mtd_info *mtd = nand_to_mtd(chip);
1633 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1634 	const u8 *data = buf;
1635 	const u8 *spare = chip->oob_poi;
1636 	int data_len = lt->data_bytes;
1637 	int spare_len = lt->spare_bytes;
1638 	int chunk, ret;
1639 	u8 status;
1640 
1641 	marvell_nfc_select_target(chip, chip->cur_cs);
1642 
1643 	/* Spare data will be written anyway, so clear it to avoid garbage */
1644 	if (!oob_required)
1645 		memset(chip->oob_poi, 0xFF, mtd->oobsize);
1646 
1647 	marvell_nfc_enable_hw_ecc(chip);
1648 
1649 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1650 		if (chunk >= lt->full_chunk_cnt) {
1651 			data_len = lt->last_data_bytes;
1652 			spare_len = lt->last_spare_bytes;
1653 		}
1654 
1655 		marvell_nfc_hw_ecc_bch_write_chunk(chip, chunk, data, data_len,
1656 						   spare, spare_len, page);
1657 		data += data_len;
1658 		spare += spare_len;
1659 
1660 		/*
1661 		 * Waiting only for CMDD or PAGED is not enough, ECC are
1662 		 * partially written. No flag is set once the operation is
1663 		 * really finished but the ND_RUN bit is cleared, so wait for it
1664 		 * before stepping into the next command.
1665 		 */
1666 		marvell_nfc_wait_ndrun(chip);
1667 	}
1668 
1669 	ret = marvell_nfc_wait_op(chip, PSEC_TO_MSEC(sdr->tPROG_max));
1670 
1671 	marvell_nfc_disable_hw_ecc(chip);
1672 
1673 	if (ret)
1674 		return ret;
1675 
1676 	/* Check write status on the chip side */
1677 	ret = nand_status_op(chip, &status);
1678 	if (ret)
1679 		return ret;
1680 
1681 	if (status & NAND_STATUS_FAIL)
1682 		return -EIO;
1683 
1684 	return 0;
1685 }
1686 
1687 static int marvell_nfc_hw_ecc_bch_write_oob_raw(struct nand_chip *chip,
1688 						int page)
1689 {
1690 	struct mtd_info *mtd = nand_to_mtd(chip);
1691 	u8 *buf = nand_get_data_buf(chip);
1692 
1693 	memset(buf, 0xFF, mtd->writesize);
1694 
1695 	return chip->ecc.write_page_raw(chip, buf, true, page);
1696 }
1697 
1698 static int marvell_nfc_hw_ecc_bch_write_oob(struct nand_chip *chip, int page)
1699 {
1700 	struct mtd_info *mtd = nand_to_mtd(chip);
1701 	u8 *buf = nand_get_data_buf(chip);
1702 
1703 	memset(buf, 0xFF, mtd->writesize);
1704 
1705 	return chip->ecc.write_page(chip, buf, true, page);
1706 }
1707 
1708 /* NAND framework ->exec_op() hooks and related helpers */
1709 static void marvell_nfc_parse_instructions(struct nand_chip *chip,
1710 					   const struct nand_subop *subop,
1711 					   struct marvell_nfc_op *nfc_op)
1712 {
1713 	const struct nand_op_instr *instr = NULL;
1714 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1715 	bool first_cmd = true;
1716 	unsigned int op_id;
1717 	int i;
1718 
1719 	/* Reset the input structure as most of its fields will be OR'ed */
1720 	memset(nfc_op, 0, sizeof(struct marvell_nfc_op));
1721 
1722 	for (op_id = 0; op_id < subop->ninstrs; op_id++) {
1723 		unsigned int offset, naddrs;
1724 		const u8 *addrs;
1725 		int len;
1726 
1727 		instr = &subop->instrs[op_id];
1728 
1729 		switch (instr->type) {
1730 		case NAND_OP_CMD_INSTR:
1731 			if (first_cmd)
1732 				nfc_op->ndcb[0] |=
1733 					NDCB0_CMD1(instr->ctx.cmd.opcode);
1734 			else
1735 				nfc_op->ndcb[0] |=
1736 					NDCB0_CMD2(instr->ctx.cmd.opcode) |
1737 					NDCB0_DBC;
1738 
1739 			nfc_op->cle_ale_delay_ns = instr->delay_ns;
1740 			first_cmd = false;
1741 			break;
1742 
1743 		case NAND_OP_ADDR_INSTR:
1744 			offset = nand_subop_get_addr_start_off(subop, op_id);
1745 			naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
1746 			addrs = &instr->ctx.addr.addrs[offset];
1747 
1748 			nfc_op->ndcb[0] |= NDCB0_ADDR_CYC(naddrs);
1749 
1750 			for (i = 0; i < min_t(unsigned int, 4, naddrs); i++)
1751 				nfc_op->ndcb[1] |= addrs[i] << (8 * i);
1752 
1753 			if (naddrs >= 5)
1754 				nfc_op->ndcb[2] |= NDCB2_ADDR5_CYC(addrs[4]);
1755 			if (naddrs >= 6)
1756 				nfc_op->ndcb[3] |= NDCB3_ADDR6_CYC(addrs[5]);
1757 			if (naddrs == 7)
1758 				nfc_op->ndcb[3] |= NDCB3_ADDR7_CYC(addrs[6]);
1759 
1760 			nfc_op->cle_ale_delay_ns = instr->delay_ns;
1761 			break;
1762 
1763 		case NAND_OP_DATA_IN_INSTR:
1764 			nfc_op->data_instr = instr;
1765 			nfc_op->data_instr_idx = op_id;
1766 			nfc_op->ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ);
1767 			if (nfc->caps->is_nfcv2) {
1768 				nfc_op->ndcb[0] |=
1769 					NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW) |
1770 					NDCB0_LEN_OVRD;
1771 				len = nand_subop_get_data_len(subop, op_id);
1772 				nfc_op->ndcb[3] |= round_up(len, FIFO_DEPTH);
1773 			}
1774 			nfc_op->data_delay_ns = instr->delay_ns;
1775 			break;
1776 
1777 		case NAND_OP_DATA_OUT_INSTR:
1778 			nfc_op->data_instr = instr;
1779 			nfc_op->data_instr_idx = op_id;
1780 			nfc_op->ndcb[0] |= NDCB0_CMD_TYPE(TYPE_WRITE);
1781 			if (nfc->caps->is_nfcv2) {
1782 				nfc_op->ndcb[0] |=
1783 					NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW) |
1784 					NDCB0_LEN_OVRD;
1785 				len = nand_subop_get_data_len(subop, op_id);
1786 				nfc_op->ndcb[3] |= round_up(len, FIFO_DEPTH);
1787 			}
1788 			nfc_op->data_delay_ns = instr->delay_ns;
1789 			break;
1790 
1791 		case NAND_OP_WAITRDY_INSTR:
1792 			nfc_op->rdy_timeout_ms = instr->ctx.waitrdy.timeout_ms;
1793 			nfc_op->rdy_delay_ns = instr->delay_ns;
1794 			break;
1795 		}
1796 	}
1797 }
1798 
1799 static int marvell_nfc_xfer_data_pio(struct nand_chip *chip,
1800 				     const struct nand_subop *subop,
1801 				     struct marvell_nfc_op *nfc_op)
1802 {
1803 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1804 	const struct nand_op_instr *instr = nfc_op->data_instr;
1805 	unsigned int op_id = nfc_op->data_instr_idx;
1806 	unsigned int len = nand_subop_get_data_len(subop, op_id);
1807 	unsigned int offset = nand_subop_get_data_start_off(subop, op_id);
1808 	bool reading = (instr->type == NAND_OP_DATA_IN_INSTR);
1809 	int ret;
1810 
1811 	if (instr->ctx.data.force_8bit)
1812 		marvell_nfc_force_byte_access(chip, true);
1813 
1814 	if (reading) {
1815 		u8 *in = instr->ctx.data.buf.in + offset;
1816 
1817 		ret = marvell_nfc_xfer_data_in_pio(nfc, in, len);
1818 	} else {
1819 		const u8 *out = instr->ctx.data.buf.out + offset;
1820 
1821 		ret = marvell_nfc_xfer_data_out_pio(nfc, out, len);
1822 	}
1823 
1824 	if (instr->ctx.data.force_8bit)
1825 		marvell_nfc_force_byte_access(chip, false);
1826 
1827 	return ret;
1828 }
1829 
1830 static int marvell_nfc_monolithic_access_exec(struct nand_chip *chip,
1831 					      const struct nand_subop *subop)
1832 {
1833 	struct marvell_nfc_op nfc_op;
1834 	bool reading;
1835 	int ret;
1836 
1837 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1838 	reading = (nfc_op.data_instr->type == NAND_OP_DATA_IN_INSTR);
1839 
1840 	ret = marvell_nfc_prepare_cmd(chip);
1841 	if (ret)
1842 		return ret;
1843 
1844 	marvell_nfc_send_cmd(chip, &nfc_op);
1845 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ | NDSR_WRDREQ,
1846 				  "RDDREQ/WRDREQ while draining raw data");
1847 	if (ret)
1848 		return ret;
1849 
1850 	cond_delay(nfc_op.cle_ale_delay_ns);
1851 
1852 	if (reading) {
1853 		if (nfc_op.rdy_timeout_ms) {
1854 			ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1855 			if (ret)
1856 				return ret;
1857 		}
1858 
1859 		cond_delay(nfc_op.rdy_delay_ns);
1860 	}
1861 
1862 	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1863 	ret = marvell_nfc_wait_cmdd(chip);
1864 	if (ret)
1865 		return ret;
1866 
1867 	cond_delay(nfc_op.data_delay_ns);
1868 
1869 	if (!reading) {
1870 		if (nfc_op.rdy_timeout_ms) {
1871 			ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1872 			if (ret)
1873 				return ret;
1874 		}
1875 
1876 		cond_delay(nfc_op.rdy_delay_ns);
1877 	}
1878 
1879 	/*
1880 	 * NDCR ND_RUN bit should be cleared automatically at the end of each
1881 	 * operation but experience shows that the behavior is buggy when it
1882 	 * comes to writes (with LEN_OVRD). Clear it by hand in this case.
1883 	 */
1884 	if (!reading) {
1885 		struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1886 
1887 		writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
1888 			       nfc->regs + NDCR);
1889 	}
1890 
1891 	return 0;
1892 }
1893 
1894 static int marvell_nfc_naked_access_exec(struct nand_chip *chip,
1895 					 const struct nand_subop *subop)
1896 {
1897 	struct marvell_nfc_op nfc_op;
1898 	int ret;
1899 
1900 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1901 
1902 	/*
1903 	 * Naked access are different in that they need to be flagged as naked
1904 	 * by the controller. Reset the controller registers fields that inform
1905 	 * on the type and refill them according to the ongoing operation.
1906 	 */
1907 	nfc_op.ndcb[0] &= ~(NDCB0_CMD_TYPE(TYPE_MASK) |
1908 			    NDCB0_CMD_XTYPE(XTYPE_MASK));
1909 	switch (subop->instrs[0].type) {
1910 	case NAND_OP_CMD_INSTR:
1911 		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_NAKED_CMD);
1912 		break;
1913 	case NAND_OP_ADDR_INSTR:
1914 		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_NAKED_ADDR);
1915 		break;
1916 	case NAND_OP_DATA_IN_INSTR:
1917 		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ) |
1918 				  NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1919 		break;
1920 	case NAND_OP_DATA_OUT_INSTR:
1921 		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_WRITE) |
1922 				  NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1923 		break;
1924 	default:
1925 		/* This should never happen */
1926 		break;
1927 	}
1928 
1929 	ret = marvell_nfc_prepare_cmd(chip);
1930 	if (ret)
1931 		return ret;
1932 
1933 	marvell_nfc_send_cmd(chip, &nfc_op);
1934 
1935 	if (!nfc_op.data_instr) {
1936 		ret = marvell_nfc_wait_cmdd(chip);
1937 		cond_delay(nfc_op.cle_ale_delay_ns);
1938 		return ret;
1939 	}
1940 
1941 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ | NDSR_WRDREQ,
1942 				  "RDDREQ/WRDREQ while draining raw data");
1943 	if (ret)
1944 		return ret;
1945 
1946 	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1947 	ret = marvell_nfc_wait_cmdd(chip);
1948 	if (ret)
1949 		return ret;
1950 
1951 	/*
1952 	 * NDCR ND_RUN bit should be cleared automatically at the end of each
1953 	 * operation but experience shows that the behavior is buggy when it
1954 	 * comes to writes (with LEN_OVRD). Clear it by hand in this case.
1955 	 */
1956 	if (subop->instrs[0].type == NAND_OP_DATA_OUT_INSTR) {
1957 		struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1958 
1959 		writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
1960 			       nfc->regs + NDCR);
1961 	}
1962 
1963 	return 0;
1964 }
1965 
1966 static int marvell_nfc_naked_waitrdy_exec(struct nand_chip *chip,
1967 					  const struct nand_subop *subop)
1968 {
1969 	struct marvell_nfc_op nfc_op;
1970 	int ret;
1971 
1972 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1973 
1974 	ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1975 	cond_delay(nfc_op.rdy_delay_ns);
1976 
1977 	return ret;
1978 }
1979 
1980 static int marvell_nfc_read_id_type_exec(struct nand_chip *chip,
1981 					 const struct nand_subop *subop)
1982 {
1983 	struct marvell_nfc_op nfc_op;
1984 	int ret;
1985 
1986 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1987 	nfc_op.ndcb[0] &= ~NDCB0_CMD_TYPE(TYPE_READ);
1988 	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ_ID);
1989 
1990 	ret = marvell_nfc_prepare_cmd(chip);
1991 	if (ret)
1992 		return ret;
1993 
1994 	marvell_nfc_send_cmd(chip, &nfc_op);
1995 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1996 				  "RDDREQ while reading ID");
1997 	if (ret)
1998 		return ret;
1999 
2000 	cond_delay(nfc_op.cle_ale_delay_ns);
2001 
2002 	if (nfc_op.rdy_timeout_ms) {
2003 		ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
2004 		if (ret)
2005 			return ret;
2006 	}
2007 
2008 	cond_delay(nfc_op.rdy_delay_ns);
2009 
2010 	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
2011 	ret = marvell_nfc_wait_cmdd(chip);
2012 	if (ret)
2013 		return ret;
2014 
2015 	cond_delay(nfc_op.data_delay_ns);
2016 
2017 	return 0;
2018 }
2019 
2020 static int marvell_nfc_read_status_exec(struct nand_chip *chip,
2021 					const struct nand_subop *subop)
2022 {
2023 	struct marvell_nfc_op nfc_op;
2024 	int ret;
2025 
2026 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
2027 	nfc_op.ndcb[0] &= ~NDCB0_CMD_TYPE(TYPE_READ);
2028 	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_STATUS);
2029 
2030 	ret = marvell_nfc_prepare_cmd(chip);
2031 	if (ret)
2032 		return ret;
2033 
2034 	marvell_nfc_send_cmd(chip, &nfc_op);
2035 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
2036 				  "RDDREQ while reading status");
2037 	if (ret)
2038 		return ret;
2039 
2040 	cond_delay(nfc_op.cle_ale_delay_ns);
2041 
2042 	if (nfc_op.rdy_timeout_ms) {
2043 		ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
2044 		if (ret)
2045 			return ret;
2046 	}
2047 
2048 	cond_delay(nfc_op.rdy_delay_ns);
2049 
2050 	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
2051 	ret = marvell_nfc_wait_cmdd(chip);
2052 	if (ret)
2053 		return ret;
2054 
2055 	cond_delay(nfc_op.data_delay_ns);
2056 
2057 	return 0;
2058 }
2059 
2060 static int marvell_nfc_reset_cmd_type_exec(struct nand_chip *chip,
2061 					   const struct nand_subop *subop)
2062 {
2063 	struct marvell_nfc_op nfc_op;
2064 	int ret;
2065 
2066 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
2067 	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_RESET);
2068 
2069 	ret = marvell_nfc_prepare_cmd(chip);
2070 	if (ret)
2071 		return ret;
2072 
2073 	marvell_nfc_send_cmd(chip, &nfc_op);
2074 	ret = marvell_nfc_wait_cmdd(chip);
2075 	if (ret)
2076 		return ret;
2077 
2078 	cond_delay(nfc_op.cle_ale_delay_ns);
2079 
2080 	ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
2081 	if (ret)
2082 		return ret;
2083 
2084 	cond_delay(nfc_op.rdy_delay_ns);
2085 
2086 	return 0;
2087 }
2088 
2089 static int marvell_nfc_erase_cmd_type_exec(struct nand_chip *chip,
2090 					   const struct nand_subop *subop)
2091 {
2092 	struct marvell_nfc_op nfc_op;
2093 	int ret;
2094 
2095 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
2096 	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_ERASE);
2097 
2098 	ret = marvell_nfc_prepare_cmd(chip);
2099 	if (ret)
2100 		return ret;
2101 
2102 	marvell_nfc_send_cmd(chip, &nfc_op);
2103 	ret = marvell_nfc_wait_cmdd(chip);
2104 	if (ret)
2105 		return ret;
2106 
2107 	cond_delay(nfc_op.cle_ale_delay_ns);
2108 
2109 	ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
2110 	if (ret)
2111 		return ret;
2112 
2113 	cond_delay(nfc_op.rdy_delay_ns);
2114 
2115 	return 0;
2116 }
2117 
2118 static const struct nand_op_parser marvell_nfcv2_op_parser = NAND_OP_PARSER(
2119 	/* Monolithic reads/writes */
2120 	NAND_OP_PARSER_PATTERN(
2121 		marvell_nfc_monolithic_access_exec,
2122 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2123 		NAND_OP_PARSER_PAT_ADDR_ELEM(true, MAX_ADDRESS_CYC_NFCV2),
2124 		NAND_OP_PARSER_PAT_CMD_ELEM(true),
2125 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
2126 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_CHUNK_SIZE)),
2127 	NAND_OP_PARSER_PATTERN(
2128 		marvell_nfc_monolithic_access_exec,
2129 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2130 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV2),
2131 		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_CHUNK_SIZE),
2132 		NAND_OP_PARSER_PAT_CMD_ELEM(true),
2133 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
2134 	/* Naked commands */
2135 	NAND_OP_PARSER_PATTERN(
2136 		marvell_nfc_naked_access_exec,
2137 		NAND_OP_PARSER_PAT_CMD_ELEM(false)),
2138 	NAND_OP_PARSER_PATTERN(
2139 		marvell_nfc_naked_access_exec,
2140 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV2)),
2141 	NAND_OP_PARSER_PATTERN(
2142 		marvell_nfc_naked_access_exec,
2143 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_CHUNK_SIZE)),
2144 	NAND_OP_PARSER_PATTERN(
2145 		marvell_nfc_naked_access_exec,
2146 		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_CHUNK_SIZE)),
2147 	NAND_OP_PARSER_PATTERN(
2148 		marvell_nfc_naked_waitrdy_exec,
2149 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2150 	);
2151 
2152 static const struct nand_op_parser marvell_nfcv1_op_parser = NAND_OP_PARSER(
2153 	/* Naked commands not supported, use a function for each pattern */
2154 	NAND_OP_PARSER_PATTERN(
2155 		marvell_nfc_read_id_type_exec,
2156 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2157 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV1),
2158 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 8)),
2159 	NAND_OP_PARSER_PATTERN(
2160 		marvell_nfc_erase_cmd_type_exec,
2161 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2162 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV1),
2163 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2164 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2165 	NAND_OP_PARSER_PATTERN(
2166 		marvell_nfc_read_status_exec,
2167 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2168 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 1)),
2169 	NAND_OP_PARSER_PATTERN(
2170 		marvell_nfc_reset_cmd_type_exec,
2171 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2172 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2173 	NAND_OP_PARSER_PATTERN(
2174 		marvell_nfc_naked_waitrdy_exec,
2175 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2176 	);
2177 
2178 static int marvell_nfc_exec_op(struct nand_chip *chip,
2179 			       const struct nand_operation *op,
2180 			       bool check_only)
2181 {
2182 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2183 
2184 	if (!check_only)
2185 		marvell_nfc_select_target(chip, op->cs);
2186 
2187 	if (nfc->caps->is_nfcv2)
2188 		return nand_op_parser_exec_op(chip, &marvell_nfcv2_op_parser,
2189 					      op, check_only);
2190 	else
2191 		return nand_op_parser_exec_op(chip, &marvell_nfcv1_op_parser,
2192 					      op, check_only);
2193 }
2194 
2195 /*
2196  * Layouts were broken in old pxa3xx_nand driver, these are supposed to be
2197  * usable.
2198  */
2199 static int marvell_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2200 				      struct mtd_oob_region *oobregion)
2201 {
2202 	struct nand_chip *chip = mtd_to_nand(mtd);
2203 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
2204 
2205 	if (section)
2206 		return -ERANGE;
2207 
2208 	oobregion->length = (lt->full_chunk_cnt * lt->ecc_bytes) +
2209 			    lt->last_ecc_bytes;
2210 	oobregion->offset = mtd->oobsize - oobregion->length;
2211 
2212 	return 0;
2213 }
2214 
2215 static int marvell_nand_ooblayout_free(struct mtd_info *mtd, int section,
2216 				       struct mtd_oob_region *oobregion)
2217 {
2218 	struct nand_chip *chip = mtd_to_nand(mtd);
2219 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
2220 
2221 	if (section)
2222 		return -ERANGE;
2223 
2224 	/*
2225 	 * Bootrom looks in bytes 0 & 5 for bad blocks for the
2226 	 * 4KB page / 4bit BCH combination.
2227 	 */
2228 	if (mtd->writesize == SZ_4K && lt->data_bytes == SZ_2K)
2229 		oobregion->offset = 6;
2230 	else
2231 		oobregion->offset = 2;
2232 
2233 	oobregion->length = (lt->full_chunk_cnt * lt->spare_bytes) +
2234 			    lt->last_spare_bytes - oobregion->offset;
2235 
2236 	return 0;
2237 }
2238 
2239 static const struct mtd_ooblayout_ops marvell_nand_ooblayout_ops = {
2240 	.ecc = marvell_nand_ooblayout_ecc,
2241 	.free = marvell_nand_ooblayout_free,
2242 };
2243 
2244 static int marvell_nand_hw_ecc_controller_init(struct mtd_info *mtd,
2245 					       struct nand_ecc_ctrl *ecc)
2246 {
2247 	struct nand_chip *chip = mtd_to_nand(mtd);
2248 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2249 	const struct marvell_hw_ecc_layout *l;
2250 	int i;
2251 
2252 	if (!nfc->caps->is_nfcv2 &&
2253 	    (mtd->writesize + mtd->oobsize > MAX_CHUNK_SIZE)) {
2254 		dev_err(nfc->dev,
2255 			"NFCv1: writesize (%d) cannot be bigger than a chunk (%d)\n",
2256 			mtd->writesize, MAX_CHUNK_SIZE - mtd->oobsize);
2257 		return -ENOTSUPP;
2258 	}
2259 
2260 	to_marvell_nand(chip)->layout = NULL;
2261 	for (i = 0; i < ARRAY_SIZE(marvell_nfc_layouts); i++) {
2262 		l = &marvell_nfc_layouts[i];
2263 		if (mtd->writesize == l->writesize &&
2264 		    ecc->size == l->chunk && ecc->strength == l->strength) {
2265 			to_marvell_nand(chip)->layout = l;
2266 			break;
2267 		}
2268 	}
2269 
2270 	if (!to_marvell_nand(chip)->layout ||
2271 	    (!nfc->caps->is_nfcv2 && ecc->strength > 1)) {
2272 		dev_err(nfc->dev,
2273 			"ECC strength %d at page size %d is not supported\n",
2274 			ecc->strength, mtd->writesize);
2275 		return -ENOTSUPP;
2276 	}
2277 
2278 	/* Special care for the layout 2k/8-bit/512B  */
2279 	if (l->writesize == 2048 && l->strength == 8) {
2280 		if (mtd->oobsize < 128) {
2281 			dev_err(nfc->dev, "Requested layout needs at least 128 OOB bytes\n");
2282 			return -ENOTSUPP;
2283 		} else {
2284 			chip->bbt_options |= NAND_BBT_NO_OOB_BBM;
2285 		}
2286 	}
2287 
2288 	mtd_set_ooblayout(mtd, &marvell_nand_ooblayout_ops);
2289 	ecc->steps = l->nchunks;
2290 	ecc->size = l->data_bytes;
2291 
2292 	if (ecc->strength == 1) {
2293 		chip->ecc.algo = NAND_ECC_ALGO_HAMMING;
2294 		ecc->read_page_raw = marvell_nfc_hw_ecc_hmg_read_page_raw;
2295 		ecc->read_page = marvell_nfc_hw_ecc_hmg_read_page;
2296 		ecc->read_oob_raw = marvell_nfc_hw_ecc_hmg_read_oob_raw;
2297 		ecc->read_oob = ecc->read_oob_raw;
2298 		ecc->write_page_raw = marvell_nfc_hw_ecc_hmg_write_page_raw;
2299 		ecc->write_page = marvell_nfc_hw_ecc_hmg_write_page;
2300 		ecc->write_oob_raw = marvell_nfc_hw_ecc_hmg_write_oob_raw;
2301 		ecc->write_oob = ecc->write_oob_raw;
2302 	} else {
2303 		chip->ecc.algo = NAND_ECC_ALGO_BCH;
2304 		ecc->strength = 16;
2305 		ecc->read_page_raw = marvell_nfc_hw_ecc_bch_read_page_raw;
2306 		ecc->read_page = marvell_nfc_hw_ecc_bch_read_page;
2307 		ecc->read_oob_raw = marvell_nfc_hw_ecc_bch_read_oob_raw;
2308 		ecc->read_oob = marvell_nfc_hw_ecc_bch_read_oob;
2309 		ecc->write_page_raw = marvell_nfc_hw_ecc_bch_write_page_raw;
2310 		ecc->write_page = marvell_nfc_hw_ecc_bch_write_page;
2311 		ecc->write_oob_raw = marvell_nfc_hw_ecc_bch_write_oob_raw;
2312 		ecc->write_oob = marvell_nfc_hw_ecc_bch_write_oob;
2313 	}
2314 
2315 	return 0;
2316 }
2317 
2318 static int marvell_nand_ecc_init(struct mtd_info *mtd,
2319 				 struct nand_ecc_ctrl *ecc)
2320 {
2321 	struct nand_chip *chip = mtd_to_nand(mtd);
2322 	const struct nand_ecc_props *requirements =
2323 		nanddev_get_ecc_requirements(&chip->base);
2324 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2325 	int ret;
2326 
2327 	if (ecc->engine_type != NAND_ECC_ENGINE_TYPE_NONE &&
2328 	    (!ecc->size || !ecc->strength)) {
2329 		if (requirements->step_size && requirements->strength) {
2330 			ecc->size = requirements->step_size;
2331 			ecc->strength = requirements->strength;
2332 		} else {
2333 			dev_info(nfc->dev,
2334 				 "No minimum ECC strength, using 1b/512B\n");
2335 			ecc->size = 512;
2336 			ecc->strength = 1;
2337 		}
2338 	}
2339 
2340 	switch (ecc->engine_type) {
2341 	case NAND_ECC_ENGINE_TYPE_ON_HOST:
2342 		ret = marvell_nand_hw_ecc_controller_init(mtd, ecc);
2343 		if (ret)
2344 			return ret;
2345 		break;
2346 	case NAND_ECC_ENGINE_TYPE_NONE:
2347 	case NAND_ECC_ENGINE_TYPE_SOFT:
2348 	case NAND_ECC_ENGINE_TYPE_ON_DIE:
2349 		if (!nfc->caps->is_nfcv2 && mtd->writesize != SZ_512 &&
2350 		    mtd->writesize != SZ_2K) {
2351 			dev_err(nfc->dev, "NFCv1 cannot write %d bytes pages\n",
2352 				mtd->writesize);
2353 			return -EINVAL;
2354 		}
2355 		break;
2356 	default:
2357 		return -EINVAL;
2358 	}
2359 
2360 	return 0;
2361 }
2362 
2363 static u8 bbt_pattern[] = {'M', 'V', 'B', 'b', 't', '0' };
2364 static u8 bbt_mirror_pattern[] = {'1', 't', 'b', 'B', 'V', 'M' };
2365 
2366 static struct nand_bbt_descr bbt_main_descr = {
2367 	.options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
2368 		   NAND_BBT_2BIT | NAND_BBT_VERSION,
2369 	.offs =	8,
2370 	.len = 6,
2371 	.veroffs = 14,
2372 	.maxblocks = 8,	/* Last 8 blocks in each chip */
2373 	.pattern = bbt_pattern
2374 };
2375 
2376 static struct nand_bbt_descr bbt_mirror_descr = {
2377 	.options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
2378 		   NAND_BBT_2BIT | NAND_BBT_VERSION,
2379 	.offs =	8,
2380 	.len = 6,
2381 	.veroffs = 14,
2382 	.maxblocks = 8,	/* Last 8 blocks in each chip */
2383 	.pattern = bbt_mirror_pattern
2384 };
2385 
2386 static int marvell_nfc_setup_interface(struct nand_chip *chip, int chipnr,
2387 				       const struct nand_interface_config *conf)
2388 {
2389 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
2390 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2391 	unsigned int period_ns = 1000000000 / clk_get_rate(nfc->core_clk) * 2;
2392 	const struct nand_sdr_timings *sdr;
2393 	struct marvell_nfc_timings nfc_tmg;
2394 	int read_delay;
2395 
2396 	sdr = nand_get_sdr_timings(conf);
2397 	if (IS_ERR(sdr))
2398 		return PTR_ERR(sdr);
2399 
2400 	if (nfc->caps->max_mode_number && nfc->caps->max_mode_number < conf->timings.mode)
2401 		return -EOPNOTSUPP;
2402 
2403 	/*
2404 	 * SDR timings are given in pico-seconds while NFC timings must be
2405 	 * expressed in NAND controller clock cycles, which is half of the
2406 	 * frequency of the accessible ECC clock retrieved by clk_get_rate().
2407 	 * This is not written anywhere in the datasheet but was observed
2408 	 * with an oscilloscope.
2409 	 *
2410 	 * NFC datasheet gives equations from which thoses calculations
2411 	 * are derived, they tend to be slightly more restrictives than the
2412 	 * given core timings and may improve the overall speed.
2413 	 */
2414 	nfc_tmg.tRP = TO_CYCLES(DIV_ROUND_UP(sdr->tRC_min, 2), period_ns) - 1;
2415 	nfc_tmg.tRH = nfc_tmg.tRP;
2416 	nfc_tmg.tWP = TO_CYCLES(DIV_ROUND_UP(sdr->tWC_min, 2), period_ns) - 1;
2417 	nfc_tmg.tWH = nfc_tmg.tWP;
2418 	nfc_tmg.tCS = TO_CYCLES(sdr->tCS_min, period_ns);
2419 	nfc_tmg.tCH = TO_CYCLES(sdr->tCH_min, period_ns) - 1;
2420 	nfc_tmg.tADL = TO_CYCLES(sdr->tADL_min, period_ns);
2421 	/*
2422 	 * Read delay is the time of propagation from SoC pins to NFC internal
2423 	 * logic. With non-EDO timings, this is MIN_RD_DEL_CNT clock cycles. In
2424 	 * EDO mode, an additional delay of tRH must be taken into account so
2425 	 * the data is sampled on the falling edge instead of the rising edge.
2426 	 */
2427 	read_delay = sdr->tRC_min >= 30000 ?
2428 		MIN_RD_DEL_CNT : MIN_RD_DEL_CNT + nfc_tmg.tRH;
2429 
2430 	nfc_tmg.tAR = TO_CYCLES(sdr->tAR_min, period_ns);
2431 	/*
2432 	 * tWHR and tRHW are supposed to be read to write delays (and vice
2433 	 * versa) but in some cases, ie. when doing a change column, they must
2434 	 * be greater than that to be sure tCCS delay is respected.
2435 	 */
2436 	nfc_tmg.tWHR = TO_CYCLES(max_t(int, sdr->tWHR_min, sdr->tCCS_min),
2437 				 period_ns) - 2;
2438 	nfc_tmg.tRHW = TO_CYCLES(max_t(int, sdr->tRHW_min, sdr->tCCS_min),
2439 				 period_ns);
2440 
2441 	/*
2442 	 * NFCv2: Use WAIT_MODE (wait for RB line), do not rely only on delays.
2443 	 * NFCv1: No WAIT_MODE, tR must be maximal.
2444 	 */
2445 	if (nfc->caps->is_nfcv2) {
2446 		nfc_tmg.tR = TO_CYCLES(sdr->tWB_max, period_ns);
2447 	} else {
2448 		nfc_tmg.tR = TO_CYCLES64(sdr->tWB_max + sdr->tR_max,
2449 					 period_ns);
2450 		if (nfc_tmg.tR + 3 > nfc_tmg.tCH)
2451 			nfc_tmg.tR = nfc_tmg.tCH - 3;
2452 		else
2453 			nfc_tmg.tR = 0;
2454 	}
2455 
2456 	if (chipnr < 0)
2457 		return 0;
2458 
2459 	marvell_nand->ndtr0 =
2460 		NDTR0_TRP(nfc_tmg.tRP) |
2461 		NDTR0_TRH(nfc_tmg.tRH) |
2462 		NDTR0_ETRP(nfc_tmg.tRP) |
2463 		NDTR0_TWP(nfc_tmg.tWP) |
2464 		NDTR0_TWH(nfc_tmg.tWH) |
2465 		NDTR0_TCS(nfc_tmg.tCS) |
2466 		NDTR0_TCH(nfc_tmg.tCH);
2467 
2468 	marvell_nand->ndtr1 =
2469 		NDTR1_TAR(nfc_tmg.tAR) |
2470 		NDTR1_TWHR(nfc_tmg.tWHR) |
2471 		NDTR1_TR(nfc_tmg.tR);
2472 
2473 	if (nfc->caps->is_nfcv2) {
2474 		marvell_nand->ndtr0 |=
2475 			NDTR0_RD_CNT_DEL(read_delay) |
2476 			NDTR0_SELCNTR |
2477 			NDTR0_TADL(nfc_tmg.tADL);
2478 
2479 		marvell_nand->ndtr1 |=
2480 			NDTR1_TRHW(nfc_tmg.tRHW) |
2481 			NDTR1_WAIT_MODE;
2482 	}
2483 
2484 	/*
2485 	 * Reset nfc->selected_chip so the next command will cause the timing
2486 	 * registers to be updated in marvell_nfc_select_target().
2487 	 */
2488 	nfc->selected_chip = NULL;
2489 
2490 	return 0;
2491 }
2492 
2493 static int marvell_nand_attach_chip(struct nand_chip *chip)
2494 {
2495 	struct mtd_info *mtd = nand_to_mtd(chip);
2496 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
2497 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2498 	struct pxa3xx_nand_platform_data *pdata = dev_get_platdata(nfc->dev);
2499 	int ret;
2500 
2501 	if (pdata && pdata->flash_bbt)
2502 		chip->bbt_options |= NAND_BBT_USE_FLASH;
2503 
2504 	if (chip->bbt_options & NAND_BBT_USE_FLASH) {
2505 		/*
2506 		 * We'll use a bad block table stored in-flash and don't
2507 		 * allow writing the bad block marker to the flash.
2508 		 */
2509 		chip->bbt_options |= NAND_BBT_NO_OOB_BBM;
2510 		chip->bbt_td = &bbt_main_descr;
2511 		chip->bbt_md = &bbt_mirror_descr;
2512 	}
2513 
2514 	/* Save the chip-specific fields of NDCR */
2515 	marvell_nand->ndcr = NDCR_PAGE_SZ(mtd->writesize);
2516 	if (chip->options & NAND_BUSWIDTH_16)
2517 		marvell_nand->ndcr |= NDCR_DWIDTH_M | NDCR_DWIDTH_C;
2518 
2519 	/*
2520 	 * On small page NANDs, only one cycle is needed to pass the
2521 	 * column address.
2522 	 */
2523 	if (mtd->writesize <= 512) {
2524 		marvell_nand->addr_cyc = 1;
2525 	} else {
2526 		marvell_nand->addr_cyc = 2;
2527 		marvell_nand->ndcr |= NDCR_RA_START;
2528 	}
2529 
2530 	/*
2531 	 * Now add the number of cycles needed to pass the row
2532 	 * address.
2533 	 *
2534 	 * Addressing a chip using CS 2 or 3 should also need the third row
2535 	 * cycle but due to inconsistance in the documentation and lack of
2536 	 * hardware to test this situation, this case is not supported.
2537 	 */
2538 	if (chip->options & NAND_ROW_ADDR_3)
2539 		marvell_nand->addr_cyc += 3;
2540 	else
2541 		marvell_nand->addr_cyc += 2;
2542 
2543 	if (pdata) {
2544 		chip->ecc.size = pdata->ecc_step_size;
2545 		chip->ecc.strength = pdata->ecc_strength;
2546 	}
2547 
2548 	ret = marvell_nand_ecc_init(mtd, &chip->ecc);
2549 	if (ret) {
2550 		dev_err(nfc->dev, "ECC init failed: %d\n", ret);
2551 		return ret;
2552 	}
2553 
2554 	if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) {
2555 		/*
2556 		 * Subpage write not available with hardware ECC, prohibit also
2557 		 * subpage read as in userspace subpage access would still be
2558 		 * allowed and subpage write, if used, would lead to numerous
2559 		 * uncorrectable ECC errors.
2560 		 */
2561 		chip->options |= NAND_NO_SUBPAGE_WRITE;
2562 	}
2563 
2564 	if (pdata || nfc->caps->legacy_of_bindings) {
2565 		/*
2566 		 * We keep the MTD name unchanged to avoid breaking platforms
2567 		 * where the MTD cmdline parser is used and the bootloader
2568 		 * has not been updated to use the new naming scheme.
2569 		 */
2570 		mtd->name = "pxa3xx_nand-0";
2571 	} else if (!mtd->name) {
2572 		/*
2573 		 * If the new bindings are used and the bootloader has not been
2574 		 * updated to pass a new mtdparts parameter on the cmdline, you
2575 		 * should define the following property in your NAND node, ie:
2576 		 *
2577 		 *	label = "main-storage";
2578 		 *
2579 		 * This way, mtd->name will be set by the core when
2580 		 * nand_set_flash_node() is called.
2581 		 */
2582 		mtd->name = devm_kasprintf(nfc->dev, GFP_KERNEL,
2583 					   "%s:nand.%d", dev_name(nfc->dev),
2584 					   marvell_nand->sels[0].cs);
2585 		if (!mtd->name) {
2586 			dev_err(nfc->dev, "Failed to allocate mtd->name\n");
2587 			return -ENOMEM;
2588 		}
2589 	}
2590 
2591 	return 0;
2592 }
2593 
2594 static const struct nand_controller_ops marvell_nand_controller_ops = {
2595 	.attach_chip = marvell_nand_attach_chip,
2596 	.exec_op = marvell_nfc_exec_op,
2597 	.setup_interface = marvell_nfc_setup_interface,
2598 };
2599 
2600 static int marvell_nand_chip_init(struct device *dev, struct marvell_nfc *nfc,
2601 				  struct device_node *np)
2602 {
2603 	struct pxa3xx_nand_platform_data *pdata = dev_get_platdata(dev);
2604 	struct marvell_nand_chip *marvell_nand;
2605 	struct mtd_info *mtd;
2606 	struct nand_chip *chip;
2607 	int nsels, ret, i;
2608 	u32 cs, rb;
2609 
2610 	/*
2611 	 * The legacy "num-cs" property indicates the number of CS on the only
2612 	 * chip connected to the controller (legacy bindings does not support
2613 	 * more than one chip). The CS and RB pins are always the #0.
2614 	 *
2615 	 * When not using legacy bindings, a couple of "reg" and "nand-rb"
2616 	 * properties must be filled. For each chip, expressed as a subnode,
2617 	 * "reg" points to the CS lines and "nand-rb" to the RB line.
2618 	 */
2619 	if (pdata || nfc->caps->legacy_of_bindings) {
2620 		nsels = 1;
2621 	} else {
2622 		nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
2623 		if (nsels <= 0) {
2624 			dev_err(dev, "missing/invalid reg property\n");
2625 			return -EINVAL;
2626 		}
2627 	}
2628 
2629 	/* Alloc the nand chip structure */
2630 	marvell_nand = devm_kzalloc(dev,
2631 				    struct_size(marvell_nand, sels, nsels),
2632 				    GFP_KERNEL);
2633 	if (!marvell_nand) {
2634 		dev_err(dev, "could not allocate chip structure\n");
2635 		return -ENOMEM;
2636 	}
2637 
2638 	marvell_nand->nsels = nsels;
2639 	marvell_nand->selected_die = -1;
2640 
2641 	for (i = 0; i < nsels; i++) {
2642 		if (pdata || nfc->caps->legacy_of_bindings) {
2643 			/*
2644 			 * Legacy bindings use the CS lines in natural
2645 			 * order (0, 1, ...)
2646 			 */
2647 			cs = i;
2648 		} else {
2649 			/* Retrieve CS id */
2650 			ret = of_property_read_u32_index(np, "reg", i, &cs);
2651 			if (ret) {
2652 				dev_err(dev, "could not retrieve reg property: %d\n",
2653 					ret);
2654 				return ret;
2655 			}
2656 		}
2657 
2658 		if (cs >= nfc->caps->max_cs_nb) {
2659 			dev_err(dev, "invalid reg value: %u (max CS = %d)\n",
2660 				cs, nfc->caps->max_cs_nb);
2661 			return -EINVAL;
2662 		}
2663 
2664 		if (test_and_set_bit(cs, &nfc->assigned_cs)) {
2665 			dev_err(dev, "CS %d already assigned\n", cs);
2666 			return -EINVAL;
2667 		}
2668 
2669 		/*
2670 		 * The cs variable represents the chip select id, which must be
2671 		 * converted in bit fields for NDCB0 and NDCB2 to select the
2672 		 * right chip. Unfortunately, due to a lack of information on
2673 		 * the subject and incoherent documentation, the user should not
2674 		 * use CS1 and CS3 at all as asserting them is not supported in
2675 		 * a reliable way (due to multiplexing inside ADDR5 field).
2676 		 */
2677 		marvell_nand->sels[i].cs = cs;
2678 		switch (cs) {
2679 		case 0:
2680 		case 2:
2681 			marvell_nand->sels[i].ndcb0_csel = 0;
2682 			break;
2683 		case 1:
2684 		case 3:
2685 			marvell_nand->sels[i].ndcb0_csel = NDCB0_CSEL;
2686 			break;
2687 		default:
2688 			return -EINVAL;
2689 		}
2690 
2691 		/* Retrieve RB id */
2692 		if (pdata || nfc->caps->legacy_of_bindings) {
2693 			/* Legacy bindings always use RB #0 */
2694 			rb = 0;
2695 		} else {
2696 			ret = of_property_read_u32_index(np, "nand-rb", i,
2697 							 &rb);
2698 			if (ret) {
2699 				dev_err(dev,
2700 					"could not retrieve RB property: %d\n",
2701 					ret);
2702 				return ret;
2703 			}
2704 		}
2705 
2706 		if (rb >= nfc->caps->max_rb_nb) {
2707 			dev_err(dev, "invalid reg value: %u (max RB = %d)\n",
2708 				rb, nfc->caps->max_rb_nb);
2709 			return -EINVAL;
2710 		}
2711 
2712 		marvell_nand->sels[i].rb = rb;
2713 	}
2714 
2715 	chip = &marvell_nand->chip;
2716 	chip->controller = &nfc->controller;
2717 	nand_set_flash_node(chip, np);
2718 
2719 	if (of_property_read_bool(np, "marvell,nand-keep-config"))
2720 		chip->options |= NAND_KEEP_TIMINGS;
2721 
2722 	mtd = nand_to_mtd(chip);
2723 	mtd->dev.parent = dev;
2724 
2725 	/*
2726 	 * Save a reference value for timing registers before
2727 	 * ->setup_interface() is called.
2728 	 */
2729 	marvell_nand->ndtr0 = readl_relaxed(nfc->regs + NDTR0);
2730 	marvell_nand->ndtr1 = readl_relaxed(nfc->regs + NDTR1);
2731 
2732 	chip->options |= NAND_BUSWIDTH_AUTO;
2733 
2734 	ret = nand_scan(chip, marvell_nand->nsels);
2735 	if (ret) {
2736 		dev_err(dev, "could not scan the nand chip\n");
2737 		return ret;
2738 	}
2739 
2740 	if (pdata)
2741 		/* Legacy bindings support only one chip */
2742 		ret = mtd_device_register(mtd, pdata->parts, pdata->nr_parts);
2743 	else
2744 		ret = mtd_device_register(mtd, NULL, 0);
2745 	if (ret) {
2746 		dev_err(dev, "failed to register mtd device: %d\n", ret);
2747 		nand_cleanup(chip);
2748 		return ret;
2749 	}
2750 
2751 	list_add_tail(&marvell_nand->node, &nfc->chips);
2752 
2753 	return 0;
2754 }
2755 
2756 static void marvell_nand_chips_cleanup(struct marvell_nfc *nfc)
2757 {
2758 	struct marvell_nand_chip *entry, *temp;
2759 	struct nand_chip *chip;
2760 	int ret;
2761 
2762 	list_for_each_entry_safe(entry, temp, &nfc->chips, node) {
2763 		chip = &entry->chip;
2764 		ret = mtd_device_unregister(nand_to_mtd(chip));
2765 		WARN_ON(ret);
2766 		nand_cleanup(chip);
2767 		list_del(&entry->node);
2768 	}
2769 }
2770 
2771 static int marvell_nand_chips_init(struct device *dev, struct marvell_nfc *nfc)
2772 {
2773 	struct device_node *np = dev->of_node;
2774 	struct device_node *nand_np;
2775 	int max_cs = nfc->caps->max_cs_nb;
2776 	int nchips;
2777 	int ret;
2778 
2779 	if (!np)
2780 		nchips = 1;
2781 	else
2782 		nchips = of_get_child_count(np);
2783 
2784 	if (nchips > max_cs) {
2785 		dev_err(dev, "too many NAND chips: %d (max = %d CS)\n", nchips,
2786 			max_cs);
2787 		return -EINVAL;
2788 	}
2789 
2790 	/*
2791 	 * Legacy bindings do not use child nodes to exhibit NAND chip
2792 	 * properties and layout. Instead, NAND properties are mixed with the
2793 	 * controller ones, and partitions are defined as direct subnodes of the
2794 	 * NAND controller node.
2795 	 */
2796 	if (nfc->caps->legacy_of_bindings) {
2797 		ret = marvell_nand_chip_init(dev, nfc, np);
2798 		return ret;
2799 	}
2800 
2801 	for_each_child_of_node(np, nand_np) {
2802 		ret = marvell_nand_chip_init(dev, nfc, nand_np);
2803 		if (ret) {
2804 			of_node_put(nand_np);
2805 			goto cleanup_chips;
2806 		}
2807 	}
2808 
2809 	return 0;
2810 
2811 cleanup_chips:
2812 	marvell_nand_chips_cleanup(nfc);
2813 
2814 	return ret;
2815 }
2816 
2817 static int marvell_nfc_init_dma(struct marvell_nfc *nfc)
2818 {
2819 	struct platform_device *pdev = container_of(nfc->dev,
2820 						    struct platform_device,
2821 						    dev);
2822 	struct dma_slave_config config = {};
2823 	struct resource *r;
2824 	int ret;
2825 
2826 	if (!IS_ENABLED(CONFIG_PXA_DMA)) {
2827 		dev_warn(nfc->dev,
2828 			 "DMA not enabled in configuration\n");
2829 		return -ENOTSUPP;
2830 	}
2831 
2832 	ret = dma_set_mask_and_coherent(nfc->dev, DMA_BIT_MASK(32));
2833 	if (ret)
2834 		return ret;
2835 
2836 	nfc->dma_chan =	dma_request_chan(nfc->dev, "data");
2837 	if (IS_ERR(nfc->dma_chan)) {
2838 		ret = PTR_ERR(nfc->dma_chan);
2839 		nfc->dma_chan = NULL;
2840 		return dev_err_probe(nfc->dev, ret, "DMA channel request failed\n");
2841 	}
2842 
2843 	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2844 	if (!r) {
2845 		ret = -ENXIO;
2846 		goto release_channel;
2847 	}
2848 
2849 	config.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2850 	config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2851 	config.src_addr = r->start + NDDB;
2852 	config.dst_addr = r->start + NDDB;
2853 	config.src_maxburst = 32;
2854 	config.dst_maxburst = 32;
2855 	ret = dmaengine_slave_config(nfc->dma_chan, &config);
2856 	if (ret < 0) {
2857 		dev_err(nfc->dev, "Failed to configure DMA channel\n");
2858 		goto release_channel;
2859 	}
2860 
2861 	/*
2862 	 * DMA must act on length multiple of 32 and this length may be
2863 	 * bigger than the destination buffer. Use this buffer instead
2864 	 * for DMA transfers and then copy the desired amount of data to
2865 	 * the provided buffer.
2866 	 */
2867 	nfc->dma_buf = kmalloc(MAX_CHUNK_SIZE, GFP_KERNEL | GFP_DMA);
2868 	if (!nfc->dma_buf) {
2869 		ret = -ENOMEM;
2870 		goto release_channel;
2871 	}
2872 
2873 	nfc->use_dma = true;
2874 
2875 	return 0;
2876 
2877 release_channel:
2878 	dma_release_channel(nfc->dma_chan);
2879 	nfc->dma_chan = NULL;
2880 
2881 	return ret;
2882 }
2883 
2884 static void marvell_nfc_reset(struct marvell_nfc *nfc)
2885 {
2886 	/*
2887 	 * ECC operations and interruptions are only enabled when specifically
2888 	 * needed. ECC shall not be activated in the early stages (fails probe).
2889 	 * Arbiter flag, even if marked as "reserved", must be set (empirical).
2890 	 * SPARE_EN bit must always be set or ECC bytes will not be at the same
2891 	 * offset in the read page and this will fail the protection.
2892 	 */
2893 	writel_relaxed(NDCR_ALL_INT | NDCR_ND_ARB_EN | NDCR_SPARE_EN |
2894 		       NDCR_RD_ID_CNT(NFCV1_READID_LEN), nfc->regs + NDCR);
2895 	writel_relaxed(0xFFFFFFFF, nfc->regs + NDSR);
2896 	writel_relaxed(0, nfc->regs + NDECCCTRL);
2897 }
2898 
2899 static int marvell_nfc_init(struct marvell_nfc *nfc)
2900 {
2901 	struct device_node *np = nfc->dev->of_node;
2902 
2903 	/*
2904 	 * Some SoCs like A7k/A8k need to enable manually the NAND
2905 	 * controller, gated clocks and reset bits to avoid being bootloader
2906 	 * dependent. This is done through the use of the System Functions
2907 	 * registers.
2908 	 */
2909 	if (nfc->caps->need_system_controller) {
2910 		struct regmap *sysctrl_base =
2911 			syscon_regmap_lookup_by_phandle(np,
2912 							"marvell,system-controller");
2913 
2914 		if (IS_ERR(sysctrl_base))
2915 			return PTR_ERR(sysctrl_base);
2916 
2917 		regmap_write(sysctrl_base, GENCONF_SOC_DEVICE_MUX,
2918 			     GENCONF_SOC_DEVICE_MUX_NFC_EN |
2919 			     GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST |
2920 			     GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST |
2921 			     GENCONF_SOC_DEVICE_MUX_NFC_INT_EN |
2922 			     GENCONF_SOC_DEVICE_MUX_NFC_DEVBUS_ARB_EN);
2923 
2924 		regmap_update_bits(sysctrl_base, GENCONF_CLK_GATING_CTRL,
2925 				   GENCONF_CLK_GATING_CTRL_ND_GATE,
2926 				   GENCONF_CLK_GATING_CTRL_ND_GATE);
2927 	}
2928 
2929 	/* Configure the DMA if appropriate */
2930 	if (!nfc->caps->is_nfcv2)
2931 		marvell_nfc_init_dma(nfc);
2932 
2933 	marvell_nfc_reset(nfc);
2934 
2935 	return 0;
2936 }
2937 
2938 static int marvell_nfc_probe(struct platform_device *pdev)
2939 {
2940 	struct device *dev = &pdev->dev;
2941 	struct marvell_nfc *nfc;
2942 	int ret;
2943 	int irq;
2944 
2945 	nfc = devm_kzalloc(&pdev->dev, sizeof(struct marvell_nfc),
2946 			   GFP_KERNEL);
2947 	if (!nfc)
2948 		return -ENOMEM;
2949 
2950 	nfc->dev = dev;
2951 	nand_controller_init(&nfc->controller);
2952 	nfc->controller.ops = &marvell_nand_controller_ops;
2953 	INIT_LIST_HEAD(&nfc->chips);
2954 
2955 	nfc->regs = devm_platform_ioremap_resource(pdev, 0);
2956 	if (IS_ERR(nfc->regs))
2957 		return PTR_ERR(nfc->regs);
2958 
2959 	irq = platform_get_irq(pdev, 0);
2960 	if (irq < 0)
2961 		return irq;
2962 
2963 	nfc->core_clk = devm_clk_get(&pdev->dev, "core");
2964 
2965 	/* Managed the legacy case (when the first clock was not named) */
2966 	if (nfc->core_clk == ERR_PTR(-ENOENT))
2967 		nfc->core_clk = devm_clk_get(&pdev->dev, NULL);
2968 
2969 	if (IS_ERR(nfc->core_clk))
2970 		return PTR_ERR(nfc->core_clk);
2971 
2972 	ret = clk_prepare_enable(nfc->core_clk);
2973 	if (ret)
2974 		return ret;
2975 
2976 	nfc->reg_clk = devm_clk_get(&pdev->dev, "reg");
2977 	if (IS_ERR(nfc->reg_clk)) {
2978 		if (PTR_ERR(nfc->reg_clk) != -ENOENT) {
2979 			ret = PTR_ERR(nfc->reg_clk);
2980 			goto unprepare_core_clk;
2981 		}
2982 
2983 		nfc->reg_clk = NULL;
2984 	}
2985 
2986 	ret = clk_prepare_enable(nfc->reg_clk);
2987 	if (ret)
2988 		goto unprepare_core_clk;
2989 
2990 	marvell_nfc_disable_int(nfc, NDCR_ALL_INT);
2991 	marvell_nfc_clear_int(nfc, NDCR_ALL_INT);
2992 	ret = devm_request_irq(dev, irq, marvell_nfc_isr,
2993 			       0, "marvell-nfc", nfc);
2994 	if (ret)
2995 		goto unprepare_reg_clk;
2996 
2997 	/* Get NAND controller capabilities */
2998 	if (pdev->id_entry)
2999 		nfc->caps = (void *)pdev->id_entry->driver_data;
3000 	else
3001 		nfc->caps = of_device_get_match_data(&pdev->dev);
3002 
3003 	if (!nfc->caps) {
3004 		dev_err(dev, "Could not retrieve NFC caps\n");
3005 		ret = -EINVAL;
3006 		goto unprepare_reg_clk;
3007 	}
3008 
3009 	/* Init the controller and then probe the chips */
3010 	ret = marvell_nfc_init(nfc);
3011 	if (ret)
3012 		goto unprepare_reg_clk;
3013 
3014 	platform_set_drvdata(pdev, nfc);
3015 
3016 	ret = marvell_nand_chips_init(dev, nfc);
3017 	if (ret)
3018 		goto release_dma;
3019 
3020 	return 0;
3021 
3022 release_dma:
3023 	if (nfc->use_dma)
3024 		dma_release_channel(nfc->dma_chan);
3025 unprepare_reg_clk:
3026 	clk_disable_unprepare(nfc->reg_clk);
3027 unprepare_core_clk:
3028 	clk_disable_unprepare(nfc->core_clk);
3029 
3030 	return ret;
3031 }
3032 
3033 static void marvell_nfc_remove(struct platform_device *pdev)
3034 {
3035 	struct marvell_nfc *nfc = platform_get_drvdata(pdev);
3036 
3037 	marvell_nand_chips_cleanup(nfc);
3038 
3039 	if (nfc->use_dma) {
3040 		dmaengine_terminate_all(nfc->dma_chan);
3041 		dma_release_channel(nfc->dma_chan);
3042 	}
3043 
3044 	clk_disable_unprepare(nfc->reg_clk);
3045 	clk_disable_unprepare(nfc->core_clk);
3046 }
3047 
3048 static int __maybe_unused marvell_nfc_suspend(struct device *dev)
3049 {
3050 	struct marvell_nfc *nfc = dev_get_drvdata(dev);
3051 	struct marvell_nand_chip *chip;
3052 
3053 	list_for_each_entry(chip, &nfc->chips, node)
3054 		marvell_nfc_wait_ndrun(&chip->chip);
3055 
3056 	clk_disable_unprepare(nfc->reg_clk);
3057 	clk_disable_unprepare(nfc->core_clk);
3058 
3059 	return 0;
3060 }
3061 
3062 static int __maybe_unused marvell_nfc_resume(struct device *dev)
3063 {
3064 	struct marvell_nfc *nfc = dev_get_drvdata(dev);
3065 	int ret;
3066 
3067 	ret = clk_prepare_enable(nfc->core_clk);
3068 	if (ret < 0)
3069 		return ret;
3070 
3071 	ret = clk_prepare_enable(nfc->reg_clk);
3072 	if (ret < 0) {
3073 		clk_disable_unprepare(nfc->core_clk);
3074 		return ret;
3075 	}
3076 
3077 	/*
3078 	 * Reset nfc->selected_chip so the next command will cause the timing
3079 	 * registers to be restored in marvell_nfc_select_target().
3080 	 */
3081 	nfc->selected_chip = NULL;
3082 
3083 	/* Reset registers that have lost their contents */
3084 	marvell_nfc_reset(nfc);
3085 
3086 	return 0;
3087 }
3088 
3089 static const struct dev_pm_ops marvell_nfc_pm_ops = {
3090 	SET_SYSTEM_SLEEP_PM_OPS(marvell_nfc_suspend, marvell_nfc_resume)
3091 };
3092 
3093 static const struct marvell_nfc_caps marvell_armada_8k_nfc_caps = {
3094 	.max_cs_nb = 4,
3095 	.max_rb_nb = 2,
3096 	.need_system_controller = true,
3097 	.is_nfcv2 = true,
3098 };
3099 
3100 static const struct marvell_nfc_caps marvell_ac5_caps = {
3101 	.max_cs_nb = 2,
3102 	.max_rb_nb = 1,
3103 	.is_nfcv2 = true,
3104 	.max_mode_number = 3,
3105 };
3106 
3107 static const struct marvell_nfc_caps marvell_armada370_nfc_caps = {
3108 	.max_cs_nb = 4,
3109 	.max_rb_nb = 2,
3110 	.is_nfcv2 = true,
3111 };
3112 
3113 static const struct marvell_nfc_caps marvell_pxa3xx_nfc_caps = {
3114 	.max_cs_nb = 2,
3115 	.max_rb_nb = 1,
3116 	.use_dma = true,
3117 };
3118 
3119 static const struct marvell_nfc_caps marvell_armada_8k_nfc_legacy_caps = {
3120 	.max_cs_nb = 4,
3121 	.max_rb_nb = 2,
3122 	.need_system_controller = true,
3123 	.legacy_of_bindings = true,
3124 	.is_nfcv2 = true,
3125 };
3126 
3127 static const struct marvell_nfc_caps marvell_armada370_nfc_legacy_caps = {
3128 	.max_cs_nb = 4,
3129 	.max_rb_nb = 2,
3130 	.legacy_of_bindings = true,
3131 	.is_nfcv2 = true,
3132 };
3133 
3134 static const struct marvell_nfc_caps marvell_pxa3xx_nfc_legacy_caps = {
3135 	.max_cs_nb = 2,
3136 	.max_rb_nb = 1,
3137 	.legacy_of_bindings = true,
3138 	.use_dma = true,
3139 };
3140 
3141 static const struct platform_device_id marvell_nfc_platform_ids[] = {
3142 	{
3143 		.name = "pxa3xx-nand",
3144 		.driver_data = (kernel_ulong_t)&marvell_pxa3xx_nfc_legacy_caps,
3145 	},
3146 	{ /* sentinel */ },
3147 };
3148 MODULE_DEVICE_TABLE(platform, marvell_nfc_platform_ids);
3149 
3150 static const struct of_device_id marvell_nfc_of_ids[] = {
3151 	{
3152 		.compatible = "marvell,armada-8k-nand-controller",
3153 		.data = &marvell_armada_8k_nfc_caps,
3154 	},
3155 	{
3156 		.compatible = "marvell,ac5-nand-controller",
3157 		.data = &marvell_ac5_caps,
3158 	},
3159 	{
3160 		.compatible = "marvell,armada370-nand-controller",
3161 		.data = &marvell_armada370_nfc_caps,
3162 	},
3163 	{
3164 		.compatible = "marvell,pxa3xx-nand-controller",
3165 		.data = &marvell_pxa3xx_nfc_caps,
3166 	},
3167 	/* Support for old/deprecated bindings: */
3168 	{
3169 		.compatible = "marvell,armada-8k-nand",
3170 		.data = &marvell_armada_8k_nfc_legacy_caps,
3171 	},
3172 	{
3173 		.compatible = "marvell,armada370-nand",
3174 		.data = &marvell_armada370_nfc_legacy_caps,
3175 	},
3176 	{
3177 		.compatible = "marvell,pxa3xx-nand",
3178 		.data = &marvell_pxa3xx_nfc_legacy_caps,
3179 	},
3180 	{ /* sentinel */ },
3181 };
3182 MODULE_DEVICE_TABLE(of, marvell_nfc_of_ids);
3183 
3184 static struct platform_driver marvell_nfc_driver = {
3185 	.driver	= {
3186 		.name		= "marvell-nfc",
3187 		.of_match_table = marvell_nfc_of_ids,
3188 		.pm		= &marvell_nfc_pm_ops,
3189 	},
3190 	.id_table = marvell_nfc_platform_ids,
3191 	.probe = marvell_nfc_probe,
3192 	.remove_new = marvell_nfc_remove,
3193 };
3194 module_platform_driver(marvell_nfc_driver);
3195 
3196 MODULE_LICENSE("GPL");
3197 MODULE_DESCRIPTION("Marvell NAND controller driver");
3198