1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Marvell NAND flash controller driver
4  *
5  * Copyright (C) 2017 Marvell
6  * Author: Miquel RAYNAL <miquel.raynal@free-electrons.com>
7  *
8  *
9  * This NAND controller driver handles two versions of the hardware,
10  * one is called NFCv1 and is available on PXA SoCs and the other is
11  * called NFCv2 and is available on Armada SoCs.
12  *
13  * The main visible difference is that NFCv1 only has Hamming ECC
14  * capabilities, while NFCv2 also embeds a BCH ECC engine. Also, DMA
15  * is not used with NFCv2.
16  *
17  * The ECC layouts are depicted in details in Marvell AN-379, but here
18  * is a brief description.
19  *
20  * When using Hamming, the data is split in 512B chunks (either 1, 2
21  * or 4) and each chunk will have its own ECC "digest" of 6B at the
22  * beginning of the OOB area and eventually the remaining free OOB
23  * bytes (also called "spare" bytes in the driver). This engine
24  * corrects up to 1 bit per chunk and detects reliably an error if
25  * there are at most 2 bitflips. Here is the page layout used by the
26  * controller when Hamming is chosen:
27  *
28  * +-------------------------------------------------------------+
29  * | Data 1 | ... | Data N | ECC 1 | ... | ECCN | Free OOB bytes |
30  * +-------------------------------------------------------------+
31  *
32  * When using the BCH engine, there are N identical (data + free OOB +
33  * ECC) sections and potentially an extra one to deal with
34  * configurations where the chosen (data + free OOB + ECC) sizes do
35  * not align with the page (data + OOB) size. ECC bytes are always
36  * 30B per ECC chunk. Here is the page layout used by the controller
37  * when BCH is chosen:
38  *
39  * +-----------------------------------------
40  * | Data 1 | Free OOB bytes 1 | ECC 1 | ...
41  * +-----------------------------------------
42  *
43  *      -------------------------------------------
44  *       ... | Data N | Free OOB bytes N | ECC N |
45  *      -------------------------------------------
46  *
47  *           --------------------------------------------+
48  *            Last Data | Last Free OOB bytes | Last ECC |
49  *           --------------------------------------------+
50  *
51  * In both cases, the layout seen by the user is always: all data
52  * first, then all free OOB bytes and finally all ECC bytes. With BCH,
53  * ECC bytes are 30B long and are padded with 0xFF to align on 32
54  * bytes.
55  *
56  * The controller has certain limitations that are handled by the
57  * driver:
58  *   - It can only read 2k at a time. To overcome this limitation, the
59  *     driver issues data cycles on the bus, without issuing new
60  *     CMD + ADDR cycles. The Marvell term is "naked" operations.
61  *   - The ECC strength in BCH mode cannot be tuned. It is fixed 16
62  *     bits. What can be tuned is the ECC block size as long as it
63  *     stays between 512B and 2kiB. It's usually chosen based on the
64  *     chip ECC requirements. For instance, using 2kiB ECC chunks
65  *     provides 4b/512B correctability.
66  *   - The controller will always treat data bytes, free OOB bytes
67  *     and ECC bytes in that order, no matter what the real layout is
68  *     (which is usually all data then all OOB bytes). The
69  *     marvell_nfc_layouts array below contains the currently
70  *     supported layouts.
71  *   - Because of these weird layouts, the Bad Block Markers can be
72  *     located in data section. In this case, the NAND_BBT_NO_OOB_BBM
73  *     option must be set to prevent scanning/writing bad block
74  *     markers.
75  */
76 
77 #include <linux/module.h>
78 #include <linux/clk.h>
79 #include <linux/mtd/rawnand.h>
80 #include <linux/of_platform.h>
81 #include <linux/iopoll.h>
82 #include <linux/interrupt.h>
83 #include <linux/slab.h>
84 #include <linux/mfd/syscon.h>
85 #include <linux/regmap.h>
86 #include <asm/unaligned.h>
87 
88 #include <linux/dmaengine.h>
89 #include <linux/dma-mapping.h>
90 #include <linux/dma/pxa-dma.h>
91 #include <linux/platform_data/mtd-nand-pxa3xx.h>
92 
93 /* Data FIFO granularity, FIFO reads/writes must be a multiple of this length */
94 #define FIFO_DEPTH		8
95 #define FIFO_REP(x)		(x / sizeof(u32))
96 #define BCH_SEQ_READS		(32 / FIFO_DEPTH)
97 /* NFC does not support transfers of larger chunks at a time */
98 #define MAX_CHUNK_SIZE		2112
99 /* NFCv1 cannot read more that 7 bytes of ID */
100 #define NFCV1_READID_LEN	7
101 /* Polling is done at a pace of POLL_PERIOD us until POLL_TIMEOUT is reached */
102 #define POLL_PERIOD		0
103 #define POLL_TIMEOUT		100000
104 /* Interrupt maximum wait period in ms */
105 #define IRQ_TIMEOUT		1000
106 /* Latency in clock cycles between SoC pins and NFC logic */
107 #define MIN_RD_DEL_CNT		3
108 /* Maximum number of contiguous address cycles */
109 #define MAX_ADDRESS_CYC_NFCV1	5
110 #define MAX_ADDRESS_CYC_NFCV2	7
111 /* System control registers/bits to enable the NAND controller on some SoCs */
112 #define GENCONF_SOC_DEVICE_MUX	0x208
113 #define GENCONF_SOC_DEVICE_MUX_NFC_EN BIT(0)
114 #define GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST BIT(20)
115 #define GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST BIT(21)
116 #define GENCONF_SOC_DEVICE_MUX_NFC_INT_EN BIT(25)
117 #define GENCONF_SOC_DEVICE_MUX_NFC_DEVBUS_ARB_EN BIT(27)
118 #define GENCONF_CLK_GATING_CTRL	0x220
119 #define GENCONF_CLK_GATING_CTRL_ND_GATE BIT(2)
120 #define GENCONF_ND_CLK_CTRL	0x700
121 #define GENCONF_ND_CLK_CTRL_EN	BIT(0)
122 
123 /* NAND controller data flash control register */
124 #define NDCR			0x00
125 #define NDCR_ALL_INT		GENMASK(11, 0)
126 #define NDCR_CS1_CMDDM		BIT(7)
127 #define NDCR_CS0_CMDDM		BIT(8)
128 #define NDCR_RDYM		BIT(11)
129 #define NDCR_ND_ARB_EN		BIT(12)
130 #define NDCR_RA_START		BIT(15)
131 #define NDCR_RD_ID_CNT(x)	(min_t(unsigned int, x, 0x7) << 16)
132 #define NDCR_PAGE_SZ(x)		(x >= 2048 ? BIT(24) : 0)
133 #define NDCR_DWIDTH_M		BIT(26)
134 #define NDCR_DWIDTH_C		BIT(27)
135 #define NDCR_ND_RUN		BIT(28)
136 #define NDCR_DMA_EN		BIT(29)
137 #define NDCR_ECC_EN		BIT(30)
138 #define NDCR_SPARE_EN		BIT(31)
139 #define NDCR_GENERIC_FIELDS_MASK (~(NDCR_RA_START | NDCR_PAGE_SZ(2048) | \
140 				    NDCR_DWIDTH_M | NDCR_DWIDTH_C))
141 
142 /* NAND interface timing parameter 0 register */
143 #define NDTR0			0x04
144 #define NDTR0_TRP(x)		((min_t(unsigned int, x, 0xF) & 0x7) << 0)
145 #define NDTR0_TRH(x)		(min_t(unsigned int, x, 0x7) << 3)
146 #define NDTR0_ETRP(x)		((min_t(unsigned int, x, 0xF) & 0x8) << 3)
147 #define NDTR0_SEL_NRE_EDGE	BIT(7)
148 #define NDTR0_TWP(x)		(min_t(unsigned int, x, 0x7) << 8)
149 #define NDTR0_TWH(x)		(min_t(unsigned int, x, 0x7) << 11)
150 #define NDTR0_TCS(x)		(min_t(unsigned int, x, 0x7) << 16)
151 #define NDTR0_TCH(x)		(min_t(unsigned int, x, 0x7) << 19)
152 #define NDTR0_RD_CNT_DEL(x)	(min_t(unsigned int, x, 0xF) << 22)
153 #define NDTR0_SELCNTR		BIT(26)
154 #define NDTR0_TADL(x)		(min_t(unsigned int, x, 0x1F) << 27)
155 
156 /* NAND interface timing parameter 1 register */
157 #define NDTR1			0x0C
158 #define NDTR1_TAR(x)		(min_t(unsigned int, x, 0xF) << 0)
159 #define NDTR1_TWHR(x)		(min_t(unsigned int, x, 0xF) << 4)
160 #define NDTR1_TRHW(x)		(min_t(unsigned int, x / 16, 0x3) << 8)
161 #define NDTR1_PRESCALE		BIT(14)
162 #define NDTR1_WAIT_MODE		BIT(15)
163 #define NDTR1_TR(x)		(min_t(unsigned int, x, 0xFFFF) << 16)
164 
165 /* NAND controller status register */
166 #define NDSR			0x14
167 #define NDSR_WRCMDREQ		BIT(0)
168 #define NDSR_RDDREQ		BIT(1)
169 #define NDSR_WRDREQ		BIT(2)
170 #define NDSR_CORERR		BIT(3)
171 #define NDSR_UNCERR		BIT(4)
172 #define NDSR_CMDD(cs)		BIT(8 - cs)
173 #define NDSR_RDY(rb)		BIT(11 + rb)
174 #define NDSR_ERRCNT(x)		((x >> 16) & 0x1F)
175 
176 /* NAND ECC control register */
177 #define NDECCCTRL		0x28
178 #define NDECCCTRL_BCH_EN	BIT(0)
179 
180 /* NAND controller data buffer register */
181 #define NDDB			0x40
182 
183 /* NAND controller command buffer 0 register */
184 #define NDCB0			0x48
185 #define NDCB0_CMD1(x)		((x & 0xFF) << 0)
186 #define NDCB0_CMD2(x)		((x & 0xFF) << 8)
187 #define NDCB0_ADDR_CYC(x)	((x & 0x7) << 16)
188 #define NDCB0_ADDR_GET_NUM_CYC(x) (((x) >> 16) & 0x7)
189 #define NDCB0_DBC		BIT(19)
190 #define NDCB0_CMD_TYPE(x)	((x & 0x7) << 21)
191 #define NDCB0_CSEL		BIT(24)
192 #define NDCB0_RDY_BYP		BIT(27)
193 #define NDCB0_LEN_OVRD		BIT(28)
194 #define NDCB0_CMD_XTYPE(x)	((x & 0x7) << 29)
195 
196 /* NAND controller command buffer 1 register */
197 #define NDCB1			0x4C
198 #define NDCB1_COLS(x)		((x & 0xFFFF) << 0)
199 #define NDCB1_ADDRS_PAGE(x)	(x << 16)
200 
201 /* NAND controller command buffer 2 register */
202 #define NDCB2			0x50
203 #define NDCB2_ADDR5_PAGE(x)	(((x >> 16) & 0xFF) << 0)
204 #define NDCB2_ADDR5_CYC(x)	((x & 0xFF) << 0)
205 
206 /* NAND controller command buffer 3 register */
207 #define NDCB3			0x54
208 #define NDCB3_ADDR6_CYC(x)	((x & 0xFF) << 16)
209 #define NDCB3_ADDR7_CYC(x)	((x & 0xFF) << 24)
210 
211 /* NAND controller command buffer 0 register 'type' and 'xtype' fields */
212 #define TYPE_READ		0
213 #define TYPE_WRITE		1
214 #define TYPE_ERASE		2
215 #define TYPE_READ_ID		3
216 #define TYPE_STATUS		4
217 #define TYPE_RESET		5
218 #define TYPE_NAKED_CMD		6
219 #define TYPE_NAKED_ADDR		7
220 #define TYPE_MASK		7
221 #define XTYPE_MONOLITHIC_RW	0
222 #define XTYPE_LAST_NAKED_RW	1
223 #define XTYPE_FINAL_COMMAND	3
224 #define XTYPE_READ		4
225 #define XTYPE_WRITE_DISPATCH	4
226 #define XTYPE_NAKED_RW		5
227 #define XTYPE_COMMAND_DISPATCH	6
228 #define XTYPE_MASK		7
229 
230 /**
231  * struct marvell_hw_ecc_layout - layout of Marvell ECC
232  *
233  * Marvell ECC engine works differently than the others, in order to limit the
234  * size of the IP, hardware engineers chose to set a fixed strength at 16 bits
235  * per subpage, and depending on a the desired strength needed by the NAND chip,
236  * a particular layout mixing data/spare/ecc is defined, with a possible last
237  * chunk smaller that the others.
238  *
239  * @writesize:		Full page size on which the layout applies
240  * @chunk:		Desired ECC chunk size on which the layout applies
241  * @strength:		Desired ECC strength (per chunk size bytes) on which the
242  *			layout applies
243  * @nchunks:		Total number of chunks
244  * @full_chunk_cnt:	Number of full-sized chunks, which is the number of
245  *			repetitions of the pattern:
246  *			(data_bytes + spare_bytes + ecc_bytes).
247  * @data_bytes:		Number of data bytes per chunk
248  * @spare_bytes:	Number of spare bytes per chunk
249  * @ecc_bytes:		Number of ecc bytes per chunk
250  * @last_data_bytes:	Number of data bytes in the last chunk
251  * @last_spare_bytes:	Number of spare bytes in the last chunk
252  * @last_ecc_bytes:	Number of ecc bytes in the last chunk
253  */
254 struct marvell_hw_ecc_layout {
255 	/* Constraints */
256 	int writesize;
257 	int chunk;
258 	int strength;
259 	/* Corresponding layout */
260 	int nchunks;
261 	int full_chunk_cnt;
262 	int data_bytes;
263 	int spare_bytes;
264 	int ecc_bytes;
265 	int last_data_bytes;
266 	int last_spare_bytes;
267 	int last_ecc_bytes;
268 };
269 
270 #define MARVELL_LAYOUT(ws, dc, ds, nc, fcc, db, sb, eb, ldb, lsb, leb)	\
271 	{								\
272 		.writesize = ws,					\
273 		.chunk = dc,						\
274 		.strength = ds,						\
275 		.nchunks = nc,						\
276 		.full_chunk_cnt = fcc,					\
277 		.data_bytes = db,					\
278 		.spare_bytes = sb,					\
279 		.ecc_bytes = eb,					\
280 		.last_data_bytes = ldb,					\
281 		.last_spare_bytes = lsb,				\
282 		.last_ecc_bytes = leb,					\
283 	}
284 
285 /* Layouts explained in AN-379_Marvell_SoC_NFC_ECC */
286 static const struct marvell_hw_ecc_layout marvell_nfc_layouts[] = {
287 	MARVELL_LAYOUT(  512,   512,  1,  1,  1,  512,  8,  8,  0,  0,  0),
288 	MARVELL_LAYOUT( 2048,   512,  1,  1,  1, 2048, 40, 24,  0,  0,  0),
289 	MARVELL_LAYOUT( 2048,   512,  4,  1,  1, 2048, 32, 30,  0,  0,  0),
290 	MARVELL_LAYOUT( 2048,   512,  8,  2,  1, 1024,  0, 30,1024,32, 30),
291 	MARVELL_LAYOUT( 2048,   512,  8,  2,  1, 1024,  0, 30,1024,64, 30),
292 	MARVELL_LAYOUT( 2048,   512,  12, 3,  2, 704,   0, 30,640,  0, 30),
293 	MARVELL_LAYOUT( 2048,   512,  16, 5,  4, 512,   0, 30,  0, 32, 30),
294 	MARVELL_LAYOUT( 4096,   512,  4,  2,  2, 2048, 32, 30,  0,  0,  0),
295 	MARVELL_LAYOUT( 4096,   512,  8,  5,  4, 1024,  0, 30,  0, 64, 30),
296 	MARVELL_LAYOUT( 4096,   512,  12, 6,  5, 704,   0, 30,576, 32, 30),
297 	MARVELL_LAYOUT( 4096,   512,  16, 9,  8, 512,   0, 30,  0, 32, 30),
298 	MARVELL_LAYOUT( 8192,   512,  4,  4,  4, 2048,  0, 30,  0,  0,  0),
299 	MARVELL_LAYOUT( 8192,   512,  8,  9,  8, 1024,  0, 30,  0, 160, 30),
300 	MARVELL_LAYOUT( 8192,   512,  12, 12, 11, 704,  0, 30,448,  64, 30),
301 	MARVELL_LAYOUT( 8192,   512,  16, 17, 16, 512,  0, 30,  0,  32, 30),
302 };
303 
304 /**
305  * struct marvell_nand_chip_sel - CS line description
306  *
307  * The Nand Flash Controller has up to 4 CE and 2 RB pins. The CE selection
308  * is made by a field in NDCB0 register, and in another field in NDCB2 register.
309  * The datasheet describes the logic with an error: ADDR5 field is once
310  * declared at the beginning of NDCB2, and another time at its end. Because the
311  * ADDR5 field of NDCB2 may be used by other bytes, it would be more logical
312  * to use the last bit of this field instead of the first ones.
313  *
314  * @cs:			Wanted CE lane.
315  * @ndcb0_csel:		Value of the NDCB0 register with or without the flag
316  *			selecting the wanted CE lane. This is set once when
317  *			the Device Tree is probed.
318  * @rb:			Ready/Busy pin for the flash chip
319  */
320 struct marvell_nand_chip_sel {
321 	unsigned int cs;
322 	u32 ndcb0_csel;
323 	unsigned int rb;
324 };
325 
326 /**
327  * struct marvell_nand_chip - stores NAND chip device related information
328  *
329  * @chip:		Base NAND chip structure
330  * @node:		Used to store NAND chips into a list
331  * @layout:		NAND layout when using hardware ECC
332  * @ndcr:		Controller register value for this NAND chip
333  * @ndtr0:		Timing registers 0 value for this NAND chip
334  * @ndtr1:		Timing registers 1 value for this NAND chip
335  * @addr_cyc:		Amount of cycles needed to pass column address
336  * @selected_die:	Current active CS
337  * @nsels:		Number of CS lines required by the NAND chip
338  * @sels:		Array of CS lines descriptions
339  */
340 struct marvell_nand_chip {
341 	struct nand_chip chip;
342 	struct list_head node;
343 	const struct marvell_hw_ecc_layout *layout;
344 	u32 ndcr;
345 	u32 ndtr0;
346 	u32 ndtr1;
347 	int addr_cyc;
348 	int selected_die;
349 	unsigned int nsels;
350 	struct marvell_nand_chip_sel sels[];
351 };
352 
353 static inline struct marvell_nand_chip *to_marvell_nand(struct nand_chip *chip)
354 {
355 	return container_of(chip, struct marvell_nand_chip, chip);
356 }
357 
358 static inline struct marvell_nand_chip_sel *to_nand_sel(struct marvell_nand_chip
359 							*nand)
360 {
361 	return &nand->sels[nand->selected_die];
362 }
363 
364 /**
365  * struct marvell_nfc_caps - NAND controller capabilities for distinction
366  *                           between compatible strings
367  *
368  * @max_cs_nb:		Number of Chip Select lines available
369  * @max_rb_nb:		Number of Ready/Busy lines available
370  * @need_system_controller: Indicates if the SoC needs to have access to the
371  *                      system controller (ie. to enable the NAND controller)
372  * @legacy_of_bindings:	Indicates if DT parsing must be done using the old
373  *			fashion way
374  * @is_nfcv2:		NFCv2 has numerous enhancements compared to NFCv1, ie.
375  *			BCH error detection and correction algorithm,
376  *			NDCB3 register has been added
377  * @use_dma:		Use dma for data transfers
378  */
379 struct marvell_nfc_caps {
380 	unsigned int max_cs_nb;
381 	unsigned int max_rb_nb;
382 	bool need_system_controller;
383 	bool legacy_of_bindings;
384 	bool is_nfcv2;
385 	bool use_dma;
386 };
387 
388 /**
389  * struct marvell_nfc - stores Marvell NAND controller information
390  *
391  * @controller:		Base controller structure
392  * @dev:		Parent device (used to print error messages)
393  * @regs:		NAND controller registers
394  * @core_clk:		Core clock
395  * @reg_clk:		Registers clock
396  * @complete:		Completion object to wait for NAND controller events
397  * @assigned_cs:	Bitmask describing already assigned CS lines
398  * @chips:		List containing all the NAND chips attached to
399  *			this NAND controller
400  * @selected_chip:	Currently selected target chip
401  * @caps:		NAND controller capabilities for each compatible string
402  * @use_dma:		Whetner DMA is used
403  * @dma_chan:		DMA channel (NFCv1 only)
404  * @dma_buf:		32-bit aligned buffer for DMA transfers (NFCv1 only)
405  */
406 struct marvell_nfc {
407 	struct nand_controller controller;
408 	struct device *dev;
409 	void __iomem *regs;
410 	struct clk *core_clk;
411 	struct clk *reg_clk;
412 	struct completion complete;
413 	unsigned long assigned_cs;
414 	struct list_head chips;
415 	struct nand_chip *selected_chip;
416 	const struct marvell_nfc_caps *caps;
417 
418 	/* DMA (NFCv1 only) */
419 	bool use_dma;
420 	struct dma_chan *dma_chan;
421 	u8 *dma_buf;
422 };
423 
424 static inline struct marvell_nfc *to_marvell_nfc(struct nand_controller *ctrl)
425 {
426 	return container_of(ctrl, struct marvell_nfc, controller);
427 }
428 
429 /**
430  * struct marvell_nfc_timings - NAND controller timings expressed in NAND
431  *                              Controller clock cycles
432  *
433  * @tRP:		ND_nRE pulse width
434  * @tRH:		ND_nRE high duration
435  * @tWP:		ND_nWE pulse time
436  * @tWH:		ND_nWE high duration
437  * @tCS:		Enable signal setup time
438  * @tCH:		Enable signal hold time
439  * @tADL:		Address to write data delay
440  * @tAR:		ND_ALE low to ND_nRE low delay
441  * @tWHR:		ND_nWE high to ND_nRE low for status read
442  * @tRHW:		ND_nRE high duration, read to write delay
443  * @tR:			ND_nWE high to ND_nRE low for read
444  */
445 struct marvell_nfc_timings {
446 	/* NDTR0 fields */
447 	unsigned int tRP;
448 	unsigned int tRH;
449 	unsigned int tWP;
450 	unsigned int tWH;
451 	unsigned int tCS;
452 	unsigned int tCH;
453 	unsigned int tADL;
454 	/* NDTR1 fields */
455 	unsigned int tAR;
456 	unsigned int tWHR;
457 	unsigned int tRHW;
458 	unsigned int tR;
459 };
460 
461 /**
462  * TO_CYCLES() - Derives a duration in numbers of clock cycles.
463  *
464  * @ps: Duration in pico-seconds
465  * @period_ns:  Clock period in nano-seconds
466  *
467  * Convert the duration in nano-seconds, then divide by the period and
468  * return the number of clock periods.
469  */
470 #define TO_CYCLES(ps, period_ns) (DIV_ROUND_UP(ps / 1000, period_ns))
471 #define TO_CYCLES64(ps, period_ns) (DIV_ROUND_UP_ULL(div_u64(ps, 1000), \
472 						     period_ns))
473 
474 /**
475  * struct marvell_nfc_op - filled during the parsing of the ->exec_op()
476  *                         subop subset of instructions.
477  *
478  * @ndcb:		Array of values written to NDCBx registers
479  * @cle_ale_delay_ns:	Optional delay after the last CMD or ADDR cycle
480  * @rdy_timeout_ms:	Timeout for waits on Ready/Busy pin
481  * @rdy_delay_ns:	Optional delay after waiting for the RB pin
482  * @data_delay_ns:	Optional delay after the data xfer
483  * @data_instr_idx:	Index of the data instruction in the subop
484  * @data_instr:		Pointer to the data instruction in the subop
485  */
486 struct marvell_nfc_op {
487 	u32 ndcb[4];
488 	unsigned int cle_ale_delay_ns;
489 	unsigned int rdy_timeout_ms;
490 	unsigned int rdy_delay_ns;
491 	unsigned int data_delay_ns;
492 	unsigned int data_instr_idx;
493 	const struct nand_op_instr *data_instr;
494 };
495 
496 /*
497  * Internal helper to conditionnally apply a delay (from the above structure,
498  * most of the time).
499  */
500 static void cond_delay(unsigned int ns)
501 {
502 	if (!ns)
503 		return;
504 
505 	if (ns < 10000)
506 		ndelay(ns);
507 	else
508 		udelay(DIV_ROUND_UP(ns, 1000));
509 }
510 
511 /*
512  * The controller has many flags that could generate interrupts, most of them
513  * are disabled and polling is used. For the very slow signals, using interrupts
514  * may relax the CPU charge.
515  */
516 static void marvell_nfc_disable_int(struct marvell_nfc *nfc, u32 int_mask)
517 {
518 	u32 reg;
519 
520 	/* Writing 1 disables the interrupt */
521 	reg = readl_relaxed(nfc->regs + NDCR);
522 	writel_relaxed(reg | int_mask, nfc->regs + NDCR);
523 }
524 
525 static void marvell_nfc_enable_int(struct marvell_nfc *nfc, u32 int_mask)
526 {
527 	u32 reg;
528 
529 	/* Writing 0 enables the interrupt */
530 	reg = readl_relaxed(nfc->regs + NDCR);
531 	writel_relaxed(reg & ~int_mask, nfc->regs + NDCR);
532 }
533 
534 static u32 marvell_nfc_clear_int(struct marvell_nfc *nfc, u32 int_mask)
535 {
536 	u32 reg;
537 
538 	reg = readl_relaxed(nfc->regs + NDSR);
539 	writel_relaxed(int_mask, nfc->regs + NDSR);
540 
541 	return reg & int_mask;
542 }
543 
544 static void marvell_nfc_force_byte_access(struct nand_chip *chip,
545 					  bool force_8bit)
546 {
547 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
548 	u32 ndcr;
549 
550 	/*
551 	 * Callers of this function do not verify if the NAND is using a 16-bit
552 	 * an 8-bit bus for normal operations, so we need to take care of that
553 	 * here by leaving the configuration unchanged if the NAND does not have
554 	 * the NAND_BUSWIDTH_16 flag set.
555 	 */
556 	if (!(chip->options & NAND_BUSWIDTH_16))
557 		return;
558 
559 	ndcr = readl_relaxed(nfc->regs + NDCR);
560 
561 	if (force_8bit)
562 		ndcr &= ~(NDCR_DWIDTH_M | NDCR_DWIDTH_C);
563 	else
564 		ndcr |= NDCR_DWIDTH_M | NDCR_DWIDTH_C;
565 
566 	writel_relaxed(ndcr, nfc->regs + NDCR);
567 }
568 
569 static int marvell_nfc_wait_ndrun(struct nand_chip *chip)
570 {
571 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
572 	u32 val;
573 	int ret;
574 
575 	/*
576 	 * The command is being processed, wait for the ND_RUN bit to be
577 	 * cleared by the NFC. If not, we must clear it by hand.
578 	 */
579 	ret = readl_relaxed_poll_timeout(nfc->regs + NDCR, val,
580 					 (val & NDCR_ND_RUN) == 0,
581 					 POLL_PERIOD, POLL_TIMEOUT);
582 	if (ret) {
583 		dev_err(nfc->dev, "Timeout on NAND controller run mode\n");
584 		writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
585 			       nfc->regs + NDCR);
586 		return ret;
587 	}
588 
589 	return 0;
590 }
591 
592 /*
593  * Any time a command has to be sent to the controller, the following sequence
594  * has to be followed:
595  * - call marvell_nfc_prepare_cmd()
596  *      -> activate the ND_RUN bit that will kind of 'start a job'
597  *      -> wait the signal indicating the NFC is waiting for a command
598  * - send the command (cmd and address cycles)
599  * - enventually send or receive the data
600  * - call marvell_nfc_end_cmd() with the corresponding flag
601  *      -> wait the flag to be triggered or cancel the job with a timeout
602  *
603  * The following helpers are here to factorize the code a bit so that
604  * specialized functions responsible for executing the actual NAND
605  * operations do not have to replicate the same code blocks.
606  */
607 static int marvell_nfc_prepare_cmd(struct nand_chip *chip)
608 {
609 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
610 	u32 ndcr, val;
611 	int ret;
612 
613 	/* Poll ND_RUN and clear NDSR before issuing any command */
614 	ret = marvell_nfc_wait_ndrun(chip);
615 	if (ret) {
616 		dev_err(nfc->dev, "Last operation did not succeed\n");
617 		return ret;
618 	}
619 
620 	ndcr = readl_relaxed(nfc->regs + NDCR);
621 	writel_relaxed(readl(nfc->regs + NDSR), nfc->regs + NDSR);
622 
623 	/* Assert ND_RUN bit and wait the NFC to be ready */
624 	writel_relaxed(ndcr | NDCR_ND_RUN, nfc->regs + NDCR);
625 	ret = readl_relaxed_poll_timeout(nfc->regs + NDSR, val,
626 					 val & NDSR_WRCMDREQ,
627 					 POLL_PERIOD, POLL_TIMEOUT);
628 	if (ret) {
629 		dev_err(nfc->dev, "Timeout on WRCMDRE\n");
630 		return -ETIMEDOUT;
631 	}
632 
633 	/* Command may be written, clear WRCMDREQ status bit */
634 	writel_relaxed(NDSR_WRCMDREQ, nfc->regs + NDSR);
635 
636 	return 0;
637 }
638 
639 static void marvell_nfc_send_cmd(struct nand_chip *chip,
640 				 struct marvell_nfc_op *nfc_op)
641 {
642 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
643 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
644 
645 	dev_dbg(nfc->dev, "\nNDCR:  0x%08x\n"
646 		"NDCB0: 0x%08x\nNDCB1: 0x%08x\nNDCB2: 0x%08x\nNDCB3: 0x%08x\n",
647 		(u32)readl_relaxed(nfc->regs + NDCR), nfc_op->ndcb[0],
648 		nfc_op->ndcb[1], nfc_op->ndcb[2], nfc_op->ndcb[3]);
649 
650 	writel_relaxed(to_nand_sel(marvell_nand)->ndcb0_csel | nfc_op->ndcb[0],
651 		       nfc->regs + NDCB0);
652 	writel_relaxed(nfc_op->ndcb[1], nfc->regs + NDCB0);
653 	writel(nfc_op->ndcb[2], nfc->regs + NDCB0);
654 
655 	/*
656 	 * Write NDCB0 four times only if LEN_OVRD is set or if ADDR6 or ADDR7
657 	 * fields are used (only available on NFCv2).
658 	 */
659 	if (nfc_op->ndcb[0] & NDCB0_LEN_OVRD ||
660 	    NDCB0_ADDR_GET_NUM_CYC(nfc_op->ndcb[0]) >= 6) {
661 		if (!WARN_ON_ONCE(!nfc->caps->is_nfcv2))
662 			writel(nfc_op->ndcb[3], nfc->regs + NDCB0);
663 	}
664 }
665 
666 static int marvell_nfc_end_cmd(struct nand_chip *chip, int flag,
667 			       const char *label)
668 {
669 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
670 	u32 val;
671 	int ret;
672 
673 	ret = readl_relaxed_poll_timeout(nfc->regs + NDSR, val,
674 					 val & flag,
675 					 POLL_PERIOD, POLL_TIMEOUT);
676 
677 	if (ret) {
678 		dev_err(nfc->dev, "Timeout on %s (NDSR: 0x%08x)\n",
679 			label, val);
680 		if (nfc->dma_chan)
681 			dmaengine_terminate_all(nfc->dma_chan);
682 		return ret;
683 	}
684 
685 	/*
686 	 * DMA function uses this helper to poll on CMDD bits without wanting
687 	 * them to be cleared.
688 	 */
689 	if (nfc->use_dma && (readl_relaxed(nfc->regs + NDCR) & NDCR_DMA_EN))
690 		return 0;
691 
692 	writel_relaxed(flag, nfc->regs + NDSR);
693 
694 	return 0;
695 }
696 
697 static int marvell_nfc_wait_cmdd(struct nand_chip *chip)
698 {
699 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
700 	int cs_flag = NDSR_CMDD(to_nand_sel(marvell_nand)->ndcb0_csel);
701 
702 	return marvell_nfc_end_cmd(chip, cs_flag, "CMDD");
703 }
704 
705 static int marvell_nfc_poll_status(struct marvell_nfc *nfc, u32 mask,
706 				   u32 expected_val, unsigned long timeout_ms)
707 {
708 	unsigned long limit;
709 	u32 st;
710 
711 	limit = jiffies + msecs_to_jiffies(timeout_ms);
712 	do {
713 		st = readl_relaxed(nfc->regs + NDSR);
714 		if (st & NDSR_RDY(1))
715 			st |= NDSR_RDY(0);
716 
717 		if ((st & mask) == expected_val)
718 			return 0;
719 
720 		cpu_relax();
721 	} while (time_after(limit, jiffies));
722 
723 	return -ETIMEDOUT;
724 }
725 
726 static int marvell_nfc_wait_op(struct nand_chip *chip, unsigned int timeout_ms)
727 {
728 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
729 	struct mtd_info *mtd = nand_to_mtd(chip);
730 	u32 pending;
731 	int ret;
732 
733 	/* Timeout is expressed in ms */
734 	if (!timeout_ms)
735 		timeout_ms = IRQ_TIMEOUT;
736 
737 	if (mtd->oops_panic_write) {
738 		ret = marvell_nfc_poll_status(nfc, NDSR_RDY(0),
739 					      NDSR_RDY(0),
740 					      timeout_ms);
741 	} else {
742 		init_completion(&nfc->complete);
743 
744 		marvell_nfc_enable_int(nfc, NDCR_RDYM);
745 		ret = wait_for_completion_timeout(&nfc->complete,
746 						  msecs_to_jiffies(timeout_ms));
747 		marvell_nfc_disable_int(nfc, NDCR_RDYM);
748 	}
749 	pending = marvell_nfc_clear_int(nfc, NDSR_RDY(0) | NDSR_RDY(1));
750 
751 	/*
752 	 * In case the interrupt was not served in the required time frame,
753 	 * check if the ISR was not served or if something went actually wrong.
754 	 */
755 	if (!ret && !pending) {
756 		dev_err(nfc->dev, "Timeout waiting for RB signal\n");
757 		return -ETIMEDOUT;
758 	}
759 
760 	return 0;
761 }
762 
763 static void marvell_nfc_select_target(struct nand_chip *chip,
764 				      unsigned int die_nr)
765 {
766 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
767 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
768 	u32 ndcr_generic;
769 
770 	/*
771 	 * Reset the NDCR register to a clean state for this particular chip,
772 	 * also clear ND_RUN bit.
773 	 */
774 	ndcr_generic = readl_relaxed(nfc->regs + NDCR) &
775 		       NDCR_GENERIC_FIELDS_MASK & ~NDCR_ND_RUN;
776 	writel_relaxed(ndcr_generic | marvell_nand->ndcr, nfc->regs + NDCR);
777 
778 	/* Also reset the interrupt status register */
779 	marvell_nfc_clear_int(nfc, NDCR_ALL_INT);
780 
781 	if (chip == nfc->selected_chip && die_nr == marvell_nand->selected_die)
782 		return;
783 
784 	writel_relaxed(marvell_nand->ndtr0, nfc->regs + NDTR0);
785 	writel_relaxed(marvell_nand->ndtr1, nfc->regs + NDTR1);
786 
787 	nfc->selected_chip = chip;
788 	marvell_nand->selected_die = die_nr;
789 }
790 
791 static irqreturn_t marvell_nfc_isr(int irq, void *dev_id)
792 {
793 	struct marvell_nfc *nfc = dev_id;
794 	u32 st = readl_relaxed(nfc->regs + NDSR);
795 	u32 ien = (~readl_relaxed(nfc->regs + NDCR)) & NDCR_ALL_INT;
796 
797 	/*
798 	 * RDY interrupt mask is one bit in NDCR while there are two status
799 	 * bit in NDSR (RDY[cs0/cs2] and RDY[cs1/cs3]).
800 	 */
801 	if (st & NDSR_RDY(1))
802 		st |= NDSR_RDY(0);
803 
804 	if (!(st & ien))
805 		return IRQ_NONE;
806 
807 	marvell_nfc_disable_int(nfc, st & NDCR_ALL_INT);
808 
809 	if (st & (NDSR_RDY(0) | NDSR_RDY(1)))
810 		complete(&nfc->complete);
811 
812 	return IRQ_HANDLED;
813 }
814 
815 /* HW ECC related functions */
816 static void marvell_nfc_enable_hw_ecc(struct nand_chip *chip)
817 {
818 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
819 	u32 ndcr = readl_relaxed(nfc->regs + NDCR);
820 
821 	if (!(ndcr & NDCR_ECC_EN)) {
822 		writel_relaxed(ndcr | NDCR_ECC_EN, nfc->regs + NDCR);
823 
824 		/*
825 		 * When enabling BCH, set threshold to 0 to always know the
826 		 * number of corrected bitflips.
827 		 */
828 		if (chip->ecc.algo == NAND_ECC_ALGO_BCH)
829 			writel_relaxed(NDECCCTRL_BCH_EN, nfc->regs + NDECCCTRL);
830 	}
831 }
832 
833 static void marvell_nfc_disable_hw_ecc(struct nand_chip *chip)
834 {
835 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
836 	u32 ndcr = readl_relaxed(nfc->regs + NDCR);
837 
838 	if (ndcr & NDCR_ECC_EN) {
839 		writel_relaxed(ndcr & ~NDCR_ECC_EN, nfc->regs + NDCR);
840 		if (chip->ecc.algo == NAND_ECC_ALGO_BCH)
841 			writel_relaxed(0, nfc->regs + NDECCCTRL);
842 	}
843 }
844 
845 /* DMA related helpers */
846 static void marvell_nfc_enable_dma(struct marvell_nfc *nfc)
847 {
848 	u32 reg;
849 
850 	reg = readl_relaxed(nfc->regs + NDCR);
851 	writel_relaxed(reg | NDCR_DMA_EN, nfc->regs + NDCR);
852 }
853 
854 static void marvell_nfc_disable_dma(struct marvell_nfc *nfc)
855 {
856 	u32 reg;
857 
858 	reg = readl_relaxed(nfc->regs + NDCR);
859 	writel_relaxed(reg & ~NDCR_DMA_EN, nfc->regs + NDCR);
860 }
861 
862 /* Read/write PIO/DMA accessors */
863 static int marvell_nfc_xfer_data_dma(struct marvell_nfc *nfc,
864 				     enum dma_data_direction direction,
865 				     unsigned int len)
866 {
867 	unsigned int dma_len = min_t(int, ALIGN(len, 32), MAX_CHUNK_SIZE);
868 	struct dma_async_tx_descriptor *tx;
869 	struct scatterlist sg;
870 	dma_cookie_t cookie;
871 	int ret;
872 
873 	marvell_nfc_enable_dma(nfc);
874 	/* Prepare the DMA transfer */
875 	sg_init_one(&sg, nfc->dma_buf, dma_len);
876 	ret = dma_map_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
877 	if (!ret) {
878 		dev_err(nfc->dev, "Could not map DMA S/G list\n");
879 		return -ENXIO;
880 	}
881 
882 	tx = dmaengine_prep_slave_sg(nfc->dma_chan, &sg, 1,
883 				     direction == DMA_FROM_DEVICE ?
884 				     DMA_DEV_TO_MEM : DMA_MEM_TO_DEV,
885 				     DMA_PREP_INTERRUPT);
886 	if (!tx) {
887 		dev_err(nfc->dev, "Could not prepare DMA S/G list\n");
888 		dma_unmap_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
889 		return -ENXIO;
890 	}
891 
892 	/* Do the task and wait for it to finish */
893 	cookie = dmaengine_submit(tx);
894 	ret = dma_submit_error(cookie);
895 	if (ret)
896 		return -EIO;
897 
898 	dma_async_issue_pending(nfc->dma_chan);
899 	ret = marvell_nfc_wait_cmdd(nfc->selected_chip);
900 	dma_unmap_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
901 	marvell_nfc_disable_dma(nfc);
902 	if (ret) {
903 		dev_err(nfc->dev, "Timeout waiting for DMA (status: %d)\n",
904 			dmaengine_tx_status(nfc->dma_chan, cookie, NULL));
905 		dmaengine_terminate_all(nfc->dma_chan);
906 		return -ETIMEDOUT;
907 	}
908 
909 	return 0;
910 }
911 
912 static int marvell_nfc_xfer_data_in_pio(struct marvell_nfc *nfc, u8 *in,
913 					unsigned int len)
914 {
915 	unsigned int last_len = len % FIFO_DEPTH;
916 	unsigned int last_full_offset = round_down(len, FIFO_DEPTH);
917 	int i;
918 
919 	for (i = 0; i < last_full_offset; i += FIFO_DEPTH)
920 		ioread32_rep(nfc->regs + NDDB, in + i, FIFO_REP(FIFO_DEPTH));
921 
922 	if (last_len) {
923 		u8 tmp_buf[FIFO_DEPTH];
924 
925 		ioread32_rep(nfc->regs + NDDB, tmp_buf, FIFO_REP(FIFO_DEPTH));
926 		memcpy(in + last_full_offset, tmp_buf, last_len);
927 	}
928 
929 	return 0;
930 }
931 
932 static int marvell_nfc_xfer_data_out_pio(struct marvell_nfc *nfc, const u8 *out,
933 					 unsigned int len)
934 {
935 	unsigned int last_len = len % FIFO_DEPTH;
936 	unsigned int last_full_offset = round_down(len, FIFO_DEPTH);
937 	int i;
938 
939 	for (i = 0; i < last_full_offset; i += FIFO_DEPTH)
940 		iowrite32_rep(nfc->regs + NDDB, out + i, FIFO_REP(FIFO_DEPTH));
941 
942 	if (last_len) {
943 		u8 tmp_buf[FIFO_DEPTH];
944 
945 		memcpy(tmp_buf, out + last_full_offset, last_len);
946 		iowrite32_rep(nfc->regs + NDDB, tmp_buf, FIFO_REP(FIFO_DEPTH));
947 	}
948 
949 	return 0;
950 }
951 
952 static void marvell_nfc_check_empty_chunk(struct nand_chip *chip,
953 					  u8 *data, int data_len,
954 					  u8 *spare, int spare_len,
955 					  u8 *ecc, int ecc_len,
956 					  unsigned int *max_bitflips)
957 {
958 	struct mtd_info *mtd = nand_to_mtd(chip);
959 	int bf;
960 
961 	/*
962 	 * Blank pages (all 0xFF) that have not been written may be recognized
963 	 * as bad if bitflips occur, so whenever an uncorrectable error occurs,
964 	 * check if the entire page (with ECC bytes) is actually blank or not.
965 	 */
966 	if (!data)
967 		data_len = 0;
968 	if (!spare)
969 		spare_len = 0;
970 	if (!ecc)
971 		ecc_len = 0;
972 
973 	bf = nand_check_erased_ecc_chunk(data, data_len, ecc, ecc_len,
974 					 spare, spare_len, chip->ecc.strength);
975 	if (bf < 0) {
976 		mtd->ecc_stats.failed++;
977 		return;
978 	}
979 
980 	/* Update the stats and max_bitflips */
981 	mtd->ecc_stats.corrected += bf;
982 	*max_bitflips = max_t(unsigned int, *max_bitflips, bf);
983 }
984 
985 /*
986  * Check if a chunk is correct or not according to the hardware ECC engine.
987  * mtd->ecc_stats.corrected is updated, as well as max_bitflips, however
988  * mtd->ecc_stats.failure is not, the function will instead return a non-zero
989  * value indicating that a check on the emptyness of the subpage must be
990  * performed before actually declaring the subpage as "corrupted".
991  */
992 static int marvell_nfc_hw_ecc_check_bitflips(struct nand_chip *chip,
993 					     unsigned int *max_bitflips)
994 {
995 	struct mtd_info *mtd = nand_to_mtd(chip);
996 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
997 	int bf = 0;
998 	u32 ndsr;
999 
1000 	ndsr = readl_relaxed(nfc->regs + NDSR);
1001 
1002 	/* Check uncorrectable error flag */
1003 	if (ndsr & NDSR_UNCERR) {
1004 		writel_relaxed(ndsr, nfc->regs + NDSR);
1005 
1006 		/*
1007 		 * Do not increment ->ecc_stats.failed now, instead, return a
1008 		 * non-zero value to indicate that this chunk was apparently
1009 		 * bad, and it should be check to see if it empty or not. If
1010 		 * the chunk (with ECC bytes) is not declared empty, the calling
1011 		 * function must increment the failure count.
1012 		 */
1013 		return -EBADMSG;
1014 	}
1015 
1016 	/* Check correctable error flag */
1017 	if (ndsr & NDSR_CORERR) {
1018 		writel_relaxed(ndsr, nfc->regs + NDSR);
1019 
1020 		if (chip->ecc.algo == NAND_ECC_ALGO_BCH)
1021 			bf = NDSR_ERRCNT(ndsr);
1022 		else
1023 			bf = 1;
1024 	}
1025 
1026 	/* Update the stats and max_bitflips */
1027 	mtd->ecc_stats.corrected += bf;
1028 	*max_bitflips = max_t(unsigned int, *max_bitflips, bf);
1029 
1030 	return 0;
1031 }
1032 
1033 /* Hamming read helpers */
1034 static int marvell_nfc_hw_ecc_hmg_do_read_page(struct nand_chip *chip,
1035 					       u8 *data_buf, u8 *oob_buf,
1036 					       bool raw, int page)
1037 {
1038 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1039 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1040 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1041 	struct marvell_nfc_op nfc_op = {
1042 		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_READ) |
1043 			   NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1044 			   NDCB0_DBC |
1045 			   NDCB0_CMD1(NAND_CMD_READ0) |
1046 			   NDCB0_CMD2(NAND_CMD_READSTART),
1047 		.ndcb[1] = NDCB1_ADDRS_PAGE(page),
1048 		.ndcb[2] = NDCB2_ADDR5_PAGE(page),
1049 	};
1050 	unsigned int oob_bytes = lt->spare_bytes + (raw ? lt->ecc_bytes : 0);
1051 	int ret;
1052 
1053 	/* NFCv2 needs more information about the operation being executed */
1054 	if (nfc->caps->is_nfcv2)
1055 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1056 
1057 	ret = marvell_nfc_prepare_cmd(chip);
1058 	if (ret)
1059 		return ret;
1060 
1061 	marvell_nfc_send_cmd(chip, &nfc_op);
1062 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1063 				  "RDDREQ while draining FIFO (data/oob)");
1064 	if (ret)
1065 		return ret;
1066 
1067 	/*
1068 	 * Read the page then the OOB area. Unlike what is shown in current
1069 	 * documentation, spare bytes are protected by the ECC engine, and must
1070 	 * be at the beginning of the OOB area or running this driver on legacy
1071 	 * systems will prevent the discovery of the BBM/BBT.
1072 	 */
1073 	if (nfc->use_dma) {
1074 		marvell_nfc_xfer_data_dma(nfc, DMA_FROM_DEVICE,
1075 					  lt->data_bytes + oob_bytes);
1076 		memcpy(data_buf, nfc->dma_buf, lt->data_bytes);
1077 		memcpy(oob_buf, nfc->dma_buf + lt->data_bytes, oob_bytes);
1078 	} else {
1079 		marvell_nfc_xfer_data_in_pio(nfc, data_buf, lt->data_bytes);
1080 		marvell_nfc_xfer_data_in_pio(nfc, oob_buf, oob_bytes);
1081 	}
1082 
1083 	ret = marvell_nfc_wait_cmdd(chip);
1084 	return ret;
1085 }
1086 
1087 static int marvell_nfc_hw_ecc_hmg_read_page_raw(struct nand_chip *chip, u8 *buf,
1088 						int oob_required, int page)
1089 {
1090 	marvell_nfc_select_target(chip, chip->cur_cs);
1091 	return marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi,
1092 						   true, page);
1093 }
1094 
1095 static int marvell_nfc_hw_ecc_hmg_read_page(struct nand_chip *chip, u8 *buf,
1096 					    int oob_required, int page)
1097 {
1098 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1099 	unsigned int full_sz = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1100 	int max_bitflips = 0, ret;
1101 	u8 *raw_buf;
1102 
1103 	marvell_nfc_select_target(chip, chip->cur_cs);
1104 	marvell_nfc_enable_hw_ecc(chip);
1105 	marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi, false,
1106 					    page);
1107 	ret = marvell_nfc_hw_ecc_check_bitflips(chip, &max_bitflips);
1108 	marvell_nfc_disable_hw_ecc(chip);
1109 
1110 	if (!ret)
1111 		return max_bitflips;
1112 
1113 	/*
1114 	 * When ECC failures are detected, check if the full page has been
1115 	 * written or not. Ignore the failure if it is actually empty.
1116 	 */
1117 	raw_buf = kmalloc(full_sz, GFP_KERNEL);
1118 	if (!raw_buf)
1119 		return -ENOMEM;
1120 
1121 	marvell_nfc_hw_ecc_hmg_do_read_page(chip, raw_buf, raw_buf +
1122 					    lt->data_bytes, true, page);
1123 	marvell_nfc_check_empty_chunk(chip, raw_buf, full_sz, NULL, 0, NULL, 0,
1124 				      &max_bitflips);
1125 	kfree(raw_buf);
1126 
1127 	return max_bitflips;
1128 }
1129 
1130 /*
1131  * Spare area in Hamming layouts is not protected by the ECC engine (even if
1132  * it appears before the ECC bytes when reading), the ->read_oob_raw() function
1133  * also stands for ->read_oob().
1134  */
1135 static int marvell_nfc_hw_ecc_hmg_read_oob_raw(struct nand_chip *chip, int page)
1136 {
1137 	u8 *buf = nand_get_data_buf(chip);
1138 
1139 	marvell_nfc_select_target(chip, chip->cur_cs);
1140 	return marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi,
1141 						   true, page);
1142 }
1143 
1144 /* Hamming write helpers */
1145 static int marvell_nfc_hw_ecc_hmg_do_write_page(struct nand_chip *chip,
1146 						const u8 *data_buf,
1147 						const u8 *oob_buf, bool raw,
1148 						int page)
1149 {
1150 	const struct nand_sdr_timings *sdr =
1151 		nand_get_sdr_timings(nand_get_interface_config(chip));
1152 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1153 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1154 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1155 	struct marvell_nfc_op nfc_op = {
1156 		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_WRITE) |
1157 			   NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1158 			   NDCB0_CMD1(NAND_CMD_SEQIN) |
1159 			   NDCB0_CMD2(NAND_CMD_PAGEPROG) |
1160 			   NDCB0_DBC,
1161 		.ndcb[1] = NDCB1_ADDRS_PAGE(page),
1162 		.ndcb[2] = NDCB2_ADDR5_PAGE(page),
1163 	};
1164 	unsigned int oob_bytes = lt->spare_bytes + (raw ? lt->ecc_bytes : 0);
1165 	int ret;
1166 
1167 	/* NFCv2 needs more information about the operation being executed */
1168 	if (nfc->caps->is_nfcv2)
1169 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1170 
1171 	ret = marvell_nfc_prepare_cmd(chip);
1172 	if (ret)
1173 		return ret;
1174 
1175 	marvell_nfc_send_cmd(chip, &nfc_op);
1176 	ret = marvell_nfc_end_cmd(chip, NDSR_WRDREQ,
1177 				  "WRDREQ while loading FIFO (data)");
1178 	if (ret)
1179 		return ret;
1180 
1181 	/* Write the page then the OOB area */
1182 	if (nfc->use_dma) {
1183 		memcpy(nfc->dma_buf, data_buf, lt->data_bytes);
1184 		memcpy(nfc->dma_buf + lt->data_bytes, oob_buf, oob_bytes);
1185 		marvell_nfc_xfer_data_dma(nfc, DMA_TO_DEVICE, lt->data_bytes +
1186 					  lt->ecc_bytes + lt->spare_bytes);
1187 	} else {
1188 		marvell_nfc_xfer_data_out_pio(nfc, data_buf, lt->data_bytes);
1189 		marvell_nfc_xfer_data_out_pio(nfc, oob_buf, oob_bytes);
1190 	}
1191 
1192 	ret = marvell_nfc_wait_cmdd(chip);
1193 	if (ret)
1194 		return ret;
1195 
1196 	ret = marvell_nfc_wait_op(chip,
1197 				  PSEC_TO_MSEC(sdr->tPROG_max));
1198 	return ret;
1199 }
1200 
1201 static int marvell_nfc_hw_ecc_hmg_write_page_raw(struct nand_chip *chip,
1202 						 const u8 *buf,
1203 						 int oob_required, int page)
1204 {
1205 	marvell_nfc_select_target(chip, chip->cur_cs);
1206 	return marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1207 						    true, page);
1208 }
1209 
1210 static int marvell_nfc_hw_ecc_hmg_write_page(struct nand_chip *chip,
1211 					     const u8 *buf,
1212 					     int oob_required, int page)
1213 {
1214 	int ret;
1215 
1216 	marvell_nfc_select_target(chip, chip->cur_cs);
1217 	marvell_nfc_enable_hw_ecc(chip);
1218 	ret = marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1219 						   false, page);
1220 	marvell_nfc_disable_hw_ecc(chip);
1221 
1222 	return ret;
1223 }
1224 
1225 /*
1226  * Spare area in Hamming layouts is not protected by the ECC engine (even if
1227  * it appears before the ECC bytes when reading), the ->write_oob_raw() function
1228  * also stands for ->write_oob().
1229  */
1230 static int marvell_nfc_hw_ecc_hmg_write_oob_raw(struct nand_chip *chip,
1231 						int page)
1232 {
1233 	struct mtd_info *mtd = nand_to_mtd(chip);
1234 	u8 *buf = nand_get_data_buf(chip);
1235 
1236 	memset(buf, 0xFF, mtd->writesize);
1237 
1238 	marvell_nfc_select_target(chip, chip->cur_cs);
1239 	return marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1240 						    true, page);
1241 }
1242 
1243 /* BCH read helpers */
1244 static int marvell_nfc_hw_ecc_bch_read_page_raw(struct nand_chip *chip, u8 *buf,
1245 						int oob_required, int page)
1246 {
1247 	struct mtd_info *mtd = nand_to_mtd(chip);
1248 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1249 	u8 *oob = chip->oob_poi;
1250 	int chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1251 	int ecc_offset = (lt->full_chunk_cnt * lt->spare_bytes) +
1252 		lt->last_spare_bytes;
1253 	int data_len = lt->data_bytes;
1254 	int spare_len = lt->spare_bytes;
1255 	int ecc_len = lt->ecc_bytes;
1256 	int chunk;
1257 
1258 	marvell_nfc_select_target(chip, chip->cur_cs);
1259 
1260 	if (oob_required)
1261 		memset(chip->oob_poi, 0xFF, mtd->oobsize);
1262 
1263 	nand_read_page_op(chip, page, 0, NULL, 0);
1264 
1265 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1266 		/* Update last chunk length */
1267 		if (chunk >= lt->full_chunk_cnt) {
1268 			data_len = lt->last_data_bytes;
1269 			spare_len = lt->last_spare_bytes;
1270 			ecc_len = lt->last_ecc_bytes;
1271 		}
1272 
1273 		/* Read data bytes*/
1274 		nand_change_read_column_op(chip, chunk * chunk_size,
1275 					   buf + (lt->data_bytes * chunk),
1276 					   data_len, false);
1277 
1278 		/* Read spare bytes */
1279 		nand_read_data_op(chip, oob + (lt->spare_bytes * chunk),
1280 				  spare_len, false, false);
1281 
1282 		/* Read ECC bytes */
1283 		nand_read_data_op(chip, oob + ecc_offset +
1284 				  (ALIGN(lt->ecc_bytes, 32) * chunk),
1285 				  ecc_len, false, false);
1286 	}
1287 
1288 	return 0;
1289 }
1290 
1291 static void marvell_nfc_hw_ecc_bch_read_chunk(struct nand_chip *chip, int chunk,
1292 					      u8 *data, unsigned int data_len,
1293 					      u8 *spare, unsigned int spare_len,
1294 					      int page)
1295 {
1296 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1297 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1298 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1299 	int i, ret;
1300 	struct marvell_nfc_op nfc_op = {
1301 		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_READ) |
1302 			   NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1303 			   NDCB0_LEN_OVRD,
1304 		.ndcb[1] = NDCB1_ADDRS_PAGE(page),
1305 		.ndcb[2] = NDCB2_ADDR5_PAGE(page),
1306 		.ndcb[3] = data_len + spare_len,
1307 	};
1308 
1309 	ret = marvell_nfc_prepare_cmd(chip);
1310 	if (ret)
1311 		return;
1312 
1313 	if (chunk == 0)
1314 		nfc_op.ndcb[0] |= NDCB0_DBC |
1315 				  NDCB0_CMD1(NAND_CMD_READ0) |
1316 				  NDCB0_CMD2(NAND_CMD_READSTART);
1317 
1318 	/*
1319 	 * Trigger the monolithic read on the first chunk, then naked read on
1320 	 * intermediate chunks and finally a last naked read on the last chunk.
1321 	 */
1322 	if (chunk == 0)
1323 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1324 	else if (chunk < lt->nchunks - 1)
1325 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_NAKED_RW);
1326 	else
1327 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1328 
1329 	marvell_nfc_send_cmd(chip, &nfc_op);
1330 
1331 	/*
1332 	 * According to the datasheet, when reading from NDDB
1333 	 * with BCH enabled, after each 32 bytes reads, we
1334 	 * have to make sure that the NDSR.RDDREQ bit is set.
1335 	 *
1336 	 * Drain the FIFO, 8 32-bit reads at a time, and skip
1337 	 * the polling on the last read.
1338 	 *
1339 	 * Length is a multiple of 32 bytes, hence it is a multiple of 8 too.
1340 	 */
1341 	for (i = 0; i < data_len; i += FIFO_DEPTH * BCH_SEQ_READS) {
1342 		marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1343 				    "RDDREQ while draining FIFO (data)");
1344 		marvell_nfc_xfer_data_in_pio(nfc, data,
1345 					     FIFO_DEPTH * BCH_SEQ_READS);
1346 		data += FIFO_DEPTH * BCH_SEQ_READS;
1347 	}
1348 
1349 	for (i = 0; i < spare_len; i += FIFO_DEPTH * BCH_SEQ_READS) {
1350 		marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1351 				    "RDDREQ while draining FIFO (OOB)");
1352 		marvell_nfc_xfer_data_in_pio(nfc, spare,
1353 					     FIFO_DEPTH * BCH_SEQ_READS);
1354 		spare += FIFO_DEPTH * BCH_SEQ_READS;
1355 	}
1356 }
1357 
1358 static int marvell_nfc_hw_ecc_bch_read_page(struct nand_chip *chip,
1359 					    u8 *buf, int oob_required,
1360 					    int page)
1361 {
1362 	struct mtd_info *mtd = nand_to_mtd(chip);
1363 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1364 	int data_len = lt->data_bytes, spare_len = lt->spare_bytes;
1365 	u8 *data = buf, *spare = chip->oob_poi;
1366 	int max_bitflips = 0;
1367 	u32 failure_mask = 0;
1368 	int chunk, ret;
1369 
1370 	marvell_nfc_select_target(chip, chip->cur_cs);
1371 
1372 	/*
1373 	 * With BCH, OOB is not fully used (and thus not read entirely), not
1374 	 * expected bytes could show up at the end of the OOB buffer if not
1375 	 * explicitly erased.
1376 	 */
1377 	if (oob_required)
1378 		memset(chip->oob_poi, 0xFF, mtd->oobsize);
1379 
1380 	marvell_nfc_enable_hw_ecc(chip);
1381 
1382 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1383 		/* Update length for the last chunk */
1384 		if (chunk >= lt->full_chunk_cnt) {
1385 			data_len = lt->last_data_bytes;
1386 			spare_len = lt->last_spare_bytes;
1387 		}
1388 
1389 		/* Read the chunk and detect number of bitflips */
1390 		marvell_nfc_hw_ecc_bch_read_chunk(chip, chunk, data, data_len,
1391 						  spare, spare_len, page);
1392 		ret = marvell_nfc_hw_ecc_check_bitflips(chip, &max_bitflips);
1393 		if (ret)
1394 			failure_mask |= BIT(chunk);
1395 
1396 		data += data_len;
1397 		spare += spare_len;
1398 	}
1399 
1400 	marvell_nfc_disable_hw_ecc(chip);
1401 
1402 	if (!failure_mask)
1403 		return max_bitflips;
1404 
1405 	/*
1406 	 * Please note that dumping the ECC bytes during a normal read with OOB
1407 	 * area would add a significant overhead as ECC bytes are "consumed" by
1408 	 * the controller in normal mode and must be re-read in raw mode. To
1409 	 * avoid dropping the performances, we prefer not to include them. The
1410 	 * user should re-read the page in raw mode if ECC bytes are required.
1411 	 */
1412 
1413 	/*
1414 	 * In case there is any subpage read error, we usually re-read only ECC
1415 	 * bytes in raw mode and check if the whole page is empty. In this case,
1416 	 * it is normal that the ECC check failed and we just ignore the error.
1417 	 *
1418 	 * However, it has been empirically observed that for some layouts (e.g
1419 	 * 2k page, 8b strength per 512B chunk), the controller tries to correct
1420 	 * bits and may create itself bitflips in the erased area. To overcome
1421 	 * this strange behavior, the whole page is re-read in raw mode, not
1422 	 * only the ECC bytes.
1423 	 */
1424 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1425 		int data_off_in_page, spare_off_in_page, ecc_off_in_page;
1426 		int data_off, spare_off, ecc_off;
1427 		int data_len, spare_len, ecc_len;
1428 
1429 		/* No failure reported for this chunk, move to the next one */
1430 		if (!(failure_mask & BIT(chunk)))
1431 			continue;
1432 
1433 		data_off_in_page = chunk * (lt->data_bytes + lt->spare_bytes +
1434 					    lt->ecc_bytes);
1435 		spare_off_in_page = data_off_in_page +
1436 			(chunk < lt->full_chunk_cnt ? lt->data_bytes :
1437 						      lt->last_data_bytes);
1438 		ecc_off_in_page = spare_off_in_page +
1439 			(chunk < lt->full_chunk_cnt ? lt->spare_bytes :
1440 						      lt->last_spare_bytes);
1441 
1442 		data_off = chunk * lt->data_bytes;
1443 		spare_off = chunk * lt->spare_bytes;
1444 		ecc_off = (lt->full_chunk_cnt * lt->spare_bytes) +
1445 			  lt->last_spare_bytes +
1446 			  (chunk * (lt->ecc_bytes + 2));
1447 
1448 		data_len = chunk < lt->full_chunk_cnt ? lt->data_bytes :
1449 							lt->last_data_bytes;
1450 		spare_len = chunk < lt->full_chunk_cnt ? lt->spare_bytes :
1451 							 lt->last_spare_bytes;
1452 		ecc_len = chunk < lt->full_chunk_cnt ? lt->ecc_bytes :
1453 						       lt->last_ecc_bytes;
1454 
1455 		/*
1456 		 * Only re-read the ECC bytes, unless we are using the 2k/8b
1457 		 * layout which is buggy in the sense that the ECC engine will
1458 		 * try to correct data bytes anyway, creating bitflips. In this
1459 		 * case, re-read the entire page.
1460 		 */
1461 		if (lt->writesize == 2048 && lt->strength == 8) {
1462 			nand_change_read_column_op(chip, data_off_in_page,
1463 						   buf + data_off, data_len,
1464 						   false);
1465 			nand_change_read_column_op(chip, spare_off_in_page,
1466 						   chip->oob_poi + spare_off, spare_len,
1467 						   false);
1468 		}
1469 
1470 		nand_change_read_column_op(chip, ecc_off_in_page,
1471 					   chip->oob_poi + ecc_off, ecc_len,
1472 					   false);
1473 
1474 		/* Check the entire chunk (data + spare + ecc) for emptyness */
1475 		marvell_nfc_check_empty_chunk(chip, buf + data_off, data_len,
1476 					      chip->oob_poi + spare_off, spare_len,
1477 					      chip->oob_poi + ecc_off, ecc_len,
1478 					      &max_bitflips);
1479 	}
1480 
1481 	return max_bitflips;
1482 }
1483 
1484 static int marvell_nfc_hw_ecc_bch_read_oob_raw(struct nand_chip *chip, int page)
1485 {
1486 	u8 *buf = nand_get_data_buf(chip);
1487 
1488 	return chip->ecc.read_page_raw(chip, buf, true, page);
1489 }
1490 
1491 static int marvell_nfc_hw_ecc_bch_read_oob(struct nand_chip *chip, int page)
1492 {
1493 	u8 *buf = nand_get_data_buf(chip);
1494 
1495 	return chip->ecc.read_page(chip, buf, true, page);
1496 }
1497 
1498 /* BCH write helpers */
1499 static int marvell_nfc_hw_ecc_bch_write_page_raw(struct nand_chip *chip,
1500 						 const u8 *buf,
1501 						 int oob_required, int page)
1502 {
1503 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1504 	int full_chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1505 	int data_len = lt->data_bytes;
1506 	int spare_len = lt->spare_bytes;
1507 	int ecc_len = lt->ecc_bytes;
1508 	int spare_offset = 0;
1509 	int ecc_offset = (lt->full_chunk_cnt * lt->spare_bytes) +
1510 		lt->last_spare_bytes;
1511 	int chunk;
1512 
1513 	marvell_nfc_select_target(chip, chip->cur_cs);
1514 
1515 	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1516 
1517 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1518 		if (chunk >= lt->full_chunk_cnt) {
1519 			data_len = lt->last_data_bytes;
1520 			spare_len = lt->last_spare_bytes;
1521 			ecc_len = lt->last_ecc_bytes;
1522 		}
1523 
1524 		/* Point to the column of the next chunk */
1525 		nand_change_write_column_op(chip, chunk * full_chunk_size,
1526 					    NULL, 0, false);
1527 
1528 		/* Write the data */
1529 		nand_write_data_op(chip, buf + (chunk * lt->data_bytes),
1530 				   data_len, false);
1531 
1532 		if (!oob_required)
1533 			continue;
1534 
1535 		/* Write the spare bytes */
1536 		if (spare_len)
1537 			nand_write_data_op(chip, chip->oob_poi + spare_offset,
1538 					   spare_len, false);
1539 
1540 		/* Write the ECC bytes */
1541 		if (ecc_len)
1542 			nand_write_data_op(chip, chip->oob_poi + ecc_offset,
1543 					   ecc_len, false);
1544 
1545 		spare_offset += spare_len;
1546 		ecc_offset += ALIGN(ecc_len, 32);
1547 	}
1548 
1549 	return nand_prog_page_end_op(chip);
1550 }
1551 
1552 static int
1553 marvell_nfc_hw_ecc_bch_write_chunk(struct nand_chip *chip, int chunk,
1554 				   const u8 *data, unsigned int data_len,
1555 				   const u8 *spare, unsigned int spare_len,
1556 				   int page)
1557 {
1558 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1559 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1560 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1561 	u32 xtype;
1562 	int ret;
1563 	struct marvell_nfc_op nfc_op = {
1564 		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_WRITE) | NDCB0_LEN_OVRD,
1565 		.ndcb[3] = data_len + spare_len,
1566 	};
1567 
1568 	/*
1569 	 * First operation dispatches the CMD_SEQIN command, issue the address
1570 	 * cycles and asks for the first chunk of data.
1571 	 * All operations in the middle (if any) will issue a naked write and
1572 	 * also ask for data.
1573 	 * Last operation (if any) asks for the last chunk of data through a
1574 	 * last naked write.
1575 	 */
1576 	if (chunk == 0) {
1577 		if (lt->nchunks == 1)
1578 			xtype = XTYPE_MONOLITHIC_RW;
1579 		else
1580 			xtype = XTYPE_WRITE_DISPATCH;
1581 
1582 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(xtype) |
1583 				  NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1584 				  NDCB0_CMD1(NAND_CMD_SEQIN);
1585 		nfc_op.ndcb[1] |= NDCB1_ADDRS_PAGE(page);
1586 		nfc_op.ndcb[2] |= NDCB2_ADDR5_PAGE(page);
1587 	} else if (chunk < lt->nchunks - 1) {
1588 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_NAKED_RW);
1589 	} else {
1590 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1591 	}
1592 
1593 	/* Always dispatch the PAGEPROG command on the last chunk */
1594 	if (chunk == lt->nchunks - 1)
1595 		nfc_op.ndcb[0] |= NDCB0_CMD2(NAND_CMD_PAGEPROG) | NDCB0_DBC;
1596 
1597 	ret = marvell_nfc_prepare_cmd(chip);
1598 	if (ret)
1599 		return ret;
1600 
1601 	marvell_nfc_send_cmd(chip, &nfc_op);
1602 	ret = marvell_nfc_end_cmd(chip, NDSR_WRDREQ,
1603 				  "WRDREQ while loading FIFO (data)");
1604 	if (ret)
1605 		return ret;
1606 
1607 	/* Transfer the contents */
1608 	iowrite32_rep(nfc->regs + NDDB, data, FIFO_REP(data_len));
1609 	iowrite32_rep(nfc->regs + NDDB, spare, FIFO_REP(spare_len));
1610 
1611 	return 0;
1612 }
1613 
1614 static int marvell_nfc_hw_ecc_bch_write_page(struct nand_chip *chip,
1615 					     const u8 *buf,
1616 					     int oob_required, int page)
1617 {
1618 	const struct nand_sdr_timings *sdr =
1619 		nand_get_sdr_timings(nand_get_interface_config(chip));
1620 	struct mtd_info *mtd = nand_to_mtd(chip);
1621 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1622 	const u8 *data = buf;
1623 	const u8 *spare = chip->oob_poi;
1624 	int data_len = lt->data_bytes;
1625 	int spare_len = lt->spare_bytes;
1626 	int chunk, ret;
1627 
1628 	marvell_nfc_select_target(chip, chip->cur_cs);
1629 
1630 	/* Spare data will be written anyway, so clear it to avoid garbage */
1631 	if (!oob_required)
1632 		memset(chip->oob_poi, 0xFF, mtd->oobsize);
1633 
1634 	marvell_nfc_enable_hw_ecc(chip);
1635 
1636 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1637 		if (chunk >= lt->full_chunk_cnt) {
1638 			data_len = lt->last_data_bytes;
1639 			spare_len = lt->last_spare_bytes;
1640 		}
1641 
1642 		marvell_nfc_hw_ecc_bch_write_chunk(chip, chunk, data, data_len,
1643 						   spare, spare_len, page);
1644 		data += data_len;
1645 		spare += spare_len;
1646 
1647 		/*
1648 		 * Waiting only for CMDD or PAGED is not enough, ECC are
1649 		 * partially written. No flag is set once the operation is
1650 		 * really finished but the ND_RUN bit is cleared, so wait for it
1651 		 * before stepping into the next command.
1652 		 */
1653 		marvell_nfc_wait_ndrun(chip);
1654 	}
1655 
1656 	ret = marvell_nfc_wait_op(chip, PSEC_TO_MSEC(sdr->tPROG_max));
1657 
1658 	marvell_nfc_disable_hw_ecc(chip);
1659 
1660 	if (ret)
1661 		return ret;
1662 
1663 	return 0;
1664 }
1665 
1666 static int marvell_nfc_hw_ecc_bch_write_oob_raw(struct nand_chip *chip,
1667 						int page)
1668 {
1669 	struct mtd_info *mtd = nand_to_mtd(chip);
1670 	u8 *buf = nand_get_data_buf(chip);
1671 
1672 	memset(buf, 0xFF, mtd->writesize);
1673 
1674 	return chip->ecc.write_page_raw(chip, buf, true, page);
1675 }
1676 
1677 static int marvell_nfc_hw_ecc_bch_write_oob(struct nand_chip *chip, int page)
1678 {
1679 	struct mtd_info *mtd = nand_to_mtd(chip);
1680 	u8 *buf = nand_get_data_buf(chip);
1681 
1682 	memset(buf, 0xFF, mtd->writesize);
1683 
1684 	return chip->ecc.write_page(chip, buf, true, page);
1685 }
1686 
1687 /* NAND framework ->exec_op() hooks and related helpers */
1688 static void marvell_nfc_parse_instructions(struct nand_chip *chip,
1689 					   const struct nand_subop *subop,
1690 					   struct marvell_nfc_op *nfc_op)
1691 {
1692 	const struct nand_op_instr *instr = NULL;
1693 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1694 	bool first_cmd = true;
1695 	unsigned int op_id;
1696 	int i;
1697 
1698 	/* Reset the input structure as most of its fields will be OR'ed */
1699 	memset(nfc_op, 0, sizeof(struct marvell_nfc_op));
1700 
1701 	for (op_id = 0; op_id < subop->ninstrs; op_id++) {
1702 		unsigned int offset, naddrs;
1703 		const u8 *addrs;
1704 		int len;
1705 
1706 		instr = &subop->instrs[op_id];
1707 
1708 		switch (instr->type) {
1709 		case NAND_OP_CMD_INSTR:
1710 			if (first_cmd)
1711 				nfc_op->ndcb[0] |=
1712 					NDCB0_CMD1(instr->ctx.cmd.opcode);
1713 			else
1714 				nfc_op->ndcb[0] |=
1715 					NDCB0_CMD2(instr->ctx.cmd.opcode) |
1716 					NDCB0_DBC;
1717 
1718 			nfc_op->cle_ale_delay_ns = instr->delay_ns;
1719 			first_cmd = false;
1720 			break;
1721 
1722 		case NAND_OP_ADDR_INSTR:
1723 			offset = nand_subop_get_addr_start_off(subop, op_id);
1724 			naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
1725 			addrs = &instr->ctx.addr.addrs[offset];
1726 
1727 			nfc_op->ndcb[0] |= NDCB0_ADDR_CYC(naddrs);
1728 
1729 			for (i = 0; i < min_t(unsigned int, 4, naddrs); i++)
1730 				nfc_op->ndcb[1] |= addrs[i] << (8 * i);
1731 
1732 			if (naddrs >= 5)
1733 				nfc_op->ndcb[2] |= NDCB2_ADDR5_CYC(addrs[4]);
1734 			if (naddrs >= 6)
1735 				nfc_op->ndcb[3] |= NDCB3_ADDR6_CYC(addrs[5]);
1736 			if (naddrs == 7)
1737 				nfc_op->ndcb[3] |= NDCB3_ADDR7_CYC(addrs[6]);
1738 
1739 			nfc_op->cle_ale_delay_ns = instr->delay_ns;
1740 			break;
1741 
1742 		case NAND_OP_DATA_IN_INSTR:
1743 			nfc_op->data_instr = instr;
1744 			nfc_op->data_instr_idx = op_id;
1745 			nfc_op->ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ);
1746 			if (nfc->caps->is_nfcv2) {
1747 				nfc_op->ndcb[0] |=
1748 					NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW) |
1749 					NDCB0_LEN_OVRD;
1750 				len = nand_subop_get_data_len(subop, op_id);
1751 				nfc_op->ndcb[3] |= round_up(len, FIFO_DEPTH);
1752 			}
1753 			nfc_op->data_delay_ns = instr->delay_ns;
1754 			break;
1755 
1756 		case NAND_OP_DATA_OUT_INSTR:
1757 			nfc_op->data_instr = instr;
1758 			nfc_op->data_instr_idx = op_id;
1759 			nfc_op->ndcb[0] |= NDCB0_CMD_TYPE(TYPE_WRITE);
1760 			if (nfc->caps->is_nfcv2) {
1761 				nfc_op->ndcb[0] |=
1762 					NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW) |
1763 					NDCB0_LEN_OVRD;
1764 				len = nand_subop_get_data_len(subop, op_id);
1765 				nfc_op->ndcb[3] |= round_up(len, FIFO_DEPTH);
1766 			}
1767 			nfc_op->data_delay_ns = instr->delay_ns;
1768 			break;
1769 
1770 		case NAND_OP_WAITRDY_INSTR:
1771 			nfc_op->rdy_timeout_ms = instr->ctx.waitrdy.timeout_ms;
1772 			nfc_op->rdy_delay_ns = instr->delay_ns;
1773 			break;
1774 		}
1775 	}
1776 }
1777 
1778 static int marvell_nfc_xfer_data_pio(struct nand_chip *chip,
1779 				     const struct nand_subop *subop,
1780 				     struct marvell_nfc_op *nfc_op)
1781 {
1782 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1783 	const struct nand_op_instr *instr = nfc_op->data_instr;
1784 	unsigned int op_id = nfc_op->data_instr_idx;
1785 	unsigned int len = nand_subop_get_data_len(subop, op_id);
1786 	unsigned int offset = nand_subop_get_data_start_off(subop, op_id);
1787 	bool reading = (instr->type == NAND_OP_DATA_IN_INSTR);
1788 	int ret;
1789 
1790 	if (instr->ctx.data.force_8bit)
1791 		marvell_nfc_force_byte_access(chip, true);
1792 
1793 	if (reading) {
1794 		u8 *in = instr->ctx.data.buf.in + offset;
1795 
1796 		ret = marvell_nfc_xfer_data_in_pio(nfc, in, len);
1797 	} else {
1798 		const u8 *out = instr->ctx.data.buf.out + offset;
1799 
1800 		ret = marvell_nfc_xfer_data_out_pio(nfc, out, len);
1801 	}
1802 
1803 	if (instr->ctx.data.force_8bit)
1804 		marvell_nfc_force_byte_access(chip, false);
1805 
1806 	return ret;
1807 }
1808 
1809 static int marvell_nfc_monolithic_access_exec(struct nand_chip *chip,
1810 					      const struct nand_subop *subop)
1811 {
1812 	struct marvell_nfc_op nfc_op;
1813 	bool reading;
1814 	int ret;
1815 
1816 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1817 	reading = (nfc_op.data_instr->type == NAND_OP_DATA_IN_INSTR);
1818 
1819 	ret = marvell_nfc_prepare_cmd(chip);
1820 	if (ret)
1821 		return ret;
1822 
1823 	marvell_nfc_send_cmd(chip, &nfc_op);
1824 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ | NDSR_WRDREQ,
1825 				  "RDDREQ/WRDREQ while draining raw data");
1826 	if (ret)
1827 		return ret;
1828 
1829 	cond_delay(nfc_op.cle_ale_delay_ns);
1830 
1831 	if (reading) {
1832 		if (nfc_op.rdy_timeout_ms) {
1833 			ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1834 			if (ret)
1835 				return ret;
1836 		}
1837 
1838 		cond_delay(nfc_op.rdy_delay_ns);
1839 	}
1840 
1841 	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1842 	ret = marvell_nfc_wait_cmdd(chip);
1843 	if (ret)
1844 		return ret;
1845 
1846 	cond_delay(nfc_op.data_delay_ns);
1847 
1848 	if (!reading) {
1849 		if (nfc_op.rdy_timeout_ms) {
1850 			ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1851 			if (ret)
1852 				return ret;
1853 		}
1854 
1855 		cond_delay(nfc_op.rdy_delay_ns);
1856 	}
1857 
1858 	/*
1859 	 * NDCR ND_RUN bit should be cleared automatically at the end of each
1860 	 * operation but experience shows that the behavior is buggy when it
1861 	 * comes to writes (with LEN_OVRD). Clear it by hand in this case.
1862 	 */
1863 	if (!reading) {
1864 		struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1865 
1866 		writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
1867 			       nfc->regs + NDCR);
1868 	}
1869 
1870 	return 0;
1871 }
1872 
1873 static int marvell_nfc_naked_access_exec(struct nand_chip *chip,
1874 					 const struct nand_subop *subop)
1875 {
1876 	struct marvell_nfc_op nfc_op;
1877 	int ret;
1878 
1879 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1880 
1881 	/*
1882 	 * Naked access are different in that they need to be flagged as naked
1883 	 * by the controller. Reset the controller registers fields that inform
1884 	 * on the type and refill them according to the ongoing operation.
1885 	 */
1886 	nfc_op.ndcb[0] &= ~(NDCB0_CMD_TYPE(TYPE_MASK) |
1887 			    NDCB0_CMD_XTYPE(XTYPE_MASK));
1888 	switch (subop->instrs[0].type) {
1889 	case NAND_OP_CMD_INSTR:
1890 		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_NAKED_CMD);
1891 		break;
1892 	case NAND_OP_ADDR_INSTR:
1893 		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_NAKED_ADDR);
1894 		break;
1895 	case NAND_OP_DATA_IN_INSTR:
1896 		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ) |
1897 				  NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1898 		break;
1899 	case NAND_OP_DATA_OUT_INSTR:
1900 		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_WRITE) |
1901 				  NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1902 		break;
1903 	default:
1904 		/* This should never happen */
1905 		break;
1906 	}
1907 
1908 	ret = marvell_nfc_prepare_cmd(chip);
1909 	if (ret)
1910 		return ret;
1911 
1912 	marvell_nfc_send_cmd(chip, &nfc_op);
1913 
1914 	if (!nfc_op.data_instr) {
1915 		ret = marvell_nfc_wait_cmdd(chip);
1916 		cond_delay(nfc_op.cle_ale_delay_ns);
1917 		return ret;
1918 	}
1919 
1920 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ | NDSR_WRDREQ,
1921 				  "RDDREQ/WRDREQ while draining raw data");
1922 	if (ret)
1923 		return ret;
1924 
1925 	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1926 	ret = marvell_nfc_wait_cmdd(chip);
1927 	if (ret)
1928 		return ret;
1929 
1930 	/*
1931 	 * NDCR ND_RUN bit should be cleared automatically at the end of each
1932 	 * operation but experience shows that the behavior is buggy when it
1933 	 * comes to writes (with LEN_OVRD). Clear it by hand in this case.
1934 	 */
1935 	if (subop->instrs[0].type == NAND_OP_DATA_OUT_INSTR) {
1936 		struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1937 
1938 		writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
1939 			       nfc->regs + NDCR);
1940 	}
1941 
1942 	return 0;
1943 }
1944 
1945 static int marvell_nfc_naked_waitrdy_exec(struct nand_chip *chip,
1946 					  const struct nand_subop *subop)
1947 {
1948 	struct marvell_nfc_op nfc_op;
1949 	int ret;
1950 
1951 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1952 
1953 	ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1954 	cond_delay(nfc_op.rdy_delay_ns);
1955 
1956 	return ret;
1957 }
1958 
1959 static int marvell_nfc_read_id_type_exec(struct nand_chip *chip,
1960 					 const struct nand_subop *subop)
1961 {
1962 	struct marvell_nfc_op nfc_op;
1963 	int ret;
1964 
1965 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1966 	nfc_op.ndcb[0] &= ~NDCB0_CMD_TYPE(TYPE_READ);
1967 	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ_ID);
1968 
1969 	ret = marvell_nfc_prepare_cmd(chip);
1970 	if (ret)
1971 		return ret;
1972 
1973 	marvell_nfc_send_cmd(chip, &nfc_op);
1974 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1975 				  "RDDREQ while reading ID");
1976 	if (ret)
1977 		return ret;
1978 
1979 	cond_delay(nfc_op.cle_ale_delay_ns);
1980 
1981 	if (nfc_op.rdy_timeout_ms) {
1982 		ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1983 		if (ret)
1984 			return ret;
1985 	}
1986 
1987 	cond_delay(nfc_op.rdy_delay_ns);
1988 
1989 	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1990 	ret = marvell_nfc_wait_cmdd(chip);
1991 	if (ret)
1992 		return ret;
1993 
1994 	cond_delay(nfc_op.data_delay_ns);
1995 
1996 	return 0;
1997 }
1998 
1999 static int marvell_nfc_read_status_exec(struct nand_chip *chip,
2000 					const struct nand_subop *subop)
2001 {
2002 	struct marvell_nfc_op nfc_op;
2003 	int ret;
2004 
2005 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
2006 	nfc_op.ndcb[0] &= ~NDCB0_CMD_TYPE(TYPE_READ);
2007 	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_STATUS);
2008 
2009 	ret = marvell_nfc_prepare_cmd(chip);
2010 	if (ret)
2011 		return ret;
2012 
2013 	marvell_nfc_send_cmd(chip, &nfc_op);
2014 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
2015 				  "RDDREQ while reading status");
2016 	if (ret)
2017 		return ret;
2018 
2019 	cond_delay(nfc_op.cle_ale_delay_ns);
2020 
2021 	if (nfc_op.rdy_timeout_ms) {
2022 		ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
2023 		if (ret)
2024 			return ret;
2025 	}
2026 
2027 	cond_delay(nfc_op.rdy_delay_ns);
2028 
2029 	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
2030 	ret = marvell_nfc_wait_cmdd(chip);
2031 	if (ret)
2032 		return ret;
2033 
2034 	cond_delay(nfc_op.data_delay_ns);
2035 
2036 	return 0;
2037 }
2038 
2039 static int marvell_nfc_reset_cmd_type_exec(struct nand_chip *chip,
2040 					   const struct nand_subop *subop)
2041 {
2042 	struct marvell_nfc_op nfc_op;
2043 	int ret;
2044 
2045 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
2046 	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_RESET);
2047 
2048 	ret = marvell_nfc_prepare_cmd(chip);
2049 	if (ret)
2050 		return ret;
2051 
2052 	marvell_nfc_send_cmd(chip, &nfc_op);
2053 	ret = marvell_nfc_wait_cmdd(chip);
2054 	if (ret)
2055 		return ret;
2056 
2057 	cond_delay(nfc_op.cle_ale_delay_ns);
2058 
2059 	ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
2060 	if (ret)
2061 		return ret;
2062 
2063 	cond_delay(nfc_op.rdy_delay_ns);
2064 
2065 	return 0;
2066 }
2067 
2068 static int marvell_nfc_erase_cmd_type_exec(struct nand_chip *chip,
2069 					   const struct nand_subop *subop)
2070 {
2071 	struct marvell_nfc_op nfc_op;
2072 	int ret;
2073 
2074 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
2075 	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_ERASE);
2076 
2077 	ret = marvell_nfc_prepare_cmd(chip);
2078 	if (ret)
2079 		return ret;
2080 
2081 	marvell_nfc_send_cmd(chip, &nfc_op);
2082 	ret = marvell_nfc_wait_cmdd(chip);
2083 	if (ret)
2084 		return ret;
2085 
2086 	cond_delay(nfc_op.cle_ale_delay_ns);
2087 
2088 	ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
2089 	if (ret)
2090 		return ret;
2091 
2092 	cond_delay(nfc_op.rdy_delay_ns);
2093 
2094 	return 0;
2095 }
2096 
2097 static const struct nand_op_parser marvell_nfcv2_op_parser = NAND_OP_PARSER(
2098 	/* Monolithic reads/writes */
2099 	NAND_OP_PARSER_PATTERN(
2100 		marvell_nfc_monolithic_access_exec,
2101 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2102 		NAND_OP_PARSER_PAT_ADDR_ELEM(true, MAX_ADDRESS_CYC_NFCV2),
2103 		NAND_OP_PARSER_PAT_CMD_ELEM(true),
2104 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
2105 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_CHUNK_SIZE)),
2106 	NAND_OP_PARSER_PATTERN(
2107 		marvell_nfc_monolithic_access_exec,
2108 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2109 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV2),
2110 		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_CHUNK_SIZE),
2111 		NAND_OP_PARSER_PAT_CMD_ELEM(true),
2112 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
2113 	/* Naked commands */
2114 	NAND_OP_PARSER_PATTERN(
2115 		marvell_nfc_naked_access_exec,
2116 		NAND_OP_PARSER_PAT_CMD_ELEM(false)),
2117 	NAND_OP_PARSER_PATTERN(
2118 		marvell_nfc_naked_access_exec,
2119 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV2)),
2120 	NAND_OP_PARSER_PATTERN(
2121 		marvell_nfc_naked_access_exec,
2122 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_CHUNK_SIZE)),
2123 	NAND_OP_PARSER_PATTERN(
2124 		marvell_nfc_naked_access_exec,
2125 		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_CHUNK_SIZE)),
2126 	NAND_OP_PARSER_PATTERN(
2127 		marvell_nfc_naked_waitrdy_exec,
2128 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2129 	);
2130 
2131 static const struct nand_op_parser marvell_nfcv1_op_parser = NAND_OP_PARSER(
2132 	/* Naked commands not supported, use a function for each pattern */
2133 	NAND_OP_PARSER_PATTERN(
2134 		marvell_nfc_read_id_type_exec,
2135 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2136 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV1),
2137 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 8)),
2138 	NAND_OP_PARSER_PATTERN(
2139 		marvell_nfc_erase_cmd_type_exec,
2140 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2141 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV1),
2142 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2143 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2144 	NAND_OP_PARSER_PATTERN(
2145 		marvell_nfc_read_status_exec,
2146 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2147 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 1)),
2148 	NAND_OP_PARSER_PATTERN(
2149 		marvell_nfc_reset_cmd_type_exec,
2150 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2151 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2152 	NAND_OP_PARSER_PATTERN(
2153 		marvell_nfc_naked_waitrdy_exec,
2154 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2155 	);
2156 
2157 static int marvell_nfc_exec_op(struct nand_chip *chip,
2158 			       const struct nand_operation *op,
2159 			       bool check_only)
2160 {
2161 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2162 
2163 	if (!check_only)
2164 		marvell_nfc_select_target(chip, op->cs);
2165 
2166 	if (nfc->caps->is_nfcv2)
2167 		return nand_op_parser_exec_op(chip, &marvell_nfcv2_op_parser,
2168 					      op, check_only);
2169 	else
2170 		return nand_op_parser_exec_op(chip, &marvell_nfcv1_op_parser,
2171 					      op, check_only);
2172 }
2173 
2174 /*
2175  * Layouts were broken in old pxa3xx_nand driver, these are supposed to be
2176  * usable.
2177  */
2178 static int marvell_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2179 				      struct mtd_oob_region *oobregion)
2180 {
2181 	struct nand_chip *chip = mtd_to_nand(mtd);
2182 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
2183 
2184 	if (section)
2185 		return -ERANGE;
2186 
2187 	oobregion->length = (lt->full_chunk_cnt * lt->ecc_bytes) +
2188 			    lt->last_ecc_bytes;
2189 	oobregion->offset = mtd->oobsize - oobregion->length;
2190 
2191 	return 0;
2192 }
2193 
2194 static int marvell_nand_ooblayout_free(struct mtd_info *mtd, int section,
2195 				       struct mtd_oob_region *oobregion)
2196 {
2197 	struct nand_chip *chip = mtd_to_nand(mtd);
2198 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
2199 
2200 	if (section)
2201 		return -ERANGE;
2202 
2203 	/*
2204 	 * Bootrom looks in bytes 0 & 5 for bad blocks for the
2205 	 * 4KB page / 4bit BCH combination.
2206 	 */
2207 	if (mtd->writesize == SZ_4K && lt->data_bytes == SZ_2K)
2208 		oobregion->offset = 6;
2209 	else
2210 		oobregion->offset = 2;
2211 
2212 	oobregion->length = (lt->full_chunk_cnt * lt->spare_bytes) +
2213 			    lt->last_spare_bytes - oobregion->offset;
2214 
2215 	return 0;
2216 }
2217 
2218 static const struct mtd_ooblayout_ops marvell_nand_ooblayout_ops = {
2219 	.ecc = marvell_nand_ooblayout_ecc,
2220 	.free = marvell_nand_ooblayout_free,
2221 };
2222 
2223 static int marvell_nand_hw_ecc_controller_init(struct mtd_info *mtd,
2224 					       struct nand_ecc_ctrl *ecc)
2225 {
2226 	struct nand_chip *chip = mtd_to_nand(mtd);
2227 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2228 	const struct marvell_hw_ecc_layout *l;
2229 	int i;
2230 
2231 	if (!nfc->caps->is_nfcv2 &&
2232 	    (mtd->writesize + mtd->oobsize > MAX_CHUNK_SIZE)) {
2233 		dev_err(nfc->dev,
2234 			"NFCv1: writesize (%d) cannot be bigger than a chunk (%d)\n",
2235 			mtd->writesize, MAX_CHUNK_SIZE - mtd->oobsize);
2236 		return -ENOTSUPP;
2237 	}
2238 
2239 	to_marvell_nand(chip)->layout = NULL;
2240 	for (i = 0; i < ARRAY_SIZE(marvell_nfc_layouts); i++) {
2241 		l = &marvell_nfc_layouts[i];
2242 		if (mtd->writesize == l->writesize &&
2243 		    ecc->size == l->chunk && ecc->strength == l->strength) {
2244 			to_marvell_nand(chip)->layout = l;
2245 			break;
2246 		}
2247 	}
2248 
2249 	if (!to_marvell_nand(chip)->layout ||
2250 	    (!nfc->caps->is_nfcv2 && ecc->strength > 1)) {
2251 		dev_err(nfc->dev,
2252 			"ECC strength %d at page size %d is not supported\n",
2253 			ecc->strength, mtd->writesize);
2254 		return -ENOTSUPP;
2255 	}
2256 
2257 	/* Special care for the layout 2k/8-bit/512B  */
2258 	if (l->writesize == 2048 && l->strength == 8) {
2259 		if (mtd->oobsize < 128) {
2260 			dev_err(nfc->dev, "Requested layout needs at least 128 OOB bytes\n");
2261 			return -ENOTSUPP;
2262 		} else {
2263 			chip->bbt_options |= NAND_BBT_NO_OOB_BBM;
2264 		}
2265 	}
2266 
2267 	mtd_set_ooblayout(mtd, &marvell_nand_ooblayout_ops);
2268 	ecc->steps = l->nchunks;
2269 	ecc->size = l->data_bytes;
2270 
2271 	if (ecc->strength == 1) {
2272 		chip->ecc.algo = NAND_ECC_ALGO_HAMMING;
2273 		ecc->read_page_raw = marvell_nfc_hw_ecc_hmg_read_page_raw;
2274 		ecc->read_page = marvell_nfc_hw_ecc_hmg_read_page;
2275 		ecc->read_oob_raw = marvell_nfc_hw_ecc_hmg_read_oob_raw;
2276 		ecc->read_oob = ecc->read_oob_raw;
2277 		ecc->write_page_raw = marvell_nfc_hw_ecc_hmg_write_page_raw;
2278 		ecc->write_page = marvell_nfc_hw_ecc_hmg_write_page;
2279 		ecc->write_oob_raw = marvell_nfc_hw_ecc_hmg_write_oob_raw;
2280 		ecc->write_oob = ecc->write_oob_raw;
2281 	} else {
2282 		chip->ecc.algo = NAND_ECC_ALGO_BCH;
2283 		ecc->strength = 16;
2284 		ecc->read_page_raw = marvell_nfc_hw_ecc_bch_read_page_raw;
2285 		ecc->read_page = marvell_nfc_hw_ecc_bch_read_page;
2286 		ecc->read_oob_raw = marvell_nfc_hw_ecc_bch_read_oob_raw;
2287 		ecc->read_oob = marvell_nfc_hw_ecc_bch_read_oob;
2288 		ecc->write_page_raw = marvell_nfc_hw_ecc_bch_write_page_raw;
2289 		ecc->write_page = marvell_nfc_hw_ecc_bch_write_page;
2290 		ecc->write_oob_raw = marvell_nfc_hw_ecc_bch_write_oob_raw;
2291 		ecc->write_oob = marvell_nfc_hw_ecc_bch_write_oob;
2292 	}
2293 
2294 	return 0;
2295 }
2296 
2297 static int marvell_nand_ecc_init(struct mtd_info *mtd,
2298 				 struct nand_ecc_ctrl *ecc)
2299 {
2300 	struct nand_chip *chip = mtd_to_nand(mtd);
2301 	const struct nand_ecc_props *requirements =
2302 		nanddev_get_ecc_requirements(&chip->base);
2303 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2304 	int ret;
2305 
2306 	if (ecc->engine_type != NAND_ECC_ENGINE_TYPE_NONE &&
2307 	    (!ecc->size || !ecc->strength)) {
2308 		if (requirements->step_size && requirements->strength) {
2309 			ecc->size = requirements->step_size;
2310 			ecc->strength = requirements->strength;
2311 		} else {
2312 			dev_info(nfc->dev,
2313 				 "No minimum ECC strength, using 1b/512B\n");
2314 			ecc->size = 512;
2315 			ecc->strength = 1;
2316 		}
2317 	}
2318 
2319 	switch (ecc->engine_type) {
2320 	case NAND_ECC_ENGINE_TYPE_ON_HOST:
2321 		ret = marvell_nand_hw_ecc_controller_init(mtd, ecc);
2322 		if (ret)
2323 			return ret;
2324 		break;
2325 	case NAND_ECC_ENGINE_TYPE_NONE:
2326 	case NAND_ECC_ENGINE_TYPE_SOFT:
2327 	case NAND_ECC_ENGINE_TYPE_ON_DIE:
2328 		if (!nfc->caps->is_nfcv2 && mtd->writesize != SZ_512 &&
2329 		    mtd->writesize != SZ_2K) {
2330 			dev_err(nfc->dev, "NFCv1 cannot write %d bytes pages\n",
2331 				mtd->writesize);
2332 			return -EINVAL;
2333 		}
2334 		break;
2335 	default:
2336 		return -EINVAL;
2337 	}
2338 
2339 	return 0;
2340 }
2341 
2342 static u8 bbt_pattern[] = {'M', 'V', 'B', 'b', 't', '0' };
2343 static u8 bbt_mirror_pattern[] = {'1', 't', 'b', 'B', 'V', 'M' };
2344 
2345 static struct nand_bbt_descr bbt_main_descr = {
2346 	.options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
2347 		   NAND_BBT_2BIT | NAND_BBT_VERSION,
2348 	.offs =	8,
2349 	.len = 6,
2350 	.veroffs = 14,
2351 	.maxblocks = 8,	/* Last 8 blocks in each chip */
2352 	.pattern = bbt_pattern
2353 };
2354 
2355 static struct nand_bbt_descr bbt_mirror_descr = {
2356 	.options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
2357 		   NAND_BBT_2BIT | NAND_BBT_VERSION,
2358 	.offs =	8,
2359 	.len = 6,
2360 	.veroffs = 14,
2361 	.maxblocks = 8,	/* Last 8 blocks in each chip */
2362 	.pattern = bbt_mirror_pattern
2363 };
2364 
2365 static int marvell_nfc_setup_interface(struct nand_chip *chip, int chipnr,
2366 				       const struct nand_interface_config *conf)
2367 {
2368 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
2369 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2370 	unsigned int period_ns = 1000000000 / clk_get_rate(nfc->core_clk) * 2;
2371 	const struct nand_sdr_timings *sdr;
2372 	struct marvell_nfc_timings nfc_tmg;
2373 	int read_delay;
2374 
2375 	sdr = nand_get_sdr_timings(conf);
2376 	if (IS_ERR(sdr))
2377 		return PTR_ERR(sdr);
2378 
2379 	/*
2380 	 * SDR timings are given in pico-seconds while NFC timings must be
2381 	 * expressed in NAND controller clock cycles, which is half of the
2382 	 * frequency of the accessible ECC clock retrieved by clk_get_rate().
2383 	 * This is not written anywhere in the datasheet but was observed
2384 	 * with an oscilloscope.
2385 	 *
2386 	 * NFC datasheet gives equations from which thoses calculations
2387 	 * are derived, they tend to be slightly more restrictives than the
2388 	 * given core timings and may improve the overall speed.
2389 	 */
2390 	nfc_tmg.tRP = TO_CYCLES(DIV_ROUND_UP(sdr->tRC_min, 2), period_ns) - 1;
2391 	nfc_tmg.tRH = nfc_tmg.tRP;
2392 	nfc_tmg.tWP = TO_CYCLES(DIV_ROUND_UP(sdr->tWC_min, 2), period_ns) - 1;
2393 	nfc_tmg.tWH = nfc_tmg.tWP;
2394 	nfc_tmg.tCS = TO_CYCLES(sdr->tCS_min, period_ns);
2395 	nfc_tmg.tCH = TO_CYCLES(sdr->tCH_min, period_ns) - 1;
2396 	nfc_tmg.tADL = TO_CYCLES(sdr->tADL_min, period_ns);
2397 	/*
2398 	 * Read delay is the time of propagation from SoC pins to NFC internal
2399 	 * logic. With non-EDO timings, this is MIN_RD_DEL_CNT clock cycles. In
2400 	 * EDO mode, an additional delay of tRH must be taken into account so
2401 	 * the data is sampled on the falling edge instead of the rising edge.
2402 	 */
2403 	read_delay = sdr->tRC_min >= 30000 ?
2404 		MIN_RD_DEL_CNT : MIN_RD_DEL_CNT + nfc_tmg.tRH;
2405 
2406 	nfc_tmg.tAR = TO_CYCLES(sdr->tAR_min, period_ns);
2407 	/*
2408 	 * tWHR and tRHW are supposed to be read to write delays (and vice
2409 	 * versa) but in some cases, ie. when doing a change column, they must
2410 	 * be greater than that to be sure tCCS delay is respected.
2411 	 */
2412 	nfc_tmg.tWHR = TO_CYCLES(max_t(int, sdr->tWHR_min, sdr->tCCS_min),
2413 				 period_ns) - 2;
2414 	nfc_tmg.tRHW = TO_CYCLES(max_t(int, sdr->tRHW_min, sdr->tCCS_min),
2415 				 period_ns);
2416 
2417 	/*
2418 	 * NFCv2: Use WAIT_MODE (wait for RB line), do not rely only on delays.
2419 	 * NFCv1: No WAIT_MODE, tR must be maximal.
2420 	 */
2421 	if (nfc->caps->is_nfcv2) {
2422 		nfc_tmg.tR = TO_CYCLES(sdr->tWB_max, period_ns);
2423 	} else {
2424 		nfc_tmg.tR = TO_CYCLES64(sdr->tWB_max + sdr->tR_max,
2425 					 period_ns);
2426 		if (nfc_tmg.tR + 3 > nfc_tmg.tCH)
2427 			nfc_tmg.tR = nfc_tmg.tCH - 3;
2428 		else
2429 			nfc_tmg.tR = 0;
2430 	}
2431 
2432 	if (chipnr < 0)
2433 		return 0;
2434 
2435 	marvell_nand->ndtr0 =
2436 		NDTR0_TRP(nfc_tmg.tRP) |
2437 		NDTR0_TRH(nfc_tmg.tRH) |
2438 		NDTR0_ETRP(nfc_tmg.tRP) |
2439 		NDTR0_TWP(nfc_tmg.tWP) |
2440 		NDTR0_TWH(nfc_tmg.tWH) |
2441 		NDTR0_TCS(nfc_tmg.tCS) |
2442 		NDTR0_TCH(nfc_tmg.tCH);
2443 
2444 	marvell_nand->ndtr1 =
2445 		NDTR1_TAR(nfc_tmg.tAR) |
2446 		NDTR1_TWHR(nfc_tmg.tWHR) |
2447 		NDTR1_TR(nfc_tmg.tR);
2448 
2449 	if (nfc->caps->is_nfcv2) {
2450 		marvell_nand->ndtr0 |=
2451 			NDTR0_RD_CNT_DEL(read_delay) |
2452 			NDTR0_SELCNTR |
2453 			NDTR0_TADL(nfc_tmg.tADL);
2454 
2455 		marvell_nand->ndtr1 |=
2456 			NDTR1_TRHW(nfc_tmg.tRHW) |
2457 			NDTR1_WAIT_MODE;
2458 	}
2459 
2460 	/*
2461 	 * Reset nfc->selected_chip so the next command will cause the timing
2462 	 * registers to be updated in marvell_nfc_select_target().
2463 	 */
2464 	nfc->selected_chip = NULL;
2465 
2466 	return 0;
2467 }
2468 
2469 static int marvell_nand_attach_chip(struct nand_chip *chip)
2470 {
2471 	struct mtd_info *mtd = nand_to_mtd(chip);
2472 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
2473 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2474 	struct pxa3xx_nand_platform_data *pdata = dev_get_platdata(nfc->dev);
2475 	int ret;
2476 
2477 	if (pdata && pdata->flash_bbt)
2478 		chip->bbt_options |= NAND_BBT_USE_FLASH;
2479 
2480 	if (chip->bbt_options & NAND_BBT_USE_FLASH) {
2481 		/*
2482 		 * We'll use a bad block table stored in-flash and don't
2483 		 * allow writing the bad block marker to the flash.
2484 		 */
2485 		chip->bbt_options |= NAND_BBT_NO_OOB_BBM;
2486 		chip->bbt_td = &bbt_main_descr;
2487 		chip->bbt_md = &bbt_mirror_descr;
2488 	}
2489 
2490 	/* Save the chip-specific fields of NDCR */
2491 	marvell_nand->ndcr = NDCR_PAGE_SZ(mtd->writesize);
2492 	if (chip->options & NAND_BUSWIDTH_16)
2493 		marvell_nand->ndcr |= NDCR_DWIDTH_M | NDCR_DWIDTH_C;
2494 
2495 	/*
2496 	 * On small page NANDs, only one cycle is needed to pass the
2497 	 * column address.
2498 	 */
2499 	if (mtd->writesize <= 512) {
2500 		marvell_nand->addr_cyc = 1;
2501 	} else {
2502 		marvell_nand->addr_cyc = 2;
2503 		marvell_nand->ndcr |= NDCR_RA_START;
2504 	}
2505 
2506 	/*
2507 	 * Now add the number of cycles needed to pass the row
2508 	 * address.
2509 	 *
2510 	 * Addressing a chip using CS 2 or 3 should also need the third row
2511 	 * cycle but due to inconsistance in the documentation and lack of
2512 	 * hardware to test this situation, this case is not supported.
2513 	 */
2514 	if (chip->options & NAND_ROW_ADDR_3)
2515 		marvell_nand->addr_cyc += 3;
2516 	else
2517 		marvell_nand->addr_cyc += 2;
2518 
2519 	if (pdata) {
2520 		chip->ecc.size = pdata->ecc_step_size;
2521 		chip->ecc.strength = pdata->ecc_strength;
2522 	}
2523 
2524 	ret = marvell_nand_ecc_init(mtd, &chip->ecc);
2525 	if (ret) {
2526 		dev_err(nfc->dev, "ECC init failed: %d\n", ret);
2527 		return ret;
2528 	}
2529 
2530 	if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) {
2531 		/*
2532 		 * Subpage write not available with hardware ECC, prohibit also
2533 		 * subpage read as in userspace subpage access would still be
2534 		 * allowed and subpage write, if used, would lead to numerous
2535 		 * uncorrectable ECC errors.
2536 		 */
2537 		chip->options |= NAND_NO_SUBPAGE_WRITE;
2538 	}
2539 
2540 	if (pdata || nfc->caps->legacy_of_bindings) {
2541 		/*
2542 		 * We keep the MTD name unchanged to avoid breaking platforms
2543 		 * where the MTD cmdline parser is used and the bootloader
2544 		 * has not been updated to use the new naming scheme.
2545 		 */
2546 		mtd->name = "pxa3xx_nand-0";
2547 	} else if (!mtd->name) {
2548 		/*
2549 		 * If the new bindings are used and the bootloader has not been
2550 		 * updated to pass a new mtdparts parameter on the cmdline, you
2551 		 * should define the following property in your NAND node, ie:
2552 		 *
2553 		 *	label = "main-storage";
2554 		 *
2555 		 * This way, mtd->name will be set by the core when
2556 		 * nand_set_flash_node() is called.
2557 		 */
2558 		mtd->name = devm_kasprintf(nfc->dev, GFP_KERNEL,
2559 					   "%s:nand.%d", dev_name(nfc->dev),
2560 					   marvell_nand->sels[0].cs);
2561 		if (!mtd->name) {
2562 			dev_err(nfc->dev, "Failed to allocate mtd->name\n");
2563 			return -ENOMEM;
2564 		}
2565 	}
2566 
2567 	return 0;
2568 }
2569 
2570 static const struct nand_controller_ops marvell_nand_controller_ops = {
2571 	.attach_chip = marvell_nand_attach_chip,
2572 	.exec_op = marvell_nfc_exec_op,
2573 	.setup_interface = marvell_nfc_setup_interface,
2574 };
2575 
2576 static int marvell_nand_chip_init(struct device *dev, struct marvell_nfc *nfc,
2577 				  struct device_node *np)
2578 {
2579 	struct pxa3xx_nand_platform_data *pdata = dev_get_platdata(dev);
2580 	struct marvell_nand_chip *marvell_nand;
2581 	struct mtd_info *mtd;
2582 	struct nand_chip *chip;
2583 	int nsels, ret, i;
2584 	u32 cs, rb;
2585 
2586 	/*
2587 	 * The legacy "num-cs" property indicates the number of CS on the only
2588 	 * chip connected to the controller (legacy bindings does not support
2589 	 * more than one chip). The CS and RB pins are always the #0.
2590 	 *
2591 	 * When not using legacy bindings, a couple of "reg" and "nand-rb"
2592 	 * properties must be filled. For each chip, expressed as a subnode,
2593 	 * "reg" points to the CS lines and "nand-rb" to the RB line.
2594 	 */
2595 	if (pdata || nfc->caps->legacy_of_bindings) {
2596 		nsels = 1;
2597 	} else {
2598 		nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
2599 		if (nsels <= 0) {
2600 			dev_err(dev, "missing/invalid reg property\n");
2601 			return -EINVAL;
2602 		}
2603 	}
2604 
2605 	/* Alloc the nand chip structure */
2606 	marvell_nand = devm_kzalloc(dev,
2607 				    struct_size(marvell_nand, sels, nsels),
2608 				    GFP_KERNEL);
2609 	if (!marvell_nand) {
2610 		dev_err(dev, "could not allocate chip structure\n");
2611 		return -ENOMEM;
2612 	}
2613 
2614 	marvell_nand->nsels = nsels;
2615 	marvell_nand->selected_die = -1;
2616 
2617 	for (i = 0; i < nsels; i++) {
2618 		if (pdata || nfc->caps->legacy_of_bindings) {
2619 			/*
2620 			 * Legacy bindings use the CS lines in natural
2621 			 * order (0, 1, ...)
2622 			 */
2623 			cs = i;
2624 		} else {
2625 			/* Retrieve CS id */
2626 			ret = of_property_read_u32_index(np, "reg", i, &cs);
2627 			if (ret) {
2628 				dev_err(dev, "could not retrieve reg property: %d\n",
2629 					ret);
2630 				return ret;
2631 			}
2632 		}
2633 
2634 		if (cs >= nfc->caps->max_cs_nb) {
2635 			dev_err(dev, "invalid reg value: %u (max CS = %d)\n",
2636 				cs, nfc->caps->max_cs_nb);
2637 			return -EINVAL;
2638 		}
2639 
2640 		if (test_and_set_bit(cs, &nfc->assigned_cs)) {
2641 			dev_err(dev, "CS %d already assigned\n", cs);
2642 			return -EINVAL;
2643 		}
2644 
2645 		/*
2646 		 * The cs variable represents the chip select id, which must be
2647 		 * converted in bit fields for NDCB0 and NDCB2 to select the
2648 		 * right chip. Unfortunately, due to a lack of information on
2649 		 * the subject and incoherent documentation, the user should not
2650 		 * use CS1 and CS3 at all as asserting them is not supported in
2651 		 * a reliable way (due to multiplexing inside ADDR5 field).
2652 		 */
2653 		marvell_nand->sels[i].cs = cs;
2654 		switch (cs) {
2655 		case 0:
2656 		case 2:
2657 			marvell_nand->sels[i].ndcb0_csel = 0;
2658 			break;
2659 		case 1:
2660 		case 3:
2661 			marvell_nand->sels[i].ndcb0_csel = NDCB0_CSEL;
2662 			break;
2663 		default:
2664 			return -EINVAL;
2665 		}
2666 
2667 		/* Retrieve RB id */
2668 		if (pdata || nfc->caps->legacy_of_bindings) {
2669 			/* Legacy bindings always use RB #0 */
2670 			rb = 0;
2671 		} else {
2672 			ret = of_property_read_u32_index(np, "nand-rb", i,
2673 							 &rb);
2674 			if (ret) {
2675 				dev_err(dev,
2676 					"could not retrieve RB property: %d\n",
2677 					ret);
2678 				return ret;
2679 			}
2680 		}
2681 
2682 		if (rb >= nfc->caps->max_rb_nb) {
2683 			dev_err(dev, "invalid reg value: %u (max RB = %d)\n",
2684 				rb, nfc->caps->max_rb_nb);
2685 			return -EINVAL;
2686 		}
2687 
2688 		marvell_nand->sels[i].rb = rb;
2689 	}
2690 
2691 	chip = &marvell_nand->chip;
2692 	chip->controller = &nfc->controller;
2693 	nand_set_flash_node(chip, np);
2694 
2695 	if (of_property_read_bool(np, "marvell,nand-keep-config"))
2696 		chip->options |= NAND_KEEP_TIMINGS;
2697 
2698 	mtd = nand_to_mtd(chip);
2699 	mtd->dev.parent = dev;
2700 
2701 	/*
2702 	 * Save a reference value for timing registers before
2703 	 * ->setup_interface() is called.
2704 	 */
2705 	marvell_nand->ndtr0 = readl_relaxed(nfc->regs + NDTR0);
2706 	marvell_nand->ndtr1 = readl_relaxed(nfc->regs + NDTR1);
2707 
2708 	chip->options |= NAND_BUSWIDTH_AUTO;
2709 
2710 	ret = nand_scan(chip, marvell_nand->nsels);
2711 	if (ret) {
2712 		dev_err(dev, "could not scan the nand chip\n");
2713 		return ret;
2714 	}
2715 
2716 	if (pdata)
2717 		/* Legacy bindings support only one chip */
2718 		ret = mtd_device_register(mtd, pdata->parts, pdata->nr_parts);
2719 	else
2720 		ret = mtd_device_register(mtd, NULL, 0);
2721 	if (ret) {
2722 		dev_err(dev, "failed to register mtd device: %d\n", ret);
2723 		nand_cleanup(chip);
2724 		return ret;
2725 	}
2726 
2727 	list_add_tail(&marvell_nand->node, &nfc->chips);
2728 
2729 	return 0;
2730 }
2731 
2732 static void marvell_nand_chips_cleanup(struct marvell_nfc *nfc)
2733 {
2734 	struct marvell_nand_chip *entry, *temp;
2735 	struct nand_chip *chip;
2736 	int ret;
2737 
2738 	list_for_each_entry_safe(entry, temp, &nfc->chips, node) {
2739 		chip = &entry->chip;
2740 		ret = mtd_device_unregister(nand_to_mtd(chip));
2741 		WARN_ON(ret);
2742 		nand_cleanup(chip);
2743 		list_del(&entry->node);
2744 	}
2745 }
2746 
2747 static int marvell_nand_chips_init(struct device *dev, struct marvell_nfc *nfc)
2748 {
2749 	struct device_node *np = dev->of_node;
2750 	struct device_node *nand_np;
2751 	int max_cs = nfc->caps->max_cs_nb;
2752 	int nchips;
2753 	int ret;
2754 
2755 	if (!np)
2756 		nchips = 1;
2757 	else
2758 		nchips = of_get_child_count(np);
2759 
2760 	if (nchips > max_cs) {
2761 		dev_err(dev, "too many NAND chips: %d (max = %d CS)\n", nchips,
2762 			max_cs);
2763 		return -EINVAL;
2764 	}
2765 
2766 	/*
2767 	 * Legacy bindings do not use child nodes to exhibit NAND chip
2768 	 * properties and layout. Instead, NAND properties are mixed with the
2769 	 * controller ones, and partitions are defined as direct subnodes of the
2770 	 * NAND controller node.
2771 	 */
2772 	if (nfc->caps->legacy_of_bindings) {
2773 		ret = marvell_nand_chip_init(dev, nfc, np);
2774 		return ret;
2775 	}
2776 
2777 	for_each_child_of_node(np, nand_np) {
2778 		ret = marvell_nand_chip_init(dev, nfc, nand_np);
2779 		if (ret) {
2780 			of_node_put(nand_np);
2781 			goto cleanup_chips;
2782 		}
2783 	}
2784 
2785 	return 0;
2786 
2787 cleanup_chips:
2788 	marvell_nand_chips_cleanup(nfc);
2789 
2790 	return ret;
2791 }
2792 
2793 static int marvell_nfc_init_dma(struct marvell_nfc *nfc)
2794 {
2795 	struct platform_device *pdev = container_of(nfc->dev,
2796 						    struct platform_device,
2797 						    dev);
2798 	struct dma_slave_config config = {};
2799 	struct resource *r;
2800 	int ret;
2801 
2802 	if (!IS_ENABLED(CONFIG_PXA_DMA)) {
2803 		dev_warn(nfc->dev,
2804 			 "DMA not enabled in configuration\n");
2805 		return -ENOTSUPP;
2806 	}
2807 
2808 	ret = dma_set_mask_and_coherent(nfc->dev, DMA_BIT_MASK(32));
2809 	if (ret)
2810 		return ret;
2811 
2812 	nfc->dma_chan =	dma_request_chan(nfc->dev, "data");
2813 	if (IS_ERR(nfc->dma_chan)) {
2814 		ret = PTR_ERR(nfc->dma_chan);
2815 		nfc->dma_chan = NULL;
2816 		return dev_err_probe(nfc->dev, ret, "DMA channel request failed\n");
2817 	}
2818 
2819 	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2820 	if (!r) {
2821 		ret = -ENXIO;
2822 		goto release_channel;
2823 	}
2824 
2825 	config.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2826 	config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2827 	config.src_addr = r->start + NDDB;
2828 	config.dst_addr = r->start + NDDB;
2829 	config.src_maxburst = 32;
2830 	config.dst_maxburst = 32;
2831 	ret = dmaengine_slave_config(nfc->dma_chan, &config);
2832 	if (ret < 0) {
2833 		dev_err(nfc->dev, "Failed to configure DMA channel\n");
2834 		goto release_channel;
2835 	}
2836 
2837 	/*
2838 	 * DMA must act on length multiple of 32 and this length may be
2839 	 * bigger than the destination buffer. Use this buffer instead
2840 	 * for DMA transfers and then copy the desired amount of data to
2841 	 * the provided buffer.
2842 	 */
2843 	nfc->dma_buf = kmalloc(MAX_CHUNK_SIZE, GFP_KERNEL | GFP_DMA);
2844 	if (!nfc->dma_buf) {
2845 		ret = -ENOMEM;
2846 		goto release_channel;
2847 	}
2848 
2849 	nfc->use_dma = true;
2850 
2851 	return 0;
2852 
2853 release_channel:
2854 	dma_release_channel(nfc->dma_chan);
2855 	nfc->dma_chan = NULL;
2856 
2857 	return ret;
2858 }
2859 
2860 static void marvell_nfc_reset(struct marvell_nfc *nfc)
2861 {
2862 	/*
2863 	 * ECC operations and interruptions are only enabled when specifically
2864 	 * needed. ECC shall not be activated in the early stages (fails probe).
2865 	 * Arbiter flag, even if marked as "reserved", must be set (empirical).
2866 	 * SPARE_EN bit must always be set or ECC bytes will not be at the same
2867 	 * offset in the read page and this will fail the protection.
2868 	 */
2869 	writel_relaxed(NDCR_ALL_INT | NDCR_ND_ARB_EN | NDCR_SPARE_EN |
2870 		       NDCR_RD_ID_CNT(NFCV1_READID_LEN), nfc->regs + NDCR);
2871 	writel_relaxed(0xFFFFFFFF, nfc->regs + NDSR);
2872 	writel_relaxed(0, nfc->regs + NDECCCTRL);
2873 }
2874 
2875 static int marvell_nfc_init(struct marvell_nfc *nfc)
2876 {
2877 	struct device_node *np = nfc->dev->of_node;
2878 
2879 	/*
2880 	 * Some SoCs like A7k/A8k need to enable manually the NAND
2881 	 * controller, gated clocks and reset bits to avoid being bootloader
2882 	 * dependent. This is done through the use of the System Functions
2883 	 * registers.
2884 	 */
2885 	if (nfc->caps->need_system_controller) {
2886 		struct regmap *sysctrl_base =
2887 			syscon_regmap_lookup_by_phandle(np,
2888 							"marvell,system-controller");
2889 
2890 		if (IS_ERR(sysctrl_base))
2891 			return PTR_ERR(sysctrl_base);
2892 
2893 		regmap_write(sysctrl_base, GENCONF_SOC_DEVICE_MUX,
2894 			     GENCONF_SOC_DEVICE_MUX_NFC_EN |
2895 			     GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST |
2896 			     GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST |
2897 			     GENCONF_SOC_DEVICE_MUX_NFC_INT_EN |
2898 			     GENCONF_SOC_DEVICE_MUX_NFC_DEVBUS_ARB_EN);
2899 
2900 		regmap_update_bits(sysctrl_base, GENCONF_CLK_GATING_CTRL,
2901 				   GENCONF_CLK_GATING_CTRL_ND_GATE,
2902 				   GENCONF_CLK_GATING_CTRL_ND_GATE);
2903 	}
2904 
2905 	/* Configure the DMA if appropriate */
2906 	if (!nfc->caps->is_nfcv2)
2907 		marvell_nfc_init_dma(nfc);
2908 
2909 	marvell_nfc_reset(nfc);
2910 
2911 	return 0;
2912 }
2913 
2914 static int marvell_nfc_probe(struct platform_device *pdev)
2915 {
2916 	struct device *dev = &pdev->dev;
2917 	struct marvell_nfc *nfc;
2918 	int ret;
2919 	int irq;
2920 
2921 	nfc = devm_kzalloc(&pdev->dev, sizeof(struct marvell_nfc),
2922 			   GFP_KERNEL);
2923 	if (!nfc)
2924 		return -ENOMEM;
2925 
2926 	nfc->dev = dev;
2927 	nand_controller_init(&nfc->controller);
2928 	nfc->controller.ops = &marvell_nand_controller_ops;
2929 	INIT_LIST_HEAD(&nfc->chips);
2930 
2931 	nfc->regs = devm_platform_ioremap_resource(pdev, 0);
2932 	if (IS_ERR(nfc->regs))
2933 		return PTR_ERR(nfc->regs);
2934 
2935 	irq = platform_get_irq(pdev, 0);
2936 	if (irq < 0)
2937 		return irq;
2938 
2939 	nfc->core_clk = devm_clk_get(&pdev->dev, "core");
2940 
2941 	/* Managed the legacy case (when the first clock was not named) */
2942 	if (nfc->core_clk == ERR_PTR(-ENOENT))
2943 		nfc->core_clk = devm_clk_get(&pdev->dev, NULL);
2944 
2945 	if (IS_ERR(nfc->core_clk))
2946 		return PTR_ERR(nfc->core_clk);
2947 
2948 	ret = clk_prepare_enable(nfc->core_clk);
2949 	if (ret)
2950 		return ret;
2951 
2952 	nfc->reg_clk = devm_clk_get(&pdev->dev, "reg");
2953 	if (IS_ERR(nfc->reg_clk)) {
2954 		if (PTR_ERR(nfc->reg_clk) != -ENOENT) {
2955 			ret = PTR_ERR(nfc->reg_clk);
2956 			goto unprepare_core_clk;
2957 		}
2958 
2959 		nfc->reg_clk = NULL;
2960 	}
2961 
2962 	ret = clk_prepare_enable(nfc->reg_clk);
2963 	if (ret)
2964 		goto unprepare_core_clk;
2965 
2966 	marvell_nfc_disable_int(nfc, NDCR_ALL_INT);
2967 	marvell_nfc_clear_int(nfc, NDCR_ALL_INT);
2968 	ret = devm_request_irq(dev, irq, marvell_nfc_isr,
2969 			       0, "marvell-nfc", nfc);
2970 	if (ret)
2971 		goto unprepare_reg_clk;
2972 
2973 	/* Get NAND controller capabilities */
2974 	if (pdev->id_entry)
2975 		nfc->caps = (void *)pdev->id_entry->driver_data;
2976 	else
2977 		nfc->caps = of_device_get_match_data(&pdev->dev);
2978 
2979 	if (!nfc->caps) {
2980 		dev_err(dev, "Could not retrieve NFC caps\n");
2981 		ret = -EINVAL;
2982 		goto unprepare_reg_clk;
2983 	}
2984 
2985 	/* Init the controller and then probe the chips */
2986 	ret = marvell_nfc_init(nfc);
2987 	if (ret)
2988 		goto unprepare_reg_clk;
2989 
2990 	platform_set_drvdata(pdev, nfc);
2991 
2992 	ret = marvell_nand_chips_init(dev, nfc);
2993 	if (ret)
2994 		goto release_dma;
2995 
2996 	return 0;
2997 
2998 release_dma:
2999 	if (nfc->use_dma)
3000 		dma_release_channel(nfc->dma_chan);
3001 unprepare_reg_clk:
3002 	clk_disable_unprepare(nfc->reg_clk);
3003 unprepare_core_clk:
3004 	clk_disable_unprepare(nfc->core_clk);
3005 
3006 	return ret;
3007 }
3008 
3009 static void marvell_nfc_remove(struct platform_device *pdev)
3010 {
3011 	struct marvell_nfc *nfc = platform_get_drvdata(pdev);
3012 
3013 	marvell_nand_chips_cleanup(nfc);
3014 
3015 	if (nfc->use_dma) {
3016 		dmaengine_terminate_all(nfc->dma_chan);
3017 		dma_release_channel(nfc->dma_chan);
3018 	}
3019 
3020 	clk_disable_unprepare(nfc->reg_clk);
3021 	clk_disable_unprepare(nfc->core_clk);
3022 }
3023 
3024 static int __maybe_unused marvell_nfc_suspend(struct device *dev)
3025 {
3026 	struct marvell_nfc *nfc = dev_get_drvdata(dev);
3027 	struct marvell_nand_chip *chip;
3028 
3029 	list_for_each_entry(chip, &nfc->chips, node)
3030 		marvell_nfc_wait_ndrun(&chip->chip);
3031 
3032 	clk_disable_unprepare(nfc->reg_clk);
3033 	clk_disable_unprepare(nfc->core_clk);
3034 
3035 	return 0;
3036 }
3037 
3038 static int __maybe_unused marvell_nfc_resume(struct device *dev)
3039 {
3040 	struct marvell_nfc *nfc = dev_get_drvdata(dev);
3041 	int ret;
3042 
3043 	ret = clk_prepare_enable(nfc->core_clk);
3044 	if (ret < 0)
3045 		return ret;
3046 
3047 	ret = clk_prepare_enable(nfc->reg_clk);
3048 	if (ret < 0) {
3049 		clk_disable_unprepare(nfc->core_clk);
3050 		return ret;
3051 	}
3052 
3053 	/*
3054 	 * Reset nfc->selected_chip so the next command will cause the timing
3055 	 * registers to be restored in marvell_nfc_select_target().
3056 	 */
3057 	nfc->selected_chip = NULL;
3058 
3059 	/* Reset registers that have lost their contents */
3060 	marvell_nfc_reset(nfc);
3061 
3062 	return 0;
3063 }
3064 
3065 static const struct dev_pm_ops marvell_nfc_pm_ops = {
3066 	SET_SYSTEM_SLEEP_PM_OPS(marvell_nfc_suspend, marvell_nfc_resume)
3067 };
3068 
3069 static const struct marvell_nfc_caps marvell_armada_8k_nfc_caps = {
3070 	.max_cs_nb = 4,
3071 	.max_rb_nb = 2,
3072 	.need_system_controller = true,
3073 	.is_nfcv2 = true,
3074 };
3075 
3076 static const struct marvell_nfc_caps marvell_armada370_nfc_caps = {
3077 	.max_cs_nb = 4,
3078 	.max_rb_nb = 2,
3079 	.is_nfcv2 = true,
3080 };
3081 
3082 static const struct marvell_nfc_caps marvell_pxa3xx_nfc_caps = {
3083 	.max_cs_nb = 2,
3084 	.max_rb_nb = 1,
3085 	.use_dma = true,
3086 };
3087 
3088 static const struct marvell_nfc_caps marvell_armada_8k_nfc_legacy_caps = {
3089 	.max_cs_nb = 4,
3090 	.max_rb_nb = 2,
3091 	.need_system_controller = true,
3092 	.legacy_of_bindings = true,
3093 	.is_nfcv2 = true,
3094 };
3095 
3096 static const struct marvell_nfc_caps marvell_armada370_nfc_legacy_caps = {
3097 	.max_cs_nb = 4,
3098 	.max_rb_nb = 2,
3099 	.legacy_of_bindings = true,
3100 	.is_nfcv2 = true,
3101 };
3102 
3103 static const struct marvell_nfc_caps marvell_pxa3xx_nfc_legacy_caps = {
3104 	.max_cs_nb = 2,
3105 	.max_rb_nb = 1,
3106 	.legacy_of_bindings = true,
3107 	.use_dma = true,
3108 };
3109 
3110 static const struct platform_device_id marvell_nfc_platform_ids[] = {
3111 	{
3112 		.name = "pxa3xx-nand",
3113 		.driver_data = (kernel_ulong_t)&marvell_pxa3xx_nfc_legacy_caps,
3114 	},
3115 	{ /* sentinel */ },
3116 };
3117 MODULE_DEVICE_TABLE(platform, marvell_nfc_platform_ids);
3118 
3119 static const struct of_device_id marvell_nfc_of_ids[] = {
3120 	{
3121 		.compatible = "marvell,armada-8k-nand-controller",
3122 		.data = &marvell_armada_8k_nfc_caps,
3123 	},
3124 	{
3125 		.compatible = "marvell,armada370-nand-controller",
3126 		.data = &marvell_armada370_nfc_caps,
3127 	},
3128 	{
3129 		.compatible = "marvell,pxa3xx-nand-controller",
3130 		.data = &marvell_pxa3xx_nfc_caps,
3131 	},
3132 	/* Support for old/deprecated bindings: */
3133 	{
3134 		.compatible = "marvell,armada-8k-nand",
3135 		.data = &marvell_armada_8k_nfc_legacy_caps,
3136 	},
3137 	{
3138 		.compatible = "marvell,armada370-nand",
3139 		.data = &marvell_armada370_nfc_legacy_caps,
3140 	},
3141 	{
3142 		.compatible = "marvell,pxa3xx-nand",
3143 		.data = &marvell_pxa3xx_nfc_legacy_caps,
3144 	},
3145 	{ /* sentinel */ },
3146 };
3147 MODULE_DEVICE_TABLE(of, marvell_nfc_of_ids);
3148 
3149 static struct platform_driver marvell_nfc_driver = {
3150 	.driver	= {
3151 		.name		= "marvell-nfc",
3152 		.of_match_table = marvell_nfc_of_ids,
3153 		.pm		= &marvell_nfc_pm_ops,
3154 	},
3155 	.id_table = marvell_nfc_platform_ids,
3156 	.probe = marvell_nfc_probe,
3157 	.remove_new = marvell_nfc_remove,
3158 };
3159 module_platform_driver(marvell_nfc_driver);
3160 
3161 MODULE_LICENSE("GPL");
3162 MODULE_DESCRIPTION("Marvell NAND controller driver");
3163