1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Marvell NAND flash controller driver
4  *
5  * Copyright (C) 2017 Marvell
6  * Author: Miquel RAYNAL <miquel.raynal@free-electrons.com>
7  *
8  *
9  * This NAND controller driver handles two versions of the hardware,
10  * one is called NFCv1 and is available on PXA SoCs and the other is
11  * called NFCv2 and is available on Armada SoCs.
12  *
13  * The main visible difference is that NFCv1 only has Hamming ECC
14  * capabilities, while NFCv2 also embeds a BCH ECC engine. Also, DMA
15  * is not used with NFCv2.
16  *
17  * The ECC layouts are depicted in details in Marvell AN-379, but here
18  * is a brief description.
19  *
20  * When using Hamming, the data is split in 512B chunks (either 1, 2
21  * or 4) and each chunk will have its own ECC "digest" of 6B at the
22  * beginning of the OOB area and eventually the remaining free OOB
23  * bytes (also called "spare" bytes in the driver). This engine
24  * corrects up to 1 bit per chunk and detects reliably an error if
25  * there are at most 2 bitflips. Here is the page layout used by the
26  * controller when Hamming is chosen:
27  *
28  * +-------------------------------------------------------------+
29  * | Data 1 | ... | Data N | ECC 1 | ... | ECCN | Free OOB bytes |
30  * +-------------------------------------------------------------+
31  *
32  * When using the BCH engine, there are N identical (data + free OOB +
33  * ECC) sections and potentially an extra one to deal with
34  * configurations where the chosen (data + free OOB + ECC) sizes do
35  * not align with the page (data + OOB) size. ECC bytes are always
36  * 30B per ECC chunk. Here is the page layout used by the controller
37  * when BCH is chosen:
38  *
39  * +-----------------------------------------
40  * | Data 1 | Free OOB bytes 1 | ECC 1 | ...
41  * +-----------------------------------------
42  *
43  *      -------------------------------------------
44  *       ... | Data N | Free OOB bytes N | ECC N |
45  *      -------------------------------------------
46  *
47  *           --------------------------------------------+
48  *            Last Data | Last Free OOB bytes | Last ECC |
49  *           --------------------------------------------+
50  *
51  * In both cases, the layout seen by the user is always: all data
52  * first, then all free OOB bytes and finally all ECC bytes. With BCH,
53  * ECC bytes are 30B long and are padded with 0xFF to align on 32
54  * bytes.
55  *
56  * The controller has certain limitations that are handled by the
57  * driver:
58  *   - It can only read 2k at a time. To overcome this limitation, the
59  *     driver issues data cycles on the bus, without issuing new
60  *     CMD + ADDR cycles. The Marvell term is "naked" operations.
61  *   - The ECC strength in BCH mode cannot be tuned. It is fixed 16
62  *     bits. What can be tuned is the ECC block size as long as it
63  *     stays between 512B and 2kiB. It's usually chosen based on the
64  *     chip ECC requirements. For instance, using 2kiB ECC chunks
65  *     provides 4b/512B correctability.
66  *   - The controller will always treat data bytes, free OOB bytes
67  *     and ECC bytes in that order, no matter what the real layout is
68  *     (which is usually all data then all OOB bytes). The
69  *     marvell_nfc_layouts array below contains the currently
70  *     supported layouts.
71  *   - Because of these weird layouts, the Bad Block Markers can be
72  *     located in data section. In this case, the NAND_BBT_NO_OOB_BBM
73  *     option must be set to prevent scanning/writing bad block
74  *     markers.
75  */
76 
77 #include <linux/module.h>
78 #include <linux/clk.h>
79 #include <linux/mtd/rawnand.h>
80 #include <linux/of_platform.h>
81 #include <linux/iopoll.h>
82 #include <linux/interrupt.h>
83 #include <linux/slab.h>
84 #include <linux/mfd/syscon.h>
85 #include <linux/regmap.h>
86 #include <asm/unaligned.h>
87 
88 #include <linux/dmaengine.h>
89 #include <linux/dma-mapping.h>
90 #include <linux/dma/pxa-dma.h>
91 #include <linux/platform_data/mtd-nand-pxa3xx.h>
92 
93 /* Data FIFO granularity, FIFO reads/writes must be a multiple of this length */
94 #define FIFO_DEPTH		8
95 #define FIFO_REP(x)		(x / sizeof(u32))
96 #define BCH_SEQ_READS		(32 / FIFO_DEPTH)
97 /* NFC does not support transfers of larger chunks at a time */
98 #define MAX_CHUNK_SIZE		2112
99 /* NFCv1 cannot read more that 7 bytes of ID */
100 #define NFCV1_READID_LEN	7
101 /* Polling is done at a pace of POLL_PERIOD us until POLL_TIMEOUT is reached */
102 #define POLL_PERIOD		0
103 #define POLL_TIMEOUT		100000
104 /* Interrupt maximum wait period in ms */
105 #define IRQ_TIMEOUT		1000
106 /* Latency in clock cycles between SoC pins and NFC logic */
107 #define MIN_RD_DEL_CNT		3
108 /* Maximum number of contiguous address cycles */
109 #define MAX_ADDRESS_CYC_NFCV1	5
110 #define MAX_ADDRESS_CYC_NFCV2	7
111 /* System control registers/bits to enable the NAND controller on some SoCs */
112 #define GENCONF_SOC_DEVICE_MUX	0x208
113 #define GENCONF_SOC_DEVICE_MUX_NFC_EN BIT(0)
114 #define GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST BIT(20)
115 #define GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST BIT(21)
116 #define GENCONF_SOC_DEVICE_MUX_NFC_INT_EN BIT(25)
117 #define GENCONF_CLK_GATING_CTRL	0x220
118 #define GENCONF_CLK_GATING_CTRL_ND_GATE BIT(2)
119 #define GENCONF_ND_CLK_CTRL	0x700
120 #define GENCONF_ND_CLK_CTRL_EN	BIT(0)
121 
122 /* NAND controller data flash control register */
123 #define NDCR			0x00
124 #define NDCR_ALL_INT		GENMASK(11, 0)
125 #define NDCR_CS1_CMDDM		BIT(7)
126 #define NDCR_CS0_CMDDM		BIT(8)
127 #define NDCR_RDYM		BIT(11)
128 #define NDCR_ND_ARB_EN		BIT(12)
129 #define NDCR_RA_START		BIT(15)
130 #define NDCR_RD_ID_CNT(x)	(min_t(unsigned int, x, 0x7) << 16)
131 #define NDCR_PAGE_SZ(x)		(x >= 2048 ? BIT(24) : 0)
132 #define NDCR_DWIDTH_M		BIT(26)
133 #define NDCR_DWIDTH_C		BIT(27)
134 #define NDCR_ND_RUN		BIT(28)
135 #define NDCR_DMA_EN		BIT(29)
136 #define NDCR_ECC_EN		BIT(30)
137 #define NDCR_SPARE_EN		BIT(31)
138 #define NDCR_GENERIC_FIELDS_MASK (~(NDCR_RA_START | NDCR_PAGE_SZ(2048) | \
139 				    NDCR_DWIDTH_M | NDCR_DWIDTH_C))
140 
141 /* NAND interface timing parameter 0 register */
142 #define NDTR0			0x04
143 #define NDTR0_TRP(x)		((min_t(unsigned int, x, 0xF) & 0x7) << 0)
144 #define NDTR0_TRH(x)		(min_t(unsigned int, x, 0x7) << 3)
145 #define NDTR0_ETRP(x)		((min_t(unsigned int, x, 0xF) & 0x8) << 3)
146 #define NDTR0_SEL_NRE_EDGE	BIT(7)
147 #define NDTR0_TWP(x)		(min_t(unsigned int, x, 0x7) << 8)
148 #define NDTR0_TWH(x)		(min_t(unsigned int, x, 0x7) << 11)
149 #define NDTR0_TCS(x)		(min_t(unsigned int, x, 0x7) << 16)
150 #define NDTR0_TCH(x)		(min_t(unsigned int, x, 0x7) << 19)
151 #define NDTR0_RD_CNT_DEL(x)	(min_t(unsigned int, x, 0xF) << 22)
152 #define NDTR0_SELCNTR		BIT(26)
153 #define NDTR0_TADL(x)		(min_t(unsigned int, x, 0x1F) << 27)
154 
155 /* NAND interface timing parameter 1 register */
156 #define NDTR1			0x0C
157 #define NDTR1_TAR(x)		(min_t(unsigned int, x, 0xF) << 0)
158 #define NDTR1_TWHR(x)		(min_t(unsigned int, x, 0xF) << 4)
159 #define NDTR1_TRHW(x)		(min_t(unsigned int, x / 16, 0x3) << 8)
160 #define NDTR1_PRESCALE		BIT(14)
161 #define NDTR1_WAIT_MODE		BIT(15)
162 #define NDTR1_TR(x)		(min_t(unsigned int, x, 0xFFFF) << 16)
163 
164 /* NAND controller status register */
165 #define NDSR			0x14
166 #define NDSR_WRCMDREQ		BIT(0)
167 #define NDSR_RDDREQ		BIT(1)
168 #define NDSR_WRDREQ		BIT(2)
169 #define NDSR_CORERR		BIT(3)
170 #define NDSR_UNCERR		BIT(4)
171 #define NDSR_CMDD(cs)		BIT(8 - cs)
172 #define NDSR_RDY(rb)		BIT(11 + rb)
173 #define NDSR_ERRCNT(x)		((x >> 16) & 0x1F)
174 
175 /* NAND ECC control register */
176 #define NDECCCTRL		0x28
177 #define NDECCCTRL_BCH_EN	BIT(0)
178 
179 /* NAND controller data buffer register */
180 #define NDDB			0x40
181 
182 /* NAND controller command buffer 0 register */
183 #define NDCB0			0x48
184 #define NDCB0_CMD1(x)		((x & 0xFF) << 0)
185 #define NDCB0_CMD2(x)		((x & 0xFF) << 8)
186 #define NDCB0_ADDR_CYC(x)	((x & 0x7) << 16)
187 #define NDCB0_ADDR_GET_NUM_CYC(x) (((x) >> 16) & 0x7)
188 #define NDCB0_DBC		BIT(19)
189 #define NDCB0_CMD_TYPE(x)	((x & 0x7) << 21)
190 #define NDCB0_CSEL		BIT(24)
191 #define NDCB0_RDY_BYP		BIT(27)
192 #define NDCB0_LEN_OVRD		BIT(28)
193 #define NDCB0_CMD_XTYPE(x)	((x & 0x7) << 29)
194 
195 /* NAND controller command buffer 1 register */
196 #define NDCB1			0x4C
197 #define NDCB1_COLS(x)		((x & 0xFFFF) << 0)
198 #define NDCB1_ADDRS_PAGE(x)	(x << 16)
199 
200 /* NAND controller command buffer 2 register */
201 #define NDCB2			0x50
202 #define NDCB2_ADDR5_PAGE(x)	(((x >> 16) & 0xFF) << 0)
203 #define NDCB2_ADDR5_CYC(x)	((x & 0xFF) << 0)
204 
205 /* NAND controller command buffer 3 register */
206 #define NDCB3			0x54
207 #define NDCB3_ADDR6_CYC(x)	((x & 0xFF) << 16)
208 #define NDCB3_ADDR7_CYC(x)	((x & 0xFF) << 24)
209 
210 /* NAND controller command buffer 0 register 'type' and 'xtype' fields */
211 #define TYPE_READ		0
212 #define TYPE_WRITE		1
213 #define TYPE_ERASE		2
214 #define TYPE_READ_ID		3
215 #define TYPE_STATUS		4
216 #define TYPE_RESET		5
217 #define TYPE_NAKED_CMD		6
218 #define TYPE_NAKED_ADDR		7
219 #define TYPE_MASK		7
220 #define XTYPE_MONOLITHIC_RW	0
221 #define XTYPE_LAST_NAKED_RW	1
222 #define XTYPE_FINAL_COMMAND	3
223 #define XTYPE_READ		4
224 #define XTYPE_WRITE_DISPATCH	4
225 #define XTYPE_NAKED_RW		5
226 #define XTYPE_COMMAND_DISPATCH	6
227 #define XTYPE_MASK		7
228 
229 /**
230  * struct marvell_hw_ecc_layout - layout of Marvell ECC
231  *
232  * Marvell ECC engine works differently than the others, in order to limit the
233  * size of the IP, hardware engineers chose to set a fixed strength at 16 bits
234  * per subpage, and depending on a the desired strength needed by the NAND chip,
235  * a particular layout mixing data/spare/ecc is defined, with a possible last
236  * chunk smaller that the others.
237  *
238  * @writesize:		Full page size on which the layout applies
239  * @chunk:		Desired ECC chunk size on which the layout applies
240  * @strength:		Desired ECC strength (per chunk size bytes) on which the
241  *			layout applies
242  * @nchunks:		Total number of chunks
243  * @full_chunk_cnt:	Number of full-sized chunks, which is the number of
244  *			repetitions of the pattern:
245  *			(data_bytes + spare_bytes + ecc_bytes).
246  * @data_bytes:		Number of data bytes per chunk
247  * @spare_bytes:	Number of spare bytes per chunk
248  * @ecc_bytes:		Number of ecc bytes per chunk
249  * @last_data_bytes:	Number of data bytes in the last chunk
250  * @last_spare_bytes:	Number of spare bytes in the last chunk
251  * @last_ecc_bytes:	Number of ecc bytes in the last chunk
252  */
253 struct marvell_hw_ecc_layout {
254 	/* Constraints */
255 	int writesize;
256 	int chunk;
257 	int strength;
258 	/* Corresponding layout */
259 	int nchunks;
260 	int full_chunk_cnt;
261 	int data_bytes;
262 	int spare_bytes;
263 	int ecc_bytes;
264 	int last_data_bytes;
265 	int last_spare_bytes;
266 	int last_ecc_bytes;
267 };
268 
269 #define MARVELL_LAYOUT(ws, dc, ds, nc, fcc, db, sb, eb, ldb, lsb, leb)	\
270 	{								\
271 		.writesize = ws,					\
272 		.chunk = dc,						\
273 		.strength = ds,						\
274 		.nchunks = nc,						\
275 		.full_chunk_cnt = fcc,					\
276 		.data_bytes = db,					\
277 		.spare_bytes = sb,					\
278 		.ecc_bytes = eb,					\
279 		.last_data_bytes = ldb,					\
280 		.last_spare_bytes = lsb,				\
281 		.last_ecc_bytes = leb,					\
282 	}
283 
284 /* Layouts explained in AN-379_Marvell_SoC_NFC_ECC */
285 static const struct marvell_hw_ecc_layout marvell_nfc_layouts[] = {
286 	MARVELL_LAYOUT(  512,   512,  1,  1,  1,  512,  8,  8,  0,  0,  0),
287 	MARVELL_LAYOUT( 2048,   512,  1,  1,  1, 2048, 40, 24,  0,  0,  0),
288 	MARVELL_LAYOUT( 2048,   512,  4,  1,  1, 2048, 32, 30,  0,  0,  0),
289 	MARVELL_LAYOUT( 2048,   512,  8,  2,  1, 1024,  0, 30,1024,32, 30),
290 	MARVELL_LAYOUT( 4096,   512,  4,  2,  2, 2048, 32, 30,  0,  0,  0),
291 	MARVELL_LAYOUT( 4096,   512,  8,  5,  4, 1024,  0, 30,  0, 64, 30),
292 	MARVELL_LAYOUT( 8192,   512,  4,  4,  4, 2048,  0, 30,  0,  0,  0),
293 	MARVELL_LAYOUT( 8192,   512,  8,  9,  8, 1024,  0, 30,  0, 160, 30),
294 };
295 
296 /**
297  * struct marvell_nand_chip_sel - CS line description
298  *
299  * The Nand Flash Controller has up to 4 CE and 2 RB pins. The CE selection
300  * is made by a field in NDCB0 register, and in another field in NDCB2 register.
301  * The datasheet describes the logic with an error: ADDR5 field is once
302  * declared at the beginning of NDCB2, and another time at its end. Because the
303  * ADDR5 field of NDCB2 may be used by other bytes, it would be more logical
304  * to use the last bit of this field instead of the first ones.
305  *
306  * @cs:			Wanted CE lane.
307  * @ndcb0_csel:		Value of the NDCB0 register with or without the flag
308  *			selecting the wanted CE lane. This is set once when
309  *			the Device Tree is probed.
310  * @rb:			Ready/Busy pin for the flash chip
311  */
312 struct marvell_nand_chip_sel {
313 	unsigned int cs;
314 	u32 ndcb0_csel;
315 	unsigned int rb;
316 };
317 
318 /**
319  * struct marvell_nand_chip - stores NAND chip device related information
320  *
321  * @chip:		Base NAND chip structure
322  * @node:		Used to store NAND chips into a list
323  * @layout:		NAND layout when using hardware ECC
324  * @ndcr:		Controller register value for this NAND chip
325  * @ndtr0:		Timing registers 0 value for this NAND chip
326  * @ndtr1:		Timing registers 1 value for this NAND chip
327  * @addr_cyc:		Amount of cycles needed to pass column address
328  * @selected_die:	Current active CS
329  * @nsels:		Number of CS lines required by the NAND chip
330  * @sels:		Array of CS lines descriptions
331  */
332 struct marvell_nand_chip {
333 	struct nand_chip chip;
334 	struct list_head node;
335 	const struct marvell_hw_ecc_layout *layout;
336 	u32 ndcr;
337 	u32 ndtr0;
338 	u32 ndtr1;
339 	int addr_cyc;
340 	int selected_die;
341 	unsigned int nsels;
342 	struct marvell_nand_chip_sel sels[];
343 };
344 
345 static inline struct marvell_nand_chip *to_marvell_nand(struct nand_chip *chip)
346 {
347 	return container_of(chip, struct marvell_nand_chip, chip);
348 }
349 
350 static inline struct marvell_nand_chip_sel *to_nand_sel(struct marvell_nand_chip
351 							*nand)
352 {
353 	return &nand->sels[nand->selected_die];
354 }
355 
356 /**
357  * struct marvell_nfc_caps - NAND controller capabilities for distinction
358  *                           between compatible strings
359  *
360  * @max_cs_nb:		Number of Chip Select lines available
361  * @max_rb_nb:		Number of Ready/Busy lines available
362  * @need_system_controller: Indicates if the SoC needs to have access to the
363  *                      system controller (ie. to enable the NAND controller)
364  * @legacy_of_bindings:	Indicates if DT parsing must be done using the old
365  *			fashion way
366  * @is_nfcv2:		NFCv2 has numerous enhancements compared to NFCv1, ie.
367  *			BCH error detection and correction algorithm,
368  *			NDCB3 register has been added
369  * @use_dma:		Use dma for data transfers
370  */
371 struct marvell_nfc_caps {
372 	unsigned int max_cs_nb;
373 	unsigned int max_rb_nb;
374 	bool need_system_controller;
375 	bool legacy_of_bindings;
376 	bool is_nfcv2;
377 	bool use_dma;
378 };
379 
380 /**
381  * struct marvell_nfc - stores Marvell NAND controller information
382  *
383  * @controller:		Base controller structure
384  * @dev:		Parent device (used to print error messages)
385  * @regs:		NAND controller registers
386  * @core_clk:		Core clock
387  * @reg_clk:		Registers clock
388  * @complete:		Completion object to wait for NAND controller events
389  * @assigned_cs:	Bitmask describing already assigned CS lines
390  * @chips:		List containing all the NAND chips attached to
391  *			this NAND controller
392  * @selected_chip:	Currently selected target chip
393  * @caps:		NAND controller capabilities for each compatible string
394  * @use_dma:		Whetner DMA is used
395  * @dma_chan:		DMA channel (NFCv1 only)
396  * @dma_buf:		32-bit aligned buffer for DMA transfers (NFCv1 only)
397  */
398 struct marvell_nfc {
399 	struct nand_controller controller;
400 	struct device *dev;
401 	void __iomem *regs;
402 	struct clk *core_clk;
403 	struct clk *reg_clk;
404 	struct completion complete;
405 	unsigned long assigned_cs;
406 	struct list_head chips;
407 	struct nand_chip *selected_chip;
408 	const struct marvell_nfc_caps *caps;
409 
410 	/* DMA (NFCv1 only) */
411 	bool use_dma;
412 	struct dma_chan *dma_chan;
413 	u8 *dma_buf;
414 };
415 
416 static inline struct marvell_nfc *to_marvell_nfc(struct nand_controller *ctrl)
417 {
418 	return container_of(ctrl, struct marvell_nfc, controller);
419 }
420 
421 /**
422  * struct marvell_nfc_timings - NAND controller timings expressed in NAND
423  *                              Controller clock cycles
424  *
425  * @tRP:		ND_nRE pulse width
426  * @tRH:		ND_nRE high duration
427  * @tWP:		ND_nWE pulse time
428  * @tWH:		ND_nWE high duration
429  * @tCS:		Enable signal setup time
430  * @tCH:		Enable signal hold time
431  * @tADL:		Address to write data delay
432  * @tAR:		ND_ALE low to ND_nRE low delay
433  * @tWHR:		ND_nWE high to ND_nRE low for status read
434  * @tRHW:		ND_nRE high duration, read to write delay
435  * @tR:			ND_nWE high to ND_nRE low for read
436  */
437 struct marvell_nfc_timings {
438 	/* NDTR0 fields */
439 	unsigned int tRP;
440 	unsigned int tRH;
441 	unsigned int tWP;
442 	unsigned int tWH;
443 	unsigned int tCS;
444 	unsigned int tCH;
445 	unsigned int tADL;
446 	/* NDTR1 fields */
447 	unsigned int tAR;
448 	unsigned int tWHR;
449 	unsigned int tRHW;
450 	unsigned int tR;
451 };
452 
453 /**
454  * TO_CYCLES() - Derives a duration in numbers of clock cycles.
455  *
456  * @ps: Duration in pico-seconds
457  * @period_ns:  Clock period in nano-seconds
458  *
459  * Convert the duration in nano-seconds, then divide by the period and
460  * return the number of clock periods.
461  */
462 #define TO_CYCLES(ps, period_ns) (DIV_ROUND_UP(ps / 1000, period_ns))
463 #define TO_CYCLES64(ps, period_ns) (DIV_ROUND_UP_ULL(div_u64(ps, 1000), \
464 						     period_ns))
465 
466 /**
467  * struct marvell_nfc_op - filled during the parsing of the ->exec_op()
468  *                         subop subset of instructions.
469  *
470  * @ndcb:		Array of values written to NDCBx registers
471  * @cle_ale_delay_ns:	Optional delay after the last CMD or ADDR cycle
472  * @rdy_timeout_ms:	Timeout for waits on Ready/Busy pin
473  * @rdy_delay_ns:	Optional delay after waiting for the RB pin
474  * @data_delay_ns:	Optional delay after the data xfer
475  * @data_instr_idx:	Index of the data instruction in the subop
476  * @data_instr:		Pointer to the data instruction in the subop
477  */
478 struct marvell_nfc_op {
479 	u32 ndcb[4];
480 	unsigned int cle_ale_delay_ns;
481 	unsigned int rdy_timeout_ms;
482 	unsigned int rdy_delay_ns;
483 	unsigned int data_delay_ns;
484 	unsigned int data_instr_idx;
485 	const struct nand_op_instr *data_instr;
486 };
487 
488 /*
489  * Internal helper to conditionnally apply a delay (from the above structure,
490  * most of the time).
491  */
492 static void cond_delay(unsigned int ns)
493 {
494 	if (!ns)
495 		return;
496 
497 	if (ns < 10000)
498 		ndelay(ns);
499 	else
500 		udelay(DIV_ROUND_UP(ns, 1000));
501 }
502 
503 /*
504  * The controller has many flags that could generate interrupts, most of them
505  * are disabled and polling is used. For the very slow signals, using interrupts
506  * may relax the CPU charge.
507  */
508 static void marvell_nfc_disable_int(struct marvell_nfc *nfc, u32 int_mask)
509 {
510 	u32 reg;
511 
512 	/* Writing 1 disables the interrupt */
513 	reg = readl_relaxed(nfc->regs + NDCR);
514 	writel_relaxed(reg | int_mask, nfc->regs + NDCR);
515 }
516 
517 static void marvell_nfc_enable_int(struct marvell_nfc *nfc, u32 int_mask)
518 {
519 	u32 reg;
520 
521 	/* Writing 0 enables the interrupt */
522 	reg = readl_relaxed(nfc->regs + NDCR);
523 	writel_relaxed(reg & ~int_mask, nfc->regs + NDCR);
524 }
525 
526 static u32 marvell_nfc_clear_int(struct marvell_nfc *nfc, u32 int_mask)
527 {
528 	u32 reg;
529 
530 	reg = readl_relaxed(nfc->regs + NDSR);
531 	writel_relaxed(int_mask, nfc->regs + NDSR);
532 
533 	return reg & int_mask;
534 }
535 
536 static void marvell_nfc_force_byte_access(struct nand_chip *chip,
537 					  bool force_8bit)
538 {
539 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
540 	u32 ndcr;
541 
542 	/*
543 	 * Callers of this function do not verify if the NAND is using a 16-bit
544 	 * an 8-bit bus for normal operations, so we need to take care of that
545 	 * here by leaving the configuration unchanged if the NAND does not have
546 	 * the NAND_BUSWIDTH_16 flag set.
547 	 */
548 	if (!(chip->options & NAND_BUSWIDTH_16))
549 		return;
550 
551 	ndcr = readl_relaxed(nfc->regs + NDCR);
552 
553 	if (force_8bit)
554 		ndcr &= ~(NDCR_DWIDTH_M | NDCR_DWIDTH_C);
555 	else
556 		ndcr |= NDCR_DWIDTH_M | NDCR_DWIDTH_C;
557 
558 	writel_relaxed(ndcr, nfc->regs + NDCR);
559 }
560 
561 static int marvell_nfc_wait_ndrun(struct nand_chip *chip)
562 {
563 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
564 	u32 val;
565 	int ret;
566 
567 	/*
568 	 * The command is being processed, wait for the ND_RUN bit to be
569 	 * cleared by the NFC. If not, we must clear it by hand.
570 	 */
571 	ret = readl_relaxed_poll_timeout(nfc->regs + NDCR, val,
572 					 (val & NDCR_ND_RUN) == 0,
573 					 POLL_PERIOD, POLL_TIMEOUT);
574 	if (ret) {
575 		dev_err(nfc->dev, "Timeout on NAND controller run mode\n");
576 		writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
577 			       nfc->regs + NDCR);
578 		return ret;
579 	}
580 
581 	return 0;
582 }
583 
584 /*
585  * Any time a command has to be sent to the controller, the following sequence
586  * has to be followed:
587  * - call marvell_nfc_prepare_cmd()
588  *      -> activate the ND_RUN bit that will kind of 'start a job'
589  *      -> wait the signal indicating the NFC is waiting for a command
590  * - send the command (cmd and address cycles)
591  * - enventually send or receive the data
592  * - call marvell_nfc_end_cmd() with the corresponding flag
593  *      -> wait the flag to be triggered or cancel the job with a timeout
594  *
595  * The following helpers are here to factorize the code a bit so that
596  * specialized functions responsible for executing the actual NAND
597  * operations do not have to replicate the same code blocks.
598  */
599 static int marvell_nfc_prepare_cmd(struct nand_chip *chip)
600 {
601 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
602 	u32 ndcr, val;
603 	int ret;
604 
605 	/* Poll ND_RUN and clear NDSR before issuing any command */
606 	ret = marvell_nfc_wait_ndrun(chip);
607 	if (ret) {
608 		dev_err(nfc->dev, "Last operation did not succeed\n");
609 		return ret;
610 	}
611 
612 	ndcr = readl_relaxed(nfc->regs + NDCR);
613 	writel_relaxed(readl(nfc->regs + NDSR), nfc->regs + NDSR);
614 
615 	/* Assert ND_RUN bit and wait the NFC to be ready */
616 	writel_relaxed(ndcr | NDCR_ND_RUN, nfc->regs + NDCR);
617 	ret = readl_relaxed_poll_timeout(nfc->regs + NDSR, val,
618 					 val & NDSR_WRCMDREQ,
619 					 POLL_PERIOD, POLL_TIMEOUT);
620 	if (ret) {
621 		dev_err(nfc->dev, "Timeout on WRCMDRE\n");
622 		return -ETIMEDOUT;
623 	}
624 
625 	/* Command may be written, clear WRCMDREQ status bit */
626 	writel_relaxed(NDSR_WRCMDREQ, nfc->regs + NDSR);
627 
628 	return 0;
629 }
630 
631 static void marvell_nfc_send_cmd(struct nand_chip *chip,
632 				 struct marvell_nfc_op *nfc_op)
633 {
634 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
635 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
636 
637 	dev_dbg(nfc->dev, "\nNDCR:  0x%08x\n"
638 		"NDCB0: 0x%08x\nNDCB1: 0x%08x\nNDCB2: 0x%08x\nNDCB3: 0x%08x\n",
639 		(u32)readl_relaxed(nfc->regs + NDCR), nfc_op->ndcb[0],
640 		nfc_op->ndcb[1], nfc_op->ndcb[2], nfc_op->ndcb[3]);
641 
642 	writel_relaxed(to_nand_sel(marvell_nand)->ndcb0_csel | nfc_op->ndcb[0],
643 		       nfc->regs + NDCB0);
644 	writel_relaxed(nfc_op->ndcb[1], nfc->regs + NDCB0);
645 	writel(nfc_op->ndcb[2], nfc->regs + NDCB0);
646 
647 	/*
648 	 * Write NDCB0 four times only if LEN_OVRD is set or if ADDR6 or ADDR7
649 	 * fields are used (only available on NFCv2).
650 	 */
651 	if (nfc_op->ndcb[0] & NDCB0_LEN_OVRD ||
652 	    NDCB0_ADDR_GET_NUM_CYC(nfc_op->ndcb[0]) >= 6) {
653 		if (!WARN_ON_ONCE(!nfc->caps->is_nfcv2))
654 			writel(nfc_op->ndcb[3], nfc->regs + NDCB0);
655 	}
656 }
657 
658 static int marvell_nfc_end_cmd(struct nand_chip *chip, int flag,
659 			       const char *label)
660 {
661 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
662 	u32 val;
663 	int ret;
664 
665 	ret = readl_relaxed_poll_timeout(nfc->regs + NDSR, val,
666 					 val & flag,
667 					 POLL_PERIOD, POLL_TIMEOUT);
668 
669 	if (ret) {
670 		dev_err(nfc->dev, "Timeout on %s (NDSR: 0x%08x)\n",
671 			label, val);
672 		if (nfc->dma_chan)
673 			dmaengine_terminate_all(nfc->dma_chan);
674 		return ret;
675 	}
676 
677 	/*
678 	 * DMA function uses this helper to poll on CMDD bits without wanting
679 	 * them to be cleared.
680 	 */
681 	if (nfc->use_dma && (readl_relaxed(nfc->regs + NDCR) & NDCR_DMA_EN))
682 		return 0;
683 
684 	writel_relaxed(flag, nfc->regs + NDSR);
685 
686 	return 0;
687 }
688 
689 static int marvell_nfc_wait_cmdd(struct nand_chip *chip)
690 {
691 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
692 	int cs_flag = NDSR_CMDD(to_nand_sel(marvell_nand)->ndcb0_csel);
693 
694 	return marvell_nfc_end_cmd(chip, cs_flag, "CMDD");
695 }
696 
697 static int marvell_nfc_poll_status(struct marvell_nfc *nfc, u32 mask,
698 				   u32 expected_val, unsigned long timeout_ms)
699 {
700 	unsigned long limit;
701 	u32 st;
702 
703 	limit = jiffies + msecs_to_jiffies(timeout_ms);
704 	do {
705 		st = readl_relaxed(nfc->regs + NDSR);
706 		if (st & NDSR_RDY(1))
707 			st |= NDSR_RDY(0);
708 
709 		if ((st & mask) == expected_val)
710 			return 0;
711 
712 		cpu_relax();
713 	} while (time_after(limit, jiffies));
714 
715 	return -ETIMEDOUT;
716 }
717 
718 static int marvell_nfc_wait_op(struct nand_chip *chip, unsigned int timeout_ms)
719 {
720 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
721 	struct mtd_info *mtd = nand_to_mtd(chip);
722 	u32 pending;
723 	int ret;
724 
725 	/* Timeout is expressed in ms */
726 	if (!timeout_ms)
727 		timeout_ms = IRQ_TIMEOUT;
728 
729 	if (mtd->oops_panic_write) {
730 		ret = marvell_nfc_poll_status(nfc, NDSR_RDY(0),
731 					      NDSR_RDY(0),
732 					      timeout_ms);
733 	} else {
734 		init_completion(&nfc->complete);
735 
736 		marvell_nfc_enable_int(nfc, NDCR_RDYM);
737 		ret = wait_for_completion_timeout(&nfc->complete,
738 						  msecs_to_jiffies(timeout_ms));
739 		marvell_nfc_disable_int(nfc, NDCR_RDYM);
740 	}
741 	pending = marvell_nfc_clear_int(nfc, NDSR_RDY(0) | NDSR_RDY(1));
742 
743 	/*
744 	 * In case the interrupt was not served in the required time frame,
745 	 * check if the ISR was not served or if something went actually wrong.
746 	 */
747 	if (!ret && !pending) {
748 		dev_err(nfc->dev, "Timeout waiting for RB signal\n");
749 		return -ETIMEDOUT;
750 	}
751 
752 	return 0;
753 }
754 
755 static void marvell_nfc_select_target(struct nand_chip *chip,
756 				      unsigned int die_nr)
757 {
758 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
759 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
760 	u32 ndcr_generic;
761 
762 	/*
763 	 * Reset the NDCR register to a clean state for this particular chip,
764 	 * also clear ND_RUN bit.
765 	 */
766 	ndcr_generic = readl_relaxed(nfc->regs + NDCR) &
767 		       NDCR_GENERIC_FIELDS_MASK & ~NDCR_ND_RUN;
768 	writel_relaxed(ndcr_generic | marvell_nand->ndcr, nfc->regs + NDCR);
769 
770 	/* Also reset the interrupt status register */
771 	marvell_nfc_clear_int(nfc, NDCR_ALL_INT);
772 
773 	if (chip == nfc->selected_chip && die_nr == marvell_nand->selected_die)
774 		return;
775 
776 	writel_relaxed(marvell_nand->ndtr0, nfc->regs + NDTR0);
777 	writel_relaxed(marvell_nand->ndtr1, nfc->regs + NDTR1);
778 
779 	nfc->selected_chip = chip;
780 	marvell_nand->selected_die = die_nr;
781 }
782 
783 static irqreturn_t marvell_nfc_isr(int irq, void *dev_id)
784 {
785 	struct marvell_nfc *nfc = dev_id;
786 	u32 st = readl_relaxed(nfc->regs + NDSR);
787 	u32 ien = (~readl_relaxed(nfc->regs + NDCR)) & NDCR_ALL_INT;
788 
789 	/*
790 	 * RDY interrupt mask is one bit in NDCR while there are two status
791 	 * bit in NDSR (RDY[cs0/cs2] and RDY[cs1/cs3]).
792 	 */
793 	if (st & NDSR_RDY(1))
794 		st |= NDSR_RDY(0);
795 
796 	if (!(st & ien))
797 		return IRQ_NONE;
798 
799 	marvell_nfc_disable_int(nfc, st & NDCR_ALL_INT);
800 
801 	if (st & (NDSR_RDY(0) | NDSR_RDY(1)))
802 		complete(&nfc->complete);
803 
804 	return IRQ_HANDLED;
805 }
806 
807 /* HW ECC related functions */
808 static void marvell_nfc_enable_hw_ecc(struct nand_chip *chip)
809 {
810 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
811 	u32 ndcr = readl_relaxed(nfc->regs + NDCR);
812 
813 	if (!(ndcr & NDCR_ECC_EN)) {
814 		writel_relaxed(ndcr | NDCR_ECC_EN, nfc->regs + NDCR);
815 
816 		/*
817 		 * When enabling BCH, set threshold to 0 to always know the
818 		 * number of corrected bitflips.
819 		 */
820 		if (chip->ecc.algo == NAND_ECC_ALGO_BCH)
821 			writel_relaxed(NDECCCTRL_BCH_EN, nfc->regs + NDECCCTRL);
822 	}
823 }
824 
825 static void marvell_nfc_disable_hw_ecc(struct nand_chip *chip)
826 {
827 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
828 	u32 ndcr = readl_relaxed(nfc->regs + NDCR);
829 
830 	if (ndcr & NDCR_ECC_EN) {
831 		writel_relaxed(ndcr & ~NDCR_ECC_EN, nfc->regs + NDCR);
832 		if (chip->ecc.algo == NAND_ECC_ALGO_BCH)
833 			writel_relaxed(0, nfc->regs + NDECCCTRL);
834 	}
835 }
836 
837 /* DMA related helpers */
838 static void marvell_nfc_enable_dma(struct marvell_nfc *nfc)
839 {
840 	u32 reg;
841 
842 	reg = readl_relaxed(nfc->regs + NDCR);
843 	writel_relaxed(reg | NDCR_DMA_EN, nfc->regs + NDCR);
844 }
845 
846 static void marvell_nfc_disable_dma(struct marvell_nfc *nfc)
847 {
848 	u32 reg;
849 
850 	reg = readl_relaxed(nfc->regs + NDCR);
851 	writel_relaxed(reg & ~NDCR_DMA_EN, nfc->regs + NDCR);
852 }
853 
854 /* Read/write PIO/DMA accessors */
855 static int marvell_nfc_xfer_data_dma(struct marvell_nfc *nfc,
856 				     enum dma_data_direction direction,
857 				     unsigned int len)
858 {
859 	unsigned int dma_len = min_t(int, ALIGN(len, 32), MAX_CHUNK_SIZE);
860 	struct dma_async_tx_descriptor *tx;
861 	struct scatterlist sg;
862 	dma_cookie_t cookie;
863 	int ret;
864 
865 	marvell_nfc_enable_dma(nfc);
866 	/* Prepare the DMA transfer */
867 	sg_init_one(&sg, nfc->dma_buf, dma_len);
868 	ret = dma_map_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
869 	if (!ret) {
870 		dev_err(nfc->dev, "Could not map DMA S/G list\n");
871 		return -ENXIO;
872 	}
873 
874 	tx = dmaengine_prep_slave_sg(nfc->dma_chan, &sg, 1,
875 				     direction == DMA_FROM_DEVICE ?
876 				     DMA_DEV_TO_MEM : DMA_MEM_TO_DEV,
877 				     DMA_PREP_INTERRUPT);
878 	if (!tx) {
879 		dev_err(nfc->dev, "Could not prepare DMA S/G list\n");
880 		dma_unmap_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
881 		return -ENXIO;
882 	}
883 
884 	/* Do the task and wait for it to finish */
885 	cookie = dmaengine_submit(tx);
886 	ret = dma_submit_error(cookie);
887 	if (ret)
888 		return -EIO;
889 
890 	dma_async_issue_pending(nfc->dma_chan);
891 	ret = marvell_nfc_wait_cmdd(nfc->selected_chip);
892 	dma_unmap_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
893 	marvell_nfc_disable_dma(nfc);
894 	if (ret) {
895 		dev_err(nfc->dev, "Timeout waiting for DMA (status: %d)\n",
896 			dmaengine_tx_status(nfc->dma_chan, cookie, NULL));
897 		dmaengine_terminate_all(nfc->dma_chan);
898 		return -ETIMEDOUT;
899 	}
900 
901 	return 0;
902 }
903 
904 static int marvell_nfc_xfer_data_in_pio(struct marvell_nfc *nfc, u8 *in,
905 					unsigned int len)
906 {
907 	unsigned int last_len = len % FIFO_DEPTH;
908 	unsigned int last_full_offset = round_down(len, FIFO_DEPTH);
909 	int i;
910 
911 	for (i = 0; i < last_full_offset; i += FIFO_DEPTH)
912 		ioread32_rep(nfc->regs + NDDB, in + i, FIFO_REP(FIFO_DEPTH));
913 
914 	if (last_len) {
915 		u8 tmp_buf[FIFO_DEPTH];
916 
917 		ioread32_rep(nfc->regs + NDDB, tmp_buf, FIFO_REP(FIFO_DEPTH));
918 		memcpy(in + last_full_offset, tmp_buf, last_len);
919 	}
920 
921 	return 0;
922 }
923 
924 static int marvell_nfc_xfer_data_out_pio(struct marvell_nfc *nfc, const u8 *out,
925 					 unsigned int len)
926 {
927 	unsigned int last_len = len % FIFO_DEPTH;
928 	unsigned int last_full_offset = round_down(len, FIFO_DEPTH);
929 	int i;
930 
931 	for (i = 0; i < last_full_offset; i += FIFO_DEPTH)
932 		iowrite32_rep(nfc->regs + NDDB, out + i, FIFO_REP(FIFO_DEPTH));
933 
934 	if (last_len) {
935 		u8 tmp_buf[FIFO_DEPTH];
936 
937 		memcpy(tmp_buf, out + last_full_offset, last_len);
938 		iowrite32_rep(nfc->regs + NDDB, tmp_buf, FIFO_REP(FIFO_DEPTH));
939 	}
940 
941 	return 0;
942 }
943 
944 static void marvell_nfc_check_empty_chunk(struct nand_chip *chip,
945 					  u8 *data, int data_len,
946 					  u8 *spare, int spare_len,
947 					  u8 *ecc, int ecc_len,
948 					  unsigned int *max_bitflips)
949 {
950 	struct mtd_info *mtd = nand_to_mtd(chip);
951 	int bf;
952 
953 	/*
954 	 * Blank pages (all 0xFF) that have not been written may be recognized
955 	 * as bad if bitflips occur, so whenever an uncorrectable error occurs,
956 	 * check if the entire page (with ECC bytes) is actually blank or not.
957 	 */
958 	if (!data)
959 		data_len = 0;
960 	if (!spare)
961 		spare_len = 0;
962 	if (!ecc)
963 		ecc_len = 0;
964 
965 	bf = nand_check_erased_ecc_chunk(data, data_len, ecc, ecc_len,
966 					 spare, spare_len, chip->ecc.strength);
967 	if (bf < 0) {
968 		mtd->ecc_stats.failed++;
969 		return;
970 	}
971 
972 	/* Update the stats and max_bitflips */
973 	mtd->ecc_stats.corrected += bf;
974 	*max_bitflips = max_t(unsigned int, *max_bitflips, bf);
975 }
976 
977 /*
978  * Check if a chunk is correct or not according to the hardware ECC engine.
979  * mtd->ecc_stats.corrected is updated, as well as max_bitflips, however
980  * mtd->ecc_stats.failure is not, the function will instead return a non-zero
981  * value indicating that a check on the emptyness of the subpage must be
982  * performed before actually declaring the subpage as "corrupted".
983  */
984 static int marvell_nfc_hw_ecc_check_bitflips(struct nand_chip *chip,
985 					     unsigned int *max_bitflips)
986 {
987 	struct mtd_info *mtd = nand_to_mtd(chip);
988 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
989 	int bf = 0;
990 	u32 ndsr;
991 
992 	ndsr = readl_relaxed(nfc->regs + NDSR);
993 
994 	/* Check uncorrectable error flag */
995 	if (ndsr & NDSR_UNCERR) {
996 		writel_relaxed(ndsr, nfc->regs + NDSR);
997 
998 		/*
999 		 * Do not increment ->ecc_stats.failed now, instead, return a
1000 		 * non-zero value to indicate that this chunk was apparently
1001 		 * bad, and it should be check to see if it empty or not. If
1002 		 * the chunk (with ECC bytes) is not declared empty, the calling
1003 		 * function must increment the failure count.
1004 		 */
1005 		return -EBADMSG;
1006 	}
1007 
1008 	/* Check correctable error flag */
1009 	if (ndsr & NDSR_CORERR) {
1010 		writel_relaxed(ndsr, nfc->regs + NDSR);
1011 
1012 		if (chip->ecc.algo == NAND_ECC_ALGO_BCH)
1013 			bf = NDSR_ERRCNT(ndsr);
1014 		else
1015 			bf = 1;
1016 	}
1017 
1018 	/* Update the stats and max_bitflips */
1019 	mtd->ecc_stats.corrected += bf;
1020 	*max_bitflips = max_t(unsigned int, *max_bitflips, bf);
1021 
1022 	return 0;
1023 }
1024 
1025 /* Hamming read helpers */
1026 static int marvell_nfc_hw_ecc_hmg_do_read_page(struct nand_chip *chip,
1027 					       u8 *data_buf, u8 *oob_buf,
1028 					       bool raw, int page)
1029 {
1030 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1031 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1032 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1033 	struct marvell_nfc_op nfc_op = {
1034 		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_READ) |
1035 			   NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1036 			   NDCB0_DBC |
1037 			   NDCB0_CMD1(NAND_CMD_READ0) |
1038 			   NDCB0_CMD2(NAND_CMD_READSTART),
1039 		.ndcb[1] = NDCB1_ADDRS_PAGE(page),
1040 		.ndcb[2] = NDCB2_ADDR5_PAGE(page),
1041 	};
1042 	unsigned int oob_bytes = lt->spare_bytes + (raw ? lt->ecc_bytes : 0);
1043 	int ret;
1044 
1045 	/* NFCv2 needs more information about the operation being executed */
1046 	if (nfc->caps->is_nfcv2)
1047 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1048 
1049 	ret = marvell_nfc_prepare_cmd(chip);
1050 	if (ret)
1051 		return ret;
1052 
1053 	marvell_nfc_send_cmd(chip, &nfc_op);
1054 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1055 				  "RDDREQ while draining FIFO (data/oob)");
1056 	if (ret)
1057 		return ret;
1058 
1059 	/*
1060 	 * Read the page then the OOB area. Unlike what is shown in current
1061 	 * documentation, spare bytes are protected by the ECC engine, and must
1062 	 * be at the beginning of the OOB area or running this driver on legacy
1063 	 * systems will prevent the discovery of the BBM/BBT.
1064 	 */
1065 	if (nfc->use_dma) {
1066 		marvell_nfc_xfer_data_dma(nfc, DMA_FROM_DEVICE,
1067 					  lt->data_bytes + oob_bytes);
1068 		memcpy(data_buf, nfc->dma_buf, lt->data_bytes);
1069 		memcpy(oob_buf, nfc->dma_buf + lt->data_bytes, oob_bytes);
1070 	} else {
1071 		marvell_nfc_xfer_data_in_pio(nfc, data_buf, lt->data_bytes);
1072 		marvell_nfc_xfer_data_in_pio(nfc, oob_buf, oob_bytes);
1073 	}
1074 
1075 	ret = marvell_nfc_wait_cmdd(chip);
1076 	return ret;
1077 }
1078 
1079 static int marvell_nfc_hw_ecc_hmg_read_page_raw(struct nand_chip *chip, u8 *buf,
1080 						int oob_required, int page)
1081 {
1082 	marvell_nfc_select_target(chip, chip->cur_cs);
1083 	return marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi,
1084 						   true, page);
1085 }
1086 
1087 static int marvell_nfc_hw_ecc_hmg_read_page(struct nand_chip *chip, u8 *buf,
1088 					    int oob_required, int page)
1089 {
1090 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1091 	unsigned int full_sz = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1092 	int max_bitflips = 0, ret;
1093 	u8 *raw_buf;
1094 
1095 	marvell_nfc_select_target(chip, chip->cur_cs);
1096 	marvell_nfc_enable_hw_ecc(chip);
1097 	marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi, false,
1098 					    page);
1099 	ret = marvell_nfc_hw_ecc_check_bitflips(chip, &max_bitflips);
1100 	marvell_nfc_disable_hw_ecc(chip);
1101 
1102 	if (!ret)
1103 		return max_bitflips;
1104 
1105 	/*
1106 	 * When ECC failures are detected, check if the full page has been
1107 	 * written or not. Ignore the failure if it is actually empty.
1108 	 */
1109 	raw_buf = kmalloc(full_sz, GFP_KERNEL);
1110 	if (!raw_buf)
1111 		return -ENOMEM;
1112 
1113 	marvell_nfc_hw_ecc_hmg_do_read_page(chip, raw_buf, raw_buf +
1114 					    lt->data_bytes, true, page);
1115 	marvell_nfc_check_empty_chunk(chip, raw_buf, full_sz, NULL, 0, NULL, 0,
1116 				      &max_bitflips);
1117 	kfree(raw_buf);
1118 
1119 	return max_bitflips;
1120 }
1121 
1122 /*
1123  * Spare area in Hamming layouts is not protected by the ECC engine (even if
1124  * it appears before the ECC bytes when reading), the ->read_oob_raw() function
1125  * also stands for ->read_oob().
1126  */
1127 static int marvell_nfc_hw_ecc_hmg_read_oob_raw(struct nand_chip *chip, int page)
1128 {
1129 	u8 *buf = nand_get_data_buf(chip);
1130 
1131 	marvell_nfc_select_target(chip, chip->cur_cs);
1132 	return marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi,
1133 						   true, page);
1134 }
1135 
1136 /* Hamming write helpers */
1137 static int marvell_nfc_hw_ecc_hmg_do_write_page(struct nand_chip *chip,
1138 						const u8 *data_buf,
1139 						const u8 *oob_buf, bool raw,
1140 						int page)
1141 {
1142 	const struct nand_sdr_timings *sdr =
1143 		nand_get_sdr_timings(nand_get_interface_config(chip));
1144 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1145 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1146 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1147 	struct marvell_nfc_op nfc_op = {
1148 		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_WRITE) |
1149 			   NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1150 			   NDCB0_CMD1(NAND_CMD_SEQIN) |
1151 			   NDCB0_CMD2(NAND_CMD_PAGEPROG) |
1152 			   NDCB0_DBC,
1153 		.ndcb[1] = NDCB1_ADDRS_PAGE(page),
1154 		.ndcb[2] = NDCB2_ADDR5_PAGE(page),
1155 	};
1156 	unsigned int oob_bytes = lt->spare_bytes + (raw ? lt->ecc_bytes : 0);
1157 	int ret;
1158 
1159 	/* NFCv2 needs more information about the operation being executed */
1160 	if (nfc->caps->is_nfcv2)
1161 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1162 
1163 	ret = marvell_nfc_prepare_cmd(chip);
1164 	if (ret)
1165 		return ret;
1166 
1167 	marvell_nfc_send_cmd(chip, &nfc_op);
1168 	ret = marvell_nfc_end_cmd(chip, NDSR_WRDREQ,
1169 				  "WRDREQ while loading FIFO (data)");
1170 	if (ret)
1171 		return ret;
1172 
1173 	/* Write the page then the OOB area */
1174 	if (nfc->use_dma) {
1175 		memcpy(nfc->dma_buf, data_buf, lt->data_bytes);
1176 		memcpy(nfc->dma_buf + lt->data_bytes, oob_buf, oob_bytes);
1177 		marvell_nfc_xfer_data_dma(nfc, DMA_TO_DEVICE, lt->data_bytes +
1178 					  lt->ecc_bytes + lt->spare_bytes);
1179 	} else {
1180 		marvell_nfc_xfer_data_out_pio(nfc, data_buf, lt->data_bytes);
1181 		marvell_nfc_xfer_data_out_pio(nfc, oob_buf, oob_bytes);
1182 	}
1183 
1184 	ret = marvell_nfc_wait_cmdd(chip);
1185 	if (ret)
1186 		return ret;
1187 
1188 	ret = marvell_nfc_wait_op(chip,
1189 				  PSEC_TO_MSEC(sdr->tPROG_max));
1190 	return ret;
1191 }
1192 
1193 static int marvell_nfc_hw_ecc_hmg_write_page_raw(struct nand_chip *chip,
1194 						 const u8 *buf,
1195 						 int oob_required, int page)
1196 {
1197 	marvell_nfc_select_target(chip, chip->cur_cs);
1198 	return marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1199 						    true, page);
1200 }
1201 
1202 static int marvell_nfc_hw_ecc_hmg_write_page(struct nand_chip *chip,
1203 					     const u8 *buf,
1204 					     int oob_required, int page)
1205 {
1206 	int ret;
1207 
1208 	marvell_nfc_select_target(chip, chip->cur_cs);
1209 	marvell_nfc_enable_hw_ecc(chip);
1210 	ret = marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1211 						   false, page);
1212 	marvell_nfc_disable_hw_ecc(chip);
1213 
1214 	return ret;
1215 }
1216 
1217 /*
1218  * Spare area in Hamming layouts is not protected by the ECC engine (even if
1219  * it appears before the ECC bytes when reading), the ->write_oob_raw() function
1220  * also stands for ->write_oob().
1221  */
1222 static int marvell_nfc_hw_ecc_hmg_write_oob_raw(struct nand_chip *chip,
1223 						int page)
1224 {
1225 	struct mtd_info *mtd = nand_to_mtd(chip);
1226 	u8 *buf = nand_get_data_buf(chip);
1227 
1228 	memset(buf, 0xFF, mtd->writesize);
1229 
1230 	marvell_nfc_select_target(chip, chip->cur_cs);
1231 	return marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1232 						    true, page);
1233 }
1234 
1235 /* BCH read helpers */
1236 static int marvell_nfc_hw_ecc_bch_read_page_raw(struct nand_chip *chip, u8 *buf,
1237 						int oob_required, int page)
1238 {
1239 	struct mtd_info *mtd = nand_to_mtd(chip);
1240 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1241 	u8 *oob = chip->oob_poi;
1242 	int chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1243 	int ecc_offset = (lt->full_chunk_cnt * lt->spare_bytes) +
1244 		lt->last_spare_bytes;
1245 	int data_len = lt->data_bytes;
1246 	int spare_len = lt->spare_bytes;
1247 	int ecc_len = lt->ecc_bytes;
1248 	int chunk;
1249 
1250 	marvell_nfc_select_target(chip, chip->cur_cs);
1251 
1252 	if (oob_required)
1253 		memset(chip->oob_poi, 0xFF, mtd->oobsize);
1254 
1255 	nand_read_page_op(chip, page, 0, NULL, 0);
1256 
1257 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1258 		/* Update last chunk length */
1259 		if (chunk >= lt->full_chunk_cnt) {
1260 			data_len = lt->last_data_bytes;
1261 			spare_len = lt->last_spare_bytes;
1262 			ecc_len = lt->last_ecc_bytes;
1263 		}
1264 
1265 		/* Read data bytes*/
1266 		nand_change_read_column_op(chip, chunk * chunk_size,
1267 					   buf + (lt->data_bytes * chunk),
1268 					   data_len, false);
1269 
1270 		/* Read spare bytes */
1271 		nand_read_data_op(chip, oob + (lt->spare_bytes * chunk),
1272 				  spare_len, false, false);
1273 
1274 		/* Read ECC bytes */
1275 		nand_read_data_op(chip, oob + ecc_offset +
1276 				  (ALIGN(lt->ecc_bytes, 32) * chunk),
1277 				  ecc_len, false, false);
1278 	}
1279 
1280 	return 0;
1281 }
1282 
1283 static void marvell_nfc_hw_ecc_bch_read_chunk(struct nand_chip *chip, int chunk,
1284 					      u8 *data, unsigned int data_len,
1285 					      u8 *spare, unsigned int spare_len,
1286 					      int page)
1287 {
1288 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1289 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1290 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1291 	int i, ret;
1292 	struct marvell_nfc_op nfc_op = {
1293 		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_READ) |
1294 			   NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1295 			   NDCB0_LEN_OVRD,
1296 		.ndcb[1] = NDCB1_ADDRS_PAGE(page),
1297 		.ndcb[2] = NDCB2_ADDR5_PAGE(page),
1298 		.ndcb[3] = data_len + spare_len,
1299 	};
1300 
1301 	ret = marvell_nfc_prepare_cmd(chip);
1302 	if (ret)
1303 		return;
1304 
1305 	if (chunk == 0)
1306 		nfc_op.ndcb[0] |= NDCB0_DBC |
1307 				  NDCB0_CMD1(NAND_CMD_READ0) |
1308 				  NDCB0_CMD2(NAND_CMD_READSTART);
1309 
1310 	/*
1311 	 * Trigger the monolithic read on the first chunk, then naked read on
1312 	 * intermediate chunks and finally a last naked read on the last chunk.
1313 	 */
1314 	if (chunk == 0)
1315 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1316 	else if (chunk < lt->nchunks - 1)
1317 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_NAKED_RW);
1318 	else
1319 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1320 
1321 	marvell_nfc_send_cmd(chip, &nfc_op);
1322 
1323 	/*
1324 	 * According to the datasheet, when reading from NDDB
1325 	 * with BCH enabled, after each 32 bytes reads, we
1326 	 * have to make sure that the NDSR.RDDREQ bit is set.
1327 	 *
1328 	 * Drain the FIFO, 8 32-bit reads at a time, and skip
1329 	 * the polling on the last read.
1330 	 *
1331 	 * Length is a multiple of 32 bytes, hence it is a multiple of 8 too.
1332 	 */
1333 	for (i = 0; i < data_len; i += FIFO_DEPTH * BCH_SEQ_READS) {
1334 		marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1335 				    "RDDREQ while draining FIFO (data)");
1336 		marvell_nfc_xfer_data_in_pio(nfc, data,
1337 					     FIFO_DEPTH * BCH_SEQ_READS);
1338 		data += FIFO_DEPTH * BCH_SEQ_READS;
1339 	}
1340 
1341 	for (i = 0; i < spare_len; i += FIFO_DEPTH * BCH_SEQ_READS) {
1342 		marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1343 				    "RDDREQ while draining FIFO (OOB)");
1344 		marvell_nfc_xfer_data_in_pio(nfc, spare,
1345 					     FIFO_DEPTH * BCH_SEQ_READS);
1346 		spare += FIFO_DEPTH * BCH_SEQ_READS;
1347 	}
1348 }
1349 
1350 static int marvell_nfc_hw_ecc_bch_read_page(struct nand_chip *chip,
1351 					    u8 *buf, int oob_required,
1352 					    int page)
1353 {
1354 	struct mtd_info *mtd = nand_to_mtd(chip);
1355 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1356 	int data_len = lt->data_bytes, spare_len = lt->spare_bytes;
1357 	u8 *data = buf, *spare = chip->oob_poi;
1358 	int max_bitflips = 0;
1359 	u32 failure_mask = 0;
1360 	int chunk, ret;
1361 
1362 	marvell_nfc_select_target(chip, chip->cur_cs);
1363 
1364 	/*
1365 	 * With BCH, OOB is not fully used (and thus not read entirely), not
1366 	 * expected bytes could show up at the end of the OOB buffer if not
1367 	 * explicitly erased.
1368 	 */
1369 	if (oob_required)
1370 		memset(chip->oob_poi, 0xFF, mtd->oobsize);
1371 
1372 	marvell_nfc_enable_hw_ecc(chip);
1373 
1374 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1375 		/* Update length for the last chunk */
1376 		if (chunk >= lt->full_chunk_cnt) {
1377 			data_len = lt->last_data_bytes;
1378 			spare_len = lt->last_spare_bytes;
1379 		}
1380 
1381 		/* Read the chunk and detect number of bitflips */
1382 		marvell_nfc_hw_ecc_bch_read_chunk(chip, chunk, data, data_len,
1383 						  spare, spare_len, page);
1384 		ret = marvell_nfc_hw_ecc_check_bitflips(chip, &max_bitflips);
1385 		if (ret)
1386 			failure_mask |= BIT(chunk);
1387 
1388 		data += data_len;
1389 		spare += spare_len;
1390 	}
1391 
1392 	marvell_nfc_disable_hw_ecc(chip);
1393 
1394 	if (!failure_mask)
1395 		return max_bitflips;
1396 
1397 	/*
1398 	 * Please note that dumping the ECC bytes during a normal read with OOB
1399 	 * area would add a significant overhead as ECC bytes are "consumed" by
1400 	 * the controller in normal mode and must be re-read in raw mode. To
1401 	 * avoid dropping the performances, we prefer not to include them. The
1402 	 * user should re-read the page in raw mode if ECC bytes are required.
1403 	 */
1404 
1405 	/*
1406 	 * In case there is any subpage read error, we usually re-read only ECC
1407 	 * bytes in raw mode and check if the whole page is empty. In this case,
1408 	 * it is normal that the ECC check failed and we just ignore the error.
1409 	 *
1410 	 * However, it has been empirically observed that for some layouts (e.g
1411 	 * 2k page, 8b strength per 512B chunk), the controller tries to correct
1412 	 * bits and may create itself bitflips in the erased area. To overcome
1413 	 * this strange behavior, the whole page is re-read in raw mode, not
1414 	 * only the ECC bytes.
1415 	 */
1416 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1417 		int data_off_in_page, spare_off_in_page, ecc_off_in_page;
1418 		int data_off, spare_off, ecc_off;
1419 		int data_len, spare_len, ecc_len;
1420 
1421 		/* No failure reported for this chunk, move to the next one */
1422 		if (!(failure_mask & BIT(chunk)))
1423 			continue;
1424 
1425 		data_off_in_page = chunk * (lt->data_bytes + lt->spare_bytes +
1426 					    lt->ecc_bytes);
1427 		spare_off_in_page = data_off_in_page +
1428 			(chunk < lt->full_chunk_cnt ? lt->data_bytes :
1429 						      lt->last_data_bytes);
1430 		ecc_off_in_page = spare_off_in_page +
1431 			(chunk < lt->full_chunk_cnt ? lt->spare_bytes :
1432 						      lt->last_spare_bytes);
1433 
1434 		data_off = chunk * lt->data_bytes;
1435 		spare_off = chunk * lt->spare_bytes;
1436 		ecc_off = (lt->full_chunk_cnt * lt->spare_bytes) +
1437 			  lt->last_spare_bytes +
1438 			  (chunk * (lt->ecc_bytes + 2));
1439 
1440 		data_len = chunk < lt->full_chunk_cnt ? lt->data_bytes :
1441 							lt->last_data_bytes;
1442 		spare_len = chunk < lt->full_chunk_cnt ? lt->spare_bytes :
1443 							 lt->last_spare_bytes;
1444 		ecc_len = chunk < lt->full_chunk_cnt ? lt->ecc_bytes :
1445 						       lt->last_ecc_bytes;
1446 
1447 		/*
1448 		 * Only re-read the ECC bytes, unless we are using the 2k/8b
1449 		 * layout which is buggy in the sense that the ECC engine will
1450 		 * try to correct data bytes anyway, creating bitflips. In this
1451 		 * case, re-read the entire page.
1452 		 */
1453 		if (lt->writesize == 2048 && lt->strength == 8) {
1454 			nand_change_read_column_op(chip, data_off_in_page,
1455 						   buf + data_off, data_len,
1456 						   false);
1457 			nand_change_read_column_op(chip, spare_off_in_page,
1458 						   chip->oob_poi + spare_off, spare_len,
1459 						   false);
1460 		}
1461 
1462 		nand_change_read_column_op(chip, ecc_off_in_page,
1463 					   chip->oob_poi + ecc_off, ecc_len,
1464 					   false);
1465 
1466 		/* Check the entire chunk (data + spare + ecc) for emptyness */
1467 		marvell_nfc_check_empty_chunk(chip, buf + data_off, data_len,
1468 					      chip->oob_poi + spare_off, spare_len,
1469 					      chip->oob_poi + ecc_off, ecc_len,
1470 					      &max_bitflips);
1471 	}
1472 
1473 	return max_bitflips;
1474 }
1475 
1476 static int marvell_nfc_hw_ecc_bch_read_oob_raw(struct nand_chip *chip, int page)
1477 {
1478 	u8 *buf = nand_get_data_buf(chip);
1479 
1480 	return chip->ecc.read_page_raw(chip, buf, true, page);
1481 }
1482 
1483 static int marvell_nfc_hw_ecc_bch_read_oob(struct nand_chip *chip, int page)
1484 {
1485 	u8 *buf = nand_get_data_buf(chip);
1486 
1487 	return chip->ecc.read_page(chip, buf, true, page);
1488 }
1489 
1490 /* BCH write helpers */
1491 static int marvell_nfc_hw_ecc_bch_write_page_raw(struct nand_chip *chip,
1492 						 const u8 *buf,
1493 						 int oob_required, int page)
1494 {
1495 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1496 	int full_chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1497 	int data_len = lt->data_bytes;
1498 	int spare_len = lt->spare_bytes;
1499 	int ecc_len = lt->ecc_bytes;
1500 	int spare_offset = 0;
1501 	int ecc_offset = (lt->full_chunk_cnt * lt->spare_bytes) +
1502 		lt->last_spare_bytes;
1503 	int chunk;
1504 
1505 	marvell_nfc_select_target(chip, chip->cur_cs);
1506 
1507 	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1508 
1509 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1510 		if (chunk >= lt->full_chunk_cnt) {
1511 			data_len = lt->last_data_bytes;
1512 			spare_len = lt->last_spare_bytes;
1513 			ecc_len = lt->last_ecc_bytes;
1514 		}
1515 
1516 		/* Point to the column of the next chunk */
1517 		nand_change_write_column_op(chip, chunk * full_chunk_size,
1518 					    NULL, 0, false);
1519 
1520 		/* Write the data */
1521 		nand_write_data_op(chip, buf + (chunk * lt->data_bytes),
1522 				   data_len, false);
1523 
1524 		if (!oob_required)
1525 			continue;
1526 
1527 		/* Write the spare bytes */
1528 		if (spare_len)
1529 			nand_write_data_op(chip, chip->oob_poi + spare_offset,
1530 					   spare_len, false);
1531 
1532 		/* Write the ECC bytes */
1533 		if (ecc_len)
1534 			nand_write_data_op(chip, chip->oob_poi + ecc_offset,
1535 					   ecc_len, false);
1536 
1537 		spare_offset += spare_len;
1538 		ecc_offset += ALIGN(ecc_len, 32);
1539 	}
1540 
1541 	return nand_prog_page_end_op(chip);
1542 }
1543 
1544 static int
1545 marvell_nfc_hw_ecc_bch_write_chunk(struct nand_chip *chip, int chunk,
1546 				   const u8 *data, unsigned int data_len,
1547 				   const u8 *spare, unsigned int spare_len,
1548 				   int page)
1549 {
1550 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1551 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1552 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1553 	u32 xtype;
1554 	int ret;
1555 	struct marvell_nfc_op nfc_op = {
1556 		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_WRITE) | NDCB0_LEN_OVRD,
1557 		.ndcb[3] = data_len + spare_len,
1558 	};
1559 
1560 	/*
1561 	 * First operation dispatches the CMD_SEQIN command, issue the address
1562 	 * cycles and asks for the first chunk of data.
1563 	 * All operations in the middle (if any) will issue a naked write and
1564 	 * also ask for data.
1565 	 * Last operation (if any) asks for the last chunk of data through a
1566 	 * last naked write.
1567 	 */
1568 	if (chunk == 0) {
1569 		if (lt->nchunks == 1)
1570 			xtype = XTYPE_MONOLITHIC_RW;
1571 		else
1572 			xtype = XTYPE_WRITE_DISPATCH;
1573 
1574 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(xtype) |
1575 				  NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1576 				  NDCB0_CMD1(NAND_CMD_SEQIN);
1577 		nfc_op.ndcb[1] |= NDCB1_ADDRS_PAGE(page);
1578 		nfc_op.ndcb[2] |= NDCB2_ADDR5_PAGE(page);
1579 	} else if (chunk < lt->nchunks - 1) {
1580 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_NAKED_RW);
1581 	} else {
1582 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1583 	}
1584 
1585 	/* Always dispatch the PAGEPROG command on the last chunk */
1586 	if (chunk == lt->nchunks - 1)
1587 		nfc_op.ndcb[0] |= NDCB0_CMD2(NAND_CMD_PAGEPROG) | NDCB0_DBC;
1588 
1589 	ret = marvell_nfc_prepare_cmd(chip);
1590 	if (ret)
1591 		return ret;
1592 
1593 	marvell_nfc_send_cmd(chip, &nfc_op);
1594 	ret = marvell_nfc_end_cmd(chip, NDSR_WRDREQ,
1595 				  "WRDREQ while loading FIFO (data)");
1596 	if (ret)
1597 		return ret;
1598 
1599 	/* Transfer the contents */
1600 	iowrite32_rep(nfc->regs + NDDB, data, FIFO_REP(data_len));
1601 	iowrite32_rep(nfc->regs + NDDB, spare, FIFO_REP(spare_len));
1602 
1603 	return 0;
1604 }
1605 
1606 static int marvell_nfc_hw_ecc_bch_write_page(struct nand_chip *chip,
1607 					     const u8 *buf,
1608 					     int oob_required, int page)
1609 {
1610 	const struct nand_sdr_timings *sdr =
1611 		nand_get_sdr_timings(nand_get_interface_config(chip));
1612 	struct mtd_info *mtd = nand_to_mtd(chip);
1613 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1614 	const u8 *data = buf;
1615 	const u8 *spare = chip->oob_poi;
1616 	int data_len = lt->data_bytes;
1617 	int spare_len = lt->spare_bytes;
1618 	int chunk, ret;
1619 
1620 	marvell_nfc_select_target(chip, chip->cur_cs);
1621 
1622 	/* Spare data will be written anyway, so clear it to avoid garbage */
1623 	if (!oob_required)
1624 		memset(chip->oob_poi, 0xFF, mtd->oobsize);
1625 
1626 	marvell_nfc_enable_hw_ecc(chip);
1627 
1628 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1629 		if (chunk >= lt->full_chunk_cnt) {
1630 			data_len = lt->last_data_bytes;
1631 			spare_len = lt->last_spare_bytes;
1632 		}
1633 
1634 		marvell_nfc_hw_ecc_bch_write_chunk(chip, chunk, data, data_len,
1635 						   spare, spare_len, page);
1636 		data += data_len;
1637 		spare += spare_len;
1638 
1639 		/*
1640 		 * Waiting only for CMDD or PAGED is not enough, ECC are
1641 		 * partially written. No flag is set once the operation is
1642 		 * really finished but the ND_RUN bit is cleared, so wait for it
1643 		 * before stepping into the next command.
1644 		 */
1645 		marvell_nfc_wait_ndrun(chip);
1646 	}
1647 
1648 	ret = marvell_nfc_wait_op(chip, PSEC_TO_MSEC(sdr->tPROG_max));
1649 
1650 	marvell_nfc_disable_hw_ecc(chip);
1651 
1652 	if (ret)
1653 		return ret;
1654 
1655 	return 0;
1656 }
1657 
1658 static int marvell_nfc_hw_ecc_bch_write_oob_raw(struct nand_chip *chip,
1659 						int page)
1660 {
1661 	struct mtd_info *mtd = nand_to_mtd(chip);
1662 	u8 *buf = nand_get_data_buf(chip);
1663 
1664 	memset(buf, 0xFF, mtd->writesize);
1665 
1666 	return chip->ecc.write_page_raw(chip, buf, true, page);
1667 }
1668 
1669 static int marvell_nfc_hw_ecc_bch_write_oob(struct nand_chip *chip, int page)
1670 {
1671 	struct mtd_info *mtd = nand_to_mtd(chip);
1672 	u8 *buf = nand_get_data_buf(chip);
1673 
1674 	memset(buf, 0xFF, mtd->writesize);
1675 
1676 	return chip->ecc.write_page(chip, buf, true, page);
1677 }
1678 
1679 /* NAND framework ->exec_op() hooks and related helpers */
1680 static void marvell_nfc_parse_instructions(struct nand_chip *chip,
1681 					   const struct nand_subop *subop,
1682 					   struct marvell_nfc_op *nfc_op)
1683 {
1684 	const struct nand_op_instr *instr = NULL;
1685 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1686 	bool first_cmd = true;
1687 	unsigned int op_id;
1688 	int i;
1689 
1690 	/* Reset the input structure as most of its fields will be OR'ed */
1691 	memset(nfc_op, 0, sizeof(struct marvell_nfc_op));
1692 
1693 	for (op_id = 0; op_id < subop->ninstrs; op_id++) {
1694 		unsigned int offset, naddrs;
1695 		const u8 *addrs;
1696 		int len;
1697 
1698 		instr = &subop->instrs[op_id];
1699 
1700 		switch (instr->type) {
1701 		case NAND_OP_CMD_INSTR:
1702 			if (first_cmd)
1703 				nfc_op->ndcb[0] |=
1704 					NDCB0_CMD1(instr->ctx.cmd.opcode);
1705 			else
1706 				nfc_op->ndcb[0] |=
1707 					NDCB0_CMD2(instr->ctx.cmd.opcode) |
1708 					NDCB0_DBC;
1709 
1710 			nfc_op->cle_ale_delay_ns = instr->delay_ns;
1711 			first_cmd = false;
1712 			break;
1713 
1714 		case NAND_OP_ADDR_INSTR:
1715 			offset = nand_subop_get_addr_start_off(subop, op_id);
1716 			naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
1717 			addrs = &instr->ctx.addr.addrs[offset];
1718 
1719 			nfc_op->ndcb[0] |= NDCB0_ADDR_CYC(naddrs);
1720 
1721 			for (i = 0; i < min_t(unsigned int, 4, naddrs); i++)
1722 				nfc_op->ndcb[1] |= addrs[i] << (8 * i);
1723 
1724 			if (naddrs >= 5)
1725 				nfc_op->ndcb[2] |= NDCB2_ADDR5_CYC(addrs[4]);
1726 			if (naddrs >= 6)
1727 				nfc_op->ndcb[3] |= NDCB3_ADDR6_CYC(addrs[5]);
1728 			if (naddrs == 7)
1729 				nfc_op->ndcb[3] |= NDCB3_ADDR7_CYC(addrs[6]);
1730 
1731 			nfc_op->cle_ale_delay_ns = instr->delay_ns;
1732 			break;
1733 
1734 		case NAND_OP_DATA_IN_INSTR:
1735 			nfc_op->data_instr = instr;
1736 			nfc_op->data_instr_idx = op_id;
1737 			nfc_op->ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ);
1738 			if (nfc->caps->is_nfcv2) {
1739 				nfc_op->ndcb[0] |=
1740 					NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW) |
1741 					NDCB0_LEN_OVRD;
1742 				len = nand_subop_get_data_len(subop, op_id);
1743 				nfc_op->ndcb[3] |= round_up(len, FIFO_DEPTH);
1744 			}
1745 			nfc_op->data_delay_ns = instr->delay_ns;
1746 			break;
1747 
1748 		case NAND_OP_DATA_OUT_INSTR:
1749 			nfc_op->data_instr = instr;
1750 			nfc_op->data_instr_idx = op_id;
1751 			nfc_op->ndcb[0] |= NDCB0_CMD_TYPE(TYPE_WRITE);
1752 			if (nfc->caps->is_nfcv2) {
1753 				nfc_op->ndcb[0] |=
1754 					NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW) |
1755 					NDCB0_LEN_OVRD;
1756 				len = nand_subop_get_data_len(subop, op_id);
1757 				nfc_op->ndcb[3] |= round_up(len, FIFO_DEPTH);
1758 			}
1759 			nfc_op->data_delay_ns = instr->delay_ns;
1760 			break;
1761 
1762 		case NAND_OP_WAITRDY_INSTR:
1763 			nfc_op->rdy_timeout_ms = instr->ctx.waitrdy.timeout_ms;
1764 			nfc_op->rdy_delay_ns = instr->delay_ns;
1765 			break;
1766 		}
1767 	}
1768 }
1769 
1770 static int marvell_nfc_xfer_data_pio(struct nand_chip *chip,
1771 				     const struct nand_subop *subop,
1772 				     struct marvell_nfc_op *nfc_op)
1773 {
1774 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1775 	const struct nand_op_instr *instr = nfc_op->data_instr;
1776 	unsigned int op_id = nfc_op->data_instr_idx;
1777 	unsigned int len = nand_subop_get_data_len(subop, op_id);
1778 	unsigned int offset = nand_subop_get_data_start_off(subop, op_id);
1779 	bool reading = (instr->type == NAND_OP_DATA_IN_INSTR);
1780 	int ret;
1781 
1782 	if (instr->ctx.data.force_8bit)
1783 		marvell_nfc_force_byte_access(chip, true);
1784 
1785 	if (reading) {
1786 		u8 *in = instr->ctx.data.buf.in + offset;
1787 
1788 		ret = marvell_nfc_xfer_data_in_pio(nfc, in, len);
1789 	} else {
1790 		const u8 *out = instr->ctx.data.buf.out + offset;
1791 
1792 		ret = marvell_nfc_xfer_data_out_pio(nfc, out, len);
1793 	}
1794 
1795 	if (instr->ctx.data.force_8bit)
1796 		marvell_nfc_force_byte_access(chip, false);
1797 
1798 	return ret;
1799 }
1800 
1801 static int marvell_nfc_monolithic_access_exec(struct nand_chip *chip,
1802 					      const struct nand_subop *subop)
1803 {
1804 	struct marvell_nfc_op nfc_op;
1805 	bool reading;
1806 	int ret;
1807 
1808 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1809 	reading = (nfc_op.data_instr->type == NAND_OP_DATA_IN_INSTR);
1810 
1811 	ret = marvell_nfc_prepare_cmd(chip);
1812 	if (ret)
1813 		return ret;
1814 
1815 	marvell_nfc_send_cmd(chip, &nfc_op);
1816 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ | NDSR_WRDREQ,
1817 				  "RDDREQ/WRDREQ while draining raw data");
1818 	if (ret)
1819 		return ret;
1820 
1821 	cond_delay(nfc_op.cle_ale_delay_ns);
1822 
1823 	if (reading) {
1824 		if (nfc_op.rdy_timeout_ms) {
1825 			ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1826 			if (ret)
1827 				return ret;
1828 		}
1829 
1830 		cond_delay(nfc_op.rdy_delay_ns);
1831 	}
1832 
1833 	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1834 	ret = marvell_nfc_wait_cmdd(chip);
1835 	if (ret)
1836 		return ret;
1837 
1838 	cond_delay(nfc_op.data_delay_ns);
1839 
1840 	if (!reading) {
1841 		if (nfc_op.rdy_timeout_ms) {
1842 			ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1843 			if (ret)
1844 				return ret;
1845 		}
1846 
1847 		cond_delay(nfc_op.rdy_delay_ns);
1848 	}
1849 
1850 	/*
1851 	 * NDCR ND_RUN bit should be cleared automatically at the end of each
1852 	 * operation but experience shows that the behavior is buggy when it
1853 	 * comes to writes (with LEN_OVRD). Clear it by hand in this case.
1854 	 */
1855 	if (!reading) {
1856 		struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1857 
1858 		writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
1859 			       nfc->regs + NDCR);
1860 	}
1861 
1862 	return 0;
1863 }
1864 
1865 static int marvell_nfc_naked_access_exec(struct nand_chip *chip,
1866 					 const struct nand_subop *subop)
1867 {
1868 	struct marvell_nfc_op nfc_op;
1869 	int ret;
1870 
1871 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1872 
1873 	/*
1874 	 * Naked access are different in that they need to be flagged as naked
1875 	 * by the controller. Reset the controller registers fields that inform
1876 	 * on the type and refill them according to the ongoing operation.
1877 	 */
1878 	nfc_op.ndcb[0] &= ~(NDCB0_CMD_TYPE(TYPE_MASK) |
1879 			    NDCB0_CMD_XTYPE(XTYPE_MASK));
1880 	switch (subop->instrs[0].type) {
1881 	case NAND_OP_CMD_INSTR:
1882 		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_NAKED_CMD);
1883 		break;
1884 	case NAND_OP_ADDR_INSTR:
1885 		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_NAKED_ADDR);
1886 		break;
1887 	case NAND_OP_DATA_IN_INSTR:
1888 		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ) |
1889 				  NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1890 		break;
1891 	case NAND_OP_DATA_OUT_INSTR:
1892 		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_WRITE) |
1893 				  NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1894 		break;
1895 	default:
1896 		/* This should never happen */
1897 		break;
1898 	}
1899 
1900 	ret = marvell_nfc_prepare_cmd(chip);
1901 	if (ret)
1902 		return ret;
1903 
1904 	marvell_nfc_send_cmd(chip, &nfc_op);
1905 
1906 	if (!nfc_op.data_instr) {
1907 		ret = marvell_nfc_wait_cmdd(chip);
1908 		cond_delay(nfc_op.cle_ale_delay_ns);
1909 		return ret;
1910 	}
1911 
1912 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ | NDSR_WRDREQ,
1913 				  "RDDREQ/WRDREQ while draining raw data");
1914 	if (ret)
1915 		return ret;
1916 
1917 	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1918 	ret = marvell_nfc_wait_cmdd(chip);
1919 	if (ret)
1920 		return ret;
1921 
1922 	/*
1923 	 * NDCR ND_RUN bit should be cleared automatically at the end of each
1924 	 * operation but experience shows that the behavior is buggy when it
1925 	 * comes to writes (with LEN_OVRD). Clear it by hand in this case.
1926 	 */
1927 	if (subop->instrs[0].type == NAND_OP_DATA_OUT_INSTR) {
1928 		struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1929 
1930 		writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
1931 			       nfc->regs + NDCR);
1932 	}
1933 
1934 	return 0;
1935 }
1936 
1937 static int marvell_nfc_naked_waitrdy_exec(struct nand_chip *chip,
1938 					  const struct nand_subop *subop)
1939 {
1940 	struct marvell_nfc_op nfc_op;
1941 	int ret;
1942 
1943 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1944 
1945 	ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1946 	cond_delay(nfc_op.rdy_delay_ns);
1947 
1948 	return ret;
1949 }
1950 
1951 static int marvell_nfc_read_id_type_exec(struct nand_chip *chip,
1952 					 const struct nand_subop *subop)
1953 {
1954 	struct marvell_nfc_op nfc_op;
1955 	int ret;
1956 
1957 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1958 	nfc_op.ndcb[0] &= ~NDCB0_CMD_TYPE(TYPE_READ);
1959 	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ_ID);
1960 
1961 	ret = marvell_nfc_prepare_cmd(chip);
1962 	if (ret)
1963 		return ret;
1964 
1965 	marvell_nfc_send_cmd(chip, &nfc_op);
1966 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1967 				  "RDDREQ while reading ID");
1968 	if (ret)
1969 		return ret;
1970 
1971 	cond_delay(nfc_op.cle_ale_delay_ns);
1972 
1973 	if (nfc_op.rdy_timeout_ms) {
1974 		ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1975 		if (ret)
1976 			return ret;
1977 	}
1978 
1979 	cond_delay(nfc_op.rdy_delay_ns);
1980 
1981 	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1982 	ret = marvell_nfc_wait_cmdd(chip);
1983 	if (ret)
1984 		return ret;
1985 
1986 	cond_delay(nfc_op.data_delay_ns);
1987 
1988 	return 0;
1989 }
1990 
1991 static int marvell_nfc_read_status_exec(struct nand_chip *chip,
1992 					const struct nand_subop *subop)
1993 {
1994 	struct marvell_nfc_op nfc_op;
1995 	int ret;
1996 
1997 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1998 	nfc_op.ndcb[0] &= ~NDCB0_CMD_TYPE(TYPE_READ);
1999 	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_STATUS);
2000 
2001 	ret = marvell_nfc_prepare_cmd(chip);
2002 	if (ret)
2003 		return ret;
2004 
2005 	marvell_nfc_send_cmd(chip, &nfc_op);
2006 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
2007 				  "RDDREQ while reading status");
2008 	if (ret)
2009 		return ret;
2010 
2011 	cond_delay(nfc_op.cle_ale_delay_ns);
2012 
2013 	if (nfc_op.rdy_timeout_ms) {
2014 		ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
2015 		if (ret)
2016 			return ret;
2017 	}
2018 
2019 	cond_delay(nfc_op.rdy_delay_ns);
2020 
2021 	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
2022 	ret = marvell_nfc_wait_cmdd(chip);
2023 	if (ret)
2024 		return ret;
2025 
2026 	cond_delay(nfc_op.data_delay_ns);
2027 
2028 	return 0;
2029 }
2030 
2031 static int marvell_nfc_reset_cmd_type_exec(struct nand_chip *chip,
2032 					   const struct nand_subop *subop)
2033 {
2034 	struct marvell_nfc_op nfc_op;
2035 	int ret;
2036 
2037 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
2038 	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_RESET);
2039 
2040 	ret = marvell_nfc_prepare_cmd(chip);
2041 	if (ret)
2042 		return ret;
2043 
2044 	marvell_nfc_send_cmd(chip, &nfc_op);
2045 	ret = marvell_nfc_wait_cmdd(chip);
2046 	if (ret)
2047 		return ret;
2048 
2049 	cond_delay(nfc_op.cle_ale_delay_ns);
2050 
2051 	ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
2052 	if (ret)
2053 		return ret;
2054 
2055 	cond_delay(nfc_op.rdy_delay_ns);
2056 
2057 	return 0;
2058 }
2059 
2060 static int marvell_nfc_erase_cmd_type_exec(struct nand_chip *chip,
2061 					   const struct nand_subop *subop)
2062 {
2063 	struct marvell_nfc_op nfc_op;
2064 	int ret;
2065 
2066 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
2067 	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_ERASE);
2068 
2069 	ret = marvell_nfc_prepare_cmd(chip);
2070 	if (ret)
2071 		return ret;
2072 
2073 	marvell_nfc_send_cmd(chip, &nfc_op);
2074 	ret = marvell_nfc_wait_cmdd(chip);
2075 	if (ret)
2076 		return ret;
2077 
2078 	cond_delay(nfc_op.cle_ale_delay_ns);
2079 
2080 	ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
2081 	if (ret)
2082 		return ret;
2083 
2084 	cond_delay(nfc_op.rdy_delay_ns);
2085 
2086 	return 0;
2087 }
2088 
2089 static const struct nand_op_parser marvell_nfcv2_op_parser = NAND_OP_PARSER(
2090 	/* Monolithic reads/writes */
2091 	NAND_OP_PARSER_PATTERN(
2092 		marvell_nfc_monolithic_access_exec,
2093 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2094 		NAND_OP_PARSER_PAT_ADDR_ELEM(true, MAX_ADDRESS_CYC_NFCV2),
2095 		NAND_OP_PARSER_PAT_CMD_ELEM(true),
2096 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
2097 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_CHUNK_SIZE)),
2098 	NAND_OP_PARSER_PATTERN(
2099 		marvell_nfc_monolithic_access_exec,
2100 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2101 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV2),
2102 		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_CHUNK_SIZE),
2103 		NAND_OP_PARSER_PAT_CMD_ELEM(true),
2104 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
2105 	/* Naked commands */
2106 	NAND_OP_PARSER_PATTERN(
2107 		marvell_nfc_naked_access_exec,
2108 		NAND_OP_PARSER_PAT_CMD_ELEM(false)),
2109 	NAND_OP_PARSER_PATTERN(
2110 		marvell_nfc_naked_access_exec,
2111 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV2)),
2112 	NAND_OP_PARSER_PATTERN(
2113 		marvell_nfc_naked_access_exec,
2114 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_CHUNK_SIZE)),
2115 	NAND_OP_PARSER_PATTERN(
2116 		marvell_nfc_naked_access_exec,
2117 		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_CHUNK_SIZE)),
2118 	NAND_OP_PARSER_PATTERN(
2119 		marvell_nfc_naked_waitrdy_exec,
2120 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2121 	);
2122 
2123 static const struct nand_op_parser marvell_nfcv1_op_parser = NAND_OP_PARSER(
2124 	/* Naked commands not supported, use a function for each pattern */
2125 	NAND_OP_PARSER_PATTERN(
2126 		marvell_nfc_read_id_type_exec,
2127 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2128 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV1),
2129 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 8)),
2130 	NAND_OP_PARSER_PATTERN(
2131 		marvell_nfc_erase_cmd_type_exec,
2132 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2133 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV1),
2134 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2135 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2136 	NAND_OP_PARSER_PATTERN(
2137 		marvell_nfc_read_status_exec,
2138 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2139 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 1)),
2140 	NAND_OP_PARSER_PATTERN(
2141 		marvell_nfc_reset_cmd_type_exec,
2142 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2143 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2144 	NAND_OP_PARSER_PATTERN(
2145 		marvell_nfc_naked_waitrdy_exec,
2146 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2147 	);
2148 
2149 static int marvell_nfc_exec_op(struct nand_chip *chip,
2150 			       const struct nand_operation *op,
2151 			       bool check_only)
2152 {
2153 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2154 
2155 	if (!check_only)
2156 		marvell_nfc_select_target(chip, op->cs);
2157 
2158 	if (nfc->caps->is_nfcv2)
2159 		return nand_op_parser_exec_op(chip, &marvell_nfcv2_op_parser,
2160 					      op, check_only);
2161 	else
2162 		return nand_op_parser_exec_op(chip, &marvell_nfcv1_op_parser,
2163 					      op, check_only);
2164 }
2165 
2166 /*
2167  * Layouts were broken in old pxa3xx_nand driver, these are supposed to be
2168  * usable.
2169  */
2170 static int marvell_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2171 				      struct mtd_oob_region *oobregion)
2172 {
2173 	struct nand_chip *chip = mtd_to_nand(mtd);
2174 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
2175 
2176 	if (section)
2177 		return -ERANGE;
2178 
2179 	oobregion->length = (lt->full_chunk_cnt * lt->ecc_bytes) +
2180 			    lt->last_ecc_bytes;
2181 	oobregion->offset = mtd->oobsize - oobregion->length;
2182 
2183 	return 0;
2184 }
2185 
2186 static int marvell_nand_ooblayout_free(struct mtd_info *mtd, int section,
2187 				       struct mtd_oob_region *oobregion)
2188 {
2189 	struct nand_chip *chip = mtd_to_nand(mtd);
2190 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
2191 
2192 	if (section)
2193 		return -ERANGE;
2194 
2195 	/*
2196 	 * Bootrom looks in bytes 0 & 5 for bad blocks for the
2197 	 * 4KB page / 4bit BCH combination.
2198 	 */
2199 	if (mtd->writesize == SZ_4K && lt->data_bytes == SZ_2K)
2200 		oobregion->offset = 6;
2201 	else
2202 		oobregion->offset = 2;
2203 
2204 	oobregion->length = (lt->full_chunk_cnt * lt->spare_bytes) +
2205 			    lt->last_spare_bytes - oobregion->offset;
2206 
2207 	return 0;
2208 }
2209 
2210 static const struct mtd_ooblayout_ops marvell_nand_ooblayout_ops = {
2211 	.ecc = marvell_nand_ooblayout_ecc,
2212 	.free = marvell_nand_ooblayout_free,
2213 };
2214 
2215 static int marvell_nand_hw_ecc_controller_init(struct mtd_info *mtd,
2216 					       struct nand_ecc_ctrl *ecc)
2217 {
2218 	struct nand_chip *chip = mtd_to_nand(mtd);
2219 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2220 	const struct marvell_hw_ecc_layout *l;
2221 	int i;
2222 
2223 	if (!nfc->caps->is_nfcv2 &&
2224 	    (mtd->writesize + mtd->oobsize > MAX_CHUNK_SIZE)) {
2225 		dev_err(nfc->dev,
2226 			"NFCv1: writesize (%d) cannot be bigger than a chunk (%d)\n",
2227 			mtd->writesize, MAX_CHUNK_SIZE - mtd->oobsize);
2228 		return -ENOTSUPP;
2229 	}
2230 
2231 	to_marvell_nand(chip)->layout = NULL;
2232 	for (i = 0; i < ARRAY_SIZE(marvell_nfc_layouts); i++) {
2233 		l = &marvell_nfc_layouts[i];
2234 		if (mtd->writesize == l->writesize &&
2235 		    ecc->size == l->chunk && ecc->strength == l->strength) {
2236 			to_marvell_nand(chip)->layout = l;
2237 			break;
2238 		}
2239 	}
2240 
2241 	if (!to_marvell_nand(chip)->layout ||
2242 	    (!nfc->caps->is_nfcv2 && ecc->strength > 1)) {
2243 		dev_err(nfc->dev,
2244 			"ECC strength %d at page size %d is not supported\n",
2245 			ecc->strength, mtd->writesize);
2246 		return -ENOTSUPP;
2247 	}
2248 
2249 	/* Special care for the layout 2k/8-bit/512B  */
2250 	if (l->writesize == 2048 && l->strength == 8) {
2251 		if (mtd->oobsize < 128) {
2252 			dev_err(nfc->dev, "Requested layout needs at least 128 OOB bytes\n");
2253 			return -ENOTSUPP;
2254 		} else {
2255 			chip->bbt_options |= NAND_BBT_NO_OOB_BBM;
2256 		}
2257 	}
2258 
2259 	mtd_set_ooblayout(mtd, &marvell_nand_ooblayout_ops);
2260 	ecc->steps = l->nchunks;
2261 	ecc->size = l->data_bytes;
2262 
2263 	if (ecc->strength == 1) {
2264 		chip->ecc.algo = NAND_ECC_ALGO_HAMMING;
2265 		ecc->read_page_raw = marvell_nfc_hw_ecc_hmg_read_page_raw;
2266 		ecc->read_page = marvell_nfc_hw_ecc_hmg_read_page;
2267 		ecc->read_oob_raw = marvell_nfc_hw_ecc_hmg_read_oob_raw;
2268 		ecc->read_oob = ecc->read_oob_raw;
2269 		ecc->write_page_raw = marvell_nfc_hw_ecc_hmg_write_page_raw;
2270 		ecc->write_page = marvell_nfc_hw_ecc_hmg_write_page;
2271 		ecc->write_oob_raw = marvell_nfc_hw_ecc_hmg_write_oob_raw;
2272 		ecc->write_oob = ecc->write_oob_raw;
2273 	} else {
2274 		chip->ecc.algo = NAND_ECC_ALGO_BCH;
2275 		ecc->strength = 16;
2276 		ecc->read_page_raw = marvell_nfc_hw_ecc_bch_read_page_raw;
2277 		ecc->read_page = marvell_nfc_hw_ecc_bch_read_page;
2278 		ecc->read_oob_raw = marvell_nfc_hw_ecc_bch_read_oob_raw;
2279 		ecc->read_oob = marvell_nfc_hw_ecc_bch_read_oob;
2280 		ecc->write_page_raw = marvell_nfc_hw_ecc_bch_write_page_raw;
2281 		ecc->write_page = marvell_nfc_hw_ecc_bch_write_page;
2282 		ecc->write_oob_raw = marvell_nfc_hw_ecc_bch_write_oob_raw;
2283 		ecc->write_oob = marvell_nfc_hw_ecc_bch_write_oob;
2284 	}
2285 
2286 	return 0;
2287 }
2288 
2289 static int marvell_nand_ecc_init(struct mtd_info *mtd,
2290 				 struct nand_ecc_ctrl *ecc)
2291 {
2292 	struct nand_chip *chip = mtd_to_nand(mtd);
2293 	const struct nand_ecc_props *requirements =
2294 		nanddev_get_ecc_requirements(&chip->base);
2295 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2296 	int ret;
2297 
2298 	if (ecc->engine_type != NAND_ECC_ENGINE_TYPE_NONE &&
2299 	    (!ecc->size || !ecc->strength)) {
2300 		if (requirements->step_size && requirements->strength) {
2301 			ecc->size = requirements->step_size;
2302 			ecc->strength = requirements->strength;
2303 		} else {
2304 			dev_info(nfc->dev,
2305 				 "No minimum ECC strength, using 1b/512B\n");
2306 			ecc->size = 512;
2307 			ecc->strength = 1;
2308 		}
2309 	}
2310 
2311 	switch (ecc->engine_type) {
2312 	case NAND_ECC_ENGINE_TYPE_ON_HOST:
2313 		ret = marvell_nand_hw_ecc_controller_init(mtd, ecc);
2314 		if (ret)
2315 			return ret;
2316 		break;
2317 	case NAND_ECC_ENGINE_TYPE_NONE:
2318 	case NAND_ECC_ENGINE_TYPE_SOFT:
2319 	case NAND_ECC_ENGINE_TYPE_ON_DIE:
2320 		if (!nfc->caps->is_nfcv2 && mtd->writesize != SZ_512 &&
2321 		    mtd->writesize != SZ_2K) {
2322 			dev_err(nfc->dev, "NFCv1 cannot write %d bytes pages\n",
2323 				mtd->writesize);
2324 			return -EINVAL;
2325 		}
2326 		break;
2327 	default:
2328 		return -EINVAL;
2329 	}
2330 
2331 	return 0;
2332 }
2333 
2334 static u8 bbt_pattern[] = {'M', 'V', 'B', 'b', 't', '0' };
2335 static u8 bbt_mirror_pattern[] = {'1', 't', 'b', 'B', 'V', 'M' };
2336 
2337 static struct nand_bbt_descr bbt_main_descr = {
2338 	.options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
2339 		   NAND_BBT_2BIT | NAND_BBT_VERSION,
2340 	.offs =	8,
2341 	.len = 6,
2342 	.veroffs = 14,
2343 	.maxblocks = 8,	/* Last 8 blocks in each chip */
2344 	.pattern = bbt_pattern
2345 };
2346 
2347 static struct nand_bbt_descr bbt_mirror_descr = {
2348 	.options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
2349 		   NAND_BBT_2BIT | NAND_BBT_VERSION,
2350 	.offs =	8,
2351 	.len = 6,
2352 	.veroffs = 14,
2353 	.maxblocks = 8,	/* Last 8 blocks in each chip */
2354 	.pattern = bbt_mirror_pattern
2355 };
2356 
2357 static int marvell_nfc_setup_interface(struct nand_chip *chip, int chipnr,
2358 				       const struct nand_interface_config *conf)
2359 {
2360 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
2361 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2362 	unsigned int period_ns = 1000000000 / clk_get_rate(nfc->core_clk) * 2;
2363 	const struct nand_sdr_timings *sdr;
2364 	struct marvell_nfc_timings nfc_tmg;
2365 	int read_delay;
2366 
2367 	sdr = nand_get_sdr_timings(conf);
2368 	if (IS_ERR(sdr))
2369 		return PTR_ERR(sdr);
2370 
2371 	/*
2372 	 * SDR timings are given in pico-seconds while NFC timings must be
2373 	 * expressed in NAND controller clock cycles, which is half of the
2374 	 * frequency of the accessible ECC clock retrieved by clk_get_rate().
2375 	 * This is not written anywhere in the datasheet but was observed
2376 	 * with an oscilloscope.
2377 	 *
2378 	 * NFC datasheet gives equations from which thoses calculations
2379 	 * are derived, they tend to be slightly more restrictives than the
2380 	 * given core timings and may improve the overall speed.
2381 	 */
2382 	nfc_tmg.tRP = TO_CYCLES(DIV_ROUND_UP(sdr->tRC_min, 2), period_ns) - 1;
2383 	nfc_tmg.tRH = nfc_tmg.tRP;
2384 	nfc_tmg.tWP = TO_CYCLES(DIV_ROUND_UP(sdr->tWC_min, 2), period_ns) - 1;
2385 	nfc_tmg.tWH = nfc_tmg.tWP;
2386 	nfc_tmg.tCS = TO_CYCLES(sdr->tCS_min, period_ns);
2387 	nfc_tmg.tCH = TO_CYCLES(sdr->tCH_min, period_ns) - 1;
2388 	nfc_tmg.tADL = TO_CYCLES(sdr->tADL_min, period_ns);
2389 	/*
2390 	 * Read delay is the time of propagation from SoC pins to NFC internal
2391 	 * logic. With non-EDO timings, this is MIN_RD_DEL_CNT clock cycles. In
2392 	 * EDO mode, an additional delay of tRH must be taken into account so
2393 	 * the data is sampled on the falling edge instead of the rising edge.
2394 	 */
2395 	read_delay = sdr->tRC_min >= 30000 ?
2396 		MIN_RD_DEL_CNT : MIN_RD_DEL_CNT + nfc_tmg.tRH;
2397 
2398 	nfc_tmg.tAR = TO_CYCLES(sdr->tAR_min, period_ns);
2399 	/*
2400 	 * tWHR and tRHW are supposed to be read to write delays (and vice
2401 	 * versa) but in some cases, ie. when doing a change column, they must
2402 	 * be greater than that to be sure tCCS delay is respected.
2403 	 */
2404 	nfc_tmg.tWHR = TO_CYCLES(max_t(int, sdr->tWHR_min, sdr->tCCS_min),
2405 				 period_ns) - 2;
2406 	nfc_tmg.tRHW = TO_CYCLES(max_t(int, sdr->tRHW_min, sdr->tCCS_min),
2407 				 period_ns);
2408 
2409 	/*
2410 	 * NFCv2: Use WAIT_MODE (wait for RB line), do not rely only on delays.
2411 	 * NFCv1: No WAIT_MODE, tR must be maximal.
2412 	 */
2413 	if (nfc->caps->is_nfcv2) {
2414 		nfc_tmg.tR = TO_CYCLES(sdr->tWB_max, period_ns);
2415 	} else {
2416 		nfc_tmg.tR = TO_CYCLES64(sdr->tWB_max + sdr->tR_max,
2417 					 period_ns);
2418 		if (nfc_tmg.tR + 3 > nfc_tmg.tCH)
2419 			nfc_tmg.tR = nfc_tmg.tCH - 3;
2420 		else
2421 			nfc_tmg.tR = 0;
2422 	}
2423 
2424 	if (chipnr < 0)
2425 		return 0;
2426 
2427 	marvell_nand->ndtr0 =
2428 		NDTR0_TRP(nfc_tmg.tRP) |
2429 		NDTR0_TRH(nfc_tmg.tRH) |
2430 		NDTR0_ETRP(nfc_tmg.tRP) |
2431 		NDTR0_TWP(nfc_tmg.tWP) |
2432 		NDTR0_TWH(nfc_tmg.tWH) |
2433 		NDTR0_TCS(nfc_tmg.tCS) |
2434 		NDTR0_TCH(nfc_tmg.tCH);
2435 
2436 	marvell_nand->ndtr1 =
2437 		NDTR1_TAR(nfc_tmg.tAR) |
2438 		NDTR1_TWHR(nfc_tmg.tWHR) |
2439 		NDTR1_TR(nfc_tmg.tR);
2440 
2441 	if (nfc->caps->is_nfcv2) {
2442 		marvell_nand->ndtr0 |=
2443 			NDTR0_RD_CNT_DEL(read_delay) |
2444 			NDTR0_SELCNTR |
2445 			NDTR0_TADL(nfc_tmg.tADL);
2446 
2447 		marvell_nand->ndtr1 |=
2448 			NDTR1_TRHW(nfc_tmg.tRHW) |
2449 			NDTR1_WAIT_MODE;
2450 	}
2451 
2452 	return 0;
2453 }
2454 
2455 static int marvell_nand_attach_chip(struct nand_chip *chip)
2456 {
2457 	struct mtd_info *mtd = nand_to_mtd(chip);
2458 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
2459 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2460 	struct pxa3xx_nand_platform_data *pdata = dev_get_platdata(nfc->dev);
2461 	int ret;
2462 
2463 	if (pdata && pdata->flash_bbt)
2464 		chip->bbt_options |= NAND_BBT_USE_FLASH;
2465 
2466 	if (chip->bbt_options & NAND_BBT_USE_FLASH) {
2467 		/*
2468 		 * We'll use a bad block table stored in-flash and don't
2469 		 * allow writing the bad block marker to the flash.
2470 		 */
2471 		chip->bbt_options |= NAND_BBT_NO_OOB_BBM;
2472 		chip->bbt_td = &bbt_main_descr;
2473 		chip->bbt_md = &bbt_mirror_descr;
2474 	}
2475 
2476 	/* Save the chip-specific fields of NDCR */
2477 	marvell_nand->ndcr = NDCR_PAGE_SZ(mtd->writesize);
2478 	if (chip->options & NAND_BUSWIDTH_16)
2479 		marvell_nand->ndcr |= NDCR_DWIDTH_M | NDCR_DWIDTH_C;
2480 
2481 	/*
2482 	 * On small page NANDs, only one cycle is needed to pass the
2483 	 * column address.
2484 	 */
2485 	if (mtd->writesize <= 512) {
2486 		marvell_nand->addr_cyc = 1;
2487 	} else {
2488 		marvell_nand->addr_cyc = 2;
2489 		marvell_nand->ndcr |= NDCR_RA_START;
2490 	}
2491 
2492 	/*
2493 	 * Now add the number of cycles needed to pass the row
2494 	 * address.
2495 	 *
2496 	 * Addressing a chip using CS 2 or 3 should also need the third row
2497 	 * cycle but due to inconsistance in the documentation and lack of
2498 	 * hardware to test this situation, this case is not supported.
2499 	 */
2500 	if (chip->options & NAND_ROW_ADDR_3)
2501 		marvell_nand->addr_cyc += 3;
2502 	else
2503 		marvell_nand->addr_cyc += 2;
2504 
2505 	if (pdata) {
2506 		chip->ecc.size = pdata->ecc_step_size;
2507 		chip->ecc.strength = pdata->ecc_strength;
2508 	}
2509 
2510 	ret = marvell_nand_ecc_init(mtd, &chip->ecc);
2511 	if (ret) {
2512 		dev_err(nfc->dev, "ECC init failed: %d\n", ret);
2513 		return ret;
2514 	}
2515 
2516 	if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) {
2517 		/*
2518 		 * Subpage write not available with hardware ECC, prohibit also
2519 		 * subpage read as in userspace subpage access would still be
2520 		 * allowed and subpage write, if used, would lead to numerous
2521 		 * uncorrectable ECC errors.
2522 		 */
2523 		chip->options |= NAND_NO_SUBPAGE_WRITE;
2524 	}
2525 
2526 	if (pdata || nfc->caps->legacy_of_bindings) {
2527 		/*
2528 		 * We keep the MTD name unchanged to avoid breaking platforms
2529 		 * where the MTD cmdline parser is used and the bootloader
2530 		 * has not been updated to use the new naming scheme.
2531 		 */
2532 		mtd->name = "pxa3xx_nand-0";
2533 	} else if (!mtd->name) {
2534 		/*
2535 		 * If the new bindings are used and the bootloader has not been
2536 		 * updated to pass a new mtdparts parameter on the cmdline, you
2537 		 * should define the following property in your NAND node, ie:
2538 		 *
2539 		 *	label = "main-storage";
2540 		 *
2541 		 * This way, mtd->name will be set by the core when
2542 		 * nand_set_flash_node() is called.
2543 		 */
2544 		mtd->name = devm_kasprintf(nfc->dev, GFP_KERNEL,
2545 					   "%s:nand.%d", dev_name(nfc->dev),
2546 					   marvell_nand->sels[0].cs);
2547 		if (!mtd->name) {
2548 			dev_err(nfc->dev, "Failed to allocate mtd->name\n");
2549 			return -ENOMEM;
2550 		}
2551 	}
2552 
2553 	return 0;
2554 }
2555 
2556 static const struct nand_controller_ops marvell_nand_controller_ops = {
2557 	.attach_chip = marvell_nand_attach_chip,
2558 	.exec_op = marvell_nfc_exec_op,
2559 	.setup_interface = marvell_nfc_setup_interface,
2560 };
2561 
2562 static int marvell_nand_chip_init(struct device *dev, struct marvell_nfc *nfc,
2563 				  struct device_node *np)
2564 {
2565 	struct pxa3xx_nand_platform_data *pdata = dev_get_platdata(dev);
2566 	struct marvell_nand_chip *marvell_nand;
2567 	struct mtd_info *mtd;
2568 	struct nand_chip *chip;
2569 	int nsels, ret, i;
2570 	u32 cs, rb;
2571 
2572 	/*
2573 	 * The legacy "num-cs" property indicates the number of CS on the only
2574 	 * chip connected to the controller (legacy bindings does not support
2575 	 * more than one chip). The CS and RB pins are always the #0.
2576 	 *
2577 	 * When not using legacy bindings, a couple of "reg" and "nand-rb"
2578 	 * properties must be filled. For each chip, expressed as a subnode,
2579 	 * "reg" points to the CS lines and "nand-rb" to the RB line.
2580 	 */
2581 	if (pdata || nfc->caps->legacy_of_bindings) {
2582 		nsels = 1;
2583 	} else {
2584 		nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
2585 		if (nsels <= 0) {
2586 			dev_err(dev, "missing/invalid reg property\n");
2587 			return -EINVAL;
2588 		}
2589 	}
2590 
2591 	/* Alloc the nand chip structure */
2592 	marvell_nand = devm_kzalloc(dev,
2593 				    struct_size(marvell_nand, sels, nsels),
2594 				    GFP_KERNEL);
2595 	if (!marvell_nand) {
2596 		dev_err(dev, "could not allocate chip structure\n");
2597 		return -ENOMEM;
2598 	}
2599 
2600 	marvell_nand->nsels = nsels;
2601 	marvell_nand->selected_die = -1;
2602 
2603 	for (i = 0; i < nsels; i++) {
2604 		if (pdata || nfc->caps->legacy_of_bindings) {
2605 			/*
2606 			 * Legacy bindings use the CS lines in natural
2607 			 * order (0, 1, ...)
2608 			 */
2609 			cs = i;
2610 		} else {
2611 			/* Retrieve CS id */
2612 			ret = of_property_read_u32_index(np, "reg", i, &cs);
2613 			if (ret) {
2614 				dev_err(dev, "could not retrieve reg property: %d\n",
2615 					ret);
2616 				return ret;
2617 			}
2618 		}
2619 
2620 		if (cs >= nfc->caps->max_cs_nb) {
2621 			dev_err(dev, "invalid reg value: %u (max CS = %d)\n",
2622 				cs, nfc->caps->max_cs_nb);
2623 			return -EINVAL;
2624 		}
2625 
2626 		if (test_and_set_bit(cs, &nfc->assigned_cs)) {
2627 			dev_err(dev, "CS %d already assigned\n", cs);
2628 			return -EINVAL;
2629 		}
2630 
2631 		/*
2632 		 * The cs variable represents the chip select id, which must be
2633 		 * converted in bit fields for NDCB0 and NDCB2 to select the
2634 		 * right chip. Unfortunately, due to a lack of information on
2635 		 * the subject and incoherent documentation, the user should not
2636 		 * use CS1 and CS3 at all as asserting them is not supported in
2637 		 * a reliable way (due to multiplexing inside ADDR5 field).
2638 		 */
2639 		marvell_nand->sels[i].cs = cs;
2640 		switch (cs) {
2641 		case 0:
2642 		case 2:
2643 			marvell_nand->sels[i].ndcb0_csel = 0;
2644 			break;
2645 		case 1:
2646 		case 3:
2647 			marvell_nand->sels[i].ndcb0_csel = NDCB0_CSEL;
2648 			break;
2649 		default:
2650 			return -EINVAL;
2651 		}
2652 
2653 		/* Retrieve RB id */
2654 		if (pdata || nfc->caps->legacy_of_bindings) {
2655 			/* Legacy bindings always use RB #0 */
2656 			rb = 0;
2657 		} else {
2658 			ret = of_property_read_u32_index(np, "nand-rb", i,
2659 							 &rb);
2660 			if (ret) {
2661 				dev_err(dev,
2662 					"could not retrieve RB property: %d\n",
2663 					ret);
2664 				return ret;
2665 			}
2666 		}
2667 
2668 		if (rb >= nfc->caps->max_rb_nb) {
2669 			dev_err(dev, "invalid reg value: %u (max RB = %d)\n",
2670 				rb, nfc->caps->max_rb_nb);
2671 			return -EINVAL;
2672 		}
2673 
2674 		marvell_nand->sels[i].rb = rb;
2675 	}
2676 
2677 	chip = &marvell_nand->chip;
2678 	chip->controller = &nfc->controller;
2679 	nand_set_flash_node(chip, np);
2680 
2681 	if (of_property_read_bool(np, "marvell,nand-keep-config"))
2682 		chip->options |= NAND_KEEP_TIMINGS;
2683 
2684 	mtd = nand_to_mtd(chip);
2685 	mtd->dev.parent = dev;
2686 
2687 	/*
2688 	 * Save a reference value for timing registers before
2689 	 * ->setup_interface() is called.
2690 	 */
2691 	marvell_nand->ndtr0 = readl_relaxed(nfc->regs + NDTR0);
2692 	marvell_nand->ndtr1 = readl_relaxed(nfc->regs + NDTR1);
2693 
2694 	chip->options |= NAND_BUSWIDTH_AUTO;
2695 
2696 	ret = nand_scan(chip, marvell_nand->nsels);
2697 	if (ret) {
2698 		dev_err(dev, "could not scan the nand chip\n");
2699 		return ret;
2700 	}
2701 
2702 	if (pdata)
2703 		/* Legacy bindings support only one chip */
2704 		ret = mtd_device_register(mtd, pdata->parts, pdata->nr_parts);
2705 	else
2706 		ret = mtd_device_register(mtd, NULL, 0);
2707 	if (ret) {
2708 		dev_err(dev, "failed to register mtd device: %d\n", ret);
2709 		nand_cleanup(chip);
2710 		return ret;
2711 	}
2712 
2713 	list_add_tail(&marvell_nand->node, &nfc->chips);
2714 
2715 	return 0;
2716 }
2717 
2718 static void marvell_nand_chips_cleanup(struct marvell_nfc *nfc)
2719 {
2720 	struct marvell_nand_chip *entry, *temp;
2721 	struct nand_chip *chip;
2722 	int ret;
2723 
2724 	list_for_each_entry_safe(entry, temp, &nfc->chips, node) {
2725 		chip = &entry->chip;
2726 		ret = mtd_device_unregister(nand_to_mtd(chip));
2727 		WARN_ON(ret);
2728 		nand_cleanup(chip);
2729 		list_del(&entry->node);
2730 	}
2731 }
2732 
2733 static int marvell_nand_chips_init(struct device *dev, struct marvell_nfc *nfc)
2734 {
2735 	struct device_node *np = dev->of_node;
2736 	struct device_node *nand_np;
2737 	int max_cs = nfc->caps->max_cs_nb;
2738 	int nchips;
2739 	int ret;
2740 
2741 	if (!np)
2742 		nchips = 1;
2743 	else
2744 		nchips = of_get_child_count(np);
2745 
2746 	if (nchips > max_cs) {
2747 		dev_err(dev, "too many NAND chips: %d (max = %d CS)\n", nchips,
2748 			max_cs);
2749 		return -EINVAL;
2750 	}
2751 
2752 	/*
2753 	 * Legacy bindings do not use child nodes to exhibit NAND chip
2754 	 * properties and layout. Instead, NAND properties are mixed with the
2755 	 * controller ones, and partitions are defined as direct subnodes of the
2756 	 * NAND controller node.
2757 	 */
2758 	if (nfc->caps->legacy_of_bindings) {
2759 		ret = marvell_nand_chip_init(dev, nfc, np);
2760 		return ret;
2761 	}
2762 
2763 	for_each_child_of_node(np, nand_np) {
2764 		ret = marvell_nand_chip_init(dev, nfc, nand_np);
2765 		if (ret) {
2766 			of_node_put(nand_np);
2767 			goto cleanup_chips;
2768 		}
2769 	}
2770 
2771 	return 0;
2772 
2773 cleanup_chips:
2774 	marvell_nand_chips_cleanup(nfc);
2775 
2776 	return ret;
2777 }
2778 
2779 static int marvell_nfc_init_dma(struct marvell_nfc *nfc)
2780 {
2781 	struct platform_device *pdev = container_of(nfc->dev,
2782 						    struct platform_device,
2783 						    dev);
2784 	struct dma_slave_config config = {};
2785 	struct resource *r;
2786 	int ret;
2787 
2788 	if (!IS_ENABLED(CONFIG_PXA_DMA)) {
2789 		dev_warn(nfc->dev,
2790 			 "DMA not enabled in configuration\n");
2791 		return -ENOTSUPP;
2792 	}
2793 
2794 	ret = dma_set_mask_and_coherent(nfc->dev, DMA_BIT_MASK(32));
2795 	if (ret)
2796 		return ret;
2797 
2798 	nfc->dma_chan =	dma_request_chan(nfc->dev, "data");
2799 	if (IS_ERR(nfc->dma_chan)) {
2800 		ret = PTR_ERR(nfc->dma_chan);
2801 		nfc->dma_chan = NULL;
2802 		return dev_err_probe(nfc->dev, ret, "DMA channel request failed\n");
2803 	}
2804 
2805 	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2806 	if (!r) {
2807 		ret = -ENXIO;
2808 		goto release_channel;
2809 	}
2810 
2811 	config.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2812 	config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2813 	config.src_addr = r->start + NDDB;
2814 	config.dst_addr = r->start + NDDB;
2815 	config.src_maxburst = 32;
2816 	config.dst_maxburst = 32;
2817 	ret = dmaengine_slave_config(nfc->dma_chan, &config);
2818 	if (ret < 0) {
2819 		dev_err(nfc->dev, "Failed to configure DMA channel\n");
2820 		goto release_channel;
2821 	}
2822 
2823 	/*
2824 	 * DMA must act on length multiple of 32 and this length may be
2825 	 * bigger than the destination buffer. Use this buffer instead
2826 	 * for DMA transfers and then copy the desired amount of data to
2827 	 * the provided buffer.
2828 	 */
2829 	nfc->dma_buf = kmalloc(MAX_CHUNK_SIZE, GFP_KERNEL | GFP_DMA);
2830 	if (!nfc->dma_buf) {
2831 		ret = -ENOMEM;
2832 		goto release_channel;
2833 	}
2834 
2835 	nfc->use_dma = true;
2836 
2837 	return 0;
2838 
2839 release_channel:
2840 	dma_release_channel(nfc->dma_chan);
2841 	nfc->dma_chan = NULL;
2842 
2843 	return ret;
2844 }
2845 
2846 static void marvell_nfc_reset(struct marvell_nfc *nfc)
2847 {
2848 	/*
2849 	 * ECC operations and interruptions are only enabled when specifically
2850 	 * needed. ECC shall not be activated in the early stages (fails probe).
2851 	 * Arbiter flag, even if marked as "reserved", must be set (empirical).
2852 	 * SPARE_EN bit must always be set or ECC bytes will not be at the same
2853 	 * offset in the read page and this will fail the protection.
2854 	 */
2855 	writel_relaxed(NDCR_ALL_INT | NDCR_ND_ARB_EN | NDCR_SPARE_EN |
2856 		       NDCR_RD_ID_CNT(NFCV1_READID_LEN), nfc->regs + NDCR);
2857 	writel_relaxed(0xFFFFFFFF, nfc->regs + NDSR);
2858 	writel_relaxed(0, nfc->regs + NDECCCTRL);
2859 }
2860 
2861 static int marvell_nfc_init(struct marvell_nfc *nfc)
2862 {
2863 	struct device_node *np = nfc->dev->of_node;
2864 
2865 	/*
2866 	 * Some SoCs like A7k/A8k need to enable manually the NAND
2867 	 * controller, gated clocks and reset bits to avoid being bootloader
2868 	 * dependent. This is done through the use of the System Functions
2869 	 * registers.
2870 	 */
2871 	if (nfc->caps->need_system_controller) {
2872 		struct regmap *sysctrl_base =
2873 			syscon_regmap_lookup_by_phandle(np,
2874 							"marvell,system-controller");
2875 
2876 		if (IS_ERR(sysctrl_base))
2877 			return PTR_ERR(sysctrl_base);
2878 
2879 		regmap_write(sysctrl_base, GENCONF_SOC_DEVICE_MUX,
2880 			     GENCONF_SOC_DEVICE_MUX_NFC_EN |
2881 			     GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST |
2882 			     GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST |
2883 			     GENCONF_SOC_DEVICE_MUX_NFC_INT_EN);
2884 
2885 		regmap_update_bits(sysctrl_base, GENCONF_CLK_GATING_CTRL,
2886 				   GENCONF_CLK_GATING_CTRL_ND_GATE,
2887 				   GENCONF_CLK_GATING_CTRL_ND_GATE);
2888 
2889 		regmap_update_bits(sysctrl_base, GENCONF_ND_CLK_CTRL,
2890 				   GENCONF_ND_CLK_CTRL_EN,
2891 				   GENCONF_ND_CLK_CTRL_EN);
2892 	}
2893 
2894 	/* Configure the DMA if appropriate */
2895 	if (!nfc->caps->is_nfcv2)
2896 		marvell_nfc_init_dma(nfc);
2897 
2898 	marvell_nfc_reset(nfc);
2899 
2900 	return 0;
2901 }
2902 
2903 static int marvell_nfc_probe(struct platform_device *pdev)
2904 {
2905 	struct device *dev = &pdev->dev;
2906 	struct marvell_nfc *nfc;
2907 	int ret;
2908 	int irq;
2909 
2910 	nfc = devm_kzalloc(&pdev->dev, sizeof(struct marvell_nfc),
2911 			   GFP_KERNEL);
2912 	if (!nfc)
2913 		return -ENOMEM;
2914 
2915 	nfc->dev = dev;
2916 	nand_controller_init(&nfc->controller);
2917 	nfc->controller.ops = &marvell_nand_controller_ops;
2918 	INIT_LIST_HEAD(&nfc->chips);
2919 
2920 	nfc->regs = devm_platform_ioremap_resource(pdev, 0);
2921 	if (IS_ERR(nfc->regs))
2922 		return PTR_ERR(nfc->regs);
2923 
2924 	irq = platform_get_irq(pdev, 0);
2925 	if (irq < 0)
2926 		return irq;
2927 
2928 	nfc->core_clk = devm_clk_get(&pdev->dev, "core");
2929 
2930 	/* Managed the legacy case (when the first clock was not named) */
2931 	if (nfc->core_clk == ERR_PTR(-ENOENT))
2932 		nfc->core_clk = devm_clk_get(&pdev->dev, NULL);
2933 
2934 	if (IS_ERR(nfc->core_clk))
2935 		return PTR_ERR(nfc->core_clk);
2936 
2937 	ret = clk_prepare_enable(nfc->core_clk);
2938 	if (ret)
2939 		return ret;
2940 
2941 	nfc->reg_clk = devm_clk_get(&pdev->dev, "reg");
2942 	if (IS_ERR(nfc->reg_clk)) {
2943 		if (PTR_ERR(nfc->reg_clk) != -ENOENT) {
2944 			ret = PTR_ERR(nfc->reg_clk);
2945 			goto unprepare_core_clk;
2946 		}
2947 
2948 		nfc->reg_clk = NULL;
2949 	}
2950 
2951 	ret = clk_prepare_enable(nfc->reg_clk);
2952 	if (ret)
2953 		goto unprepare_core_clk;
2954 
2955 	marvell_nfc_disable_int(nfc, NDCR_ALL_INT);
2956 	marvell_nfc_clear_int(nfc, NDCR_ALL_INT);
2957 	ret = devm_request_irq(dev, irq, marvell_nfc_isr,
2958 			       0, "marvell-nfc", nfc);
2959 	if (ret)
2960 		goto unprepare_reg_clk;
2961 
2962 	/* Get NAND controller capabilities */
2963 	if (pdev->id_entry)
2964 		nfc->caps = (void *)pdev->id_entry->driver_data;
2965 	else
2966 		nfc->caps = of_device_get_match_data(&pdev->dev);
2967 
2968 	if (!nfc->caps) {
2969 		dev_err(dev, "Could not retrieve NFC caps\n");
2970 		ret = -EINVAL;
2971 		goto unprepare_reg_clk;
2972 	}
2973 
2974 	/* Init the controller and then probe the chips */
2975 	ret = marvell_nfc_init(nfc);
2976 	if (ret)
2977 		goto unprepare_reg_clk;
2978 
2979 	platform_set_drvdata(pdev, nfc);
2980 
2981 	ret = marvell_nand_chips_init(dev, nfc);
2982 	if (ret)
2983 		goto release_dma;
2984 
2985 	return 0;
2986 
2987 release_dma:
2988 	if (nfc->use_dma)
2989 		dma_release_channel(nfc->dma_chan);
2990 unprepare_reg_clk:
2991 	clk_disable_unprepare(nfc->reg_clk);
2992 unprepare_core_clk:
2993 	clk_disable_unprepare(nfc->core_clk);
2994 
2995 	return ret;
2996 }
2997 
2998 static int marvell_nfc_remove(struct platform_device *pdev)
2999 {
3000 	struct marvell_nfc *nfc = platform_get_drvdata(pdev);
3001 
3002 	marvell_nand_chips_cleanup(nfc);
3003 
3004 	if (nfc->use_dma) {
3005 		dmaengine_terminate_all(nfc->dma_chan);
3006 		dma_release_channel(nfc->dma_chan);
3007 	}
3008 
3009 	clk_disable_unprepare(nfc->reg_clk);
3010 	clk_disable_unprepare(nfc->core_clk);
3011 
3012 	return 0;
3013 }
3014 
3015 static int __maybe_unused marvell_nfc_suspend(struct device *dev)
3016 {
3017 	struct marvell_nfc *nfc = dev_get_drvdata(dev);
3018 	struct marvell_nand_chip *chip;
3019 
3020 	list_for_each_entry(chip, &nfc->chips, node)
3021 		marvell_nfc_wait_ndrun(&chip->chip);
3022 
3023 	clk_disable_unprepare(nfc->reg_clk);
3024 	clk_disable_unprepare(nfc->core_clk);
3025 
3026 	return 0;
3027 }
3028 
3029 static int __maybe_unused marvell_nfc_resume(struct device *dev)
3030 {
3031 	struct marvell_nfc *nfc = dev_get_drvdata(dev);
3032 	int ret;
3033 
3034 	ret = clk_prepare_enable(nfc->core_clk);
3035 	if (ret < 0)
3036 		return ret;
3037 
3038 	ret = clk_prepare_enable(nfc->reg_clk);
3039 	if (ret < 0) {
3040 		clk_disable_unprepare(nfc->core_clk);
3041 		return ret;
3042 	}
3043 
3044 	/*
3045 	 * Reset nfc->selected_chip so the next command will cause the timing
3046 	 * registers to be restored in marvell_nfc_select_target().
3047 	 */
3048 	nfc->selected_chip = NULL;
3049 
3050 	/* Reset registers that have lost their contents */
3051 	marvell_nfc_reset(nfc);
3052 
3053 	return 0;
3054 }
3055 
3056 static const struct dev_pm_ops marvell_nfc_pm_ops = {
3057 	SET_SYSTEM_SLEEP_PM_OPS(marvell_nfc_suspend, marvell_nfc_resume)
3058 };
3059 
3060 static const struct marvell_nfc_caps marvell_armada_8k_nfc_caps = {
3061 	.max_cs_nb = 4,
3062 	.max_rb_nb = 2,
3063 	.need_system_controller = true,
3064 	.is_nfcv2 = true,
3065 };
3066 
3067 static const struct marvell_nfc_caps marvell_armada370_nfc_caps = {
3068 	.max_cs_nb = 4,
3069 	.max_rb_nb = 2,
3070 	.is_nfcv2 = true,
3071 };
3072 
3073 static const struct marvell_nfc_caps marvell_pxa3xx_nfc_caps = {
3074 	.max_cs_nb = 2,
3075 	.max_rb_nb = 1,
3076 	.use_dma = true,
3077 };
3078 
3079 static const struct marvell_nfc_caps marvell_armada_8k_nfc_legacy_caps = {
3080 	.max_cs_nb = 4,
3081 	.max_rb_nb = 2,
3082 	.need_system_controller = true,
3083 	.legacy_of_bindings = true,
3084 	.is_nfcv2 = true,
3085 };
3086 
3087 static const struct marvell_nfc_caps marvell_armada370_nfc_legacy_caps = {
3088 	.max_cs_nb = 4,
3089 	.max_rb_nb = 2,
3090 	.legacy_of_bindings = true,
3091 	.is_nfcv2 = true,
3092 };
3093 
3094 static const struct marvell_nfc_caps marvell_pxa3xx_nfc_legacy_caps = {
3095 	.max_cs_nb = 2,
3096 	.max_rb_nb = 1,
3097 	.legacy_of_bindings = true,
3098 	.use_dma = true,
3099 };
3100 
3101 static const struct platform_device_id marvell_nfc_platform_ids[] = {
3102 	{
3103 		.name = "pxa3xx-nand",
3104 		.driver_data = (kernel_ulong_t)&marvell_pxa3xx_nfc_legacy_caps,
3105 	},
3106 	{ /* sentinel */ },
3107 };
3108 MODULE_DEVICE_TABLE(platform, marvell_nfc_platform_ids);
3109 
3110 static const struct of_device_id marvell_nfc_of_ids[] = {
3111 	{
3112 		.compatible = "marvell,armada-8k-nand-controller",
3113 		.data = &marvell_armada_8k_nfc_caps,
3114 	},
3115 	{
3116 		.compatible = "marvell,armada370-nand-controller",
3117 		.data = &marvell_armada370_nfc_caps,
3118 	},
3119 	{
3120 		.compatible = "marvell,pxa3xx-nand-controller",
3121 		.data = &marvell_pxa3xx_nfc_caps,
3122 	},
3123 	/* Support for old/deprecated bindings: */
3124 	{
3125 		.compatible = "marvell,armada-8k-nand",
3126 		.data = &marvell_armada_8k_nfc_legacy_caps,
3127 	},
3128 	{
3129 		.compatible = "marvell,armada370-nand",
3130 		.data = &marvell_armada370_nfc_legacy_caps,
3131 	},
3132 	{
3133 		.compatible = "marvell,pxa3xx-nand",
3134 		.data = &marvell_pxa3xx_nfc_legacy_caps,
3135 	},
3136 	{ /* sentinel */ },
3137 };
3138 MODULE_DEVICE_TABLE(of, marvell_nfc_of_ids);
3139 
3140 static struct platform_driver marvell_nfc_driver = {
3141 	.driver	= {
3142 		.name		= "marvell-nfc",
3143 		.of_match_table = marvell_nfc_of_ids,
3144 		.pm		= &marvell_nfc_pm_ops,
3145 	},
3146 	.id_table = marvell_nfc_platform_ids,
3147 	.probe = marvell_nfc_probe,
3148 	.remove	= marvell_nfc_remove,
3149 };
3150 module_platform_driver(marvell_nfc_driver);
3151 
3152 MODULE_LICENSE("GPL");
3153 MODULE_DESCRIPTION("Marvell NAND controller driver");
3154