1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * Cadence NAND flash controller driver
4  *
5  * Copyright (C) 2019 Cadence
6  *
7  * Author: Piotr Sroka <piotrs@cadence.com>
8  */
9 
10 #include <linux/bitfield.h>
11 #include <linux/clk.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/dmaengine.h>
14 #include <linux/interrupt.h>
15 #include <linux/module.h>
16 #include <linux/mtd/mtd.h>
17 #include <linux/mtd/rawnand.h>
18 #include <linux/of_device.h>
19 #include <linux/iopoll.h>
20 #include <linux/slab.h>
21 
22 /*
23  * HPNFC can work in 3 modes:
24  * -  PIO - can work in master or slave DMA
25  * -  CDMA - needs Master DMA for accessing command descriptors.
26  * -  Generic mode - can use only slave DMA.
27  * CDMA and PIO modes can be used to execute only base commands.
28  * Generic mode can be used to execute any command
29  * on NAND flash memory. Driver uses CDMA mode for
30  * block erasing, page reading, page programing.
31  * Generic mode is used for executing rest of commands.
32  */
33 
34 #define MAX_ADDRESS_CYC		6
35 #define MAX_ERASE_ADDRESS_CYC	3
36 #define MAX_DATA_SIZE		0xFFFC
37 #define DMA_DATA_SIZE_ALIGN	8
38 
39 /* Register definition. */
40 /*
41  * Command register 0.
42  * Writing data to this register will initiate a new transaction
43  * of the NF controller.
44  */
45 #define CMD_REG0			0x0000
46 /* Command type field mask. */
47 #define		CMD_REG0_CT		GENMASK(31, 30)
48 /* Command type CDMA. */
49 #define		CMD_REG0_CT_CDMA	0uL
50 /* Command type generic. */
51 #define		CMD_REG0_CT_GEN		3uL
52 /* Command thread number field mask. */
53 #define		CMD_REG0_TN		GENMASK(27, 24)
54 
55 /* Command register 2. */
56 #define CMD_REG2			0x0008
57 /* Command register 3. */
58 #define CMD_REG3			0x000C
59 /* Pointer register to select which thread status will be selected. */
60 #define CMD_STATUS_PTR			0x0010
61 /* Command status register for selected thread. */
62 #define CMD_STATUS			0x0014
63 
64 /* Interrupt status register. */
65 #define INTR_STATUS			0x0110
66 #define		INTR_STATUS_SDMA_ERR	BIT(22)
67 #define		INTR_STATUS_SDMA_TRIGG	BIT(21)
68 #define		INTR_STATUS_UNSUPP_CMD	BIT(19)
69 #define		INTR_STATUS_DDMA_TERR	BIT(18)
70 #define		INTR_STATUS_CDMA_TERR	BIT(17)
71 #define		INTR_STATUS_CDMA_IDL	BIT(16)
72 
73 /* Interrupt enable register. */
74 #define INTR_ENABLE				0x0114
75 #define		INTR_ENABLE_INTR_EN		BIT(31)
76 #define		INTR_ENABLE_SDMA_ERR_EN		BIT(22)
77 #define		INTR_ENABLE_SDMA_TRIGG_EN	BIT(21)
78 #define		INTR_ENABLE_UNSUPP_CMD_EN	BIT(19)
79 #define		INTR_ENABLE_DDMA_TERR_EN	BIT(18)
80 #define		INTR_ENABLE_CDMA_TERR_EN	BIT(17)
81 #define		INTR_ENABLE_CDMA_IDLE_EN	BIT(16)
82 
83 /* Controller internal state. */
84 #define CTRL_STATUS				0x0118
85 #define		CTRL_STATUS_INIT_COMP		BIT(9)
86 #define		CTRL_STATUS_CTRL_BUSY		BIT(8)
87 
88 /* Command Engine threads state. */
89 #define TRD_STATUS				0x0120
90 
91 /* Command Engine interrupt thread error status. */
92 #define TRD_ERR_INT_STATUS			0x0128
93 /* Command Engine interrupt thread error enable. */
94 #define TRD_ERR_INT_STATUS_EN			0x0130
95 /* Command Engine interrupt thread complete status. */
96 #define TRD_COMP_INT_STATUS			0x0138
97 
98 /*
99  * Transfer config 0 register.
100  * Configures data transfer parameters.
101  */
102 #define TRAN_CFG_0				0x0400
103 /* Offset value from the beginning of the page. */
104 #define		TRAN_CFG_0_OFFSET		GENMASK(31, 16)
105 /* Numbers of sectors to transfer within singlNF device's page. */
106 #define		TRAN_CFG_0_SEC_CNT		GENMASK(7, 0)
107 
108 /*
109  * Transfer config 1 register.
110  * Configures data transfer parameters.
111  */
112 #define TRAN_CFG_1				0x0404
113 /* Size of last data sector. */
114 #define		TRAN_CFG_1_LAST_SEC_SIZE	GENMASK(31, 16)
115 /* Size of not-last data sector. */
116 #define		TRAN_CFG_1_SECTOR_SIZE		GENMASK(15, 0)
117 
118 /* ECC engine configuration register 0. */
119 #define ECC_CONFIG_0				0x0428
120 /* Correction strength. */
121 #define		ECC_CONFIG_0_CORR_STR		GENMASK(10, 8)
122 /* Enable erased pages detection mechanism. */
123 #define		ECC_CONFIG_0_ERASE_DET_EN	BIT(1)
124 /* Enable controller ECC check bits generation and correction. */
125 #define		ECC_CONFIG_0_ECC_EN		BIT(0)
126 
127 /* ECC engine configuration register 1. */
128 #define ECC_CONFIG_1				0x042C
129 
130 /* Multiplane settings register. */
131 #define MULTIPLANE_CFG				0x0434
132 /* Cache operation settings. */
133 #define CACHE_CFG				0x0438
134 
135 /* DMA settings register. */
136 #define DMA_SETINGS				0x043C
137 /* Enable SDMA error report on access unprepared slave DMA interface. */
138 #define		DMA_SETINGS_SDMA_ERR_RSP	BIT(17)
139 
140 /* Transferred data block size for the slave DMA module. */
141 #define SDMA_SIZE				0x0440
142 
143 /* Thread number associated with transferred data block
144  * for the slave DMA module.
145  */
146 #define SDMA_TRD_NUM				0x0444
147 /* Thread number mask. */
148 #define		SDMA_TRD_NUM_SDMA_TRD		GENMASK(2, 0)
149 
150 #define CONTROL_DATA_CTRL			0x0494
151 /* Thread number mask. */
152 #define		CONTROL_DATA_CTRL_SIZE		GENMASK(15, 0)
153 
154 #define CTRL_VERSION				0x800
155 #define		CTRL_VERSION_REV		GENMASK(7, 0)
156 
157 /* Available hardware features of the controller. */
158 #define CTRL_FEATURES				0x804
159 /* Support for NV-DDR2/3 work mode. */
160 #define		CTRL_FEATURES_NVDDR_2_3		BIT(28)
161 /* Support for NV-DDR work mode. */
162 #define		CTRL_FEATURES_NVDDR		BIT(27)
163 /* Support for asynchronous work mode. */
164 #define		CTRL_FEATURES_ASYNC		BIT(26)
165 /* Support for asynchronous work mode. */
166 #define		CTRL_FEATURES_N_BANKS		GENMASK(25, 24)
167 /* Slave and Master DMA data width. */
168 #define		CTRL_FEATURES_DMA_DWITH64	BIT(21)
169 /* Availability of Control Data feature.*/
170 #define		CTRL_FEATURES_CONTROL_DATA	BIT(10)
171 
172 /* BCH Engine identification register 0 - correction strengths. */
173 #define BCH_CFG_0				0x838
174 #define		BCH_CFG_0_CORR_CAP_0		GENMASK(7, 0)
175 #define		BCH_CFG_0_CORR_CAP_1		GENMASK(15, 8)
176 #define		BCH_CFG_0_CORR_CAP_2		GENMASK(23, 16)
177 #define		BCH_CFG_0_CORR_CAP_3		GENMASK(31, 24)
178 
179 /* BCH Engine identification register 1 - correction strengths. */
180 #define BCH_CFG_1				0x83C
181 #define		BCH_CFG_1_CORR_CAP_4		GENMASK(7, 0)
182 #define		BCH_CFG_1_CORR_CAP_5		GENMASK(15, 8)
183 #define		BCH_CFG_1_CORR_CAP_6		GENMASK(23, 16)
184 #define		BCH_CFG_1_CORR_CAP_7		GENMASK(31, 24)
185 
186 /* BCH Engine identification register 2 - sector sizes. */
187 #define BCH_CFG_2				0x840
188 #define		BCH_CFG_2_SECT_0		GENMASK(15, 0)
189 #define		BCH_CFG_2_SECT_1		GENMASK(31, 16)
190 
191 /* BCH Engine identification register 3. */
192 #define BCH_CFG_3				0x844
193 #define		BCH_CFG_3_METADATA_SIZE		GENMASK(23, 16)
194 
195 /* Ready/Busy# line status. */
196 #define RBN_SETINGS				0x1004
197 
198 /* Common settings. */
199 #define COMMON_SET				0x1008
200 /* 16 bit device connected to the NAND Flash interface. */
201 #define		COMMON_SET_DEVICE_16BIT		BIT(8)
202 
203 /* Skip_bytes registers. */
204 #define SKIP_BYTES_CONF				0x100C
205 #define		SKIP_BYTES_MARKER_VALUE		GENMASK(31, 16)
206 #define		SKIP_BYTES_NUM_OF_BYTES		GENMASK(7, 0)
207 
208 #define SKIP_BYTES_OFFSET			0x1010
209 #define		 SKIP_BYTES_OFFSET_VALUE	GENMASK(23, 0)
210 
211 /* Timings configuration. */
212 #define ASYNC_TOGGLE_TIMINGS			0x101c
213 #define		ASYNC_TOGGLE_TIMINGS_TRH	GENMASK(28, 24)
214 #define		ASYNC_TOGGLE_TIMINGS_TRP	GENMASK(20, 16)
215 #define		ASYNC_TOGGLE_TIMINGS_TWH	GENMASK(12, 8)
216 #define		ASYNC_TOGGLE_TIMINGS_TWP	GENMASK(4, 0)
217 
218 #define	TIMINGS0				0x1024
219 #define		TIMINGS0_TADL			GENMASK(31, 24)
220 #define		TIMINGS0_TCCS			GENMASK(23, 16)
221 #define		TIMINGS0_TWHR			GENMASK(15, 8)
222 #define		TIMINGS0_TRHW			GENMASK(7, 0)
223 
224 #define	TIMINGS1				0x1028
225 #define		TIMINGS1_TRHZ			GENMASK(31, 24)
226 #define		TIMINGS1_TWB			GENMASK(23, 16)
227 #define		TIMINGS1_TVDLY			GENMASK(7, 0)
228 
229 #define	TIMINGS2				0x102c
230 #define		TIMINGS2_TFEAT			GENMASK(25, 16)
231 #define		TIMINGS2_CS_HOLD_TIME		GENMASK(13, 8)
232 #define		TIMINGS2_CS_SETUP_TIME		GENMASK(5, 0)
233 
234 /* Configuration of the resynchronization of slave DLL of PHY. */
235 #define DLL_PHY_CTRL				0x1034
236 #define		DLL_PHY_CTRL_DLL_RST_N		BIT(24)
237 #define		DLL_PHY_CTRL_EXTENDED_WR_MODE	BIT(17)
238 #define		DLL_PHY_CTRL_EXTENDED_RD_MODE	BIT(16)
239 #define		DLL_PHY_CTRL_RS_HIGH_WAIT_CNT	GENMASK(11, 8)
240 #define		DLL_PHY_CTRL_RS_IDLE_CNT	GENMASK(7, 0)
241 
242 /* Register controlling DQ related timing. */
243 #define PHY_DQ_TIMING				0x2000
244 /* Register controlling DSQ related timing.  */
245 #define PHY_DQS_TIMING				0x2004
246 #define		PHY_DQS_TIMING_DQS_SEL_OE_END	GENMASK(3, 0)
247 #define		PHY_DQS_TIMING_PHONY_DQS_SEL	BIT(16)
248 #define		PHY_DQS_TIMING_USE_PHONY_DQS	BIT(20)
249 
250 /* Register controlling the gate and loopback control related timing. */
251 #define PHY_GATE_LPBK_CTRL			0x2008
252 #define		PHY_GATE_LPBK_CTRL_RDS		GENMASK(24, 19)
253 
254 /* Register holds the control for the master DLL logic. */
255 #define PHY_DLL_MASTER_CTRL			0x200C
256 #define		PHY_DLL_MASTER_CTRL_BYPASS_MODE	BIT(23)
257 
258 /* Register holds the control for the slave DLL logic. */
259 #define PHY_DLL_SLAVE_CTRL			0x2010
260 
261 /* This register handles the global control settings for the PHY. */
262 #define PHY_CTRL				0x2080
263 #define		PHY_CTRL_SDR_DQS		BIT(14)
264 #define		PHY_CTRL_PHONY_DQS		GENMASK(9, 4)
265 
266 /*
267  * This register handles the global control settings
268  * for the termination selects for reads.
269  */
270 #define PHY_TSEL				0x2084
271 
272 /* Generic command layout. */
273 #define GCMD_LAY_CS			GENMASK_ULL(11, 8)
274 /*
275  * This bit informs the minicotroller if it has to wait for tWB
276  * after sending the last CMD/ADDR/DATA in the sequence.
277  */
278 #define GCMD_LAY_TWB			BIT_ULL(6)
279 /* Type of generic instruction. */
280 #define GCMD_LAY_INSTR			GENMASK_ULL(5, 0)
281 
282 /* Generic CMD sequence type. */
283 #define		GCMD_LAY_INSTR_CMD	0
284 /* Generic ADDR sequence type. */
285 #define		GCMD_LAY_INSTR_ADDR	1
286 /* Generic data transfer sequence type. */
287 #define		GCMD_LAY_INSTR_DATA	2
288 
289 /* Input part of generic command type of input is command. */
290 #define GCMD_LAY_INPUT_CMD		GENMASK_ULL(23, 16)
291 
292 /* Generic command address sequence - address fields. */
293 #define GCMD_LAY_INPUT_ADDR		GENMASK_ULL(63, 16)
294 /* Generic command address sequence - address size. */
295 #define GCMD_LAY_INPUT_ADDR_SIZE	GENMASK_ULL(13, 11)
296 
297 /* Transfer direction field of generic command data sequence. */
298 #define GCMD_DIR			BIT_ULL(11)
299 /* Read transfer direction of generic command data sequence. */
300 #define		GCMD_DIR_READ		0
301 /* Write transfer direction of generic command data sequence. */
302 #define		GCMD_DIR_WRITE		1
303 
304 /* ECC enabled flag of generic command data sequence - ECC enabled. */
305 #define GCMD_ECC_EN			BIT_ULL(12)
306 /* Generic command data sequence - sector size. */
307 #define GCMD_SECT_SIZE			GENMASK_ULL(31, 16)
308 /* Generic command data sequence - sector count. */
309 #define GCMD_SECT_CNT			GENMASK_ULL(39, 32)
310 /* Generic command data sequence - last sector size. */
311 #define GCMD_LAST_SIZE			GENMASK_ULL(55, 40)
312 
313 /* CDMA descriptor fields. */
314 /* Erase command type of CDMA descriptor. */
315 #define CDMA_CT_ERASE		0x1000
316 /* Program page command type of CDMA descriptor. */
317 #define CDMA_CT_WR		0x2100
318 /* Read page command type of CDMA descriptor. */
319 #define CDMA_CT_RD		0x2200
320 
321 /* Flash pointer memory shift. */
322 #define CDMA_CFPTR_MEM_SHIFT	24
323 /* Flash pointer memory mask. */
324 #define CDMA_CFPTR_MEM		GENMASK(26, 24)
325 
326 /*
327  * Command DMA descriptor flags. If set causes issue interrupt after
328  * the completion of descriptor processing.
329  */
330 #define CDMA_CF_INT		BIT(8)
331 /*
332  * Command DMA descriptor flags - the next descriptor
333  * address field is valid and descriptor processing should continue.
334  */
335 #define CDMA_CF_CONT		BIT(9)
336 /* DMA master flag of command DMA descriptor. */
337 #define CDMA_CF_DMA_MASTER	BIT(10)
338 
339 /* Operation complete status of command descriptor. */
340 #define CDMA_CS_COMP		BIT(15)
341 /* Operation complete status of command descriptor. */
342 /* Command descriptor status - operation fail. */
343 #define CDMA_CS_FAIL		BIT(14)
344 /* Command descriptor status - page erased. */
345 #define CDMA_CS_ERP		BIT(11)
346 /* Command descriptor status - timeout occurred. */
347 #define CDMA_CS_TOUT		BIT(10)
348 /*
349  * Maximum amount of correction applied to one ECC sector.
350  * It is part of command descriptor status.
351  */
352 #define CDMA_CS_MAXERR		GENMASK(9, 2)
353 /* Command descriptor status - uncorrectable ECC error. */
354 #define CDMA_CS_UNCE		BIT(1)
355 /* Command descriptor status - descriptor error. */
356 #define CDMA_CS_ERR		BIT(0)
357 
358 /* Status of operation - OK. */
359 #define STAT_OK			0
360 /* Status of operation - FAIL. */
361 #define STAT_FAIL		2
362 /* Status of operation - uncorrectable ECC error. */
363 #define STAT_ECC_UNCORR		3
364 /* Status of operation - page erased. */
365 #define STAT_ERASED		5
366 /* Status of operation - correctable ECC error. */
367 #define STAT_ECC_CORR		6
368 /* Status of operation - unsuspected state. */
369 #define STAT_UNKNOWN		7
370 /* Status of operation - operation is not completed yet. */
371 #define STAT_BUSY		0xFF
372 
373 #define BCH_MAX_NUM_CORR_CAPS		8
374 #define BCH_MAX_NUM_SECTOR_SIZES	2
375 
376 struct cadence_nand_timings {
377 	u32 async_toggle_timings;
378 	u32 timings0;
379 	u32 timings1;
380 	u32 timings2;
381 	u32 dll_phy_ctrl;
382 	u32 phy_ctrl;
383 	u32 phy_dqs_timing;
384 	u32 phy_gate_lpbk_ctrl;
385 };
386 
387 /* Command DMA descriptor. */
388 struct cadence_nand_cdma_desc {
389 	/* Next descriptor address. */
390 	u64 next_pointer;
391 
392 	/* Flash address is a 32-bit address comprising of BANK and ROW ADDR. */
393 	u32 flash_pointer;
394 	/*field appears in HPNFC version 13*/
395 	u16 bank;
396 	u16 rsvd0;
397 
398 	/* Operation the controller needs to perform. */
399 	u16 command_type;
400 	u16 rsvd1;
401 	/* Flags for operation of this command. */
402 	u16 command_flags;
403 	u16 rsvd2;
404 
405 	/* System/host memory address required for data DMA commands. */
406 	u64 memory_pointer;
407 
408 	/* Status of operation. */
409 	u32 status;
410 	u32 rsvd3;
411 
412 	/* Address pointer to sync buffer location. */
413 	u64 sync_flag_pointer;
414 
415 	/* Controls the buffer sync mechanism. */
416 	u32 sync_arguments;
417 	u32 rsvd4;
418 
419 	/* Control data pointer. */
420 	u64 ctrl_data_ptr;
421 };
422 
423 /* Interrupt status. */
424 struct cadence_nand_irq_status {
425 	/* Thread operation complete status. */
426 	u32 trd_status;
427 	/* Thread operation error. */
428 	u32 trd_error;
429 	/* Controller status. */
430 	u32 status;
431 };
432 
433 /* Cadence NAND flash controller capabilities get from driver data. */
434 struct cadence_nand_dt_devdata {
435 	/* Skew value of the output signals of the NAND Flash interface. */
436 	u32 if_skew;
437 	/* It informs if slave DMA interface is connected to DMA engine. */
438 	unsigned int has_dma:1;
439 };
440 
441 /* Cadence NAND flash controller capabilities read from registers. */
442 struct cdns_nand_caps {
443 	/* Maximum number of banks supported by hardware. */
444 	u8 max_banks;
445 	/* Slave and Master DMA data width in bytes (4 or 8). */
446 	u8 data_dma_width;
447 	/* Control Data feature supported. */
448 	bool data_control_supp;
449 	/* Is PHY type DLL. */
450 	bool is_phy_type_dll;
451 };
452 
453 struct cdns_nand_ctrl {
454 	struct device *dev;
455 	struct nand_controller controller;
456 	struct cadence_nand_cdma_desc *cdma_desc;
457 	/* IP capability. */
458 	const struct cadence_nand_dt_devdata *caps1;
459 	struct cdns_nand_caps caps2;
460 	u8 ctrl_rev;
461 	dma_addr_t dma_cdma_desc;
462 	u8 *buf;
463 	u32 buf_size;
464 	u8 curr_corr_str_idx;
465 
466 	/* Register interface. */
467 	void __iomem *reg;
468 
469 	struct {
470 		void __iomem *virt;
471 		dma_addr_t dma;
472 	} io;
473 
474 	int irq;
475 	/* Interrupts that have happened. */
476 	struct cadence_nand_irq_status irq_status;
477 	/* Interrupts we are waiting for. */
478 	struct cadence_nand_irq_status irq_mask;
479 	struct completion complete;
480 	/* Protect irq_mask and irq_status. */
481 	spinlock_t irq_lock;
482 
483 	int ecc_strengths[BCH_MAX_NUM_CORR_CAPS];
484 	struct nand_ecc_step_info ecc_stepinfos[BCH_MAX_NUM_SECTOR_SIZES];
485 	struct nand_ecc_caps ecc_caps;
486 
487 	int curr_trans_type;
488 
489 	struct dma_chan *dmac;
490 
491 	u32 nf_clk_rate;
492 	/*
493 	 * Estimated Board delay. The value includes the total
494 	 * round trip delay for the signals and is used for deciding on values
495 	 * associated with data read capture.
496 	 */
497 	u32 board_delay;
498 
499 	struct nand_chip *selected_chip;
500 
501 	unsigned long assigned_cs;
502 	struct list_head chips;
503 	u8 bch_metadata_size;
504 };
505 
506 struct cdns_nand_chip {
507 	struct cadence_nand_timings timings;
508 	struct nand_chip chip;
509 	u8 nsels;
510 	struct list_head node;
511 
512 	/*
513 	 * part of oob area of NAND flash memory page.
514 	 * This part is available for user to read or write.
515 	 */
516 	u32 avail_oob_size;
517 
518 	/* Sector size. There are few sectors per mtd->writesize */
519 	u32 sector_size;
520 	u32 sector_count;
521 
522 	/* Offset of BBM. */
523 	u8 bbm_offs;
524 	/* Number of bytes reserved for BBM. */
525 	u8 bbm_len;
526 	/* ECC strength index. */
527 	u8 corr_str_idx;
528 
529 	u8 cs[];
530 };
531 
532 struct ecc_info {
533 	int (*calc_ecc_bytes)(int step_size, int strength);
534 	int max_step_size;
535 };
536 
537 static inline struct
538 cdns_nand_chip *to_cdns_nand_chip(struct nand_chip *chip)
539 {
540 	return container_of(chip, struct cdns_nand_chip, chip);
541 }
542 
543 static inline struct
544 cdns_nand_ctrl *to_cdns_nand_ctrl(struct nand_controller *controller)
545 {
546 	return container_of(controller, struct cdns_nand_ctrl, controller);
547 }
548 
549 static bool
550 cadence_nand_dma_buf_ok(struct cdns_nand_ctrl *cdns_ctrl, const void *buf,
551 			u32 buf_len)
552 {
553 	u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;
554 
555 	return buf && virt_addr_valid(buf) &&
556 		likely(IS_ALIGNED((uintptr_t)buf, data_dma_width)) &&
557 		likely(IS_ALIGNED(buf_len, DMA_DATA_SIZE_ALIGN));
558 }
559 
560 static int cadence_nand_wait_for_value(struct cdns_nand_ctrl *cdns_ctrl,
561 				       u32 reg_offset, u32 timeout_us,
562 				       u32 mask, bool is_clear)
563 {
564 	u32 val;
565 	int ret;
566 
567 	ret = readl_relaxed_poll_timeout(cdns_ctrl->reg + reg_offset,
568 					 val, !(val & mask) == is_clear,
569 					 10, timeout_us);
570 
571 	if (ret < 0) {
572 		dev_err(cdns_ctrl->dev,
573 			"Timeout while waiting for reg %x with mask %x is clear %d\n",
574 			reg_offset, mask, is_clear);
575 	}
576 
577 	return ret;
578 }
579 
580 static int cadence_nand_set_ecc_enable(struct cdns_nand_ctrl *cdns_ctrl,
581 				       bool enable)
582 {
583 	u32 reg;
584 
585 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
586 					1000000,
587 					CTRL_STATUS_CTRL_BUSY, true))
588 		return -ETIMEDOUT;
589 
590 	reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
591 
592 	if (enable)
593 		reg |= ECC_CONFIG_0_ECC_EN;
594 	else
595 		reg &= ~ECC_CONFIG_0_ECC_EN;
596 
597 	writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
598 
599 	return 0;
600 }
601 
602 static void cadence_nand_set_ecc_strength(struct cdns_nand_ctrl *cdns_ctrl,
603 					  u8 corr_str_idx)
604 {
605 	u32 reg;
606 
607 	if (cdns_ctrl->curr_corr_str_idx == corr_str_idx)
608 		return;
609 
610 	reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
611 	reg &= ~ECC_CONFIG_0_CORR_STR;
612 	reg |= FIELD_PREP(ECC_CONFIG_0_CORR_STR, corr_str_idx);
613 	writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
614 
615 	cdns_ctrl->curr_corr_str_idx = corr_str_idx;
616 }
617 
618 static int cadence_nand_get_ecc_strength_idx(struct cdns_nand_ctrl *cdns_ctrl,
619 					     u8 strength)
620 {
621 	int i, corr_str_idx = -1;
622 
623 	for (i = 0; i < BCH_MAX_NUM_CORR_CAPS; i++) {
624 		if (cdns_ctrl->ecc_strengths[i] == strength) {
625 			corr_str_idx = i;
626 			break;
627 		}
628 	}
629 
630 	return corr_str_idx;
631 }
632 
633 static int cadence_nand_set_skip_marker_val(struct cdns_nand_ctrl *cdns_ctrl,
634 					    u16 marker_value)
635 {
636 	u32 reg;
637 
638 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
639 					1000000,
640 					CTRL_STATUS_CTRL_BUSY, true))
641 		return -ETIMEDOUT;
642 
643 	reg = readl_relaxed(cdns_ctrl->reg + SKIP_BYTES_CONF);
644 	reg &= ~SKIP_BYTES_MARKER_VALUE;
645 	reg |= FIELD_PREP(SKIP_BYTES_MARKER_VALUE,
646 			  marker_value);
647 
648 	writel_relaxed(reg, cdns_ctrl->reg + SKIP_BYTES_CONF);
649 
650 	return 0;
651 }
652 
653 static int cadence_nand_set_skip_bytes_conf(struct cdns_nand_ctrl *cdns_ctrl,
654 					    u8 num_of_bytes,
655 					    u32 offset_value,
656 					    int enable)
657 {
658 	u32 reg, skip_bytes_offset;
659 
660 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
661 					1000000,
662 					CTRL_STATUS_CTRL_BUSY, true))
663 		return -ETIMEDOUT;
664 
665 	if (!enable) {
666 		num_of_bytes = 0;
667 		offset_value = 0;
668 	}
669 
670 	reg = readl_relaxed(cdns_ctrl->reg + SKIP_BYTES_CONF);
671 	reg &= ~SKIP_BYTES_NUM_OF_BYTES;
672 	reg |= FIELD_PREP(SKIP_BYTES_NUM_OF_BYTES,
673 			  num_of_bytes);
674 	skip_bytes_offset = FIELD_PREP(SKIP_BYTES_OFFSET_VALUE,
675 				       offset_value);
676 
677 	writel_relaxed(reg, cdns_ctrl->reg + SKIP_BYTES_CONF);
678 	writel_relaxed(skip_bytes_offset, cdns_ctrl->reg + SKIP_BYTES_OFFSET);
679 
680 	return 0;
681 }
682 
683 /* Functions enables/disables hardware detection of erased data */
684 static void cadence_nand_set_erase_detection(struct cdns_nand_ctrl *cdns_ctrl,
685 					     bool enable,
686 					     u8 bitflips_threshold)
687 {
688 	u32 reg;
689 
690 	reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
691 
692 	if (enable)
693 		reg |= ECC_CONFIG_0_ERASE_DET_EN;
694 	else
695 		reg &= ~ECC_CONFIG_0_ERASE_DET_EN;
696 
697 	writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
698 	writel_relaxed(bitflips_threshold, cdns_ctrl->reg + ECC_CONFIG_1);
699 }
700 
701 static int cadence_nand_set_access_width16(struct cdns_nand_ctrl *cdns_ctrl,
702 					   bool bit_bus16)
703 {
704 	u32 reg;
705 
706 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
707 					1000000,
708 					CTRL_STATUS_CTRL_BUSY, true))
709 		return -ETIMEDOUT;
710 
711 	reg = readl_relaxed(cdns_ctrl->reg + COMMON_SET);
712 
713 	if (!bit_bus16)
714 		reg &= ~COMMON_SET_DEVICE_16BIT;
715 	else
716 		reg |= COMMON_SET_DEVICE_16BIT;
717 	writel_relaxed(reg, cdns_ctrl->reg + COMMON_SET);
718 
719 	return 0;
720 }
721 
722 static void
723 cadence_nand_clear_interrupt(struct cdns_nand_ctrl *cdns_ctrl,
724 			     struct cadence_nand_irq_status *irq_status)
725 {
726 	writel_relaxed(irq_status->status, cdns_ctrl->reg + INTR_STATUS);
727 	writel_relaxed(irq_status->trd_status,
728 		       cdns_ctrl->reg + TRD_COMP_INT_STATUS);
729 	writel_relaxed(irq_status->trd_error,
730 		       cdns_ctrl->reg + TRD_ERR_INT_STATUS);
731 }
732 
733 static void
734 cadence_nand_read_int_status(struct cdns_nand_ctrl *cdns_ctrl,
735 			     struct cadence_nand_irq_status *irq_status)
736 {
737 	irq_status->status = readl_relaxed(cdns_ctrl->reg + INTR_STATUS);
738 	irq_status->trd_status = readl_relaxed(cdns_ctrl->reg
739 					       + TRD_COMP_INT_STATUS);
740 	irq_status->trd_error = readl_relaxed(cdns_ctrl->reg
741 					      + TRD_ERR_INT_STATUS);
742 }
743 
744 static u32 irq_detected(struct cdns_nand_ctrl *cdns_ctrl,
745 			struct cadence_nand_irq_status *irq_status)
746 {
747 	cadence_nand_read_int_status(cdns_ctrl, irq_status);
748 
749 	return irq_status->status || irq_status->trd_status ||
750 		irq_status->trd_error;
751 }
752 
753 static void cadence_nand_reset_irq(struct cdns_nand_ctrl *cdns_ctrl)
754 {
755 	unsigned long flags;
756 
757 	spin_lock_irqsave(&cdns_ctrl->irq_lock, flags);
758 	memset(&cdns_ctrl->irq_status, 0, sizeof(cdns_ctrl->irq_status));
759 	memset(&cdns_ctrl->irq_mask, 0, sizeof(cdns_ctrl->irq_mask));
760 	spin_unlock_irqrestore(&cdns_ctrl->irq_lock, flags);
761 }
762 
763 /*
764  * This is the interrupt service routine. It handles all interrupts
765  * sent to this device.
766  */
767 static irqreturn_t cadence_nand_isr(int irq, void *dev_id)
768 {
769 	struct cdns_nand_ctrl *cdns_ctrl = dev_id;
770 	struct cadence_nand_irq_status irq_status;
771 	irqreturn_t result = IRQ_NONE;
772 
773 	spin_lock(&cdns_ctrl->irq_lock);
774 
775 	if (irq_detected(cdns_ctrl, &irq_status)) {
776 		/* Handle interrupt. */
777 		/* First acknowledge it. */
778 		cadence_nand_clear_interrupt(cdns_ctrl, &irq_status);
779 		/* Status in the device context for someone to read. */
780 		cdns_ctrl->irq_status.status |= irq_status.status;
781 		cdns_ctrl->irq_status.trd_status |= irq_status.trd_status;
782 		cdns_ctrl->irq_status.trd_error |= irq_status.trd_error;
783 		/* Notify anyone who cares that it happened. */
784 		complete(&cdns_ctrl->complete);
785 		/* Tell the OS that we've handled this. */
786 		result = IRQ_HANDLED;
787 	}
788 	spin_unlock(&cdns_ctrl->irq_lock);
789 
790 	return result;
791 }
792 
793 static void cadence_nand_set_irq_mask(struct cdns_nand_ctrl *cdns_ctrl,
794 				      struct cadence_nand_irq_status *irq_mask)
795 {
796 	writel_relaxed(INTR_ENABLE_INTR_EN | irq_mask->status,
797 		       cdns_ctrl->reg + INTR_ENABLE);
798 
799 	writel_relaxed(irq_mask->trd_error,
800 		       cdns_ctrl->reg + TRD_ERR_INT_STATUS_EN);
801 }
802 
803 static void
804 cadence_nand_wait_for_irq(struct cdns_nand_ctrl *cdns_ctrl,
805 			  struct cadence_nand_irq_status *irq_mask,
806 			  struct cadence_nand_irq_status *irq_status)
807 {
808 	unsigned long timeout = msecs_to_jiffies(10000);
809 	unsigned long time_left;
810 
811 	time_left = wait_for_completion_timeout(&cdns_ctrl->complete,
812 						timeout);
813 
814 	*irq_status = cdns_ctrl->irq_status;
815 	if (time_left == 0) {
816 		/* Timeout error. */
817 		dev_err(cdns_ctrl->dev, "timeout occurred:\n");
818 		dev_err(cdns_ctrl->dev, "\tstatus = 0x%x, mask = 0x%x\n",
819 			irq_status->status, irq_mask->status);
820 		dev_err(cdns_ctrl->dev,
821 			"\ttrd_status = 0x%x, trd_status mask = 0x%x\n",
822 			irq_status->trd_status, irq_mask->trd_status);
823 		dev_err(cdns_ctrl->dev,
824 			"\t trd_error = 0x%x, trd_error mask = 0x%x\n",
825 			irq_status->trd_error, irq_mask->trd_error);
826 	}
827 }
828 
829 /* Execute generic command on NAND controller. */
830 static int cadence_nand_generic_cmd_send(struct cdns_nand_ctrl *cdns_ctrl,
831 					 u8 chip_nr,
832 					 u64 mini_ctrl_cmd)
833 {
834 	u32 mini_ctrl_cmd_l, mini_ctrl_cmd_h, reg;
835 
836 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_CS, chip_nr);
837 	mini_ctrl_cmd_l = mini_ctrl_cmd & 0xFFFFFFFF;
838 	mini_ctrl_cmd_h = mini_ctrl_cmd >> 32;
839 
840 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
841 					1000000,
842 					CTRL_STATUS_CTRL_BUSY, true))
843 		return -ETIMEDOUT;
844 
845 	cadence_nand_reset_irq(cdns_ctrl);
846 
847 	writel_relaxed(mini_ctrl_cmd_l, cdns_ctrl->reg + CMD_REG2);
848 	writel_relaxed(mini_ctrl_cmd_h, cdns_ctrl->reg + CMD_REG3);
849 
850 	/* Select generic command. */
851 	reg = FIELD_PREP(CMD_REG0_CT, CMD_REG0_CT_GEN);
852 	/* Thread number. */
853 	reg |= FIELD_PREP(CMD_REG0_TN, 0);
854 
855 	/* Issue command. */
856 	writel_relaxed(reg, cdns_ctrl->reg + CMD_REG0);
857 
858 	return 0;
859 }
860 
861 /* Wait for data on slave DMA interface. */
862 static int cadence_nand_wait_on_sdma(struct cdns_nand_ctrl *cdns_ctrl,
863 				     u8 *out_sdma_trd,
864 				     u32 *out_sdma_size)
865 {
866 	struct cadence_nand_irq_status irq_mask, irq_status;
867 
868 	irq_mask.trd_status = 0;
869 	irq_mask.trd_error = 0;
870 	irq_mask.status = INTR_STATUS_SDMA_TRIGG
871 		| INTR_STATUS_SDMA_ERR
872 		| INTR_STATUS_UNSUPP_CMD;
873 
874 	cadence_nand_set_irq_mask(cdns_ctrl, &irq_mask);
875 	cadence_nand_wait_for_irq(cdns_ctrl, &irq_mask, &irq_status);
876 	if (irq_status.status == 0) {
877 		dev_err(cdns_ctrl->dev, "Timeout while waiting for SDMA\n");
878 		return -ETIMEDOUT;
879 	}
880 
881 	if (irq_status.status & INTR_STATUS_SDMA_TRIGG) {
882 		*out_sdma_size = readl_relaxed(cdns_ctrl->reg + SDMA_SIZE);
883 		*out_sdma_trd  = readl_relaxed(cdns_ctrl->reg + SDMA_TRD_NUM);
884 		*out_sdma_trd =
885 			FIELD_GET(SDMA_TRD_NUM_SDMA_TRD, *out_sdma_trd);
886 	} else {
887 		dev_err(cdns_ctrl->dev, "SDMA error - irq_status %x\n",
888 			irq_status.status);
889 		return -EIO;
890 	}
891 
892 	return 0;
893 }
894 
895 static void cadence_nand_get_caps(struct cdns_nand_ctrl *cdns_ctrl)
896 {
897 	u32  reg;
898 
899 	reg = readl_relaxed(cdns_ctrl->reg + CTRL_FEATURES);
900 
901 	cdns_ctrl->caps2.max_banks = 1 << FIELD_GET(CTRL_FEATURES_N_BANKS, reg);
902 
903 	if (FIELD_GET(CTRL_FEATURES_DMA_DWITH64, reg))
904 		cdns_ctrl->caps2.data_dma_width = 8;
905 	else
906 		cdns_ctrl->caps2.data_dma_width = 4;
907 
908 	if (reg & CTRL_FEATURES_CONTROL_DATA)
909 		cdns_ctrl->caps2.data_control_supp = true;
910 
911 	if (reg & (CTRL_FEATURES_NVDDR_2_3
912 		   | CTRL_FEATURES_NVDDR))
913 		cdns_ctrl->caps2.is_phy_type_dll = true;
914 }
915 
916 /* Prepare CDMA descriptor. */
917 static void
918 cadence_nand_cdma_desc_prepare(struct cdns_nand_ctrl *cdns_ctrl,
919 			       char nf_mem, u32 flash_ptr, dma_addr_t mem_ptr,
920 				   dma_addr_t ctrl_data_ptr, u16 ctype)
921 {
922 	struct cadence_nand_cdma_desc *cdma_desc = cdns_ctrl->cdma_desc;
923 
924 	memset(cdma_desc, 0, sizeof(struct cadence_nand_cdma_desc));
925 
926 	/* Set fields for one descriptor. */
927 	cdma_desc->flash_pointer = flash_ptr;
928 	if (cdns_ctrl->ctrl_rev >= 13)
929 		cdma_desc->bank = nf_mem;
930 	else
931 		cdma_desc->flash_pointer |= (nf_mem << CDMA_CFPTR_MEM_SHIFT);
932 
933 	cdma_desc->command_flags |= CDMA_CF_DMA_MASTER;
934 	cdma_desc->command_flags  |= CDMA_CF_INT;
935 
936 	cdma_desc->memory_pointer = mem_ptr;
937 	cdma_desc->status = 0;
938 	cdma_desc->sync_flag_pointer = 0;
939 	cdma_desc->sync_arguments = 0;
940 
941 	cdma_desc->command_type = ctype;
942 	cdma_desc->ctrl_data_ptr = ctrl_data_ptr;
943 }
944 
945 static u8 cadence_nand_check_desc_error(struct cdns_nand_ctrl *cdns_ctrl,
946 					u32 desc_status)
947 {
948 	if (desc_status & CDMA_CS_ERP)
949 		return STAT_ERASED;
950 
951 	if (desc_status & CDMA_CS_UNCE)
952 		return STAT_ECC_UNCORR;
953 
954 	if (desc_status & CDMA_CS_ERR) {
955 		dev_err(cdns_ctrl->dev, ":CDMA desc error flag detected.\n");
956 		return STAT_FAIL;
957 	}
958 
959 	if (FIELD_GET(CDMA_CS_MAXERR, desc_status))
960 		return STAT_ECC_CORR;
961 
962 	return STAT_FAIL;
963 }
964 
965 static int cadence_nand_cdma_finish(struct cdns_nand_ctrl *cdns_ctrl)
966 {
967 	struct cadence_nand_cdma_desc *desc_ptr = cdns_ctrl->cdma_desc;
968 	u8 status = STAT_BUSY;
969 
970 	if (desc_ptr->status & CDMA_CS_FAIL) {
971 		status = cadence_nand_check_desc_error(cdns_ctrl,
972 						       desc_ptr->status);
973 		dev_err(cdns_ctrl->dev, ":CDMA error %x\n", desc_ptr->status);
974 	} else if (desc_ptr->status & CDMA_CS_COMP) {
975 		/* Descriptor finished with no errors. */
976 		if (desc_ptr->command_flags & CDMA_CF_CONT) {
977 			dev_info(cdns_ctrl->dev, "DMA unsupported flag is set");
978 			status = STAT_UNKNOWN;
979 		} else {
980 			/* Last descriptor.  */
981 			status = STAT_OK;
982 		}
983 	}
984 
985 	return status;
986 }
987 
988 static int cadence_nand_cdma_send(struct cdns_nand_ctrl *cdns_ctrl,
989 				  u8 thread)
990 {
991 	u32 reg;
992 	int status;
993 
994 	/* Wait for thread ready. */
995 	status = cadence_nand_wait_for_value(cdns_ctrl, TRD_STATUS,
996 					     1000000,
997 					     BIT(thread), true);
998 	if (status)
999 		return status;
1000 
1001 	cadence_nand_reset_irq(cdns_ctrl);
1002 	reinit_completion(&cdns_ctrl->complete);
1003 
1004 	writel_relaxed((u32)cdns_ctrl->dma_cdma_desc,
1005 		       cdns_ctrl->reg + CMD_REG2);
1006 	writel_relaxed(0, cdns_ctrl->reg + CMD_REG3);
1007 
1008 	/* Select CDMA mode. */
1009 	reg = FIELD_PREP(CMD_REG0_CT, CMD_REG0_CT_CDMA);
1010 	/* Thread number. */
1011 	reg |= FIELD_PREP(CMD_REG0_TN, thread);
1012 	/* Issue command. */
1013 	writel_relaxed(reg, cdns_ctrl->reg + CMD_REG0);
1014 
1015 	return 0;
1016 }
1017 
1018 /* Send SDMA command and wait for finish. */
1019 static u32
1020 cadence_nand_cdma_send_and_wait(struct cdns_nand_ctrl *cdns_ctrl,
1021 				u8 thread)
1022 {
1023 	struct cadence_nand_irq_status irq_mask, irq_status = {0};
1024 	int status;
1025 
1026 	irq_mask.trd_status = BIT(thread);
1027 	irq_mask.trd_error = BIT(thread);
1028 	irq_mask.status = INTR_STATUS_CDMA_TERR;
1029 
1030 	cadence_nand_set_irq_mask(cdns_ctrl, &irq_mask);
1031 
1032 	status = cadence_nand_cdma_send(cdns_ctrl, thread);
1033 	if (status)
1034 		return status;
1035 
1036 	cadence_nand_wait_for_irq(cdns_ctrl, &irq_mask, &irq_status);
1037 
1038 	if (irq_status.status == 0 && irq_status.trd_status == 0 &&
1039 	    irq_status.trd_error == 0) {
1040 		dev_err(cdns_ctrl->dev, "CDMA command timeout\n");
1041 		return -ETIMEDOUT;
1042 	}
1043 	if (irq_status.status & irq_mask.status) {
1044 		dev_err(cdns_ctrl->dev, "CDMA command failed\n");
1045 		return -EIO;
1046 	}
1047 
1048 	return 0;
1049 }
1050 
1051 /*
1052  * ECC size depends on configured ECC strength and on maximum supported
1053  * ECC step size.
1054  */
1055 static int cadence_nand_calc_ecc_bytes(int max_step_size, int strength)
1056 {
1057 	int nbytes = DIV_ROUND_UP(fls(8 * max_step_size) * strength, 8);
1058 
1059 	return ALIGN(nbytes, 2);
1060 }
1061 
1062 #define CADENCE_NAND_CALC_ECC_BYTES(max_step_size) \
1063 	static int \
1064 	cadence_nand_calc_ecc_bytes_##max_step_size(int step_size, \
1065 						    int strength)\
1066 	{\
1067 		return cadence_nand_calc_ecc_bytes(max_step_size, strength);\
1068 	}
1069 
1070 CADENCE_NAND_CALC_ECC_BYTES(256)
1071 CADENCE_NAND_CALC_ECC_BYTES(512)
1072 CADENCE_NAND_CALC_ECC_BYTES(1024)
1073 CADENCE_NAND_CALC_ECC_BYTES(2048)
1074 CADENCE_NAND_CALC_ECC_BYTES(4096)
1075 
1076 /* Function reads BCH capabilities. */
1077 static int cadence_nand_read_bch_caps(struct cdns_nand_ctrl *cdns_ctrl)
1078 {
1079 	struct nand_ecc_caps *ecc_caps = &cdns_ctrl->ecc_caps;
1080 	int max_step_size = 0, nstrengths, i;
1081 	u32 reg;
1082 
1083 	reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_3);
1084 	cdns_ctrl->bch_metadata_size = FIELD_GET(BCH_CFG_3_METADATA_SIZE, reg);
1085 	if (cdns_ctrl->bch_metadata_size < 4) {
1086 		dev_err(cdns_ctrl->dev,
1087 			"Driver needs at least 4 bytes of BCH meta data\n");
1088 		return -EIO;
1089 	}
1090 
1091 	reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_0);
1092 	cdns_ctrl->ecc_strengths[0] = FIELD_GET(BCH_CFG_0_CORR_CAP_0, reg);
1093 	cdns_ctrl->ecc_strengths[1] = FIELD_GET(BCH_CFG_0_CORR_CAP_1, reg);
1094 	cdns_ctrl->ecc_strengths[2] = FIELD_GET(BCH_CFG_0_CORR_CAP_2, reg);
1095 	cdns_ctrl->ecc_strengths[3] = FIELD_GET(BCH_CFG_0_CORR_CAP_3, reg);
1096 
1097 	reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_1);
1098 	cdns_ctrl->ecc_strengths[4] = FIELD_GET(BCH_CFG_1_CORR_CAP_4, reg);
1099 	cdns_ctrl->ecc_strengths[5] = FIELD_GET(BCH_CFG_1_CORR_CAP_5, reg);
1100 	cdns_ctrl->ecc_strengths[6] = FIELD_GET(BCH_CFG_1_CORR_CAP_6, reg);
1101 	cdns_ctrl->ecc_strengths[7] = FIELD_GET(BCH_CFG_1_CORR_CAP_7, reg);
1102 
1103 	reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_2);
1104 	cdns_ctrl->ecc_stepinfos[0].stepsize =
1105 		FIELD_GET(BCH_CFG_2_SECT_0, reg);
1106 
1107 	cdns_ctrl->ecc_stepinfos[1].stepsize =
1108 		FIELD_GET(BCH_CFG_2_SECT_1, reg);
1109 
1110 	nstrengths = 0;
1111 	for (i = 0; i < BCH_MAX_NUM_CORR_CAPS; i++) {
1112 		if (cdns_ctrl->ecc_strengths[i] != 0)
1113 			nstrengths++;
1114 	}
1115 
1116 	ecc_caps->nstepinfos = 0;
1117 	for (i = 0; i < BCH_MAX_NUM_SECTOR_SIZES; i++) {
1118 		/* ECC strengths are common for all step infos. */
1119 		cdns_ctrl->ecc_stepinfos[i].nstrengths = nstrengths;
1120 		cdns_ctrl->ecc_stepinfos[i].strengths =
1121 			cdns_ctrl->ecc_strengths;
1122 
1123 		if (cdns_ctrl->ecc_stepinfos[i].stepsize != 0)
1124 			ecc_caps->nstepinfos++;
1125 
1126 		if (cdns_ctrl->ecc_stepinfos[i].stepsize > max_step_size)
1127 			max_step_size = cdns_ctrl->ecc_stepinfos[i].stepsize;
1128 	}
1129 	ecc_caps->stepinfos = &cdns_ctrl->ecc_stepinfos[0];
1130 
1131 	switch (max_step_size) {
1132 	case 256:
1133 		ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_256;
1134 		break;
1135 	case 512:
1136 		ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_512;
1137 		break;
1138 	case 1024:
1139 		ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_1024;
1140 		break;
1141 	case 2048:
1142 		ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_2048;
1143 		break;
1144 	case 4096:
1145 		ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_4096;
1146 		break;
1147 	default:
1148 		dev_err(cdns_ctrl->dev,
1149 			"Unsupported sector size(ecc step size) %d\n",
1150 			max_step_size);
1151 		return -EIO;
1152 	}
1153 
1154 	return 0;
1155 }
1156 
1157 /* Hardware initialization. */
1158 static int cadence_nand_hw_init(struct cdns_nand_ctrl *cdns_ctrl)
1159 {
1160 	int status;
1161 	u32 reg;
1162 
1163 	status = cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
1164 					     1000000,
1165 					     CTRL_STATUS_INIT_COMP, false);
1166 	if (status)
1167 		return status;
1168 
1169 	reg = readl_relaxed(cdns_ctrl->reg + CTRL_VERSION);
1170 	cdns_ctrl->ctrl_rev = FIELD_GET(CTRL_VERSION_REV, reg);
1171 
1172 	dev_info(cdns_ctrl->dev,
1173 		 "%s: cadence nand controller version reg %x\n",
1174 		 __func__, reg);
1175 
1176 	/* Disable cache and multiplane. */
1177 	writel_relaxed(0, cdns_ctrl->reg + MULTIPLANE_CFG);
1178 	writel_relaxed(0, cdns_ctrl->reg + CACHE_CFG);
1179 
1180 	/* Clear all interrupts. */
1181 	writel_relaxed(0xFFFFFFFF, cdns_ctrl->reg + INTR_STATUS);
1182 
1183 	cadence_nand_get_caps(cdns_ctrl);
1184 	if (cadence_nand_read_bch_caps(cdns_ctrl))
1185 		return -EIO;
1186 
1187 #ifndef CONFIG_64BIT
1188 	if (cdns_ctrl->caps2.data_dma_width == 8) {
1189 		dev_err(cdns_ctrl->dev,
1190 			"cannot access 64-bit dma on !64-bit architectures");
1191 		return -EIO;
1192 	}
1193 #endif
1194 
1195 	/*
1196 	 * Set IO width access to 8.
1197 	 * It is because during SW device discovering width access
1198 	 * is expected to be 8.
1199 	 */
1200 	status = cadence_nand_set_access_width16(cdns_ctrl, false);
1201 
1202 	return status;
1203 }
1204 
1205 #define TT_MAIN_OOB_AREAS	2
1206 #define TT_RAW_PAGE		3
1207 #define TT_BBM			4
1208 #define TT_MAIN_OOB_AREA_EXT	5
1209 
1210 /* Prepare size of data to transfer. */
1211 static void
1212 cadence_nand_prepare_data_size(struct nand_chip *chip,
1213 			       int transfer_type)
1214 {
1215 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1216 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1217 	struct mtd_info *mtd = nand_to_mtd(chip);
1218 	u32 sec_size = 0, offset = 0, sec_cnt = 1;
1219 	u32 last_sec_size = cdns_chip->sector_size;
1220 	u32 data_ctrl_size = 0;
1221 	u32 reg = 0;
1222 
1223 	if (cdns_ctrl->curr_trans_type == transfer_type)
1224 		return;
1225 
1226 	switch (transfer_type) {
1227 	case TT_MAIN_OOB_AREA_EXT:
1228 		sec_cnt = cdns_chip->sector_count;
1229 		sec_size = cdns_chip->sector_size;
1230 		data_ctrl_size = cdns_chip->avail_oob_size;
1231 		break;
1232 	case TT_MAIN_OOB_AREAS:
1233 		sec_cnt = cdns_chip->sector_count;
1234 		last_sec_size = cdns_chip->sector_size
1235 			+ cdns_chip->avail_oob_size;
1236 		sec_size = cdns_chip->sector_size;
1237 		break;
1238 	case TT_RAW_PAGE:
1239 		last_sec_size = mtd->writesize + mtd->oobsize;
1240 		break;
1241 	case TT_BBM:
1242 		offset = mtd->writesize + cdns_chip->bbm_offs;
1243 		last_sec_size = 8;
1244 		break;
1245 	}
1246 
1247 	reg = 0;
1248 	reg |= FIELD_PREP(TRAN_CFG_0_OFFSET, offset);
1249 	reg |= FIELD_PREP(TRAN_CFG_0_SEC_CNT, sec_cnt);
1250 	writel_relaxed(reg, cdns_ctrl->reg + TRAN_CFG_0);
1251 
1252 	reg = 0;
1253 	reg |= FIELD_PREP(TRAN_CFG_1_LAST_SEC_SIZE, last_sec_size);
1254 	reg |= FIELD_PREP(TRAN_CFG_1_SECTOR_SIZE, sec_size);
1255 	writel_relaxed(reg, cdns_ctrl->reg + TRAN_CFG_1);
1256 
1257 	if (cdns_ctrl->caps2.data_control_supp) {
1258 		reg = readl_relaxed(cdns_ctrl->reg + CONTROL_DATA_CTRL);
1259 		reg &= ~CONTROL_DATA_CTRL_SIZE;
1260 		reg |= FIELD_PREP(CONTROL_DATA_CTRL_SIZE, data_ctrl_size);
1261 		writel_relaxed(reg, cdns_ctrl->reg + CONTROL_DATA_CTRL);
1262 	}
1263 
1264 	cdns_ctrl->curr_trans_type = transfer_type;
1265 }
1266 
1267 static int
1268 cadence_nand_cdma_transfer(struct cdns_nand_ctrl *cdns_ctrl, u8 chip_nr,
1269 			   int page, void *buf, void *ctrl_dat, u32 buf_size,
1270 			   u32 ctrl_dat_size, enum dma_data_direction dir,
1271 			   bool with_ecc)
1272 {
1273 	dma_addr_t dma_buf, dma_ctrl_dat = 0;
1274 	u8 thread_nr = chip_nr;
1275 	int status;
1276 	u16 ctype;
1277 
1278 	if (dir == DMA_FROM_DEVICE)
1279 		ctype = CDMA_CT_RD;
1280 	else
1281 		ctype = CDMA_CT_WR;
1282 
1283 	cadence_nand_set_ecc_enable(cdns_ctrl, with_ecc);
1284 
1285 	dma_buf = dma_map_single(cdns_ctrl->dev, buf, buf_size, dir);
1286 	if (dma_mapping_error(cdns_ctrl->dev, dma_buf)) {
1287 		dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1288 		return -EIO;
1289 	}
1290 
1291 	if (ctrl_dat && ctrl_dat_size) {
1292 		dma_ctrl_dat = dma_map_single(cdns_ctrl->dev, ctrl_dat,
1293 					      ctrl_dat_size, dir);
1294 		if (dma_mapping_error(cdns_ctrl->dev, dma_ctrl_dat)) {
1295 			dma_unmap_single(cdns_ctrl->dev, dma_buf,
1296 					 buf_size, dir);
1297 			dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1298 			return -EIO;
1299 		}
1300 	}
1301 
1302 	cadence_nand_cdma_desc_prepare(cdns_ctrl, chip_nr, page,
1303 				       dma_buf, dma_ctrl_dat, ctype);
1304 
1305 	status = cadence_nand_cdma_send_and_wait(cdns_ctrl, thread_nr);
1306 
1307 	dma_unmap_single(cdns_ctrl->dev, dma_buf,
1308 			 buf_size, dir);
1309 
1310 	if (ctrl_dat && ctrl_dat_size)
1311 		dma_unmap_single(cdns_ctrl->dev, dma_ctrl_dat,
1312 				 ctrl_dat_size, dir);
1313 	if (status)
1314 		return status;
1315 
1316 	return cadence_nand_cdma_finish(cdns_ctrl);
1317 }
1318 
1319 static void cadence_nand_set_timings(struct cdns_nand_ctrl *cdns_ctrl,
1320 				     struct cadence_nand_timings *t)
1321 {
1322 	writel_relaxed(t->async_toggle_timings,
1323 		       cdns_ctrl->reg + ASYNC_TOGGLE_TIMINGS);
1324 	writel_relaxed(t->timings0, cdns_ctrl->reg + TIMINGS0);
1325 	writel_relaxed(t->timings1, cdns_ctrl->reg + TIMINGS1);
1326 	writel_relaxed(t->timings2, cdns_ctrl->reg + TIMINGS2);
1327 
1328 	if (cdns_ctrl->caps2.is_phy_type_dll)
1329 		writel_relaxed(t->dll_phy_ctrl, cdns_ctrl->reg + DLL_PHY_CTRL);
1330 
1331 	writel_relaxed(t->phy_ctrl, cdns_ctrl->reg + PHY_CTRL);
1332 
1333 	if (cdns_ctrl->caps2.is_phy_type_dll) {
1334 		writel_relaxed(0, cdns_ctrl->reg + PHY_TSEL);
1335 		writel_relaxed(2, cdns_ctrl->reg + PHY_DQ_TIMING);
1336 		writel_relaxed(t->phy_dqs_timing,
1337 			       cdns_ctrl->reg + PHY_DQS_TIMING);
1338 		writel_relaxed(t->phy_gate_lpbk_ctrl,
1339 			       cdns_ctrl->reg + PHY_GATE_LPBK_CTRL);
1340 		writel_relaxed(PHY_DLL_MASTER_CTRL_BYPASS_MODE,
1341 			       cdns_ctrl->reg + PHY_DLL_MASTER_CTRL);
1342 		writel_relaxed(0, cdns_ctrl->reg + PHY_DLL_SLAVE_CTRL);
1343 	}
1344 }
1345 
1346 static int cadence_nand_select_target(struct nand_chip *chip)
1347 {
1348 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1349 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1350 
1351 	if (chip == cdns_ctrl->selected_chip)
1352 		return 0;
1353 
1354 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
1355 					1000000,
1356 					CTRL_STATUS_CTRL_BUSY, true))
1357 		return -ETIMEDOUT;
1358 
1359 	cadence_nand_set_timings(cdns_ctrl, &cdns_chip->timings);
1360 
1361 	cadence_nand_set_ecc_strength(cdns_ctrl,
1362 				      cdns_chip->corr_str_idx);
1363 
1364 	cadence_nand_set_erase_detection(cdns_ctrl, true,
1365 					 chip->ecc.strength);
1366 
1367 	cdns_ctrl->curr_trans_type = -1;
1368 	cdns_ctrl->selected_chip = chip;
1369 
1370 	return 0;
1371 }
1372 
1373 static int cadence_nand_erase(struct nand_chip *chip, u32 page)
1374 {
1375 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1376 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1377 	int status;
1378 	u8 thread_nr = cdns_chip->cs[chip->cur_cs];
1379 
1380 	cadence_nand_cdma_desc_prepare(cdns_ctrl,
1381 				       cdns_chip->cs[chip->cur_cs],
1382 				       page, 0, 0,
1383 				       CDMA_CT_ERASE);
1384 	status = cadence_nand_cdma_send_and_wait(cdns_ctrl, thread_nr);
1385 	if (status) {
1386 		dev_err(cdns_ctrl->dev, "erase operation failed\n");
1387 		return -EIO;
1388 	}
1389 
1390 	status = cadence_nand_cdma_finish(cdns_ctrl);
1391 	if (status)
1392 		return status;
1393 
1394 	return 0;
1395 }
1396 
1397 static int cadence_nand_read_bbm(struct nand_chip *chip, int page, u8 *buf)
1398 {
1399 	int status;
1400 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1401 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1402 	struct mtd_info *mtd = nand_to_mtd(chip);
1403 
1404 	cadence_nand_prepare_data_size(chip, TT_BBM);
1405 
1406 	cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1407 
1408 	/*
1409 	 * Read only bad block marker from offset
1410 	 * defined by a memory manufacturer.
1411 	 */
1412 	status = cadence_nand_cdma_transfer(cdns_ctrl,
1413 					    cdns_chip->cs[chip->cur_cs],
1414 					    page, cdns_ctrl->buf, NULL,
1415 					    mtd->oobsize,
1416 					    0, DMA_FROM_DEVICE, false);
1417 	if (status) {
1418 		dev_err(cdns_ctrl->dev, "read BBM failed\n");
1419 		return -EIO;
1420 	}
1421 
1422 	memcpy(buf + cdns_chip->bbm_offs, cdns_ctrl->buf, cdns_chip->bbm_len);
1423 
1424 	return 0;
1425 }
1426 
1427 static int cadence_nand_write_page(struct nand_chip *chip,
1428 				   const u8 *buf, int oob_required,
1429 				   int page)
1430 {
1431 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1432 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1433 	struct mtd_info *mtd = nand_to_mtd(chip);
1434 	int status;
1435 	u16 marker_val = 0xFFFF;
1436 
1437 	status = cadence_nand_select_target(chip);
1438 	if (status)
1439 		return status;
1440 
1441 	cadence_nand_set_skip_bytes_conf(cdns_ctrl, cdns_chip->bbm_len,
1442 					 mtd->writesize
1443 					 + cdns_chip->bbm_offs,
1444 					 1);
1445 
1446 	if (oob_required) {
1447 		marker_val = *(u16 *)(chip->oob_poi
1448 				      + cdns_chip->bbm_offs);
1449 	} else {
1450 		/* Set oob data to 0xFF. */
1451 		memset(cdns_ctrl->buf + mtd->writesize, 0xFF,
1452 		       cdns_chip->avail_oob_size);
1453 	}
1454 
1455 	cadence_nand_set_skip_marker_val(cdns_ctrl, marker_val);
1456 
1457 	cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREA_EXT);
1458 
1459 	if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, mtd->writesize) &&
1460 	    cdns_ctrl->caps2.data_control_supp) {
1461 		u8 *oob;
1462 
1463 		if (oob_required)
1464 			oob = chip->oob_poi;
1465 		else
1466 			oob = cdns_ctrl->buf + mtd->writesize;
1467 
1468 		status = cadence_nand_cdma_transfer(cdns_ctrl,
1469 						    cdns_chip->cs[chip->cur_cs],
1470 						    page, (void *)buf, oob,
1471 						    mtd->writesize,
1472 						    cdns_chip->avail_oob_size,
1473 						    DMA_TO_DEVICE, true);
1474 		if (status) {
1475 			dev_err(cdns_ctrl->dev, "write page failed\n");
1476 			return -EIO;
1477 		}
1478 
1479 		return 0;
1480 	}
1481 
1482 	if (oob_required) {
1483 		/* Transfer the data to the oob area. */
1484 		memcpy(cdns_ctrl->buf + mtd->writesize, chip->oob_poi,
1485 		       cdns_chip->avail_oob_size);
1486 	}
1487 
1488 	memcpy(cdns_ctrl->buf, buf, mtd->writesize);
1489 
1490 	cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREAS);
1491 
1492 	return cadence_nand_cdma_transfer(cdns_ctrl,
1493 					  cdns_chip->cs[chip->cur_cs],
1494 					  page, cdns_ctrl->buf, NULL,
1495 					  mtd->writesize
1496 					  + cdns_chip->avail_oob_size,
1497 					  0, DMA_TO_DEVICE, true);
1498 }
1499 
1500 static int cadence_nand_write_oob(struct nand_chip *chip, int page)
1501 {
1502 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1503 	struct mtd_info *mtd = nand_to_mtd(chip);
1504 
1505 	memset(cdns_ctrl->buf, 0xFF, mtd->writesize);
1506 
1507 	return cadence_nand_write_page(chip, cdns_ctrl->buf, 1, page);
1508 }
1509 
1510 static int cadence_nand_write_page_raw(struct nand_chip *chip,
1511 				       const u8 *buf, int oob_required,
1512 				       int page)
1513 {
1514 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1515 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1516 	struct mtd_info *mtd = nand_to_mtd(chip);
1517 	int writesize = mtd->writesize;
1518 	int oobsize = mtd->oobsize;
1519 	int ecc_steps = chip->ecc.steps;
1520 	int ecc_size = chip->ecc.size;
1521 	int ecc_bytes = chip->ecc.bytes;
1522 	void *tmp_buf = cdns_ctrl->buf;
1523 	int oob_skip = cdns_chip->bbm_len;
1524 	size_t size = writesize + oobsize;
1525 	int i, pos, len;
1526 	int status = 0;
1527 
1528 	status = cadence_nand_select_target(chip);
1529 	if (status)
1530 		return status;
1531 
1532 	/*
1533 	 * Fill the buffer with 0xff first except the full page transfer.
1534 	 * This simplifies the logic.
1535 	 */
1536 	if (!buf || !oob_required)
1537 		memset(tmp_buf, 0xff, size);
1538 
1539 	cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1540 
1541 	/* Arrange the buffer for syndrome payload/ecc layout. */
1542 	if (buf) {
1543 		for (i = 0; i < ecc_steps; i++) {
1544 			pos = i * (ecc_size + ecc_bytes);
1545 			len = ecc_size;
1546 
1547 			if (pos >= writesize)
1548 				pos += oob_skip;
1549 			else if (pos + len > writesize)
1550 				len = writesize - pos;
1551 
1552 			memcpy(tmp_buf + pos, buf, len);
1553 			buf += len;
1554 			if (len < ecc_size) {
1555 				len = ecc_size - len;
1556 				memcpy(tmp_buf + writesize + oob_skip, buf,
1557 				       len);
1558 				buf += len;
1559 			}
1560 		}
1561 	}
1562 
1563 	if (oob_required) {
1564 		const u8 *oob = chip->oob_poi;
1565 		u32 oob_data_offset = (cdns_chip->sector_count - 1) *
1566 			(cdns_chip->sector_size + chip->ecc.bytes)
1567 			+ cdns_chip->sector_size + oob_skip;
1568 
1569 		/* BBM at the beginning of the OOB area. */
1570 		memcpy(tmp_buf + writesize, oob, oob_skip);
1571 
1572 		/* OOB free. */
1573 		memcpy(tmp_buf + oob_data_offset, oob,
1574 		       cdns_chip->avail_oob_size);
1575 		oob += cdns_chip->avail_oob_size;
1576 
1577 		/* OOB ECC. */
1578 		for (i = 0; i < ecc_steps; i++) {
1579 			pos = ecc_size + i * (ecc_size + ecc_bytes);
1580 			if (i == (ecc_steps - 1))
1581 				pos += cdns_chip->avail_oob_size;
1582 
1583 			len = ecc_bytes;
1584 
1585 			if (pos >= writesize)
1586 				pos += oob_skip;
1587 			else if (pos + len > writesize)
1588 				len = writesize - pos;
1589 
1590 			memcpy(tmp_buf + pos, oob, len);
1591 			oob += len;
1592 			if (len < ecc_bytes) {
1593 				len = ecc_bytes - len;
1594 				memcpy(tmp_buf + writesize + oob_skip, oob,
1595 				       len);
1596 				oob += len;
1597 			}
1598 		}
1599 	}
1600 
1601 	cadence_nand_prepare_data_size(chip, TT_RAW_PAGE);
1602 
1603 	return cadence_nand_cdma_transfer(cdns_ctrl,
1604 					  cdns_chip->cs[chip->cur_cs],
1605 					  page, cdns_ctrl->buf, NULL,
1606 					  mtd->writesize +
1607 					  mtd->oobsize,
1608 					  0, DMA_TO_DEVICE, false);
1609 }
1610 
1611 static int cadence_nand_write_oob_raw(struct nand_chip *chip,
1612 				      int page)
1613 {
1614 	return cadence_nand_write_page_raw(chip, NULL, true, page);
1615 }
1616 
1617 static int cadence_nand_read_page(struct nand_chip *chip,
1618 				  u8 *buf, int oob_required, int page)
1619 {
1620 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1621 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1622 	struct mtd_info *mtd = nand_to_mtd(chip);
1623 	int status = 0;
1624 	int ecc_err_count = 0;
1625 
1626 	status = cadence_nand_select_target(chip);
1627 	if (status)
1628 		return status;
1629 
1630 	cadence_nand_set_skip_bytes_conf(cdns_ctrl, cdns_chip->bbm_len,
1631 					 mtd->writesize
1632 					 + cdns_chip->bbm_offs, 1);
1633 
1634 	/*
1635 	 * If data buffer can be accessed by DMA and data_control feature
1636 	 * is supported then transfer data and oob directly.
1637 	 */
1638 	if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, mtd->writesize) &&
1639 	    cdns_ctrl->caps2.data_control_supp) {
1640 		u8 *oob;
1641 
1642 		if (oob_required)
1643 			oob = chip->oob_poi;
1644 		else
1645 			oob = cdns_ctrl->buf + mtd->writesize;
1646 
1647 		cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREA_EXT);
1648 		status = cadence_nand_cdma_transfer(cdns_ctrl,
1649 						    cdns_chip->cs[chip->cur_cs],
1650 						    page, buf, oob,
1651 						    mtd->writesize,
1652 						    cdns_chip->avail_oob_size,
1653 						    DMA_FROM_DEVICE, true);
1654 	/* Otherwise use bounce buffer. */
1655 	} else {
1656 		cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREAS);
1657 		status = cadence_nand_cdma_transfer(cdns_ctrl,
1658 						    cdns_chip->cs[chip->cur_cs],
1659 						    page, cdns_ctrl->buf,
1660 						    NULL, mtd->writesize
1661 						    + cdns_chip->avail_oob_size,
1662 						    0, DMA_FROM_DEVICE, true);
1663 
1664 		memcpy(buf, cdns_ctrl->buf, mtd->writesize);
1665 		if (oob_required)
1666 			memcpy(chip->oob_poi,
1667 			       cdns_ctrl->buf + mtd->writesize,
1668 			       mtd->oobsize);
1669 	}
1670 
1671 	switch (status) {
1672 	case STAT_ECC_UNCORR:
1673 		mtd->ecc_stats.failed++;
1674 		ecc_err_count++;
1675 		break;
1676 	case STAT_ECC_CORR:
1677 		ecc_err_count = FIELD_GET(CDMA_CS_MAXERR,
1678 					  cdns_ctrl->cdma_desc->status);
1679 		mtd->ecc_stats.corrected += ecc_err_count;
1680 		break;
1681 	case STAT_ERASED:
1682 	case STAT_OK:
1683 		break;
1684 	default:
1685 		dev_err(cdns_ctrl->dev, "read page failed\n");
1686 		return -EIO;
1687 	}
1688 
1689 	if (oob_required)
1690 		if (cadence_nand_read_bbm(chip, page, chip->oob_poi))
1691 			return -EIO;
1692 
1693 	return ecc_err_count;
1694 }
1695 
1696 /* Reads OOB data from the device. */
1697 static int cadence_nand_read_oob(struct nand_chip *chip, int page)
1698 {
1699 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1700 
1701 	return cadence_nand_read_page(chip, cdns_ctrl->buf, 1, page);
1702 }
1703 
1704 static int cadence_nand_read_page_raw(struct nand_chip *chip,
1705 				      u8 *buf, int oob_required, int page)
1706 {
1707 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1708 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1709 	struct mtd_info *mtd = nand_to_mtd(chip);
1710 	int oob_skip = cdns_chip->bbm_len;
1711 	int writesize = mtd->writesize;
1712 	int ecc_steps = chip->ecc.steps;
1713 	int ecc_size = chip->ecc.size;
1714 	int ecc_bytes = chip->ecc.bytes;
1715 	void *tmp_buf = cdns_ctrl->buf;
1716 	int i, pos, len;
1717 	int status = 0;
1718 
1719 	status = cadence_nand_select_target(chip);
1720 	if (status)
1721 		return status;
1722 
1723 	cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1724 
1725 	cadence_nand_prepare_data_size(chip, TT_RAW_PAGE);
1726 	status = cadence_nand_cdma_transfer(cdns_ctrl,
1727 					    cdns_chip->cs[chip->cur_cs],
1728 					    page, cdns_ctrl->buf, NULL,
1729 					    mtd->writesize
1730 					    + mtd->oobsize,
1731 					    0, DMA_FROM_DEVICE, false);
1732 
1733 	switch (status) {
1734 	case STAT_ERASED:
1735 	case STAT_OK:
1736 		break;
1737 	default:
1738 		dev_err(cdns_ctrl->dev, "read raw page failed\n");
1739 		return -EIO;
1740 	}
1741 
1742 	/* Arrange the buffer for syndrome payload/ecc layout. */
1743 	if (buf) {
1744 		for (i = 0; i < ecc_steps; i++) {
1745 			pos = i * (ecc_size + ecc_bytes);
1746 			len = ecc_size;
1747 
1748 			if (pos >= writesize)
1749 				pos += oob_skip;
1750 			else if (pos + len > writesize)
1751 				len = writesize - pos;
1752 
1753 			memcpy(buf, tmp_buf + pos, len);
1754 			buf += len;
1755 			if (len < ecc_size) {
1756 				len = ecc_size - len;
1757 				memcpy(buf, tmp_buf + writesize + oob_skip,
1758 				       len);
1759 				buf += len;
1760 			}
1761 		}
1762 	}
1763 
1764 	if (oob_required) {
1765 		u8 *oob = chip->oob_poi;
1766 		u32 oob_data_offset = (cdns_chip->sector_count - 1) *
1767 			(cdns_chip->sector_size + chip->ecc.bytes)
1768 			+ cdns_chip->sector_size + oob_skip;
1769 
1770 		/* OOB free. */
1771 		memcpy(oob, tmp_buf + oob_data_offset,
1772 		       cdns_chip->avail_oob_size);
1773 
1774 		/* BBM at the beginning of the OOB area. */
1775 		memcpy(oob, tmp_buf + writesize, oob_skip);
1776 
1777 		oob += cdns_chip->avail_oob_size;
1778 
1779 		/* OOB ECC */
1780 		for (i = 0; i < ecc_steps; i++) {
1781 			pos = ecc_size + i * (ecc_size + ecc_bytes);
1782 			len = ecc_bytes;
1783 
1784 			if (i == (ecc_steps - 1))
1785 				pos += cdns_chip->avail_oob_size;
1786 
1787 			if (pos >= writesize)
1788 				pos += oob_skip;
1789 			else if (pos + len > writesize)
1790 				len = writesize - pos;
1791 
1792 			memcpy(oob, tmp_buf + pos, len);
1793 			oob += len;
1794 			if (len < ecc_bytes) {
1795 				len = ecc_bytes - len;
1796 				memcpy(oob, tmp_buf + writesize + oob_skip,
1797 				       len);
1798 				oob += len;
1799 			}
1800 		}
1801 	}
1802 
1803 	return 0;
1804 }
1805 
1806 static int cadence_nand_read_oob_raw(struct nand_chip *chip,
1807 				     int page)
1808 {
1809 	return cadence_nand_read_page_raw(chip, NULL, true, page);
1810 }
1811 
1812 static void cadence_nand_slave_dma_transfer_finished(void *data)
1813 {
1814 	struct completion *finished = data;
1815 
1816 	complete(finished);
1817 }
1818 
1819 static int cadence_nand_slave_dma_transfer(struct cdns_nand_ctrl *cdns_ctrl,
1820 					   void *buf,
1821 					   dma_addr_t dev_dma, size_t len,
1822 					   enum dma_data_direction dir)
1823 {
1824 	DECLARE_COMPLETION_ONSTACK(finished);
1825 	struct dma_chan *chan;
1826 	struct dma_device *dma_dev;
1827 	dma_addr_t src_dma, dst_dma, buf_dma;
1828 	struct dma_async_tx_descriptor *tx;
1829 	dma_cookie_t cookie;
1830 
1831 	chan = cdns_ctrl->dmac;
1832 	dma_dev = chan->device;
1833 
1834 	buf_dma = dma_map_single(dma_dev->dev, buf, len, dir);
1835 	if (dma_mapping_error(dma_dev->dev, buf_dma)) {
1836 		dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1837 		goto err;
1838 	}
1839 
1840 	if (dir == DMA_FROM_DEVICE) {
1841 		src_dma = cdns_ctrl->io.dma;
1842 		dst_dma = buf_dma;
1843 	} else {
1844 		src_dma = buf_dma;
1845 		dst_dma = cdns_ctrl->io.dma;
1846 	}
1847 
1848 	tx = dmaengine_prep_dma_memcpy(cdns_ctrl->dmac, dst_dma, src_dma, len,
1849 				       DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
1850 	if (!tx) {
1851 		dev_err(cdns_ctrl->dev, "Failed to prepare DMA memcpy\n");
1852 		goto err_unmap;
1853 	}
1854 
1855 	tx->callback = cadence_nand_slave_dma_transfer_finished;
1856 	tx->callback_param = &finished;
1857 
1858 	cookie = dmaengine_submit(tx);
1859 	if (dma_submit_error(cookie)) {
1860 		dev_err(cdns_ctrl->dev, "Failed to do DMA tx_submit\n");
1861 		goto err_unmap;
1862 	}
1863 
1864 	dma_async_issue_pending(cdns_ctrl->dmac);
1865 	wait_for_completion(&finished);
1866 
1867 	dma_unmap_single(cdns_ctrl->dev, buf_dma, len, dir);
1868 
1869 	return 0;
1870 
1871 err_unmap:
1872 	dma_unmap_single(cdns_ctrl->dev, buf_dma, len, dir);
1873 
1874 err:
1875 	dev_dbg(cdns_ctrl->dev, "Fall back to CPU I/O\n");
1876 
1877 	return -EIO;
1878 }
1879 
1880 static int cadence_nand_read_buf(struct cdns_nand_ctrl *cdns_ctrl,
1881 				 u8 *buf, int len)
1882 {
1883 	u8 thread_nr = 0;
1884 	u32 sdma_size;
1885 	int status;
1886 
1887 	/* Wait until slave DMA interface is ready to data transfer. */
1888 	status = cadence_nand_wait_on_sdma(cdns_ctrl, &thread_nr, &sdma_size);
1889 	if (status)
1890 		return status;
1891 
1892 	if (!cdns_ctrl->caps1->has_dma) {
1893 		u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;
1894 
1895 		int len_in_words = (data_dma_width == 4) ? len >> 2 : len >> 3;
1896 
1897 		/* read alingment data */
1898 		if (data_dma_width == 4)
1899 			ioread32_rep(cdns_ctrl->io.virt, buf, len_in_words);
1900 #ifdef CONFIG_64BIT
1901 		else
1902 			readsq(cdns_ctrl->io.virt, buf, len_in_words);
1903 #endif
1904 
1905 		if (sdma_size > len) {
1906 			int read_bytes = (data_dma_width == 4) ?
1907 				len_in_words << 2 : len_in_words << 3;
1908 
1909 			/* read rest data from slave DMA interface if any */
1910 			if (data_dma_width == 4)
1911 				ioread32_rep(cdns_ctrl->io.virt,
1912 					     cdns_ctrl->buf,
1913 					     sdma_size / 4 - len_in_words);
1914 #ifdef CONFIG_64BIT
1915 			else
1916 				readsq(cdns_ctrl->io.virt, cdns_ctrl->buf,
1917 				       sdma_size / 8 - len_in_words);
1918 #endif
1919 
1920 			/* copy rest of data */
1921 			memcpy(buf + read_bytes, cdns_ctrl->buf,
1922 			       len - read_bytes);
1923 		}
1924 		return 0;
1925 	}
1926 
1927 	if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, len)) {
1928 		status = cadence_nand_slave_dma_transfer(cdns_ctrl, buf,
1929 							 cdns_ctrl->io.dma,
1930 							 len, DMA_FROM_DEVICE);
1931 		if (status == 0)
1932 			return 0;
1933 
1934 		dev_warn(cdns_ctrl->dev,
1935 			 "Slave DMA transfer failed. Try again using bounce buffer.");
1936 	}
1937 
1938 	/* If DMA transfer is not possible or failed then use bounce buffer. */
1939 	status = cadence_nand_slave_dma_transfer(cdns_ctrl, cdns_ctrl->buf,
1940 						 cdns_ctrl->io.dma,
1941 						 sdma_size, DMA_FROM_DEVICE);
1942 
1943 	if (status) {
1944 		dev_err(cdns_ctrl->dev, "Slave DMA transfer failed");
1945 		return status;
1946 	}
1947 
1948 	memcpy(buf, cdns_ctrl->buf, len);
1949 
1950 	return 0;
1951 }
1952 
1953 static int cadence_nand_write_buf(struct cdns_nand_ctrl *cdns_ctrl,
1954 				  const u8 *buf, int len)
1955 {
1956 	u8 thread_nr = 0;
1957 	u32 sdma_size;
1958 	int status;
1959 
1960 	/* Wait until slave DMA interface is ready to data transfer. */
1961 	status = cadence_nand_wait_on_sdma(cdns_ctrl, &thread_nr, &sdma_size);
1962 	if (status)
1963 		return status;
1964 
1965 	if (!cdns_ctrl->caps1->has_dma) {
1966 		u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;
1967 
1968 		int len_in_words = (data_dma_width == 4) ? len >> 2 : len >> 3;
1969 
1970 		if (data_dma_width == 4)
1971 			iowrite32_rep(cdns_ctrl->io.virt, buf, len_in_words);
1972 #ifdef CONFIG_64BIT
1973 		else
1974 			writesq(cdns_ctrl->io.virt, buf, len_in_words);
1975 #endif
1976 
1977 		if (sdma_size > len) {
1978 			int written_bytes = (data_dma_width == 4) ?
1979 				len_in_words << 2 : len_in_words << 3;
1980 
1981 			/* copy rest of data */
1982 			memcpy(cdns_ctrl->buf, buf + written_bytes,
1983 			       len - written_bytes);
1984 
1985 			/* write all expected by nand controller data */
1986 			if (data_dma_width == 4)
1987 				iowrite32_rep(cdns_ctrl->io.virt,
1988 					      cdns_ctrl->buf,
1989 					      sdma_size / 4 - len_in_words);
1990 #ifdef CONFIG_64BIT
1991 			else
1992 				writesq(cdns_ctrl->io.virt, cdns_ctrl->buf,
1993 					sdma_size / 8 - len_in_words);
1994 #endif
1995 		}
1996 
1997 		return 0;
1998 	}
1999 
2000 	if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, len)) {
2001 		status = cadence_nand_slave_dma_transfer(cdns_ctrl, (void *)buf,
2002 							 cdns_ctrl->io.dma,
2003 							 len, DMA_TO_DEVICE);
2004 		if (status == 0)
2005 			return 0;
2006 
2007 		dev_warn(cdns_ctrl->dev,
2008 			 "Slave DMA transfer failed. Try again using bounce buffer.");
2009 	}
2010 
2011 	/* If DMA transfer is not possible or failed then use bounce buffer. */
2012 	memcpy(cdns_ctrl->buf, buf, len);
2013 
2014 	status = cadence_nand_slave_dma_transfer(cdns_ctrl, cdns_ctrl->buf,
2015 						 cdns_ctrl->io.dma,
2016 						 sdma_size, DMA_TO_DEVICE);
2017 
2018 	if (status)
2019 		dev_err(cdns_ctrl->dev, "Slave DMA transfer failed");
2020 
2021 	return status;
2022 }
2023 
2024 static int cadence_nand_force_byte_access(struct nand_chip *chip,
2025 					  bool force_8bit)
2026 {
2027 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2028 
2029 	/*
2030 	 * Callers of this function do not verify if the NAND is using a 16-bit
2031 	 * an 8-bit bus for normal operations, so we need to take care of that
2032 	 * here by leaving the configuration unchanged if the NAND does not have
2033 	 * the NAND_BUSWIDTH_16 flag set.
2034 	 */
2035 	if (!(chip->options & NAND_BUSWIDTH_16))
2036 		return 0;
2037 
2038 	return cadence_nand_set_access_width16(cdns_ctrl, !force_8bit);
2039 }
2040 
2041 static int cadence_nand_cmd_opcode(struct nand_chip *chip,
2042 				   const struct nand_subop *subop)
2043 {
2044 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2045 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2046 	const struct nand_op_instr *instr;
2047 	unsigned int op_id = 0;
2048 	u64 mini_ctrl_cmd = 0;
2049 	int ret;
2050 
2051 	instr = &subop->instrs[op_id];
2052 
2053 	if (instr->delay_ns > 0)
2054 		mini_ctrl_cmd |= GCMD_LAY_TWB;
2055 
2056 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2057 				    GCMD_LAY_INSTR_CMD);
2058 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_CMD,
2059 				    instr->ctx.cmd.opcode);
2060 
2061 	ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2062 					    cdns_chip->cs[chip->cur_cs],
2063 					    mini_ctrl_cmd);
2064 	if (ret)
2065 		dev_err(cdns_ctrl->dev, "send cmd %x failed\n",
2066 			instr->ctx.cmd.opcode);
2067 
2068 	return ret;
2069 }
2070 
2071 static int cadence_nand_cmd_address(struct nand_chip *chip,
2072 				    const struct nand_subop *subop)
2073 {
2074 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2075 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2076 	const struct nand_op_instr *instr;
2077 	unsigned int op_id = 0;
2078 	u64 mini_ctrl_cmd = 0;
2079 	unsigned int offset, naddrs;
2080 	u64 address = 0;
2081 	const u8 *addrs;
2082 	int ret;
2083 	int i;
2084 
2085 	instr = &subop->instrs[op_id];
2086 
2087 	if (instr->delay_ns > 0)
2088 		mini_ctrl_cmd |= GCMD_LAY_TWB;
2089 
2090 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2091 				    GCMD_LAY_INSTR_ADDR);
2092 
2093 	offset = nand_subop_get_addr_start_off(subop, op_id);
2094 	naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
2095 	addrs = &instr->ctx.addr.addrs[offset];
2096 
2097 	for (i = 0; i < naddrs; i++)
2098 		address |= (u64)addrs[i] << (8 * i);
2099 
2100 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_ADDR,
2101 				    address);
2102 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_ADDR_SIZE,
2103 				    naddrs - 1);
2104 
2105 	ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2106 					    cdns_chip->cs[chip->cur_cs],
2107 					    mini_ctrl_cmd);
2108 	if (ret)
2109 		dev_err(cdns_ctrl->dev, "send address %llx failed\n", address);
2110 
2111 	return ret;
2112 }
2113 
2114 static int cadence_nand_cmd_erase(struct nand_chip *chip,
2115 				  const struct nand_subop *subop)
2116 {
2117 	unsigned int op_id;
2118 
2119 	if (subop->instrs[0].ctx.cmd.opcode == NAND_CMD_ERASE1) {
2120 		int i;
2121 		const struct nand_op_instr *instr = NULL;
2122 		unsigned int offset, naddrs;
2123 		const u8 *addrs;
2124 		u32 page = 0;
2125 
2126 		instr = &subop->instrs[1];
2127 		offset = nand_subop_get_addr_start_off(subop, 1);
2128 		naddrs = nand_subop_get_num_addr_cyc(subop, 1);
2129 		addrs = &instr->ctx.addr.addrs[offset];
2130 
2131 		for (i = 0; i < naddrs; i++)
2132 			page |= (u32)addrs[i] << (8 * i);
2133 
2134 		return cadence_nand_erase(chip, page);
2135 	}
2136 
2137 	/*
2138 	 * If it is not an erase operation then handle operation
2139 	 * by calling exec_op function.
2140 	 */
2141 	for (op_id = 0; op_id < subop->ninstrs; op_id++) {
2142 		int ret;
2143 		const struct nand_operation nand_op = {
2144 			.cs = chip->cur_cs,
2145 			.instrs =  &subop->instrs[op_id],
2146 			.ninstrs = 1};
2147 		ret = chip->controller->ops->exec_op(chip, &nand_op, false);
2148 		if (ret)
2149 			return ret;
2150 	}
2151 
2152 	return 0;
2153 }
2154 
2155 static int cadence_nand_cmd_data(struct nand_chip *chip,
2156 				 const struct nand_subop *subop)
2157 {
2158 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2159 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2160 	const struct nand_op_instr *instr;
2161 	unsigned int offset, op_id = 0;
2162 	u64 mini_ctrl_cmd = 0;
2163 	int len = 0;
2164 	int ret;
2165 
2166 	instr = &subop->instrs[op_id];
2167 
2168 	if (instr->delay_ns > 0)
2169 		mini_ctrl_cmd |= GCMD_LAY_TWB;
2170 
2171 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2172 				    GCMD_LAY_INSTR_DATA);
2173 
2174 	if (instr->type == NAND_OP_DATA_OUT_INSTR)
2175 		mini_ctrl_cmd |= FIELD_PREP(GCMD_DIR,
2176 					    GCMD_DIR_WRITE);
2177 
2178 	len = nand_subop_get_data_len(subop, op_id);
2179 	offset = nand_subop_get_data_start_off(subop, op_id);
2180 	mini_ctrl_cmd |= FIELD_PREP(GCMD_SECT_CNT, 1);
2181 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAST_SIZE, len);
2182 	if (instr->ctx.data.force_8bit) {
2183 		ret = cadence_nand_force_byte_access(chip, true);
2184 		if (ret) {
2185 			dev_err(cdns_ctrl->dev,
2186 				"cannot change byte access generic data cmd failed\n");
2187 			return ret;
2188 		}
2189 	}
2190 
2191 	ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2192 					    cdns_chip->cs[chip->cur_cs],
2193 					    mini_ctrl_cmd);
2194 	if (ret) {
2195 		dev_err(cdns_ctrl->dev, "send generic data cmd failed\n");
2196 		return ret;
2197 	}
2198 
2199 	if (instr->type == NAND_OP_DATA_IN_INSTR) {
2200 		void *buf = instr->ctx.data.buf.in + offset;
2201 
2202 		ret = cadence_nand_read_buf(cdns_ctrl, buf, len);
2203 	} else {
2204 		const void *buf = instr->ctx.data.buf.out + offset;
2205 
2206 		ret = cadence_nand_write_buf(cdns_ctrl, buf, len);
2207 	}
2208 
2209 	if (ret) {
2210 		dev_err(cdns_ctrl->dev, "data transfer failed for generic command\n");
2211 		return ret;
2212 	}
2213 
2214 	if (instr->ctx.data.force_8bit) {
2215 		ret = cadence_nand_force_byte_access(chip, false);
2216 		if (ret) {
2217 			dev_err(cdns_ctrl->dev,
2218 				"cannot change byte access generic data cmd failed\n");
2219 		}
2220 	}
2221 
2222 	return ret;
2223 }
2224 
2225 static int cadence_nand_cmd_waitrdy(struct nand_chip *chip,
2226 				    const struct nand_subop *subop)
2227 {
2228 	int status;
2229 	unsigned int op_id = 0;
2230 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2231 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2232 	const struct nand_op_instr *instr = &subop->instrs[op_id];
2233 	u32 timeout_us = instr->ctx.waitrdy.timeout_ms * 1000;
2234 
2235 	status = cadence_nand_wait_for_value(cdns_ctrl, RBN_SETINGS,
2236 					     timeout_us,
2237 					     BIT(cdns_chip->cs[chip->cur_cs]),
2238 					     false);
2239 	return status;
2240 }
2241 
2242 static const struct nand_op_parser cadence_nand_op_parser = NAND_OP_PARSER(
2243 	NAND_OP_PARSER_PATTERN(
2244 		cadence_nand_cmd_erase,
2245 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2246 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ERASE_ADDRESS_CYC),
2247 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2248 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2249 	NAND_OP_PARSER_PATTERN(
2250 		cadence_nand_cmd_opcode,
2251 		NAND_OP_PARSER_PAT_CMD_ELEM(false)),
2252 	NAND_OP_PARSER_PATTERN(
2253 		cadence_nand_cmd_address,
2254 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC)),
2255 	NAND_OP_PARSER_PATTERN(
2256 		cadence_nand_cmd_data,
2257 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_DATA_SIZE)),
2258 	NAND_OP_PARSER_PATTERN(
2259 		cadence_nand_cmd_data,
2260 		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_DATA_SIZE)),
2261 	NAND_OP_PARSER_PATTERN(
2262 		cadence_nand_cmd_waitrdy,
2263 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false))
2264 	);
2265 
2266 static int cadence_nand_exec_op(struct nand_chip *chip,
2267 				const struct nand_operation *op,
2268 				bool check_only)
2269 {
2270 	if (!check_only) {
2271 		int status = cadence_nand_select_target(chip);
2272 
2273 		if (status)
2274 			return status;
2275 	}
2276 
2277 	return nand_op_parser_exec_op(chip, &cadence_nand_op_parser, op,
2278 				      check_only);
2279 }
2280 
2281 static int cadence_nand_ooblayout_free(struct mtd_info *mtd, int section,
2282 				       struct mtd_oob_region *oobregion)
2283 {
2284 	struct nand_chip *chip = mtd_to_nand(mtd);
2285 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2286 
2287 	if (section)
2288 		return -ERANGE;
2289 
2290 	oobregion->offset = cdns_chip->bbm_len;
2291 	oobregion->length = cdns_chip->avail_oob_size
2292 		- cdns_chip->bbm_len;
2293 
2294 	return 0;
2295 }
2296 
2297 static int cadence_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2298 				      struct mtd_oob_region *oobregion)
2299 {
2300 	struct nand_chip *chip = mtd_to_nand(mtd);
2301 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2302 
2303 	if (section)
2304 		return -ERANGE;
2305 
2306 	oobregion->offset = cdns_chip->avail_oob_size;
2307 	oobregion->length = chip->ecc.total;
2308 
2309 	return 0;
2310 }
2311 
2312 static const struct mtd_ooblayout_ops cadence_nand_ooblayout_ops = {
2313 	.free = cadence_nand_ooblayout_free,
2314 	.ecc = cadence_nand_ooblayout_ecc,
2315 };
2316 
2317 static int calc_cycl(u32 timing, u32 clock)
2318 {
2319 	if (timing == 0 || clock == 0)
2320 		return 0;
2321 
2322 	if ((timing % clock) > 0)
2323 		return timing / clock;
2324 	else
2325 		return timing / clock - 1;
2326 }
2327 
2328 /* Calculate max data valid window. */
2329 static inline u32 calc_tdvw_max(u32 trp_cnt, u32 clk_period, u32 trhoh_min,
2330 				u32 board_delay_skew_min, u32 ext_mode)
2331 {
2332 	if (ext_mode == 0)
2333 		clk_period /= 2;
2334 
2335 	return (trp_cnt + 1) * clk_period + trhoh_min +
2336 		board_delay_skew_min;
2337 }
2338 
2339 /* Calculate data valid window. */
2340 static inline u32 calc_tdvw(u32 trp_cnt, u32 clk_period, u32 trhoh_min,
2341 			    u32 trea_max, u32 ext_mode)
2342 {
2343 	if (ext_mode == 0)
2344 		clk_period /= 2;
2345 
2346 	return (trp_cnt + 1) * clk_period + trhoh_min - trea_max;
2347 }
2348 
2349 static int
2350 cadence_nand_setup_interface(struct nand_chip *chip, int chipnr,
2351 			     const struct nand_interface_config *conf)
2352 {
2353 	const struct nand_sdr_timings *sdr;
2354 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2355 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2356 	struct cadence_nand_timings *t = &cdns_chip->timings;
2357 	u32 reg;
2358 	u32 board_delay = cdns_ctrl->board_delay;
2359 	u32 clk_period = DIV_ROUND_DOWN_ULL(1000000000000ULL,
2360 					    cdns_ctrl->nf_clk_rate);
2361 	u32 tceh_cnt, tcs_cnt, tadl_cnt, tccs_cnt;
2362 	u32 tfeat_cnt, trhz_cnt, tvdly_cnt;
2363 	u32 trhw_cnt, twb_cnt, twh_cnt = 0, twhr_cnt;
2364 	u32 twp_cnt = 0, trp_cnt = 0, trh_cnt = 0;
2365 	u32 if_skew = cdns_ctrl->caps1->if_skew;
2366 	u32 board_delay_skew_min = board_delay - if_skew;
2367 	u32 board_delay_skew_max = board_delay + if_skew;
2368 	u32 dqs_sampl_res, phony_dqs_mod;
2369 	u32 tdvw, tdvw_min, tdvw_max;
2370 	u32 ext_rd_mode, ext_wr_mode;
2371 	u32 dll_phy_dqs_timing = 0, phony_dqs_timing = 0, rd_del_sel = 0;
2372 	u32 sampling_point;
2373 
2374 	sdr = nand_get_sdr_timings(conf);
2375 	if (IS_ERR(sdr))
2376 		return PTR_ERR(sdr);
2377 
2378 	memset(t, 0, sizeof(*t));
2379 	/* Sampling point calculation. */
2380 
2381 	if (cdns_ctrl->caps2.is_phy_type_dll)
2382 		phony_dqs_mod = 2;
2383 	else
2384 		phony_dqs_mod = 1;
2385 
2386 	dqs_sampl_res = clk_period / phony_dqs_mod;
2387 
2388 	tdvw_min = sdr->tREA_max + board_delay_skew_max;
2389 	/*
2390 	 * The idea of those calculation is to get the optimum value
2391 	 * for tRP and tRH timings. If it is NOT possible to sample data
2392 	 * with optimal tRP/tRH settings, the parameters will be extended.
2393 	 * If clk_period is 50ns (the lowest value) this condition is met
2394 	 * for SDR timing modes 1, 2, 3, 4 and 5.
2395 	 * If clk_period is 20ns the condition is met only for SDR timing
2396 	 * mode 5.
2397 	 */
2398 	if (sdr->tRC_min <= clk_period &&
2399 	    sdr->tRP_min <= (clk_period / 2) &&
2400 	    sdr->tREH_min <= (clk_period / 2)) {
2401 		/* Performance mode. */
2402 		ext_rd_mode = 0;
2403 		tdvw = calc_tdvw(trp_cnt, clk_period, sdr->tRHOH_min,
2404 				 sdr->tREA_max, ext_rd_mode);
2405 		tdvw_max = calc_tdvw_max(trp_cnt, clk_period, sdr->tRHOH_min,
2406 					 board_delay_skew_min,
2407 					 ext_rd_mode);
2408 		/*
2409 		 * Check if data valid window and sampling point can be found
2410 		 * and is not on the edge (ie. we have hold margin).
2411 		 * If not extend the tRP timings.
2412 		 */
2413 		if (tdvw > 0) {
2414 			if (tdvw_max <= tdvw_min ||
2415 			    (tdvw_max % dqs_sampl_res) == 0) {
2416 				/*
2417 				 * No valid sampling point so the RE pulse need
2418 				 * to be widen widening by half clock cycle.
2419 				 */
2420 				ext_rd_mode = 1;
2421 			}
2422 		} else {
2423 			/*
2424 			 * There is no valid window
2425 			 * to be able to sample data the tRP need to be widen.
2426 			 * Very safe calculations are performed here.
2427 			 */
2428 			trp_cnt = (sdr->tREA_max + board_delay_skew_max
2429 				   + dqs_sampl_res) / clk_period;
2430 			ext_rd_mode = 1;
2431 		}
2432 
2433 	} else {
2434 		/* Extended read mode. */
2435 		u32 trh;
2436 
2437 		ext_rd_mode = 1;
2438 		trp_cnt = calc_cycl(sdr->tRP_min, clk_period);
2439 		trh = sdr->tRC_min - ((trp_cnt + 1) * clk_period);
2440 		if (sdr->tREH_min >= trh)
2441 			trh_cnt = calc_cycl(sdr->tREH_min, clk_period);
2442 		else
2443 			trh_cnt = calc_cycl(trh, clk_period);
2444 
2445 		tdvw = calc_tdvw(trp_cnt, clk_period, sdr->tRHOH_min,
2446 				 sdr->tREA_max, ext_rd_mode);
2447 		/*
2448 		 * Check if data valid window and sampling point can be found
2449 		 * or if it is at the edge check if previous is valid
2450 		 * - if not extend the tRP timings.
2451 		 */
2452 		if (tdvw > 0) {
2453 			tdvw_max = calc_tdvw_max(trp_cnt, clk_period,
2454 						 sdr->tRHOH_min,
2455 						 board_delay_skew_min,
2456 						 ext_rd_mode);
2457 
2458 			if ((((tdvw_max / dqs_sampl_res)
2459 			      * dqs_sampl_res) <= tdvw_min) ||
2460 			    (((tdvw_max % dqs_sampl_res) == 0) &&
2461 			     (((tdvw_max / dqs_sampl_res - 1)
2462 			       * dqs_sampl_res) <= tdvw_min))) {
2463 				/*
2464 				 * Data valid window width is lower than
2465 				 * sampling resolution and do not hit any
2466 				 * sampling point to be sure the sampling point
2467 				 * will be found the RE low pulse width will be
2468 				 *  extended by one clock cycle.
2469 				 */
2470 				trp_cnt = trp_cnt + 1;
2471 			}
2472 		} else {
2473 			/*
2474 			 * There is no valid window to be able to sample data.
2475 			 * The tRP need to be widen.
2476 			 * Very safe calculations are performed here.
2477 			 */
2478 			trp_cnt = (sdr->tREA_max + board_delay_skew_max
2479 				   + dqs_sampl_res) / clk_period;
2480 		}
2481 	}
2482 
2483 	tdvw_max = calc_tdvw_max(trp_cnt, clk_period,
2484 				 sdr->tRHOH_min,
2485 				 board_delay_skew_min, ext_rd_mode);
2486 
2487 	if (sdr->tWC_min <= clk_period &&
2488 	    (sdr->tWP_min + if_skew) <= (clk_period / 2) &&
2489 	    (sdr->tWH_min + if_skew) <= (clk_period / 2)) {
2490 		ext_wr_mode = 0;
2491 	} else {
2492 		u32 twh;
2493 
2494 		ext_wr_mode = 1;
2495 		twp_cnt = calc_cycl(sdr->tWP_min + if_skew, clk_period);
2496 		if ((twp_cnt + 1) * clk_period < (sdr->tALS_min + if_skew))
2497 			twp_cnt = calc_cycl(sdr->tALS_min + if_skew,
2498 					    clk_period);
2499 
2500 		twh = (sdr->tWC_min - (twp_cnt + 1) * clk_period);
2501 		if (sdr->tWH_min >= twh)
2502 			twh = sdr->tWH_min;
2503 
2504 		twh_cnt = calc_cycl(twh + if_skew, clk_period);
2505 	}
2506 
2507 	reg = FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TRH, trh_cnt);
2508 	reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TRP, trp_cnt);
2509 	reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TWH, twh_cnt);
2510 	reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TWP, twp_cnt);
2511 	t->async_toggle_timings = reg;
2512 	dev_dbg(cdns_ctrl->dev, "ASYNC_TOGGLE_TIMINGS_SDR\t%x\n", reg);
2513 
2514 	tadl_cnt = calc_cycl((sdr->tADL_min + if_skew), clk_period);
2515 	tccs_cnt = calc_cycl((sdr->tCCS_min + if_skew), clk_period);
2516 	twhr_cnt = calc_cycl((sdr->tWHR_min + if_skew), clk_period);
2517 	trhw_cnt = calc_cycl((sdr->tRHW_min + if_skew), clk_period);
2518 	reg = FIELD_PREP(TIMINGS0_TADL, tadl_cnt);
2519 
2520 	/*
2521 	 * If timing exceeds delay field in timing register
2522 	 * then use maximum value.
2523 	 */
2524 	if (FIELD_FIT(TIMINGS0_TCCS, tccs_cnt))
2525 		reg |= FIELD_PREP(TIMINGS0_TCCS, tccs_cnt);
2526 	else
2527 		reg |= TIMINGS0_TCCS;
2528 
2529 	reg |= FIELD_PREP(TIMINGS0_TWHR, twhr_cnt);
2530 	reg |= FIELD_PREP(TIMINGS0_TRHW, trhw_cnt);
2531 	t->timings0 = reg;
2532 	dev_dbg(cdns_ctrl->dev, "TIMINGS0_SDR\t%x\n", reg);
2533 
2534 	/* The following is related to single signal so skew is not needed. */
2535 	trhz_cnt = calc_cycl(sdr->tRHZ_max, clk_period);
2536 	trhz_cnt = trhz_cnt + 1;
2537 	twb_cnt = calc_cycl((sdr->tWB_max + board_delay), clk_period);
2538 	/*
2539 	 * Because of the two stage syncflop the value must be increased by 3
2540 	 * first value is related with sync, second value is related
2541 	 * with output if delay.
2542 	 */
2543 	twb_cnt = twb_cnt + 3 + 5;
2544 	/*
2545 	 * The following is related to the we edge of the random data input
2546 	 * sequence so skew is not needed.
2547 	 */
2548 	tvdly_cnt = calc_cycl(500000 + if_skew, clk_period);
2549 	reg = FIELD_PREP(TIMINGS1_TRHZ, trhz_cnt);
2550 	reg |= FIELD_PREP(TIMINGS1_TWB, twb_cnt);
2551 	reg |= FIELD_PREP(TIMINGS1_TVDLY, tvdly_cnt);
2552 	t->timings1 = reg;
2553 	dev_dbg(cdns_ctrl->dev, "TIMINGS1_SDR\t%x\n", reg);
2554 
2555 	tfeat_cnt = calc_cycl(sdr->tFEAT_max, clk_period);
2556 	if (tfeat_cnt < twb_cnt)
2557 		tfeat_cnt = twb_cnt;
2558 
2559 	tceh_cnt = calc_cycl(sdr->tCEH_min, clk_period);
2560 	tcs_cnt = calc_cycl((sdr->tCS_min + if_skew), clk_period);
2561 
2562 	reg = FIELD_PREP(TIMINGS2_TFEAT, tfeat_cnt);
2563 	reg |= FIELD_PREP(TIMINGS2_CS_HOLD_TIME, tceh_cnt);
2564 	reg |= FIELD_PREP(TIMINGS2_CS_SETUP_TIME, tcs_cnt);
2565 	t->timings2 = reg;
2566 	dev_dbg(cdns_ctrl->dev, "TIMINGS2_SDR\t%x\n", reg);
2567 
2568 	if (cdns_ctrl->caps2.is_phy_type_dll) {
2569 		reg = DLL_PHY_CTRL_DLL_RST_N;
2570 		if (ext_wr_mode)
2571 			reg |= DLL_PHY_CTRL_EXTENDED_WR_MODE;
2572 		if (ext_rd_mode)
2573 			reg |= DLL_PHY_CTRL_EXTENDED_RD_MODE;
2574 
2575 		reg |= FIELD_PREP(DLL_PHY_CTRL_RS_HIGH_WAIT_CNT, 7);
2576 		reg |= FIELD_PREP(DLL_PHY_CTRL_RS_IDLE_CNT, 7);
2577 		t->dll_phy_ctrl = reg;
2578 		dev_dbg(cdns_ctrl->dev, "DLL_PHY_CTRL_SDR\t%x\n", reg);
2579 	}
2580 
2581 	/* Sampling point calculation. */
2582 	if ((tdvw_max % dqs_sampl_res) > 0)
2583 		sampling_point = tdvw_max / dqs_sampl_res;
2584 	else
2585 		sampling_point = (tdvw_max / dqs_sampl_res - 1);
2586 
2587 	if (sampling_point * dqs_sampl_res > tdvw_min) {
2588 		dll_phy_dqs_timing =
2589 			FIELD_PREP(PHY_DQS_TIMING_DQS_SEL_OE_END, 4);
2590 		dll_phy_dqs_timing |= PHY_DQS_TIMING_USE_PHONY_DQS;
2591 		phony_dqs_timing = sampling_point / phony_dqs_mod;
2592 
2593 		if ((sampling_point % 2) > 0) {
2594 			dll_phy_dqs_timing |= PHY_DQS_TIMING_PHONY_DQS_SEL;
2595 			if ((tdvw_max % dqs_sampl_res) == 0)
2596 				/*
2597 				 * Calculation for sampling point at the edge
2598 				 * of data and being odd number.
2599 				 */
2600 				phony_dqs_timing = (tdvw_max / dqs_sampl_res)
2601 					/ phony_dqs_mod - 1;
2602 
2603 			if (!cdns_ctrl->caps2.is_phy_type_dll)
2604 				phony_dqs_timing--;
2605 
2606 		} else {
2607 			phony_dqs_timing--;
2608 		}
2609 		rd_del_sel = phony_dqs_timing + 3;
2610 	} else {
2611 		dev_warn(cdns_ctrl->dev,
2612 			 "ERROR : cannot find valid sampling point\n");
2613 	}
2614 
2615 	reg = FIELD_PREP(PHY_CTRL_PHONY_DQS, phony_dqs_timing);
2616 	if (cdns_ctrl->caps2.is_phy_type_dll)
2617 		reg  |= PHY_CTRL_SDR_DQS;
2618 	t->phy_ctrl = reg;
2619 	dev_dbg(cdns_ctrl->dev, "PHY_CTRL_REG_SDR\t%x\n", reg);
2620 
2621 	if (cdns_ctrl->caps2.is_phy_type_dll) {
2622 		dev_dbg(cdns_ctrl->dev, "PHY_TSEL_REG_SDR\t%x\n", 0);
2623 		dev_dbg(cdns_ctrl->dev, "PHY_DQ_TIMING_REG_SDR\t%x\n", 2);
2624 		dev_dbg(cdns_ctrl->dev, "PHY_DQS_TIMING_REG_SDR\t%x\n",
2625 			dll_phy_dqs_timing);
2626 		t->phy_dqs_timing = dll_phy_dqs_timing;
2627 
2628 		reg = FIELD_PREP(PHY_GATE_LPBK_CTRL_RDS, rd_del_sel);
2629 		dev_dbg(cdns_ctrl->dev, "PHY_GATE_LPBK_CTRL_REG_SDR\t%x\n",
2630 			reg);
2631 		t->phy_gate_lpbk_ctrl = reg;
2632 
2633 		dev_dbg(cdns_ctrl->dev, "PHY_DLL_MASTER_CTRL_REG_SDR\t%lx\n",
2634 			PHY_DLL_MASTER_CTRL_BYPASS_MODE);
2635 		dev_dbg(cdns_ctrl->dev, "PHY_DLL_SLAVE_CTRL_REG_SDR\t%x\n", 0);
2636 	}
2637 
2638 	return 0;
2639 }
2640 
2641 static int cadence_nand_attach_chip(struct nand_chip *chip)
2642 {
2643 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2644 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2645 	u32 ecc_size;
2646 	struct mtd_info *mtd = nand_to_mtd(chip);
2647 	int ret;
2648 
2649 	if (chip->options & NAND_BUSWIDTH_16) {
2650 		ret = cadence_nand_set_access_width16(cdns_ctrl, true);
2651 		if (ret)
2652 			return ret;
2653 	}
2654 
2655 	chip->bbt_options |= NAND_BBT_USE_FLASH;
2656 	chip->bbt_options |= NAND_BBT_NO_OOB;
2657 	chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2658 
2659 	chip->options |= NAND_NO_SUBPAGE_WRITE;
2660 
2661 	cdns_chip->bbm_offs = chip->badblockpos;
2662 	cdns_chip->bbm_offs &= ~0x01;
2663 	/* this value should be even number */
2664 	cdns_chip->bbm_len = 2;
2665 
2666 	ret = nand_ecc_choose_conf(chip,
2667 				   &cdns_ctrl->ecc_caps,
2668 				   mtd->oobsize - cdns_chip->bbm_len);
2669 	if (ret) {
2670 		dev_err(cdns_ctrl->dev, "ECC configuration failed\n");
2671 		return ret;
2672 	}
2673 
2674 	dev_dbg(cdns_ctrl->dev,
2675 		"chosen ECC settings: step=%d, strength=%d, bytes=%d\n",
2676 		chip->ecc.size, chip->ecc.strength, chip->ecc.bytes);
2677 
2678 	/* Error correction configuration. */
2679 	cdns_chip->sector_size = chip->ecc.size;
2680 	cdns_chip->sector_count = mtd->writesize / cdns_chip->sector_size;
2681 	ecc_size = cdns_chip->sector_count * chip->ecc.bytes;
2682 
2683 	cdns_chip->avail_oob_size = mtd->oobsize - ecc_size;
2684 
2685 	if (cdns_chip->avail_oob_size > cdns_ctrl->bch_metadata_size)
2686 		cdns_chip->avail_oob_size = cdns_ctrl->bch_metadata_size;
2687 
2688 	if ((cdns_chip->avail_oob_size + cdns_chip->bbm_len + ecc_size)
2689 	    > mtd->oobsize)
2690 		cdns_chip->avail_oob_size -= 4;
2691 
2692 	ret = cadence_nand_get_ecc_strength_idx(cdns_ctrl, chip->ecc.strength);
2693 	if (ret < 0)
2694 		return -EINVAL;
2695 
2696 	cdns_chip->corr_str_idx = (u8)ret;
2697 
2698 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
2699 					1000000,
2700 					CTRL_STATUS_CTRL_BUSY, true))
2701 		return -ETIMEDOUT;
2702 
2703 	cadence_nand_set_ecc_strength(cdns_ctrl,
2704 				      cdns_chip->corr_str_idx);
2705 
2706 	cadence_nand_set_erase_detection(cdns_ctrl, true,
2707 					 chip->ecc.strength);
2708 
2709 	/* Override the default read operations. */
2710 	chip->ecc.read_page = cadence_nand_read_page;
2711 	chip->ecc.read_page_raw = cadence_nand_read_page_raw;
2712 	chip->ecc.write_page = cadence_nand_write_page;
2713 	chip->ecc.write_page_raw = cadence_nand_write_page_raw;
2714 	chip->ecc.read_oob = cadence_nand_read_oob;
2715 	chip->ecc.write_oob = cadence_nand_write_oob;
2716 	chip->ecc.read_oob_raw = cadence_nand_read_oob_raw;
2717 	chip->ecc.write_oob_raw = cadence_nand_write_oob_raw;
2718 
2719 	if ((mtd->writesize + mtd->oobsize) > cdns_ctrl->buf_size)
2720 		cdns_ctrl->buf_size = mtd->writesize + mtd->oobsize;
2721 
2722 	/* Is 32-bit DMA supported? */
2723 	ret = dma_set_mask(cdns_ctrl->dev, DMA_BIT_MASK(32));
2724 	if (ret) {
2725 		dev_err(cdns_ctrl->dev, "no usable DMA configuration\n");
2726 		return ret;
2727 	}
2728 
2729 	mtd_set_ooblayout(mtd, &cadence_nand_ooblayout_ops);
2730 
2731 	return 0;
2732 }
2733 
2734 static const struct nand_controller_ops cadence_nand_controller_ops = {
2735 	.attach_chip = cadence_nand_attach_chip,
2736 	.exec_op = cadence_nand_exec_op,
2737 	.setup_interface = cadence_nand_setup_interface,
2738 };
2739 
2740 static int cadence_nand_chip_init(struct cdns_nand_ctrl *cdns_ctrl,
2741 				  struct device_node *np)
2742 {
2743 	struct cdns_nand_chip *cdns_chip;
2744 	struct mtd_info *mtd;
2745 	struct nand_chip *chip;
2746 	int nsels, ret, i;
2747 	u32 cs;
2748 
2749 	nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
2750 	if (nsels <= 0) {
2751 		dev_err(cdns_ctrl->dev, "missing/invalid reg property\n");
2752 		return -EINVAL;
2753 	}
2754 
2755 	/* Allocate the nand chip structure. */
2756 	cdns_chip = devm_kzalloc(cdns_ctrl->dev, sizeof(*cdns_chip) +
2757 				 (nsels * sizeof(u8)),
2758 				 GFP_KERNEL);
2759 	if (!cdns_chip) {
2760 		dev_err(cdns_ctrl->dev, "could not allocate chip structure\n");
2761 		return -ENOMEM;
2762 	}
2763 
2764 	cdns_chip->nsels = nsels;
2765 
2766 	for (i = 0; i < nsels; i++) {
2767 		/* Retrieve CS id. */
2768 		ret = of_property_read_u32_index(np, "reg", i, &cs);
2769 		if (ret) {
2770 			dev_err(cdns_ctrl->dev,
2771 				"could not retrieve reg property: %d\n",
2772 				ret);
2773 			return ret;
2774 		}
2775 
2776 		if (cs >= cdns_ctrl->caps2.max_banks) {
2777 			dev_err(cdns_ctrl->dev,
2778 				"invalid reg value: %u (max CS = %d)\n",
2779 				cs, cdns_ctrl->caps2.max_banks);
2780 			return -EINVAL;
2781 		}
2782 
2783 		if (test_and_set_bit(cs, &cdns_ctrl->assigned_cs)) {
2784 			dev_err(cdns_ctrl->dev,
2785 				"CS %d already assigned\n", cs);
2786 			return -EINVAL;
2787 		}
2788 
2789 		cdns_chip->cs[i] = cs;
2790 	}
2791 
2792 	chip = &cdns_chip->chip;
2793 	chip->controller = &cdns_ctrl->controller;
2794 	nand_set_flash_node(chip, np);
2795 
2796 	mtd = nand_to_mtd(chip);
2797 	mtd->dev.parent = cdns_ctrl->dev;
2798 
2799 	/*
2800 	 * Default to HW ECC engine mode. If the nand-ecc-mode property is given
2801 	 * in the DT node, this entry will be overwritten in nand_scan_ident().
2802 	 */
2803 	chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2804 
2805 	ret = nand_scan(chip, cdns_chip->nsels);
2806 	if (ret) {
2807 		dev_err(cdns_ctrl->dev, "could not scan the nand chip\n");
2808 		return ret;
2809 	}
2810 
2811 	ret = mtd_device_register(mtd, NULL, 0);
2812 	if (ret) {
2813 		dev_err(cdns_ctrl->dev,
2814 			"failed to register mtd device: %d\n", ret);
2815 		nand_cleanup(chip);
2816 		return ret;
2817 	}
2818 
2819 	list_add_tail(&cdns_chip->node, &cdns_ctrl->chips);
2820 
2821 	return 0;
2822 }
2823 
2824 static void cadence_nand_chips_cleanup(struct cdns_nand_ctrl *cdns_ctrl)
2825 {
2826 	struct cdns_nand_chip *entry, *temp;
2827 	struct nand_chip *chip;
2828 	int ret;
2829 
2830 	list_for_each_entry_safe(entry, temp, &cdns_ctrl->chips, node) {
2831 		chip = &entry->chip;
2832 		ret = mtd_device_unregister(nand_to_mtd(chip));
2833 		WARN_ON(ret);
2834 		nand_cleanup(chip);
2835 		list_del(&entry->node);
2836 	}
2837 }
2838 
2839 static int cadence_nand_chips_init(struct cdns_nand_ctrl *cdns_ctrl)
2840 {
2841 	struct device_node *np = cdns_ctrl->dev->of_node;
2842 	struct device_node *nand_np;
2843 	int max_cs = cdns_ctrl->caps2.max_banks;
2844 	int nchips, ret;
2845 
2846 	nchips = of_get_child_count(np);
2847 
2848 	if (nchips > max_cs) {
2849 		dev_err(cdns_ctrl->dev,
2850 			"too many NAND chips: %d (max = %d CS)\n",
2851 			nchips, max_cs);
2852 		return -EINVAL;
2853 	}
2854 
2855 	for_each_child_of_node(np, nand_np) {
2856 		ret = cadence_nand_chip_init(cdns_ctrl, nand_np);
2857 		if (ret) {
2858 			of_node_put(nand_np);
2859 			cadence_nand_chips_cleanup(cdns_ctrl);
2860 			return ret;
2861 		}
2862 	}
2863 
2864 	return 0;
2865 }
2866 
2867 static void
2868 cadence_nand_irq_cleanup(int irqnum, struct cdns_nand_ctrl *cdns_ctrl)
2869 {
2870 	/* Disable interrupts. */
2871 	writel_relaxed(INTR_ENABLE_INTR_EN, cdns_ctrl->reg + INTR_ENABLE);
2872 }
2873 
2874 static int cadence_nand_init(struct cdns_nand_ctrl *cdns_ctrl)
2875 {
2876 	dma_cap_mask_t mask;
2877 	int ret;
2878 
2879 	cdns_ctrl->cdma_desc = dma_alloc_coherent(cdns_ctrl->dev,
2880 						  sizeof(*cdns_ctrl->cdma_desc),
2881 						  &cdns_ctrl->dma_cdma_desc,
2882 						  GFP_KERNEL);
2883 	if (!cdns_ctrl->dma_cdma_desc)
2884 		return -ENOMEM;
2885 
2886 	cdns_ctrl->buf_size = SZ_16K;
2887 	cdns_ctrl->buf = kmalloc(cdns_ctrl->buf_size, GFP_KERNEL);
2888 	if (!cdns_ctrl->buf) {
2889 		ret = -ENOMEM;
2890 		goto free_buf_desc;
2891 	}
2892 
2893 	if (devm_request_irq(cdns_ctrl->dev, cdns_ctrl->irq, cadence_nand_isr,
2894 			     IRQF_SHARED, "cadence-nand-controller",
2895 			     cdns_ctrl)) {
2896 		dev_err(cdns_ctrl->dev, "Unable to allocate IRQ\n");
2897 		ret = -ENODEV;
2898 		goto free_buf;
2899 	}
2900 
2901 	spin_lock_init(&cdns_ctrl->irq_lock);
2902 	init_completion(&cdns_ctrl->complete);
2903 
2904 	ret = cadence_nand_hw_init(cdns_ctrl);
2905 	if (ret)
2906 		goto disable_irq;
2907 
2908 	dma_cap_zero(mask);
2909 	dma_cap_set(DMA_MEMCPY, mask);
2910 
2911 	if (cdns_ctrl->caps1->has_dma) {
2912 		cdns_ctrl->dmac = dma_request_channel(mask, NULL, NULL);
2913 		if (!cdns_ctrl->dmac) {
2914 			dev_err(cdns_ctrl->dev,
2915 				"Unable to get a DMA channel\n");
2916 			ret = -EBUSY;
2917 			goto disable_irq;
2918 		}
2919 	}
2920 
2921 	nand_controller_init(&cdns_ctrl->controller);
2922 	INIT_LIST_HEAD(&cdns_ctrl->chips);
2923 
2924 	cdns_ctrl->controller.ops = &cadence_nand_controller_ops;
2925 	cdns_ctrl->curr_corr_str_idx = 0xFF;
2926 
2927 	ret = cadence_nand_chips_init(cdns_ctrl);
2928 	if (ret) {
2929 		dev_err(cdns_ctrl->dev, "Failed to register MTD: %d\n",
2930 			ret);
2931 		goto dma_release_chnl;
2932 	}
2933 
2934 	kfree(cdns_ctrl->buf);
2935 	cdns_ctrl->buf = kzalloc(cdns_ctrl->buf_size, GFP_KERNEL);
2936 	if (!cdns_ctrl->buf) {
2937 		ret = -ENOMEM;
2938 		goto dma_release_chnl;
2939 	}
2940 
2941 	return 0;
2942 
2943 dma_release_chnl:
2944 	if (cdns_ctrl->dmac)
2945 		dma_release_channel(cdns_ctrl->dmac);
2946 
2947 disable_irq:
2948 	cadence_nand_irq_cleanup(cdns_ctrl->irq, cdns_ctrl);
2949 
2950 free_buf:
2951 	kfree(cdns_ctrl->buf);
2952 
2953 free_buf_desc:
2954 	dma_free_coherent(cdns_ctrl->dev, sizeof(struct cadence_nand_cdma_desc),
2955 			  cdns_ctrl->cdma_desc, cdns_ctrl->dma_cdma_desc);
2956 
2957 	return ret;
2958 }
2959 
2960 /* Driver exit point. */
2961 static void cadence_nand_remove(struct cdns_nand_ctrl *cdns_ctrl)
2962 {
2963 	cadence_nand_chips_cleanup(cdns_ctrl);
2964 	cadence_nand_irq_cleanup(cdns_ctrl->irq, cdns_ctrl);
2965 	kfree(cdns_ctrl->buf);
2966 	dma_free_coherent(cdns_ctrl->dev, sizeof(struct cadence_nand_cdma_desc),
2967 			  cdns_ctrl->cdma_desc, cdns_ctrl->dma_cdma_desc);
2968 
2969 	if (cdns_ctrl->dmac)
2970 		dma_release_channel(cdns_ctrl->dmac);
2971 }
2972 
2973 struct cadence_nand_dt {
2974 	struct cdns_nand_ctrl cdns_ctrl;
2975 	struct clk *clk;
2976 };
2977 
2978 static const struct cadence_nand_dt_devdata cadence_nand_default = {
2979 	.if_skew = 0,
2980 	.has_dma = 1,
2981 };
2982 
2983 static const struct of_device_id cadence_nand_dt_ids[] = {
2984 	{
2985 		.compatible = "cdns,hp-nfc",
2986 		.data = &cadence_nand_default
2987 	}, {}
2988 };
2989 
2990 MODULE_DEVICE_TABLE(of, cadence_nand_dt_ids);
2991 
2992 static int cadence_nand_dt_probe(struct platform_device *ofdev)
2993 {
2994 	struct resource *res;
2995 	struct cadence_nand_dt *dt;
2996 	struct cdns_nand_ctrl *cdns_ctrl;
2997 	int ret;
2998 	const struct of_device_id *of_id;
2999 	const struct cadence_nand_dt_devdata *devdata;
3000 	u32 val;
3001 
3002 	of_id = of_match_device(cadence_nand_dt_ids, &ofdev->dev);
3003 	if (of_id) {
3004 		ofdev->id_entry = of_id->data;
3005 		devdata = of_id->data;
3006 	} else {
3007 		pr_err("Failed to find the right device id.\n");
3008 		return -ENOMEM;
3009 	}
3010 
3011 	dt = devm_kzalloc(&ofdev->dev, sizeof(*dt), GFP_KERNEL);
3012 	if (!dt)
3013 		return -ENOMEM;
3014 
3015 	cdns_ctrl = &dt->cdns_ctrl;
3016 	cdns_ctrl->caps1 = devdata;
3017 
3018 	cdns_ctrl->dev = &ofdev->dev;
3019 	cdns_ctrl->irq = platform_get_irq(ofdev, 0);
3020 	if (cdns_ctrl->irq < 0)
3021 		return cdns_ctrl->irq;
3022 
3023 	dev_info(cdns_ctrl->dev, "IRQ: nr %d\n", cdns_ctrl->irq);
3024 
3025 	cdns_ctrl->reg = devm_platform_ioremap_resource(ofdev, 0);
3026 	if (IS_ERR(cdns_ctrl->reg))
3027 		return PTR_ERR(cdns_ctrl->reg);
3028 
3029 	cdns_ctrl->io.virt = devm_platform_get_and_ioremap_resource(ofdev, 1, &res);
3030 	if (IS_ERR(cdns_ctrl->io.virt))
3031 		return PTR_ERR(cdns_ctrl->io.virt);
3032 	cdns_ctrl->io.dma = res->start;
3033 
3034 	dt->clk = devm_clk_get(cdns_ctrl->dev, "nf_clk");
3035 	if (IS_ERR(dt->clk))
3036 		return PTR_ERR(dt->clk);
3037 
3038 	cdns_ctrl->nf_clk_rate = clk_get_rate(dt->clk);
3039 
3040 	ret = of_property_read_u32(ofdev->dev.of_node,
3041 				   "cdns,board-delay-ps", &val);
3042 	if (ret) {
3043 		val = 4830;
3044 		dev_info(cdns_ctrl->dev,
3045 			 "missing cdns,board-delay-ps property, %d was set\n",
3046 			 val);
3047 	}
3048 	cdns_ctrl->board_delay = val;
3049 
3050 	ret = cadence_nand_init(cdns_ctrl);
3051 	if (ret)
3052 		return ret;
3053 
3054 	platform_set_drvdata(ofdev, dt);
3055 	return 0;
3056 }
3057 
3058 static int cadence_nand_dt_remove(struct platform_device *ofdev)
3059 {
3060 	struct cadence_nand_dt *dt = platform_get_drvdata(ofdev);
3061 
3062 	cadence_nand_remove(&dt->cdns_ctrl);
3063 
3064 	return 0;
3065 }
3066 
3067 static struct platform_driver cadence_nand_dt_driver = {
3068 	.probe		= cadence_nand_dt_probe,
3069 	.remove		= cadence_nand_dt_remove,
3070 	.driver		= {
3071 		.name	= "cadence-nand-controller",
3072 		.of_match_table = cadence_nand_dt_ids,
3073 	},
3074 };
3075 
3076 module_platform_driver(cadence_nand_dt_driver);
3077 
3078 MODULE_AUTHOR("Piotr Sroka <piotrs@cadence.com>");
3079 MODULE_LICENSE("GPL v2");
3080 MODULE_DESCRIPTION("Driver for Cadence NAND flash controller");
3081 
3082