1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * Cadence NAND flash controller driver
4  *
5  * Copyright (C) 2019 Cadence
6  *
7  * Author: Piotr Sroka <piotrs@cadence.com>
8  */
9 
10 #include <linux/bitfield.h>
11 #include <linux/clk.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/dmaengine.h>
14 #include <linux/interrupt.h>
15 #include <linux/module.h>
16 #include <linux/mtd/mtd.h>
17 #include <linux/mtd/rawnand.h>
18 #include <linux/of_device.h>
19 #include <linux/iopoll.h>
20 #include <linux/slab.h>
21 
22 /*
23  * HPNFC can work in 3 modes:
24  * -  PIO - can work in master or slave DMA
25  * -  CDMA - needs Master DMA for accessing command descriptors.
26  * -  Generic mode - can use only slave DMA.
27  * CDMA and PIO modes can be used to execute only base commands.
28  * Generic mode can be used to execute any command
29  * on NAND flash memory. Driver uses CDMA mode for
30  * block erasing, page reading, page programing.
31  * Generic mode is used for executing rest of commands.
32  */
33 
34 #define MAX_ADDRESS_CYC		6
35 #define MAX_ERASE_ADDRESS_CYC	3
36 #define MAX_DATA_SIZE		0xFFFC
37 #define DMA_DATA_SIZE_ALIGN	8
38 
39 /* Register definition. */
40 /*
41  * Command register 0.
42  * Writing data to this register will initiate a new transaction
43  * of the NF controller.
44  */
45 #define CMD_REG0			0x0000
46 /* Command type field mask. */
47 #define		CMD_REG0_CT		GENMASK(31, 30)
48 /* Command type CDMA. */
49 #define		CMD_REG0_CT_CDMA	0uL
50 /* Command type generic. */
51 #define		CMD_REG0_CT_GEN		3uL
52 /* Command thread number field mask. */
53 #define		CMD_REG0_TN		GENMASK(27, 24)
54 
55 /* Command register 2. */
56 #define CMD_REG2			0x0008
57 /* Command register 3. */
58 #define CMD_REG3			0x000C
59 /* Pointer register to select which thread status will be selected. */
60 #define CMD_STATUS_PTR			0x0010
61 /* Command status register for selected thread. */
62 #define CMD_STATUS			0x0014
63 
64 /* Interrupt status register. */
65 #define INTR_STATUS			0x0110
66 #define		INTR_STATUS_SDMA_ERR	BIT(22)
67 #define		INTR_STATUS_SDMA_TRIGG	BIT(21)
68 #define		INTR_STATUS_UNSUPP_CMD	BIT(19)
69 #define		INTR_STATUS_DDMA_TERR	BIT(18)
70 #define		INTR_STATUS_CDMA_TERR	BIT(17)
71 #define		INTR_STATUS_CDMA_IDL	BIT(16)
72 
73 /* Interrupt enable register. */
74 #define INTR_ENABLE				0x0114
75 #define		INTR_ENABLE_INTR_EN		BIT(31)
76 #define		INTR_ENABLE_SDMA_ERR_EN		BIT(22)
77 #define		INTR_ENABLE_SDMA_TRIGG_EN	BIT(21)
78 #define		INTR_ENABLE_UNSUPP_CMD_EN	BIT(19)
79 #define		INTR_ENABLE_DDMA_TERR_EN	BIT(18)
80 #define		INTR_ENABLE_CDMA_TERR_EN	BIT(17)
81 #define		INTR_ENABLE_CDMA_IDLE_EN	BIT(16)
82 
83 /* Controller internal state. */
84 #define CTRL_STATUS				0x0118
85 #define		CTRL_STATUS_INIT_COMP		BIT(9)
86 #define		CTRL_STATUS_CTRL_BUSY		BIT(8)
87 
88 /* Command Engine threads state. */
89 #define TRD_STATUS				0x0120
90 
91 /* Command Engine interrupt thread error status. */
92 #define TRD_ERR_INT_STATUS			0x0128
93 /* Command Engine interrupt thread error enable. */
94 #define TRD_ERR_INT_STATUS_EN			0x0130
95 /* Command Engine interrupt thread complete status. */
96 #define TRD_COMP_INT_STATUS			0x0138
97 
98 /*
99  * Transfer config 0 register.
100  * Configures data transfer parameters.
101  */
102 #define TRAN_CFG_0				0x0400
103 /* Offset value from the beginning of the page. */
104 #define		TRAN_CFG_0_OFFSET		GENMASK(31, 16)
105 /* Numbers of sectors to transfer within singlNF device's page. */
106 #define		TRAN_CFG_0_SEC_CNT		GENMASK(7, 0)
107 
108 /*
109  * Transfer config 1 register.
110  * Configures data transfer parameters.
111  */
112 #define TRAN_CFG_1				0x0404
113 /* Size of last data sector. */
114 #define		TRAN_CFG_1_LAST_SEC_SIZE	GENMASK(31, 16)
115 /* Size of not-last data sector. */
116 #define		TRAN_CFG_1_SECTOR_SIZE		GENMASK(15, 0)
117 
118 /* ECC engine configuration register 0. */
119 #define ECC_CONFIG_0				0x0428
120 /* Correction strength. */
121 #define		ECC_CONFIG_0_CORR_STR		GENMASK(10, 8)
122 /* Enable erased pages detection mechanism. */
123 #define		ECC_CONFIG_0_ERASE_DET_EN	BIT(1)
124 /* Enable controller ECC check bits generation and correction. */
125 #define		ECC_CONFIG_0_ECC_EN		BIT(0)
126 
127 /* ECC engine configuration register 1. */
128 #define ECC_CONFIG_1				0x042C
129 
130 /* Multiplane settings register. */
131 #define MULTIPLANE_CFG				0x0434
132 /* Cache operation settings. */
133 #define CACHE_CFG				0x0438
134 
135 /* DMA settings register. */
136 #define DMA_SETINGS				0x043C
137 /* Enable SDMA error report on access unprepared slave DMA interface. */
138 #define		DMA_SETINGS_SDMA_ERR_RSP	BIT(17)
139 
140 /* Transferred data block size for the slave DMA module. */
141 #define SDMA_SIZE				0x0440
142 
143 /* Thread number associated with transferred data block
144  * for the slave DMA module.
145  */
146 #define SDMA_TRD_NUM				0x0444
147 /* Thread number mask. */
148 #define		SDMA_TRD_NUM_SDMA_TRD		GENMASK(2, 0)
149 
150 #define CONTROL_DATA_CTRL			0x0494
151 /* Thread number mask. */
152 #define		CONTROL_DATA_CTRL_SIZE		GENMASK(15, 0)
153 
154 #define CTRL_VERSION				0x800
155 #define		CTRL_VERSION_REV		GENMASK(7, 0)
156 
157 /* Available hardware features of the controller. */
158 #define CTRL_FEATURES				0x804
159 /* Support for NV-DDR2/3 work mode. */
160 #define		CTRL_FEATURES_NVDDR_2_3		BIT(28)
161 /* Support for NV-DDR work mode. */
162 #define		CTRL_FEATURES_NVDDR		BIT(27)
163 /* Support for asynchronous work mode. */
164 #define		CTRL_FEATURES_ASYNC		BIT(26)
165 /* Support for asynchronous work mode. */
166 #define		CTRL_FEATURES_N_BANKS		GENMASK(25, 24)
167 /* Slave and Master DMA data width. */
168 #define		CTRL_FEATURES_DMA_DWITH64	BIT(21)
169 /* Availability of Control Data feature.*/
170 #define		CTRL_FEATURES_CONTROL_DATA	BIT(10)
171 
172 /* BCH Engine identification register 0 - correction strengths. */
173 #define BCH_CFG_0				0x838
174 #define		BCH_CFG_0_CORR_CAP_0		GENMASK(7, 0)
175 #define		BCH_CFG_0_CORR_CAP_1		GENMASK(15, 8)
176 #define		BCH_CFG_0_CORR_CAP_2		GENMASK(23, 16)
177 #define		BCH_CFG_0_CORR_CAP_3		GENMASK(31, 24)
178 
179 /* BCH Engine identification register 1 - correction strengths. */
180 #define BCH_CFG_1				0x83C
181 #define		BCH_CFG_1_CORR_CAP_4		GENMASK(7, 0)
182 #define		BCH_CFG_1_CORR_CAP_5		GENMASK(15, 8)
183 #define		BCH_CFG_1_CORR_CAP_6		GENMASK(23, 16)
184 #define		BCH_CFG_1_CORR_CAP_7		GENMASK(31, 24)
185 
186 /* BCH Engine identification register 2 - sector sizes. */
187 #define BCH_CFG_2				0x840
188 #define		BCH_CFG_2_SECT_0		GENMASK(15, 0)
189 #define		BCH_CFG_2_SECT_1		GENMASK(31, 16)
190 
191 /* BCH Engine identification register 3. */
192 #define BCH_CFG_3				0x844
193 #define		BCH_CFG_3_METADATA_SIZE		GENMASK(23, 16)
194 
195 /* Ready/Busy# line status. */
196 #define RBN_SETINGS				0x1004
197 
198 /* Common settings. */
199 #define COMMON_SET				0x1008
200 /* 16 bit device connected to the NAND Flash interface. */
201 #define		COMMON_SET_DEVICE_16BIT		BIT(8)
202 
203 /* Skip_bytes registers. */
204 #define SKIP_BYTES_CONF				0x100C
205 #define		SKIP_BYTES_MARKER_VALUE		GENMASK(31, 16)
206 #define		SKIP_BYTES_NUM_OF_BYTES		GENMASK(7, 0)
207 
208 #define SKIP_BYTES_OFFSET			0x1010
209 #define		 SKIP_BYTES_OFFSET_VALUE	GENMASK(23, 0)
210 
211 /* Timings configuration. */
212 #define ASYNC_TOGGLE_TIMINGS			0x101c
213 #define		ASYNC_TOGGLE_TIMINGS_TRH	GENMASK(28, 24)
214 #define		ASYNC_TOGGLE_TIMINGS_TRP	GENMASK(20, 16)
215 #define		ASYNC_TOGGLE_TIMINGS_TWH	GENMASK(12, 8)
216 #define		ASYNC_TOGGLE_TIMINGS_TWP	GENMASK(4, 0)
217 
218 #define	TIMINGS0				0x1024
219 #define		TIMINGS0_TADL			GENMASK(31, 24)
220 #define		TIMINGS0_TCCS			GENMASK(23, 16)
221 #define		TIMINGS0_TWHR			GENMASK(15, 8)
222 #define		TIMINGS0_TRHW			GENMASK(7, 0)
223 
224 #define	TIMINGS1				0x1028
225 #define		TIMINGS1_TRHZ			GENMASK(31, 24)
226 #define		TIMINGS1_TWB			GENMASK(23, 16)
227 #define		TIMINGS1_TVDLY			GENMASK(7, 0)
228 
229 #define	TIMINGS2				0x102c
230 #define		TIMINGS2_TFEAT			GENMASK(25, 16)
231 #define		TIMINGS2_CS_HOLD_TIME		GENMASK(13, 8)
232 #define		TIMINGS2_CS_SETUP_TIME		GENMASK(5, 0)
233 
234 /* Configuration of the resynchronization of slave DLL of PHY. */
235 #define DLL_PHY_CTRL				0x1034
236 #define		DLL_PHY_CTRL_DLL_RST_N		BIT(24)
237 #define		DLL_PHY_CTRL_EXTENDED_WR_MODE	BIT(17)
238 #define		DLL_PHY_CTRL_EXTENDED_RD_MODE	BIT(16)
239 #define		DLL_PHY_CTRL_RS_HIGH_WAIT_CNT	GENMASK(11, 8)
240 #define		DLL_PHY_CTRL_RS_IDLE_CNT	GENMASK(7, 0)
241 
242 /* Register controlling DQ related timing. */
243 #define PHY_DQ_TIMING				0x2000
244 /* Register controlling DSQ related timing.  */
245 #define PHY_DQS_TIMING				0x2004
246 #define		PHY_DQS_TIMING_DQS_SEL_OE_END	GENMASK(3, 0)
247 #define		PHY_DQS_TIMING_PHONY_DQS_SEL	BIT(16)
248 #define		PHY_DQS_TIMING_USE_PHONY_DQS	BIT(20)
249 
250 /* Register controlling the gate and loopback control related timing. */
251 #define PHY_GATE_LPBK_CTRL			0x2008
252 #define		PHY_GATE_LPBK_CTRL_RDS		GENMASK(24, 19)
253 
254 /* Register holds the control for the master DLL logic. */
255 #define PHY_DLL_MASTER_CTRL			0x200C
256 #define		PHY_DLL_MASTER_CTRL_BYPASS_MODE	BIT(23)
257 
258 /* Register holds the control for the slave DLL logic. */
259 #define PHY_DLL_SLAVE_CTRL			0x2010
260 
261 /* This register handles the global control settings for the PHY. */
262 #define PHY_CTRL				0x2080
263 #define		PHY_CTRL_SDR_DQS		BIT(14)
264 #define		PHY_CTRL_PHONY_DQS		GENMASK(9, 4)
265 
266 /*
267  * This register handles the global control settings
268  * for the termination selects for reads.
269  */
270 #define PHY_TSEL				0x2084
271 
272 /* Generic command layout. */
273 #define GCMD_LAY_CS			GENMASK_ULL(11, 8)
274 /*
275  * This bit informs the minicotroller if it has to wait for tWB
276  * after sending the last CMD/ADDR/DATA in the sequence.
277  */
278 #define GCMD_LAY_TWB			BIT_ULL(6)
279 /* Type of generic instruction. */
280 #define GCMD_LAY_INSTR			GENMASK_ULL(5, 0)
281 
282 /* Generic CMD sequence type. */
283 #define		GCMD_LAY_INSTR_CMD	0
284 /* Generic ADDR sequence type. */
285 #define		GCMD_LAY_INSTR_ADDR	1
286 /* Generic data transfer sequence type. */
287 #define		GCMD_LAY_INSTR_DATA	2
288 
289 /* Input part of generic command type of input is command. */
290 #define GCMD_LAY_INPUT_CMD		GENMASK_ULL(23, 16)
291 
292 /* Generic command address sequence - address fields. */
293 #define GCMD_LAY_INPUT_ADDR		GENMASK_ULL(63, 16)
294 /* Generic command address sequence - address size. */
295 #define GCMD_LAY_INPUT_ADDR_SIZE	GENMASK_ULL(13, 11)
296 
297 /* Transfer direction field of generic command data sequence. */
298 #define GCMD_DIR			BIT_ULL(11)
299 /* Read transfer direction of generic command data sequence. */
300 #define		GCMD_DIR_READ		0
301 /* Write transfer direction of generic command data sequence. */
302 #define		GCMD_DIR_WRITE		1
303 
304 /* ECC enabled flag of generic command data sequence - ECC enabled. */
305 #define GCMD_ECC_EN			BIT_ULL(12)
306 /* Generic command data sequence - sector size. */
307 #define GCMD_SECT_SIZE			GENMASK_ULL(31, 16)
308 /* Generic command data sequence - sector count. */
309 #define GCMD_SECT_CNT			GENMASK_ULL(39, 32)
310 /* Generic command data sequence - last sector size. */
311 #define GCMD_LAST_SIZE			GENMASK_ULL(55, 40)
312 
313 /* CDMA descriptor fields. */
314 /* Erase command type of CDMA descriptor. */
315 #define CDMA_CT_ERASE		0x1000
316 /* Program page command type of CDMA descriptor. */
317 #define CDMA_CT_WR		0x2100
318 /* Read page command type of CDMA descriptor. */
319 #define CDMA_CT_RD		0x2200
320 
321 /* Flash pointer memory shift. */
322 #define CDMA_CFPTR_MEM_SHIFT	24
323 /* Flash pointer memory mask. */
324 #define CDMA_CFPTR_MEM		GENMASK(26, 24)
325 
326 /*
327  * Command DMA descriptor flags. If set causes issue interrupt after
328  * the completion of descriptor processing.
329  */
330 #define CDMA_CF_INT		BIT(8)
331 /*
332  * Command DMA descriptor flags - the next descriptor
333  * address field is valid and descriptor processing should continue.
334  */
335 #define CDMA_CF_CONT		BIT(9)
336 /* DMA master flag of command DMA descriptor. */
337 #define CDMA_CF_DMA_MASTER	BIT(10)
338 
339 /* Operation complete status of command descriptor. */
340 #define CDMA_CS_COMP		BIT(15)
341 /* Operation complete status of command descriptor. */
342 /* Command descriptor status - operation fail. */
343 #define CDMA_CS_FAIL		BIT(14)
344 /* Command descriptor status - page erased. */
345 #define CDMA_CS_ERP		BIT(11)
346 /* Command descriptor status - timeout occurred. */
347 #define CDMA_CS_TOUT		BIT(10)
348 /*
349  * Maximum amount of correction applied to one ECC sector.
350  * It is part of command descriptor status.
351  */
352 #define CDMA_CS_MAXERR		GENMASK(9, 2)
353 /* Command descriptor status - uncorrectable ECC error. */
354 #define CDMA_CS_UNCE		BIT(1)
355 /* Command descriptor status - descriptor error. */
356 #define CDMA_CS_ERR		BIT(0)
357 
358 /* Status of operation - OK. */
359 #define STAT_OK			0
360 /* Status of operation - FAIL. */
361 #define STAT_FAIL		2
362 /* Status of operation - uncorrectable ECC error. */
363 #define STAT_ECC_UNCORR		3
364 /* Status of operation - page erased. */
365 #define STAT_ERASED		5
366 /* Status of operation - correctable ECC error. */
367 #define STAT_ECC_CORR		6
368 /* Status of operation - unsuspected state. */
369 #define STAT_UNKNOWN		7
370 /* Status of operation - operation is not completed yet. */
371 #define STAT_BUSY		0xFF
372 
373 #define BCH_MAX_NUM_CORR_CAPS		8
374 #define BCH_MAX_NUM_SECTOR_SIZES	2
375 
376 struct cadence_nand_timings {
377 	u32 async_toggle_timings;
378 	u32 timings0;
379 	u32 timings1;
380 	u32 timings2;
381 	u32 dll_phy_ctrl;
382 	u32 phy_ctrl;
383 	u32 phy_dqs_timing;
384 	u32 phy_gate_lpbk_ctrl;
385 };
386 
387 /* Command DMA descriptor. */
388 struct cadence_nand_cdma_desc {
389 	/* Next descriptor address. */
390 	u64 next_pointer;
391 
392 	/* Flash address is a 32-bit address comprising of BANK and ROW ADDR. */
393 	u32 flash_pointer;
394 	/*field appears in HPNFC version 13*/
395 	u16 bank;
396 	u16 rsvd0;
397 
398 	/* Operation the controller needs to perform. */
399 	u16 command_type;
400 	u16 rsvd1;
401 	/* Flags for operation of this command. */
402 	u16 command_flags;
403 	u16 rsvd2;
404 
405 	/* System/host memory address required for data DMA commands. */
406 	u64 memory_pointer;
407 
408 	/* Status of operation. */
409 	u32 status;
410 	u32 rsvd3;
411 
412 	/* Address pointer to sync buffer location. */
413 	u64 sync_flag_pointer;
414 
415 	/* Controls the buffer sync mechanism. */
416 	u32 sync_arguments;
417 	u32 rsvd4;
418 
419 	/* Control data pointer. */
420 	u64 ctrl_data_ptr;
421 };
422 
423 /* Interrupt status. */
424 struct cadence_nand_irq_status {
425 	/* Thread operation complete status. */
426 	u32 trd_status;
427 	/* Thread operation error. */
428 	u32 trd_error;
429 	/* Controller status. */
430 	u32 status;
431 };
432 
433 /* Cadence NAND flash controller capabilities get from driver data. */
434 struct cadence_nand_dt_devdata {
435 	/* Skew value of the output signals of the NAND Flash interface. */
436 	u32 if_skew;
437 	/* It informs if slave DMA interface is connected to DMA engine. */
438 	unsigned int has_dma:1;
439 };
440 
441 /* Cadence NAND flash controller capabilities read from registers. */
442 struct cdns_nand_caps {
443 	/* Maximum number of banks supported by hardware. */
444 	u8 max_banks;
445 	/* Slave and Master DMA data width in bytes (4 or 8). */
446 	u8 data_dma_width;
447 	/* Control Data feature supported. */
448 	bool data_control_supp;
449 	/* Is PHY type DLL. */
450 	bool is_phy_type_dll;
451 };
452 
453 struct cdns_nand_ctrl {
454 	struct device *dev;
455 	struct nand_controller controller;
456 	struct cadence_nand_cdma_desc *cdma_desc;
457 	/* IP capability. */
458 	const struct cadence_nand_dt_devdata *caps1;
459 	struct cdns_nand_caps caps2;
460 	u8 ctrl_rev;
461 	dma_addr_t dma_cdma_desc;
462 	u8 *buf;
463 	u32 buf_size;
464 	u8 curr_corr_str_idx;
465 
466 	/* Register interface. */
467 	void __iomem *reg;
468 
469 	struct {
470 		void __iomem *virt;
471 		dma_addr_t dma;
472 	} io;
473 
474 	int irq;
475 	/* Interrupts that have happened. */
476 	struct cadence_nand_irq_status irq_status;
477 	/* Interrupts we are waiting for. */
478 	struct cadence_nand_irq_status irq_mask;
479 	struct completion complete;
480 	/* Protect irq_mask and irq_status. */
481 	spinlock_t irq_lock;
482 
483 	int ecc_strengths[BCH_MAX_NUM_CORR_CAPS];
484 	struct nand_ecc_step_info ecc_stepinfos[BCH_MAX_NUM_SECTOR_SIZES];
485 	struct nand_ecc_caps ecc_caps;
486 
487 	int curr_trans_type;
488 
489 	struct dma_chan *dmac;
490 
491 	u32 nf_clk_rate;
492 	/*
493 	 * Estimated Board delay. The value includes the total
494 	 * round trip delay for the signals and is used for deciding on values
495 	 * associated with data read capture.
496 	 */
497 	u32 board_delay;
498 
499 	struct nand_chip *selected_chip;
500 
501 	unsigned long assigned_cs;
502 	struct list_head chips;
503 	u8 bch_metadata_size;
504 };
505 
506 struct cdns_nand_chip {
507 	struct cadence_nand_timings timings;
508 	struct nand_chip chip;
509 	u8 nsels;
510 	struct list_head node;
511 
512 	/*
513 	 * part of oob area of NAND flash memory page.
514 	 * This part is available for user to read or write.
515 	 */
516 	u32 avail_oob_size;
517 
518 	/* Sector size. There are few sectors per mtd->writesize */
519 	u32 sector_size;
520 	u32 sector_count;
521 
522 	/* Offset of BBM. */
523 	u8 bbm_offs;
524 	/* Number of bytes reserved for BBM. */
525 	u8 bbm_len;
526 	/* ECC strength index. */
527 	u8 corr_str_idx;
528 
529 	u8 cs[];
530 };
531 
532 struct ecc_info {
533 	int (*calc_ecc_bytes)(int step_size, int strength);
534 	int max_step_size;
535 };
536 
537 static inline struct
538 cdns_nand_chip *to_cdns_nand_chip(struct nand_chip *chip)
539 {
540 	return container_of(chip, struct cdns_nand_chip, chip);
541 }
542 
543 static inline struct
544 cdns_nand_ctrl *to_cdns_nand_ctrl(struct nand_controller *controller)
545 {
546 	return container_of(controller, struct cdns_nand_ctrl, controller);
547 }
548 
549 static bool
550 cadence_nand_dma_buf_ok(struct cdns_nand_ctrl *cdns_ctrl, const void *buf,
551 			u32 buf_len)
552 {
553 	u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;
554 
555 	return buf && virt_addr_valid(buf) &&
556 		likely(IS_ALIGNED((uintptr_t)buf, data_dma_width)) &&
557 		likely(IS_ALIGNED(buf_len, DMA_DATA_SIZE_ALIGN));
558 }
559 
560 static int cadence_nand_wait_for_value(struct cdns_nand_ctrl *cdns_ctrl,
561 				       u32 reg_offset, u32 timeout_us,
562 				       u32 mask, bool is_clear)
563 {
564 	u32 val;
565 	int ret;
566 
567 	ret = readl_relaxed_poll_timeout(cdns_ctrl->reg + reg_offset,
568 					 val, !(val & mask) == is_clear,
569 					 10, timeout_us);
570 
571 	if (ret < 0) {
572 		dev_err(cdns_ctrl->dev,
573 			"Timeout while waiting for reg %x with mask %x is clear %d\n",
574 			reg_offset, mask, is_clear);
575 	}
576 
577 	return ret;
578 }
579 
580 static int cadence_nand_set_ecc_enable(struct cdns_nand_ctrl *cdns_ctrl,
581 				       bool enable)
582 {
583 	u32 reg;
584 
585 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
586 					1000000,
587 					CTRL_STATUS_CTRL_BUSY, true))
588 		return -ETIMEDOUT;
589 
590 	reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
591 
592 	if (enable)
593 		reg |= ECC_CONFIG_0_ECC_EN;
594 	else
595 		reg &= ~ECC_CONFIG_0_ECC_EN;
596 
597 	writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
598 
599 	return 0;
600 }
601 
602 static void cadence_nand_set_ecc_strength(struct cdns_nand_ctrl *cdns_ctrl,
603 					  u8 corr_str_idx)
604 {
605 	u32 reg;
606 
607 	if (cdns_ctrl->curr_corr_str_idx == corr_str_idx)
608 		return;
609 
610 	reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
611 	reg &= ~ECC_CONFIG_0_CORR_STR;
612 	reg |= FIELD_PREP(ECC_CONFIG_0_CORR_STR, corr_str_idx);
613 	writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
614 
615 	cdns_ctrl->curr_corr_str_idx = corr_str_idx;
616 }
617 
618 static int cadence_nand_get_ecc_strength_idx(struct cdns_nand_ctrl *cdns_ctrl,
619 					     u8 strength)
620 {
621 	int i, corr_str_idx = -1;
622 
623 	for (i = 0; i < BCH_MAX_NUM_CORR_CAPS; i++) {
624 		if (cdns_ctrl->ecc_strengths[i] == strength) {
625 			corr_str_idx = i;
626 			break;
627 		}
628 	}
629 
630 	return corr_str_idx;
631 }
632 
633 static int cadence_nand_set_skip_marker_val(struct cdns_nand_ctrl *cdns_ctrl,
634 					    u16 marker_value)
635 {
636 	u32 reg;
637 
638 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
639 					1000000,
640 					CTRL_STATUS_CTRL_BUSY, true))
641 		return -ETIMEDOUT;
642 
643 	reg = readl_relaxed(cdns_ctrl->reg + SKIP_BYTES_CONF);
644 	reg &= ~SKIP_BYTES_MARKER_VALUE;
645 	reg |= FIELD_PREP(SKIP_BYTES_MARKER_VALUE,
646 			  marker_value);
647 
648 	writel_relaxed(reg, cdns_ctrl->reg + SKIP_BYTES_CONF);
649 
650 	return 0;
651 }
652 
653 static int cadence_nand_set_skip_bytes_conf(struct cdns_nand_ctrl *cdns_ctrl,
654 					    u8 num_of_bytes,
655 					    u32 offset_value,
656 					    int enable)
657 {
658 	u32 reg, skip_bytes_offset;
659 
660 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
661 					1000000,
662 					CTRL_STATUS_CTRL_BUSY, true))
663 		return -ETIMEDOUT;
664 
665 	if (!enable) {
666 		num_of_bytes = 0;
667 		offset_value = 0;
668 	}
669 
670 	reg = readl_relaxed(cdns_ctrl->reg + SKIP_BYTES_CONF);
671 	reg &= ~SKIP_BYTES_NUM_OF_BYTES;
672 	reg |= FIELD_PREP(SKIP_BYTES_NUM_OF_BYTES,
673 			  num_of_bytes);
674 	skip_bytes_offset = FIELD_PREP(SKIP_BYTES_OFFSET_VALUE,
675 				       offset_value);
676 
677 	writel_relaxed(reg, cdns_ctrl->reg + SKIP_BYTES_CONF);
678 	writel_relaxed(skip_bytes_offset, cdns_ctrl->reg + SKIP_BYTES_OFFSET);
679 
680 	return 0;
681 }
682 
683 /* Functions enables/disables hardware detection of erased data */
684 static void cadence_nand_set_erase_detection(struct cdns_nand_ctrl *cdns_ctrl,
685 					     bool enable,
686 					     u8 bitflips_threshold)
687 {
688 	u32 reg;
689 
690 	reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
691 
692 	if (enable)
693 		reg |= ECC_CONFIG_0_ERASE_DET_EN;
694 	else
695 		reg &= ~ECC_CONFIG_0_ERASE_DET_EN;
696 
697 	writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
698 	writel_relaxed(bitflips_threshold, cdns_ctrl->reg + ECC_CONFIG_1);
699 }
700 
701 static int cadence_nand_set_access_width16(struct cdns_nand_ctrl *cdns_ctrl,
702 					   bool bit_bus16)
703 {
704 	u32 reg;
705 
706 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
707 					1000000,
708 					CTRL_STATUS_CTRL_BUSY, true))
709 		return -ETIMEDOUT;
710 
711 	reg = readl_relaxed(cdns_ctrl->reg + COMMON_SET);
712 
713 	if (!bit_bus16)
714 		reg &= ~COMMON_SET_DEVICE_16BIT;
715 	else
716 		reg |= COMMON_SET_DEVICE_16BIT;
717 	writel_relaxed(reg, cdns_ctrl->reg + COMMON_SET);
718 
719 	return 0;
720 }
721 
722 static void
723 cadence_nand_clear_interrupt(struct cdns_nand_ctrl *cdns_ctrl,
724 			     struct cadence_nand_irq_status *irq_status)
725 {
726 	writel_relaxed(irq_status->status, cdns_ctrl->reg + INTR_STATUS);
727 	writel_relaxed(irq_status->trd_status,
728 		       cdns_ctrl->reg + TRD_COMP_INT_STATUS);
729 	writel_relaxed(irq_status->trd_error,
730 		       cdns_ctrl->reg + TRD_ERR_INT_STATUS);
731 }
732 
733 static void
734 cadence_nand_read_int_status(struct cdns_nand_ctrl *cdns_ctrl,
735 			     struct cadence_nand_irq_status *irq_status)
736 {
737 	irq_status->status = readl_relaxed(cdns_ctrl->reg + INTR_STATUS);
738 	irq_status->trd_status = readl_relaxed(cdns_ctrl->reg
739 					       + TRD_COMP_INT_STATUS);
740 	irq_status->trd_error = readl_relaxed(cdns_ctrl->reg
741 					      + TRD_ERR_INT_STATUS);
742 }
743 
744 static u32 irq_detected(struct cdns_nand_ctrl *cdns_ctrl,
745 			struct cadence_nand_irq_status *irq_status)
746 {
747 	cadence_nand_read_int_status(cdns_ctrl, irq_status);
748 
749 	return irq_status->status || irq_status->trd_status ||
750 		irq_status->trd_error;
751 }
752 
753 static void cadence_nand_reset_irq(struct cdns_nand_ctrl *cdns_ctrl)
754 {
755 	unsigned long flags;
756 
757 	spin_lock_irqsave(&cdns_ctrl->irq_lock, flags);
758 	memset(&cdns_ctrl->irq_status, 0, sizeof(cdns_ctrl->irq_status));
759 	memset(&cdns_ctrl->irq_mask, 0, sizeof(cdns_ctrl->irq_mask));
760 	spin_unlock_irqrestore(&cdns_ctrl->irq_lock, flags);
761 }
762 
763 /*
764  * This is the interrupt service routine. It handles all interrupts
765  * sent to this device.
766  */
767 static irqreturn_t cadence_nand_isr(int irq, void *dev_id)
768 {
769 	struct cdns_nand_ctrl *cdns_ctrl = dev_id;
770 	struct cadence_nand_irq_status irq_status;
771 	irqreturn_t result = IRQ_NONE;
772 
773 	spin_lock(&cdns_ctrl->irq_lock);
774 
775 	if (irq_detected(cdns_ctrl, &irq_status)) {
776 		/* Handle interrupt. */
777 		/* First acknowledge it. */
778 		cadence_nand_clear_interrupt(cdns_ctrl, &irq_status);
779 		/* Status in the device context for someone to read. */
780 		cdns_ctrl->irq_status.status |= irq_status.status;
781 		cdns_ctrl->irq_status.trd_status |= irq_status.trd_status;
782 		cdns_ctrl->irq_status.trd_error |= irq_status.trd_error;
783 		/* Notify anyone who cares that it happened. */
784 		complete(&cdns_ctrl->complete);
785 		/* Tell the OS that we've handled this. */
786 		result = IRQ_HANDLED;
787 	}
788 	spin_unlock(&cdns_ctrl->irq_lock);
789 
790 	return result;
791 }
792 
793 static void cadence_nand_set_irq_mask(struct cdns_nand_ctrl *cdns_ctrl,
794 				      struct cadence_nand_irq_status *irq_mask)
795 {
796 	writel_relaxed(INTR_ENABLE_INTR_EN | irq_mask->status,
797 		       cdns_ctrl->reg + INTR_ENABLE);
798 
799 	writel_relaxed(irq_mask->trd_error,
800 		       cdns_ctrl->reg + TRD_ERR_INT_STATUS_EN);
801 }
802 
803 static void
804 cadence_nand_wait_for_irq(struct cdns_nand_ctrl *cdns_ctrl,
805 			  struct cadence_nand_irq_status *irq_mask,
806 			  struct cadence_nand_irq_status *irq_status)
807 {
808 	unsigned long timeout = msecs_to_jiffies(10000);
809 	unsigned long time_left;
810 
811 	time_left = wait_for_completion_timeout(&cdns_ctrl->complete,
812 						timeout);
813 
814 	*irq_status = cdns_ctrl->irq_status;
815 	if (time_left == 0) {
816 		/* Timeout error. */
817 		dev_err(cdns_ctrl->dev, "timeout occurred:\n");
818 		dev_err(cdns_ctrl->dev, "\tstatus = 0x%x, mask = 0x%x\n",
819 			irq_status->status, irq_mask->status);
820 		dev_err(cdns_ctrl->dev,
821 			"\ttrd_status = 0x%x, trd_status mask = 0x%x\n",
822 			irq_status->trd_status, irq_mask->trd_status);
823 		dev_err(cdns_ctrl->dev,
824 			"\t trd_error = 0x%x, trd_error mask = 0x%x\n",
825 			irq_status->trd_error, irq_mask->trd_error);
826 	}
827 }
828 
829 /* Execute generic command on NAND controller. */
830 static int cadence_nand_generic_cmd_send(struct cdns_nand_ctrl *cdns_ctrl,
831 					 u8 chip_nr,
832 					 u64 mini_ctrl_cmd)
833 {
834 	u32 mini_ctrl_cmd_l, mini_ctrl_cmd_h, reg;
835 
836 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_CS, chip_nr);
837 	mini_ctrl_cmd_l = mini_ctrl_cmd & 0xFFFFFFFF;
838 	mini_ctrl_cmd_h = mini_ctrl_cmd >> 32;
839 
840 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
841 					1000000,
842 					CTRL_STATUS_CTRL_BUSY, true))
843 		return -ETIMEDOUT;
844 
845 	cadence_nand_reset_irq(cdns_ctrl);
846 
847 	writel_relaxed(mini_ctrl_cmd_l, cdns_ctrl->reg + CMD_REG2);
848 	writel_relaxed(mini_ctrl_cmd_h, cdns_ctrl->reg + CMD_REG3);
849 
850 	/* Select generic command. */
851 	reg = FIELD_PREP(CMD_REG0_CT, CMD_REG0_CT_GEN);
852 	/* Thread number. */
853 	reg |= FIELD_PREP(CMD_REG0_TN, 0);
854 
855 	/* Issue command. */
856 	writel_relaxed(reg, cdns_ctrl->reg + CMD_REG0);
857 
858 	return 0;
859 }
860 
861 /* Wait for data on slave DMA interface. */
862 static int cadence_nand_wait_on_sdma(struct cdns_nand_ctrl *cdns_ctrl,
863 				     u8 *out_sdma_trd,
864 				     u32 *out_sdma_size)
865 {
866 	struct cadence_nand_irq_status irq_mask, irq_status;
867 
868 	irq_mask.trd_status = 0;
869 	irq_mask.trd_error = 0;
870 	irq_mask.status = INTR_STATUS_SDMA_TRIGG
871 		| INTR_STATUS_SDMA_ERR
872 		| INTR_STATUS_UNSUPP_CMD;
873 
874 	cadence_nand_set_irq_mask(cdns_ctrl, &irq_mask);
875 	cadence_nand_wait_for_irq(cdns_ctrl, &irq_mask, &irq_status);
876 	if (irq_status.status == 0) {
877 		dev_err(cdns_ctrl->dev, "Timeout while waiting for SDMA\n");
878 		return -ETIMEDOUT;
879 	}
880 
881 	if (irq_status.status & INTR_STATUS_SDMA_TRIGG) {
882 		*out_sdma_size = readl_relaxed(cdns_ctrl->reg + SDMA_SIZE);
883 		*out_sdma_trd  = readl_relaxed(cdns_ctrl->reg + SDMA_TRD_NUM);
884 		*out_sdma_trd =
885 			FIELD_GET(SDMA_TRD_NUM_SDMA_TRD, *out_sdma_trd);
886 	} else {
887 		dev_err(cdns_ctrl->dev, "SDMA error - irq_status %x\n",
888 			irq_status.status);
889 		return -EIO;
890 	}
891 
892 	return 0;
893 }
894 
895 static void cadence_nand_get_caps(struct cdns_nand_ctrl *cdns_ctrl)
896 {
897 	u32  reg;
898 
899 	reg = readl_relaxed(cdns_ctrl->reg + CTRL_FEATURES);
900 
901 	cdns_ctrl->caps2.max_banks = 1 << FIELD_GET(CTRL_FEATURES_N_BANKS, reg);
902 
903 	if (FIELD_GET(CTRL_FEATURES_DMA_DWITH64, reg))
904 		cdns_ctrl->caps2.data_dma_width = 8;
905 	else
906 		cdns_ctrl->caps2.data_dma_width = 4;
907 
908 	if (reg & CTRL_FEATURES_CONTROL_DATA)
909 		cdns_ctrl->caps2.data_control_supp = true;
910 
911 	if (reg & (CTRL_FEATURES_NVDDR_2_3
912 		   | CTRL_FEATURES_NVDDR))
913 		cdns_ctrl->caps2.is_phy_type_dll = true;
914 }
915 
916 /* Prepare CDMA descriptor. */
917 static void
918 cadence_nand_cdma_desc_prepare(struct cdns_nand_ctrl *cdns_ctrl,
919 			       char nf_mem, u32 flash_ptr, dma_addr_t mem_ptr,
920 				   dma_addr_t ctrl_data_ptr, u16 ctype)
921 {
922 	struct cadence_nand_cdma_desc *cdma_desc = cdns_ctrl->cdma_desc;
923 
924 	memset(cdma_desc, 0, sizeof(struct cadence_nand_cdma_desc));
925 
926 	/* Set fields for one descriptor. */
927 	cdma_desc->flash_pointer = flash_ptr;
928 	if (cdns_ctrl->ctrl_rev >= 13)
929 		cdma_desc->bank = nf_mem;
930 	else
931 		cdma_desc->flash_pointer |= (nf_mem << CDMA_CFPTR_MEM_SHIFT);
932 
933 	cdma_desc->command_flags |= CDMA_CF_DMA_MASTER;
934 	cdma_desc->command_flags  |= CDMA_CF_INT;
935 
936 	cdma_desc->memory_pointer = mem_ptr;
937 	cdma_desc->status = 0;
938 	cdma_desc->sync_flag_pointer = 0;
939 	cdma_desc->sync_arguments = 0;
940 
941 	cdma_desc->command_type = ctype;
942 	cdma_desc->ctrl_data_ptr = ctrl_data_ptr;
943 }
944 
945 static u8 cadence_nand_check_desc_error(struct cdns_nand_ctrl *cdns_ctrl,
946 					u32 desc_status)
947 {
948 	if (desc_status & CDMA_CS_ERP)
949 		return STAT_ERASED;
950 
951 	if (desc_status & CDMA_CS_UNCE)
952 		return STAT_ECC_UNCORR;
953 
954 	if (desc_status & CDMA_CS_ERR) {
955 		dev_err(cdns_ctrl->dev, ":CDMA desc error flag detected.\n");
956 		return STAT_FAIL;
957 	}
958 
959 	if (FIELD_GET(CDMA_CS_MAXERR, desc_status))
960 		return STAT_ECC_CORR;
961 
962 	return STAT_FAIL;
963 }
964 
965 static int cadence_nand_cdma_finish(struct cdns_nand_ctrl *cdns_ctrl)
966 {
967 	struct cadence_nand_cdma_desc *desc_ptr = cdns_ctrl->cdma_desc;
968 	u8 status = STAT_BUSY;
969 
970 	if (desc_ptr->status & CDMA_CS_FAIL) {
971 		status = cadence_nand_check_desc_error(cdns_ctrl,
972 						       desc_ptr->status);
973 		dev_err(cdns_ctrl->dev, ":CDMA error %x\n", desc_ptr->status);
974 	} else if (desc_ptr->status & CDMA_CS_COMP) {
975 		/* Descriptor finished with no errors. */
976 		if (desc_ptr->command_flags & CDMA_CF_CONT) {
977 			dev_info(cdns_ctrl->dev, "DMA unsupported flag is set");
978 			status = STAT_UNKNOWN;
979 		} else {
980 			/* Last descriptor.  */
981 			status = STAT_OK;
982 		}
983 	}
984 
985 	return status;
986 }
987 
988 static int cadence_nand_cdma_send(struct cdns_nand_ctrl *cdns_ctrl,
989 				  u8 thread)
990 {
991 	u32 reg;
992 	int status;
993 
994 	/* Wait for thread ready. */
995 	status = cadence_nand_wait_for_value(cdns_ctrl, TRD_STATUS,
996 					     1000000,
997 					     BIT(thread), true);
998 	if (status)
999 		return status;
1000 
1001 	cadence_nand_reset_irq(cdns_ctrl);
1002 	reinit_completion(&cdns_ctrl->complete);
1003 
1004 	writel_relaxed((u32)cdns_ctrl->dma_cdma_desc,
1005 		       cdns_ctrl->reg + CMD_REG2);
1006 	writel_relaxed(0, cdns_ctrl->reg + CMD_REG3);
1007 
1008 	/* Select CDMA mode. */
1009 	reg = FIELD_PREP(CMD_REG0_CT, CMD_REG0_CT_CDMA);
1010 	/* Thread number. */
1011 	reg |= FIELD_PREP(CMD_REG0_TN, thread);
1012 	/* Issue command. */
1013 	writel_relaxed(reg, cdns_ctrl->reg + CMD_REG0);
1014 
1015 	return 0;
1016 }
1017 
1018 /* Send SDMA command and wait for finish. */
1019 static u32
1020 cadence_nand_cdma_send_and_wait(struct cdns_nand_ctrl *cdns_ctrl,
1021 				u8 thread)
1022 {
1023 	struct cadence_nand_irq_status irq_mask, irq_status = {0};
1024 	int status;
1025 
1026 	irq_mask.trd_status = BIT(thread);
1027 	irq_mask.trd_error = BIT(thread);
1028 	irq_mask.status = INTR_STATUS_CDMA_TERR;
1029 
1030 	cadence_nand_set_irq_mask(cdns_ctrl, &irq_mask);
1031 
1032 	status = cadence_nand_cdma_send(cdns_ctrl, thread);
1033 	if (status)
1034 		return status;
1035 
1036 	cadence_nand_wait_for_irq(cdns_ctrl, &irq_mask, &irq_status);
1037 
1038 	if (irq_status.status == 0 && irq_status.trd_status == 0 &&
1039 	    irq_status.trd_error == 0) {
1040 		dev_err(cdns_ctrl->dev, "CDMA command timeout\n");
1041 		return -ETIMEDOUT;
1042 	}
1043 	if (irq_status.status & irq_mask.status) {
1044 		dev_err(cdns_ctrl->dev, "CDMA command failed\n");
1045 		return -EIO;
1046 	}
1047 
1048 	return 0;
1049 }
1050 
1051 /*
1052  * ECC size depends on configured ECC strength and on maximum supported
1053  * ECC step size.
1054  */
1055 static int cadence_nand_calc_ecc_bytes(int max_step_size, int strength)
1056 {
1057 	int nbytes = DIV_ROUND_UP(fls(8 * max_step_size) * strength, 8);
1058 
1059 	return ALIGN(nbytes, 2);
1060 }
1061 
1062 #define CADENCE_NAND_CALC_ECC_BYTES(max_step_size) \
1063 	static int \
1064 	cadence_nand_calc_ecc_bytes_##max_step_size(int step_size, \
1065 						    int strength)\
1066 	{\
1067 		return cadence_nand_calc_ecc_bytes(max_step_size, strength);\
1068 	}
1069 
1070 CADENCE_NAND_CALC_ECC_BYTES(256)
1071 CADENCE_NAND_CALC_ECC_BYTES(512)
1072 CADENCE_NAND_CALC_ECC_BYTES(1024)
1073 CADENCE_NAND_CALC_ECC_BYTES(2048)
1074 CADENCE_NAND_CALC_ECC_BYTES(4096)
1075 
1076 /* Function reads BCH capabilities. */
1077 static int cadence_nand_read_bch_caps(struct cdns_nand_ctrl *cdns_ctrl)
1078 {
1079 	struct nand_ecc_caps *ecc_caps = &cdns_ctrl->ecc_caps;
1080 	int max_step_size = 0, nstrengths, i;
1081 	u32 reg;
1082 
1083 	reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_3);
1084 	cdns_ctrl->bch_metadata_size = FIELD_GET(BCH_CFG_3_METADATA_SIZE, reg);
1085 	if (cdns_ctrl->bch_metadata_size < 4) {
1086 		dev_err(cdns_ctrl->dev,
1087 			"Driver needs at least 4 bytes of BCH meta data\n");
1088 		return -EIO;
1089 	}
1090 
1091 	reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_0);
1092 	cdns_ctrl->ecc_strengths[0] = FIELD_GET(BCH_CFG_0_CORR_CAP_0, reg);
1093 	cdns_ctrl->ecc_strengths[1] = FIELD_GET(BCH_CFG_0_CORR_CAP_1, reg);
1094 	cdns_ctrl->ecc_strengths[2] = FIELD_GET(BCH_CFG_0_CORR_CAP_2, reg);
1095 	cdns_ctrl->ecc_strengths[3] = FIELD_GET(BCH_CFG_0_CORR_CAP_3, reg);
1096 
1097 	reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_1);
1098 	cdns_ctrl->ecc_strengths[4] = FIELD_GET(BCH_CFG_1_CORR_CAP_4, reg);
1099 	cdns_ctrl->ecc_strengths[5] = FIELD_GET(BCH_CFG_1_CORR_CAP_5, reg);
1100 	cdns_ctrl->ecc_strengths[6] = FIELD_GET(BCH_CFG_1_CORR_CAP_6, reg);
1101 	cdns_ctrl->ecc_strengths[7] = FIELD_GET(BCH_CFG_1_CORR_CAP_7, reg);
1102 
1103 	reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_2);
1104 	cdns_ctrl->ecc_stepinfos[0].stepsize =
1105 		FIELD_GET(BCH_CFG_2_SECT_0, reg);
1106 
1107 	cdns_ctrl->ecc_stepinfos[1].stepsize =
1108 		FIELD_GET(BCH_CFG_2_SECT_1, reg);
1109 
1110 	nstrengths = 0;
1111 	for (i = 0; i < BCH_MAX_NUM_CORR_CAPS; i++) {
1112 		if (cdns_ctrl->ecc_strengths[i] != 0)
1113 			nstrengths++;
1114 	}
1115 
1116 	ecc_caps->nstepinfos = 0;
1117 	for (i = 0; i < BCH_MAX_NUM_SECTOR_SIZES; i++) {
1118 		/* ECC strengths are common for all step infos. */
1119 		cdns_ctrl->ecc_stepinfos[i].nstrengths = nstrengths;
1120 		cdns_ctrl->ecc_stepinfos[i].strengths =
1121 			cdns_ctrl->ecc_strengths;
1122 
1123 		if (cdns_ctrl->ecc_stepinfos[i].stepsize != 0)
1124 			ecc_caps->nstepinfos++;
1125 
1126 		if (cdns_ctrl->ecc_stepinfos[i].stepsize > max_step_size)
1127 			max_step_size = cdns_ctrl->ecc_stepinfos[i].stepsize;
1128 	}
1129 	ecc_caps->stepinfos = &cdns_ctrl->ecc_stepinfos[0];
1130 
1131 	switch (max_step_size) {
1132 	case 256:
1133 		ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_256;
1134 		break;
1135 	case 512:
1136 		ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_512;
1137 		break;
1138 	case 1024:
1139 		ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_1024;
1140 		break;
1141 	case 2048:
1142 		ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_2048;
1143 		break;
1144 	case 4096:
1145 		ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_4096;
1146 		break;
1147 	default:
1148 		dev_err(cdns_ctrl->dev,
1149 			"Unsupported sector size(ecc step size) %d\n",
1150 			max_step_size);
1151 		return -EIO;
1152 	}
1153 
1154 	return 0;
1155 }
1156 
1157 /* Hardware initialization. */
1158 static int cadence_nand_hw_init(struct cdns_nand_ctrl *cdns_ctrl)
1159 {
1160 	int status;
1161 	u32 reg;
1162 
1163 	status = cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
1164 					     1000000,
1165 					     CTRL_STATUS_INIT_COMP, false);
1166 	if (status)
1167 		return status;
1168 
1169 	reg = readl_relaxed(cdns_ctrl->reg + CTRL_VERSION);
1170 	cdns_ctrl->ctrl_rev = FIELD_GET(CTRL_VERSION_REV, reg);
1171 
1172 	dev_info(cdns_ctrl->dev,
1173 		 "%s: cadence nand controller version reg %x\n",
1174 		 __func__, reg);
1175 
1176 	/* Disable cache and multiplane. */
1177 	writel_relaxed(0, cdns_ctrl->reg + MULTIPLANE_CFG);
1178 	writel_relaxed(0, cdns_ctrl->reg + CACHE_CFG);
1179 
1180 	/* Clear all interrupts. */
1181 	writel_relaxed(0xFFFFFFFF, cdns_ctrl->reg + INTR_STATUS);
1182 
1183 	cadence_nand_get_caps(cdns_ctrl);
1184 	if (cadence_nand_read_bch_caps(cdns_ctrl))
1185 		return -EIO;
1186 
1187 	/*
1188 	 * Set IO width access to 8.
1189 	 * It is because during SW device discovering width access
1190 	 * is expected to be 8.
1191 	 */
1192 	status = cadence_nand_set_access_width16(cdns_ctrl, false);
1193 
1194 	return status;
1195 }
1196 
1197 #define TT_MAIN_OOB_AREAS	2
1198 #define TT_RAW_PAGE		3
1199 #define TT_BBM			4
1200 #define TT_MAIN_OOB_AREA_EXT	5
1201 
1202 /* Prepare size of data to transfer. */
1203 static void
1204 cadence_nand_prepare_data_size(struct nand_chip *chip,
1205 			       int transfer_type)
1206 {
1207 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1208 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1209 	struct mtd_info *mtd = nand_to_mtd(chip);
1210 	u32 sec_size = 0, offset = 0, sec_cnt = 1;
1211 	u32 last_sec_size = cdns_chip->sector_size;
1212 	u32 data_ctrl_size = 0;
1213 	u32 reg = 0;
1214 
1215 	if (cdns_ctrl->curr_trans_type == transfer_type)
1216 		return;
1217 
1218 	switch (transfer_type) {
1219 	case TT_MAIN_OOB_AREA_EXT:
1220 		sec_cnt = cdns_chip->sector_count;
1221 		sec_size = cdns_chip->sector_size;
1222 		data_ctrl_size = cdns_chip->avail_oob_size;
1223 		break;
1224 	case TT_MAIN_OOB_AREAS:
1225 		sec_cnt = cdns_chip->sector_count;
1226 		last_sec_size = cdns_chip->sector_size
1227 			+ cdns_chip->avail_oob_size;
1228 		sec_size = cdns_chip->sector_size;
1229 		break;
1230 	case TT_RAW_PAGE:
1231 		last_sec_size = mtd->writesize + mtd->oobsize;
1232 		break;
1233 	case TT_BBM:
1234 		offset = mtd->writesize + cdns_chip->bbm_offs;
1235 		last_sec_size = 8;
1236 		break;
1237 	}
1238 
1239 	reg = 0;
1240 	reg |= FIELD_PREP(TRAN_CFG_0_OFFSET, offset);
1241 	reg |= FIELD_PREP(TRAN_CFG_0_SEC_CNT, sec_cnt);
1242 	writel_relaxed(reg, cdns_ctrl->reg + TRAN_CFG_0);
1243 
1244 	reg = 0;
1245 	reg |= FIELD_PREP(TRAN_CFG_1_LAST_SEC_SIZE, last_sec_size);
1246 	reg |= FIELD_PREP(TRAN_CFG_1_SECTOR_SIZE, sec_size);
1247 	writel_relaxed(reg, cdns_ctrl->reg + TRAN_CFG_1);
1248 
1249 	if (cdns_ctrl->caps2.data_control_supp) {
1250 		reg = readl_relaxed(cdns_ctrl->reg + CONTROL_DATA_CTRL);
1251 		reg &= ~CONTROL_DATA_CTRL_SIZE;
1252 		reg |= FIELD_PREP(CONTROL_DATA_CTRL_SIZE, data_ctrl_size);
1253 		writel_relaxed(reg, cdns_ctrl->reg + CONTROL_DATA_CTRL);
1254 	}
1255 
1256 	cdns_ctrl->curr_trans_type = transfer_type;
1257 }
1258 
1259 static int
1260 cadence_nand_cdma_transfer(struct cdns_nand_ctrl *cdns_ctrl, u8 chip_nr,
1261 			   int page, void *buf, void *ctrl_dat, u32 buf_size,
1262 			   u32 ctrl_dat_size, enum dma_data_direction dir,
1263 			   bool with_ecc)
1264 {
1265 	dma_addr_t dma_buf, dma_ctrl_dat = 0;
1266 	u8 thread_nr = chip_nr;
1267 	int status;
1268 	u16 ctype;
1269 
1270 	if (dir == DMA_FROM_DEVICE)
1271 		ctype = CDMA_CT_RD;
1272 	else
1273 		ctype = CDMA_CT_WR;
1274 
1275 	cadence_nand_set_ecc_enable(cdns_ctrl, with_ecc);
1276 
1277 	dma_buf = dma_map_single(cdns_ctrl->dev, buf, buf_size, dir);
1278 	if (dma_mapping_error(cdns_ctrl->dev, dma_buf)) {
1279 		dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1280 		return -EIO;
1281 	}
1282 
1283 	if (ctrl_dat && ctrl_dat_size) {
1284 		dma_ctrl_dat = dma_map_single(cdns_ctrl->dev, ctrl_dat,
1285 					      ctrl_dat_size, dir);
1286 		if (dma_mapping_error(cdns_ctrl->dev, dma_ctrl_dat)) {
1287 			dma_unmap_single(cdns_ctrl->dev, dma_buf,
1288 					 buf_size, dir);
1289 			dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1290 			return -EIO;
1291 		}
1292 	}
1293 
1294 	cadence_nand_cdma_desc_prepare(cdns_ctrl, chip_nr, page,
1295 				       dma_buf, dma_ctrl_dat, ctype);
1296 
1297 	status = cadence_nand_cdma_send_and_wait(cdns_ctrl, thread_nr);
1298 
1299 	dma_unmap_single(cdns_ctrl->dev, dma_buf,
1300 			 buf_size, dir);
1301 
1302 	if (ctrl_dat && ctrl_dat_size)
1303 		dma_unmap_single(cdns_ctrl->dev, dma_ctrl_dat,
1304 				 ctrl_dat_size, dir);
1305 	if (status)
1306 		return status;
1307 
1308 	return cadence_nand_cdma_finish(cdns_ctrl);
1309 }
1310 
1311 static void cadence_nand_set_timings(struct cdns_nand_ctrl *cdns_ctrl,
1312 				     struct cadence_nand_timings *t)
1313 {
1314 	writel_relaxed(t->async_toggle_timings,
1315 		       cdns_ctrl->reg + ASYNC_TOGGLE_TIMINGS);
1316 	writel_relaxed(t->timings0, cdns_ctrl->reg + TIMINGS0);
1317 	writel_relaxed(t->timings1, cdns_ctrl->reg + TIMINGS1);
1318 	writel_relaxed(t->timings2, cdns_ctrl->reg + TIMINGS2);
1319 
1320 	if (cdns_ctrl->caps2.is_phy_type_dll)
1321 		writel_relaxed(t->dll_phy_ctrl, cdns_ctrl->reg + DLL_PHY_CTRL);
1322 
1323 	writel_relaxed(t->phy_ctrl, cdns_ctrl->reg + PHY_CTRL);
1324 
1325 	if (cdns_ctrl->caps2.is_phy_type_dll) {
1326 		writel_relaxed(0, cdns_ctrl->reg + PHY_TSEL);
1327 		writel_relaxed(2, cdns_ctrl->reg + PHY_DQ_TIMING);
1328 		writel_relaxed(t->phy_dqs_timing,
1329 			       cdns_ctrl->reg + PHY_DQS_TIMING);
1330 		writel_relaxed(t->phy_gate_lpbk_ctrl,
1331 			       cdns_ctrl->reg + PHY_GATE_LPBK_CTRL);
1332 		writel_relaxed(PHY_DLL_MASTER_CTRL_BYPASS_MODE,
1333 			       cdns_ctrl->reg + PHY_DLL_MASTER_CTRL);
1334 		writel_relaxed(0, cdns_ctrl->reg + PHY_DLL_SLAVE_CTRL);
1335 	}
1336 }
1337 
1338 static int cadence_nand_select_target(struct nand_chip *chip)
1339 {
1340 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1341 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1342 
1343 	if (chip == cdns_ctrl->selected_chip)
1344 		return 0;
1345 
1346 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
1347 					1000000,
1348 					CTRL_STATUS_CTRL_BUSY, true))
1349 		return -ETIMEDOUT;
1350 
1351 	cadence_nand_set_timings(cdns_ctrl, &cdns_chip->timings);
1352 
1353 	cadence_nand_set_ecc_strength(cdns_ctrl,
1354 				      cdns_chip->corr_str_idx);
1355 
1356 	cadence_nand_set_erase_detection(cdns_ctrl, true,
1357 					 chip->ecc.strength);
1358 
1359 	cdns_ctrl->curr_trans_type = -1;
1360 	cdns_ctrl->selected_chip = chip;
1361 
1362 	return 0;
1363 }
1364 
1365 static int cadence_nand_erase(struct nand_chip *chip, u32 page)
1366 {
1367 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1368 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1369 	int status;
1370 	u8 thread_nr = cdns_chip->cs[chip->cur_cs];
1371 
1372 	cadence_nand_cdma_desc_prepare(cdns_ctrl,
1373 				       cdns_chip->cs[chip->cur_cs],
1374 				       page, 0, 0,
1375 				       CDMA_CT_ERASE);
1376 	status = cadence_nand_cdma_send_and_wait(cdns_ctrl, thread_nr);
1377 	if (status) {
1378 		dev_err(cdns_ctrl->dev, "erase operation failed\n");
1379 		return -EIO;
1380 	}
1381 
1382 	status = cadence_nand_cdma_finish(cdns_ctrl);
1383 	if (status)
1384 		return status;
1385 
1386 	return 0;
1387 }
1388 
1389 static int cadence_nand_read_bbm(struct nand_chip *chip, int page, u8 *buf)
1390 {
1391 	int status;
1392 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1393 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1394 	struct mtd_info *mtd = nand_to_mtd(chip);
1395 
1396 	cadence_nand_prepare_data_size(chip, TT_BBM);
1397 
1398 	cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1399 
1400 	/*
1401 	 * Read only bad block marker from offset
1402 	 * defined by a memory manufacturer.
1403 	 */
1404 	status = cadence_nand_cdma_transfer(cdns_ctrl,
1405 					    cdns_chip->cs[chip->cur_cs],
1406 					    page, cdns_ctrl->buf, NULL,
1407 					    mtd->oobsize,
1408 					    0, DMA_FROM_DEVICE, false);
1409 	if (status) {
1410 		dev_err(cdns_ctrl->dev, "read BBM failed\n");
1411 		return -EIO;
1412 	}
1413 
1414 	memcpy(buf + cdns_chip->bbm_offs, cdns_ctrl->buf, cdns_chip->bbm_len);
1415 
1416 	return 0;
1417 }
1418 
1419 static int cadence_nand_write_page(struct nand_chip *chip,
1420 				   const u8 *buf, int oob_required,
1421 				   int page)
1422 {
1423 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1424 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1425 	struct mtd_info *mtd = nand_to_mtd(chip);
1426 	int status;
1427 	u16 marker_val = 0xFFFF;
1428 
1429 	status = cadence_nand_select_target(chip);
1430 	if (status)
1431 		return status;
1432 
1433 	cadence_nand_set_skip_bytes_conf(cdns_ctrl, cdns_chip->bbm_len,
1434 					 mtd->writesize
1435 					 + cdns_chip->bbm_offs,
1436 					 1);
1437 
1438 	if (oob_required) {
1439 		marker_val = *(u16 *)(chip->oob_poi
1440 				      + cdns_chip->bbm_offs);
1441 	} else {
1442 		/* Set oob data to 0xFF. */
1443 		memset(cdns_ctrl->buf + mtd->writesize, 0xFF,
1444 		       cdns_chip->avail_oob_size);
1445 	}
1446 
1447 	cadence_nand_set_skip_marker_val(cdns_ctrl, marker_val);
1448 
1449 	cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREA_EXT);
1450 
1451 	if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, mtd->writesize) &&
1452 	    cdns_ctrl->caps2.data_control_supp) {
1453 		u8 *oob;
1454 
1455 		if (oob_required)
1456 			oob = chip->oob_poi;
1457 		else
1458 			oob = cdns_ctrl->buf + mtd->writesize;
1459 
1460 		status = cadence_nand_cdma_transfer(cdns_ctrl,
1461 						    cdns_chip->cs[chip->cur_cs],
1462 						    page, (void *)buf, oob,
1463 						    mtd->writesize,
1464 						    cdns_chip->avail_oob_size,
1465 						    DMA_TO_DEVICE, true);
1466 		if (status) {
1467 			dev_err(cdns_ctrl->dev, "write page failed\n");
1468 			return -EIO;
1469 		}
1470 
1471 		return 0;
1472 	}
1473 
1474 	if (oob_required) {
1475 		/* Transfer the data to the oob area. */
1476 		memcpy(cdns_ctrl->buf + mtd->writesize, chip->oob_poi,
1477 		       cdns_chip->avail_oob_size);
1478 	}
1479 
1480 	memcpy(cdns_ctrl->buf, buf, mtd->writesize);
1481 
1482 	cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREAS);
1483 
1484 	return cadence_nand_cdma_transfer(cdns_ctrl,
1485 					  cdns_chip->cs[chip->cur_cs],
1486 					  page, cdns_ctrl->buf, NULL,
1487 					  mtd->writesize
1488 					  + cdns_chip->avail_oob_size,
1489 					  0, DMA_TO_DEVICE, true);
1490 }
1491 
1492 static int cadence_nand_write_oob(struct nand_chip *chip, int page)
1493 {
1494 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1495 	struct mtd_info *mtd = nand_to_mtd(chip);
1496 
1497 	memset(cdns_ctrl->buf, 0xFF, mtd->writesize);
1498 
1499 	return cadence_nand_write_page(chip, cdns_ctrl->buf, 1, page);
1500 }
1501 
1502 static int cadence_nand_write_page_raw(struct nand_chip *chip,
1503 				       const u8 *buf, int oob_required,
1504 				       int page)
1505 {
1506 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1507 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1508 	struct mtd_info *mtd = nand_to_mtd(chip);
1509 	int writesize = mtd->writesize;
1510 	int oobsize = mtd->oobsize;
1511 	int ecc_steps = chip->ecc.steps;
1512 	int ecc_size = chip->ecc.size;
1513 	int ecc_bytes = chip->ecc.bytes;
1514 	void *tmp_buf = cdns_ctrl->buf;
1515 	int oob_skip = cdns_chip->bbm_len;
1516 	size_t size = writesize + oobsize;
1517 	int i, pos, len;
1518 	int status = 0;
1519 
1520 	status = cadence_nand_select_target(chip);
1521 	if (status)
1522 		return status;
1523 
1524 	/*
1525 	 * Fill the buffer with 0xff first except the full page transfer.
1526 	 * This simplifies the logic.
1527 	 */
1528 	if (!buf || !oob_required)
1529 		memset(tmp_buf, 0xff, size);
1530 
1531 	cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1532 
1533 	/* Arrange the buffer for syndrome payload/ecc layout. */
1534 	if (buf) {
1535 		for (i = 0; i < ecc_steps; i++) {
1536 			pos = i * (ecc_size + ecc_bytes);
1537 			len = ecc_size;
1538 
1539 			if (pos >= writesize)
1540 				pos += oob_skip;
1541 			else if (pos + len > writesize)
1542 				len = writesize - pos;
1543 
1544 			memcpy(tmp_buf + pos, buf, len);
1545 			buf += len;
1546 			if (len < ecc_size) {
1547 				len = ecc_size - len;
1548 				memcpy(tmp_buf + writesize + oob_skip, buf,
1549 				       len);
1550 				buf += len;
1551 			}
1552 		}
1553 	}
1554 
1555 	if (oob_required) {
1556 		const u8 *oob = chip->oob_poi;
1557 		u32 oob_data_offset = (cdns_chip->sector_count - 1) *
1558 			(cdns_chip->sector_size + chip->ecc.bytes)
1559 			+ cdns_chip->sector_size + oob_skip;
1560 
1561 		/* BBM at the beginning of the OOB area. */
1562 		memcpy(tmp_buf + writesize, oob, oob_skip);
1563 
1564 		/* OOB free. */
1565 		memcpy(tmp_buf + oob_data_offset, oob,
1566 		       cdns_chip->avail_oob_size);
1567 		oob += cdns_chip->avail_oob_size;
1568 
1569 		/* OOB ECC. */
1570 		for (i = 0; i < ecc_steps; i++) {
1571 			pos = ecc_size + i * (ecc_size + ecc_bytes);
1572 			if (i == (ecc_steps - 1))
1573 				pos += cdns_chip->avail_oob_size;
1574 
1575 			len = ecc_bytes;
1576 
1577 			if (pos >= writesize)
1578 				pos += oob_skip;
1579 			else if (pos + len > writesize)
1580 				len = writesize - pos;
1581 
1582 			memcpy(tmp_buf + pos, oob, len);
1583 			oob += len;
1584 			if (len < ecc_bytes) {
1585 				len = ecc_bytes - len;
1586 				memcpy(tmp_buf + writesize + oob_skip, oob,
1587 				       len);
1588 				oob += len;
1589 			}
1590 		}
1591 	}
1592 
1593 	cadence_nand_prepare_data_size(chip, TT_RAW_PAGE);
1594 
1595 	return cadence_nand_cdma_transfer(cdns_ctrl,
1596 					  cdns_chip->cs[chip->cur_cs],
1597 					  page, cdns_ctrl->buf, NULL,
1598 					  mtd->writesize +
1599 					  mtd->oobsize,
1600 					  0, DMA_TO_DEVICE, false);
1601 }
1602 
1603 static int cadence_nand_write_oob_raw(struct nand_chip *chip,
1604 				      int page)
1605 {
1606 	return cadence_nand_write_page_raw(chip, NULL, true, page);
1607 }
1608 
1609 static int cadence_nand_read_page(struct nand_chip *chip,
1610 				  u8 *buf, int oob_required, int page)
1611 {
1612 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1613 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1614 	struct mtd_info *mtd = nand_to_mtd(chip);
1615 	int status = 0;
1616 	int ecc_err_count = 0;
1617 
1618 	status = cadence_nand_select_target(chip);
1619 	if (status)
1620 		return status;
1621 
1622 	cadence_nand_set_skip_bytes_conf(cdns_ctrl, cdns_chip->bbm_len,
1623 					 mtd->writesize
1624 					 + cdns_chip->bbm_offs, 1);
1625 
1626 	/*
1627 	 * If data buffer can be accessed by DMA and data_control feature
1628 	 * is supported then transfer data and oob directly.
1629 	 */
1630 	if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, mtd->writesize) &&
1631 	    cdns_ctrl->caps2.data_control_supp) {
1632 		u8 *oob;
1633 
1634 		if (oob_required)
1635 			oob = chip->oob_poi;
1636 		else
1637 			oob = cdns_ctrl->buf + mtd->writesize;
1638 
1639 		cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREA_EXT);
1640 		status = cadence_nand_cdma_transfer(cdns_ctrl,
1641 						    cdns_chip->cs[chip->cur_cs],
1642 						    page, buf, oob,
1643 						    mtd->writesize,
1644 						    cdns_chip->avail_oob_size,
1645 						    DMA_FROM_DEVICE, true);
1646 	/* Otherwise use bounce buffer. */
1647 	} else {
1648 		cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREAS);
1649 		status = cadence_nand_cdma_transfer(cdns_ctrl,
1650 						    cdns_chip->cs[chip->cur_cs],
1651 						    page, cdns_ctrl->buf,
1652 						    NULL, mtd->writesize
1653 						    + cdns_chip->avail_oob_size,
1654 						    0, DMA_FROM_DEVICE, true);
1655 
1656 		memcpy(buf, cdns_ctrl->buf, mtd->writesize);
1657 		if (oob_required)
1658 			memcpy(chip->oob_poi,
1659 			       cdns_ctrl->buf + mtd->writesize,
1660 			       mtd->oobsize);
1661 	}
1662 
1663 	switch (status) {
1664 	case STAT_ECC_UNCORR:
1665 		mtd->ecc_stats.failed++;
1666 		ecc_err_count++;
1667 		break;
1668 	case STAT_ECC_CORR:
1669 		ecc_err_count = FIELD_GET(CDMA_CS_MAXERR,
1670 					  cdns_ctrl->cdma_desc->status);
1671 		mtd->ecc_stats.corrected += ecc_err_count;
1672 		break;
1673 	case STAT_ERASED:
1674 	case STAT_OK:
1675 		break;
1676 	default:
1677 		dev_err(cdns_ctrl->dev, "read page failed\n");
1678 		return -EIO;
1679 	}
1680 
1681 	if (oob_required)
1682 		if (cadence_nand_read_bbm(chip, page, chip->oob_poi))
1683 			return -EIO;
1684 
1685 	return ecc_err_count;
1686 }
1687 
1688 /* Reads OOB data from the device. */
1689 static int cadence_nand_read_oob(struct nand_chip *chip, int page)
1690 {
1691 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1692 
1693 	return cadence_nand_read_page(chip, cdns_ctrl->buf, 1, page);
1694 }
1695 
1696 static int cadence_nand_read_page_raw(struct nand_chip *chip,
1697 				      u8 *buf, int oob_required, int page)
1698 {
1699 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1700 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1701 	struct mtd_info *mtd = nand_to_mtd(chip);
1702 	int oob_skip = cdns_chip->bbm_len;
1703 	int writesize = mtd->writesize;
1704 	int ecc_steps = chip->ecc.steps;
1705 	int ecc_size = chip->ecc.size;
1706 	int ecc_bytes = chip->ecc.bytes;
1707 	void *tmp_buf = cdns_ctrl->buf;
1708 	int i, pos, len;
1709 	int status = 0;
1710 
1711 	status = cadence_nand_select_target(chip);
1712 	if (status)
1713 		return status;
1714 
1715 	cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1716 
1717 	cadence_nand_prepare_data_size(chip, TT_RAW_PAGE);
1718 	status = cadence_nand_cdma_transfer(cdns_ctrl,
1719 					    cdns_chip->cs[chip->cur_cs],
1720 					    page, cdns_ctrl->buf, NULL,
1721 					    mtd->writesize
1722 					    + mtd->oobsize,
1723 					    0, DMA_FROM_DEVICE, false);
1724 
1725 	switch (status) {
1726 	case STAT_ERASED:
1727 	case STAT_OK:
1728 		break;
1729 	default:
1730 		dev_err(cdns_ctrl->dev, "read raw page failed\n");
1731 		return -EIO;
1732 	}
1733 
1734 	/* Arrange the buffer for syndrome payload/ecc layout. */
1735 	if (buf) {
1736 		for (i = 0; i < ecc_steps; i++) {
1737 			pos = i * (ecc_size + ecc_bytes);
1738 			len = ecc_size;
1739 
1740 			if (pos >= writesize)
1741 				pos += oob_skip;
1742 			else if (pos + len > writesize)
1743 				len = writesize - pos;
1744 
1745 			memcpy(buf, tmp_buf + pos, len);
1746 			buf += len;
1747 			if (len < ecc_size) {
1748 				len = ecc_size - len;
1749 				memcpy(buf, tmp_buf + writesize + oob_skip,
1750 				       len);
1751 				buf += len;
1752 			}
1753 		}
1754 	}
1755 
1756 	if (oob_required) {
1757 		u8 *oob = chip->oob_poi;
1758 		u32 oob_data_offset = (cdns_chip->sector_count - 1) *
1759 			(cdns_chip->sector_size + chip->ecc.bytes)
1760 			+ cdns_chip->sector_size + oob_skip;
1761 
1762 		/* OOB free. */
1763 		memcpy(oob, tmp_buf + oob_data_offset,
1764 		       cdns_chip->avail_oob_size);
1765 
1766 		/* BBM at the beginning of the OOB area. */
1767 		memcpy(oob, tmp_buf + writesize, oob_skip);
1768 
1769 		oob += cdns_chip->avail_oob_size;
1770 
1771 		/* OOB ECC */
1772 		for (i = 0; i < ecc_steps; i++) {
1773 			pos = ecc_size + i * (ecc_size + ecc_bytes);
1774 			len = ecc_bytes;
1775 
1776 			if (i == (ecc_steps - 1))
1777 				pos += cdns_chip->avail_oob_size;
1778 
1779 			if (pos >= writesize)
1780 				pos += oob_skip;
1781 			else if (pos + len > writesize)
1782 				len = writesize - pos;
1783 
1784 			memcpy(oob, tmp_buf + pos, len);
1785 			oob += len;
1786 			if (len < ecc_bytes) {
1787 				len = ecc_bytes - len;
1788 				memcpy(oob, tmp_buf + writesize + oob_skip,
1789 				       len);
1790 				oob += len;
1791 			}
1792 		}
1793 	}
1794 
1795 	return 0;
1796 }
1797 
1798 static int cadence_nand_read_oob_raw(struct nand_chip *chip,
1799 				     int page)
1800 {
1801 	return cadence_nand_read_page_raw(chip, NULL, true, page);
1802 }
1803 
1804 static void cadence_nand_slave_dma_transfer_finished(void *data)
1805 {
1806 	struct completion *finished = data;
1807 
1808 	complete(finished);
1809 }
1810 
1811 static int cadence_nand_slave_dma_transfer(struct cdns_nand_ctrl *cdns_ctrl,
1812 					   void *buf,
1813 					   dma_addr_t dev_dma, size_t len,
1814 					   enum dma_data_direction dir)
1815 {
1816 	DECLARE_COMPLETION_ONSTACK(finished);
1817 	struct dma_chan *chan;
1818 	struct dma_device *dma_dev;
1819 	dma_addr_t src_dma, dst_dma, buf_dma;
1820 	struct dma_async_tx_descriptor *tx;
1821 	dma_cookie_t cookie;
1822 
1823 	chan = cdns_ctrl->dmac;
1824 	dma_dev = chan->device;
1825 
1826 	buf_dma = dma_map_single(dma_dev->dev, buf, len, dir);
1827 	if (dma_mapping_error(dma_dev->dev, buf_dma)) {
1828 		dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1829 		goto err;
1830 	}
1831 
1832 	if (dir == DMA_FROM_DEVICE) {
1833 		src_dma = cdns_ctrl->io.dma;
1834 		dst_dma = buf_dma;
1835 	} else {
1836 		src_dma = buf_dma;
1837 		dst_dma = cdns_ctrl->io.dma;
1838 	}
1839 
1840 	tx = dmaengine_prep_dma_memcpy(cdns_ctrl->dmac, dst_dma, src_dma, len,
1841 				       DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
1842 	if (!tx) {
1843 		dev_err(cdns_ctrl->dev, "Failed to prepare DMA memcpy\n");
1844 		goto err_unmap;
1845 	}
1846 
1847 	tx->callback = cadence_nand_slave_dma_transfer_finished;
1848 	tx->callback_param = &finished;
1849 
1850 	cookie = dmaengine_submit(tx);
1851 	if (dma_submit_error(cookie)) {
1852 		dev_err(cdns_ctrl->dev, "Failed to do DMA tx_submit\n");
1853 		goto err_unmap;
1854 	}
1855 
1856 	dma_async_issue_pending(cdns_ctrl->dmac);
1857 	wait_for_completion(&finished);
1858 
1859 	dma_unmap_single(cdns_ctrl->dev, buf_dma, len, dir);
1860 
1861 	return 0;
1862 
1863 err_unmap:
1864 	dma_unmap_single(cdns_ctrl->dev, buf_dma, len, dir);
1865 
1866 err:
1867 	dev_dbg(cdns_ctrl->dev, "Fall back to CPU I/O\n");
1868 
1869 	return -EIO;
1870 }
1871 
1872 static int cadence_nand_read_buf(struct cdns_nand_ctrl *cdns_ctrl,
1873 				 u8 *buf, int len)
1874 {
1875 	u8 thread_nr = 0;
1876 	u32 sdma_size;
1877 	int status;
1878 
1879 	/* Wait until slave DMA interface is ready to data transfer. */
1880 	status = cadence_nand_wait_on_sdma(cdns_ctrl, &thread_nr, &sdma_size);
1881 	if (status)
1882 		return status;
1883 
1884 	if (!cdns_ctrl->caps1->has_dma) {
1885 		int len_in_words = len >> 2;
1886 
1887 		/* read alingment data */
1888 		ioread32_rep(cdns_ctrl->io.virt, buf, len_in_words);
1889 		if (sdma_size > len) {
1890 			/* read rest data from slave DMA interface if any */
1891 			ioread32_rep(cdns_ctrl->io.virt, cdns_ctrl->buf,
1892 				     sdma_size / 4 - len_in_words);
1893 			/* copy rest of data */
1894 			memcpy(buf + (len_in_words << 2), cdns_ctrl->buf,
1895 			       len - (len_in_words << 2));
1896 		}
1897 		return 0;
1898 	}
1899 
1900 	if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, len)) {
1901 		status = cadence_nand_slave_dma_transfer(cdns_ctrl, buf,
1902 							 cdns_ctrl->io.dma,
1903 							 len, DMA_FROM_DEVICE);
1904 		if (status == 0)
1905 			return 0;
1906 
1907 		dev_warn(cdns_ctrl->dev,
1908 			 "Slave DMA transfer failed. Try again using bounce buffer.");
1909 	}
1910 
1911 	/* If DMA transfer is not possible or failed then use bounce buffer. */
1912 	status = cadence_nand_slave_dma_transfer(cdns_ctrl, cdns_ctrl->buf,
1913 						 cdns_ctrl->io.dma,
1914 						 sdma_size, DMA_FROM_DEVICE);
1915 
1916 	if (status) {
1917 		dev_err(cdns_ctrl->dev, "Slave DMA transfer failed");
1918 		return status;
1919 	}
1920 
1921 	memcpy(buf, cdns_ctrl->buf, len);
1922 
1923 	return 0;
1924 }
1925 
1926 static int cadence_nand_write_buf(struct cdns_nand_ctrl *cdns_ctrl,
1927 				  const u8 *buf, int len)
1928 {
1929 	u8 thread_nr = 0;
1930 	u32 sdma_size;
1931 	int status;
1932 
1933 	/* Wait until slave DMA interface is ready to data transfer. */
1934 	status = cadence_nand_wait_on_sdma(cdns_ctrl, &thread_nr, &sdma_size);
1935 	if (status)
1936 		return status;
1937 
1938 	if (!cdns_ctrl->caps1->has_dma) {
1939 		int len_in_words = len >> 2;
1940 
1941 		iowrite32_rep(cdns_ctrl->io.virt, buf, len_in_words);
1942 		if (sdma_size > len) {
1943 			/* copy rest of data */
1944 			memcpy(cdns_ctrl->buf, buf + (len_in_words << 2),
1945 			       len - (len_in_words << 2));
1946 			/* write all expected by nand controller data */
1947 			iowrite32_rep(cdns_ctrl->io.virt, cdns_ctrl->buf,
1948 				      sdma_size / 4 - len_in_words);
1949 		}
1950 
1951 		return 0;
1952 	}
1953 
1954 	if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, len)) {
1955 		status = cadence_nand_slave_dma_transfer(cdns_ctrl, (void *)buf,
1956 							 cdns_ctrl->io.dma,
1957 							 len, DMA_TO_DEVICE);
1958 		if (status == 0)
1959 			return 0;
1960 
1961 		dev_warn(cdns_ctrl->dev,
1962 			 "Slave DMA transfer failed. Try again using bounce buffer.");
1963 	}
1964 
1965 	/* If DMA transfer is not possible or failed then use bounce buffer. */
1966 	memcpy(cdns_ctrl->buf, buf, len);
1967 
1968 	status = cadence_nand_slave_dma_transfer(cdns_ctrl, cdns_ctrl->buf,
1969 						 cdns_ctrl->io.dma,
1970 						 sdma_size, DMA_TO_DEVICE);
1971 
1972 	if (status)
1973 		dev_err(cdns_ctrl->dev, "Slave DMA transfer failed");
1974 
1975 	return status;
1976 }
1977 
1978 static int cadence_nand_force_byte_access(struct nand_chip *chip,
1979 					  bool force_8bit)
1980 {
1981 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1982 	int status;
1983 
1984 	/*
1985 	 * Callers of this function do not verify if the NAND is using a 16-bit
1986 	 * an 8-bit bus for normal operations, so we need to take care of that
1987 	 * here by leaving the configuration unchanged if the NAND does not have
1988 	 * the NAND_BUSWIDTH_16 flag set.
1989 	 */
1990 	if (!(chip->options & NAND_BUSWIDTH_16))
1991 		return 0;
1992 
1993 	status = cadence_nand_set_access_width16(cdns_ctrl, !force_8bit);
1994 
1995 	return status;
1996 }
1997 
1998 static int cadence_nand_cmd_opcode(struct nand_chip *chip,
1999 				   const struct nand_subop *subop)
2000 {
2001 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2002 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2003 	const struct nand_op_instr *instr;
2004 	unsigned int op_id = 0;
2005 	u64 mini_ctrl_cmd = 0;
2006 	int ret;
2007 
2008 	instr = &subop->instrs[op_id];
2009 
2010 	if (instr->delay_ns > 0)
2011 		mini_ctrl_cmd |= GCMD_LAY_TWB;
2012 
2013 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2014 				    GCMD_LAY_INSTR_CMD);
2015 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_CMD,
2016 				    instr->ctx.cmd.opcode);
2017 
2018 	ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2019 					    cdns_chip->cs[chip->cur_cs],
2020 					    mini_ctrl_cmd);
2021 	if (ret)
2022 		dev_err(cdns_ctrl->dev, "send cmd %x failed\n",
2023 			instr->ctx.cmd.opcode);
2024 
2025 	return ret;
2026 }
2027 
2028 static int cadence_nand_cmd_address(struct nand_chip *chip,
2029 				    const struct nand_subop *subop)
2030 {
2031 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2032 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2033 	const struct nand_op_instr *instr;
2034 	unsigned int op_id = 0;
2035 	u64 mini_ctrl_cmd = 0;
2036 	unsigned int offset, naddrs;
2037 	u64 address = 0;
2038 	const u8 *addrs;
2039 	int ret;
2040 	int i;
2041 
2042 	instr = &subop->instrs[op_id];
2043 
2044 	if (instr->delay_ns > 0)
2045 		mini_ctrl_cmd |= GCMD_LAY_TWB;
2046 
2047 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2048 				    GCMD_LAY_INSTR_ADDR);
2049 
2050 	offset = nand_subop_get_addr_start_off(subop, op_id);
2051 	naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
2052 	addrs = &instr->ctx.addr.addrs[offset];
2053 
2054 	for (i = 0; i < naddrs; i++)
2055 		address |= (u64)addrs[i] << (8 * i);
2056 
2057 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_ADDR,
2058 				    address);
2059 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_ADDR_SIZE,
2060 				    naddrs - 1);
2061 
2062 	ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2063 					    cdns_chip->cs[chip->cur_cs],
2064 					    mini_ctrl_cmd);
2065 	if (ret)
2066 		dev_err(cdns_ctrl->dev, "send address %llx failed\n", address);
2067 
2068 	return ret;
2069 }
2070 
2071 static int cadence_nand_cmd_erase(struct nand_chip *chip,
2072 				  const struct nand_subop *subop)
2073 {
2074 	unsigned int op_id;
2075 
2076 	if (subop->instrs[0].ctx.cmd.opcode == NAND_CMD_ERASE1) {
2077 		int i;
2078 		const struct nand_op_instr *instr = NULL;
2079 		unsigned int offset, naddrs;
2080 		const u8 *addrs;
2081 		u32 page = 0;
2082 
2083 		instr = &subop->instrs[1];
2084 		offset = nand_subop_get_addr_start_off(subop, 1);
2085 		naddrs = nand_subop_get_num_addr_cyc(subop, 1);
2086 		addrs = &instr->ctx.addr.addrs[offset];
2087 
2088 		for (i = 0; i < naddrs; i++)
2089 			page |= (u32)addrs[i] << (8 * i);
2090 
2091 		return cadence_nand_erase(chip, page);
2092 	}
2093 
2094 	/*
2095 	 * If it is not an erase operation then handle operation
2096 	 * by calling exec_op function.
2097 	 */
2098 	for (op_id = 0; op_id < subop->ninstrs; op_id++) {
2099 		int ret;
2100 		const struct nand_operation nand_op = {
2101 			.cs = chip->cur_cs,
2102 			.instrs =  &subop->instrs[op_id],
2103 			.ninstrs = 1};
2104 		ret = chip->controller->ops->exec_op(chip, &nand_op, false);
2105 		if (ret)
2106 			return ret;
2107 	}
2108 
2109 	return 0;
2110 }
2111 
2112 static int cadence_nand_cmd_data(struct nand_chip *chip,
2113 				 const struct nand_subop *subop)
2114 {
2115 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2116 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2117 	const struct nand_op_instr *instr;
2118 	unsigned int offset, op_id = 0;
2119 	u64 mini_ctrl_cmd = 0;
2120 	int len = 0;
2121 	int ret;
2122 
2123 	instr = &subop->instrs[op_id];
2124 
2125 	if (instr->delay_ns > 0)
2126 		mini_ctrl_cmd |= GCMD_LAY_TWB;
2127 
2128 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2129 				    GCMD_LAY_INSTR_DATA);
2130 
2131 	if (instr->type == NAND_OP_DATA_OUT_INSTR)
2132 		mini_ctrl_cmd |= FIELD_PREP(GCMD_DIR,
2133 					    GCMD_DIR_WRITE);
2134 
2135 	len = nand_subop_get_data_len(subop, op_id);
2136 	offset = nand_subop_get_data_start_off(subop, op_id);
2137 	mini_ctrl_cmd |= FIELD_PREP(GCMD_SECT_CNT, 1);
2138 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAST_SIZE, len);
2139 	if (instr->ctx.data.force_8bit) {
2140 		ret = cadence_nand_force_byte_access(chip, true);
2141 		if (ret) {
2142 			dev_err(cdns_ctrl->dev,
2143 				"cannot change byte access generic data cmd failed\n");
2144 			return ret;
2145 		}
2146 	}
2147 
2148 	ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2149 					    cdns_chip->cs[chip->cur_cs],
2150 					    mini_ctrl_cmd);
2151 	if (ret) {
2152 		dev_err(cdns_ctrl->dev, "send generic data cmd failed\n");
2153 		return ret;
2154 	}
2155 
2156 	if (instr->type == NAND_OP_DATA_IN_INSTR) {
2157 		void *buf = instr->ctx.data.buf.in + offset;
2158 
2159 		ret = cadence_nand_read_buf(cdns_ctrl, buf, len);
2160 	} else {
2161 		const void *buf = instr->ctx.data.buf.out + offset;
2162 
2163 		ret = cadence_nand_write_buf(cdns_ctrl, buf, len);
2164 	}
2165 
2166 	if (ret) {
2167 		dev_err(cdns_ctrl->dev, "data transfer failed for generic command\n");
2168 		return ret;
2169 	}
2170 
2171 	if (instr->ctx.data.force_8bit) {
2172 		ret = cadence_nand_force_byte_access(chip, false);
2173 		if (ret) {
2174 			dev_err(cdns_ctrl->dev,
2175 				"cannot change byte access generic data cmd failed\n");
2176 		}
2177 	}
2178 
2179 	return ret;
2180 }
2181 
2182 static int cadence_nand_cmd_waitrdy(struct nand_chip *chip,
2183 				    const struct nand_subop *subop)
2184 {
2185 	int status;
2186 	unsigned int op_id = 0;
2187 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2188 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2189 	const struct nand_op_instr *instr = &subop->instrs[op_id];
2190 	u32 timeout_us = instr->ctx.waitrdy.timeout_ms * 1000;
2191 
2192 	status = cadence_nand_wait_for_value(cdns_ctrl, RBN_SETINGS,
2193 					     timeout_us,
2194 					     BIT(cdns_chip->cs[chip->cur_cs]),
2195 					     false);
2196 	return status;
2197 }
2198 
2199 static const struct nand_op_parser cadence_nand_op_parser = NAND_OP_PARSER(
2200 	NAND_OP_PARSER_PATTERN(
2201 		cadence_nand_cmd_erase,
2202 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2203 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ERASE_ADDRESS_CYC),
2204 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2205 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2206 	NAND_OP_PARSER_PATTERN(
2207 		cadence_nand_cmd_opcode,
2208 		NAND_OP_PARSER_PAT_CMD_ELEM(false)),
2209 	NAND_OP_PARSER_PATTERN(
2210 		cadence_nand_cmd_address,
2211 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC)),
2212 	NAND_OP_PARSER_PATTERN(
2213 		cadence_nand_cmd_data,
2214 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_DATA_SIZE)),
2215 	NAND_OP_PARSER_PATTERN(
2216 		cadence_nand_cmd_data,
2217 		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_DATA_SIZE)),
2218 	NAND_OP_PARSER_PATTERN(
2219 		cadence_nand_cmd_waitrdy,
2220 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false))
2221 	);
2222 
2223 static int cadence_nand_exec_op(struct nand_chip *chip,
2224 				const struct nand_operation *op,
2225 				bool check_only)
2226 {
2227 	if (!check_only) {
2228 		int status = cadence_nand_select_target(chip);
2229 
2230 		if (status)
2231 			return status;
2232 	}
2233 
2234 	return nand_op_parser_exec_op(chip, &cadence_nand_op_parser, op,
2235 				      check_only);
2236 }
2237 
2238 static int cadence_nand_ooblayout_free(struct mtd_info *mtd, int section,
2239 				       struct mtd_oob_region *oobregion)
2240 {
2241 	struct nand_chip *chip = mtd_to_nand(mtd);
2242 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2243 
2244 	if (section)
2245 		return -ERANGE;
2246 
2247 	oobregion->offset = cdns_chip->bbm_len;
2248 	oobregion->length = cdns_chip->avail_oob_size
2249 		- cdns_chip->bbm_len;
2250 
2251 	return 0;
2252 }
2253 
2254 static int cadence_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2255 				      struct mtd_oob_region *oobregion)
2256 {
2257 	struct nand_chip *chip = mtd_to_nand(mtd);
2258 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2259 
2260 	if (section)
2261 		return -ERANGE;
2262 
2263 	oobregion->offset = cdns_chip->avail_oob_size;
2264 	oobregion->length = chip->ecc.total;
2265 
2266 	return 0;
2267 }
2268 
2269 static const struct mtd_ooblayout_ops cadence_nand_ooblayout_ops = {
2270 	.free = cadence_nand_ooblayout_free,
2271 	.ecc = cadence_nand_ooblayout_ecc,
2272 };
2273 
2274 static int calc_cycl(u32 timing, u32 clock)
2275 {
2276 	if (timing == 0 || clock == 0)
2277 		return 0;
2278 
2279 	if ((timing % clock) > 0)
2280 		return timing / clock;
2281 	else
2282 		return timing / clock - 1;
2283 }
2284 
2285 /* Calculate max data valid window. */
2286 static inline u32 calc_tdvw_max(u32 trp_cnt, u32 clk_period, u32 trhoh_min,
2287 				u32 board_delay_skew_min, u32 ext_mode)
2288 {
2289 	if (ext_mode == 0)
2290 		clk_period /= 2;
2291 
2292 	return (trp_cnt + 1) * clk_period + trhoh_min +
2293 		board_delay_skew_min;
2294 }
2295 
2296 /* Calculate data valid window. */
2297 static inline u32 calc_tdvw(u32 trp_cnt, u32 clk_period, u32 trhoh_min,
2298 			    u32 trea_max, u32 ext_mode)
2299 {
2300 	if (ext_mode == 0)
2301 		clk_period /= 2;
2302 
2303 	return (trp_cnt + 1) * clk_period + trhoh_min - trea_max;
2304 }
2305 
2306 static int
2307 cadence_nand_setup_interface(struct nand_chip *chip, int chipnr,
2308 			     const struct nand_interface_config *conf)
2309 {
2310 	const struct nand_sdr_timings *sdr;
2311 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2312 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2313 	struct cadence_nand_timings *t = &cdns_chip->timings;
2314 	u32 reg;
2315 	u32 board_delay = cdns_ctrl->board_delay;
2316 	u32 clk_period = DIV_ROUND_DOWN_ULL(1000000000000ULL,
2317 					    cdns_ctrl->nf_clk_rate);
2318 	u32 tceh_cnt, tcs_cnt, tadl_cnt, tccs_cnt;
2319 	u32 tfeat_cnt, trhz_cnt, tvdly_cnt;
2320 	u32 trhw_cnt, twb_cnt, twh_cnt = 0, twhr_cnt;
2321 	u32 twp_cnt = 0, trp_cnt = 0, trh_cnt = 0;
2322 	u32 if_skew = cdns_ctrl->caps1->if_skew;
2323 	u32 board_delay_skew_min = board_delay - if_skew;
2324 	u32 board_delay_skew_max = board_delay + if_skew;
2325 	u32 dqs_sampl_res, phony_dqs_mod;
2326 	u32 tdvw, tdvw_min, tdvw_max;
2327 	u32 ext_rd_mode, ext_wr_mode;
2328 	u32 dll_phy_dqs_timing = 0, phony_dqs_timing = 0, rd_del_sel = 0;
2329 	u32 sampling_point;
2330 
2331 	sdr = nand_get_sdr_timings(conf);
2332 	if (IS_ERR(sdr))
2333 		return PTR_ERR(sdr);
2334 
2335 	memset(t, 0, sizeof(*t));
2336 	/* Sampling point calculation. */
2337 
2338 	if (cdns_ctrl->caps2.is_phy_type_dll)
2339 		phony_dqs_mod = 2;
2340 	else
2341 		phony_dqs_mod = 1;
2342 
2343 	dqs_sampl_res = clk_period / phony_dqs_mod;
2344 
2345 	tdvw_min = sdr->tREA_max + board_delay_skew_max;
2346 	/*
2347 	 * The idea of those calculation is to get the optimum value
2348 	 * for tRP and tRH timings. If it is NOT possible to sample data
2349 	 * with optimal tRP/tRH settings, the parameters will be extended.
2350 	 * If clk_period is 50ns (the lowest value) this condition is met
2351 	 * for SDR timing modes 1, 2, 3, 4 and 5.
2352 	 * If clk_period is 20ns the condition is met only for SDR timing
2353 	 * mode 5.
2354 	 */
2355 	if (sdr->tRC_min <= clk_period &&
2356 	    sdr->tRP_min <= (clk_period / 2) &&
2357 	    sdr->tREH_min <= (clk_period / 2)) {
2358 		/* Performance mode. */
2359 		ext_rd_mode = 0;
2360 		tdvw = calc_tdvw(trp_cnt, clk_period, sdr->tRHOH_min,
2361 				 sdr->tREA_max, ext_rd_mode);
2362 		tdvw_max = calc_tdvw_max(trp_cnt, clk_period, sdr->tRHOH_min,
2363 					 board_delay_skew_min,
2364 					 ext_rd_mode);
2365 		/*
2366 		 * Check if data valid window and sampling point can be found
2367 		 * and is not on the edge (ie. we have hold margin).
2368 		 * If not extend the tRP timings.
2369 		 */
2370 		if (tdvw > 0) {
2371 			if (tdvw_max <= tdvw_min ||
2372 			    (tdvw_max % dqs_sampl_res) == 0) {
2373 				/*
2374 				 * No valid sampling point so the RE pulse need
2375 				 * to be widen widening by half clock cycle.
2376 				 */
2377 				ext_rd_mode = 1;
2378 			}
2379 		} else {
2380 			/*
2381 			 * There is no valid window
2382 			 * to be able to sample data the tRP need to be widen.
2383 			 * Very safe calculations are performed here.
2384 			 */
2385 			trp_cnt = (sdr->tREA_max + board_delay_skew_max
2386 				   + dqs_sampl_res) / clk_period;
2387 			ext_rd_mode = 1;
2388 		}
2389 
2390 	} else {
2391 		/* Extended read mode. */
2392 		u32 trh;
2393 
2394 		ext_rd_mode = 1;
2395 		trp_cnt = calc_cycl(sdr->tRP_min, clk_period);
2396 		trh = sdr->tRC_min - ((trp_cnt + 1) * clk_period);
2397 		if (sdr->tREH_min >= trh)
2398 			trh_cnt = calc_cycl(sdr->tREH_min, clk_period);
2399 		else
2400 			trh_cnt = calc_cycl(trh, clk_period);
2401 
2402 		tdvw = calc_tdvw(trp_cnt, clk_period, sdr->tRHOH_min,
2403 				 sdr->tREA_max, ext_rd_mode);
2404 		/*
2405 		 * Check if data valid window and sampling point can be found
2406 		 * or if it is at the edge check if previous is valid
2407 		 * - if not extend the tRP timings.
2408 		 */
2409 		if (tdvw > 0) {
2410 			tdvw_max = calc_tdvw_max(trp_cnt, clk_period,
2411 						 sdr->tRHOH_min,
2412 						 board_delay_skew_min,
2413 						 ext_rd_mode);
2414 
2415 			if ((((tdvw_max / dqs_sampl_res)
2416 			      * dqs_sampl_res) <= tdvw_min) ||
2417 			    (((tdvw_max % dqs_sampl_res) == 0) &&
2418 			     (((tdvw_max / dqs_sampl_res - 1)
2419 			       * dqs_sampl_res) <= tdvw_min))) {
2420 				/*
2421 				 * Data valid window width is lower than
2422 				 * sampling resolution and do not hit any
2423 				 * sampling point to be sure the sampling point
2424 				 * will be found the RE low pulse width will be
2425 				 *  extended by one clock cycle.
2426 				 */
2427 				trp_cnt = trp_cnt + 1;
2428 			}
2429 		} else {
2430 			/*
2431 			 * There is no valid window to be able to sample data.
2432 			 * The tRP need to be widen.
2433 			 * Very safe calculations are performed here.
2434 			 */
2435 			trp_cnt = (sdr->tREA_max + board_delay_skew_max
2436 				   + dqs_sampl_res) / clk_period;
2437 		}
2438 	}
2439 
2440 	tdvw_max = calc_tdvw_max(trp_cnt, clk_period,
2441 				 sdr->tRHOH_min,
2442 				 board_delay_skew_min, ext_rd_mode);
2443 
2444 	if (sdr->tWC_min <= clk_period &&
2445 	    (sdr->tWP_min + if_skew) <= (clk_period / 2) &&
2446 	    (sdr->tWH_min + if_skew) <= (clk_period / 2)) {
2447 		ext_wr_mode = 0;
2448 	} else {
2449 		u32 twh;
2450 
2451 		ext_wr_mode = 1;
2452 		twp_cnt = calc_cycl(sdr->tWP_min + if_skew, clk_period);
2453 		if ((twp_cnt + 1) * clk_period < (sdr->tALS_min + if_skew))
2454 			twp_cnt = calc_cycl(sdr->tALS_min + if_skew,
2455 					    clk_period);
2456 
2457 		twh = (sdr->tWC_min - (twp_cnt + 1) * clk_period);
2458 		if (sdr->tWH_min >= twh)
2459 			twh = sdr->tWH_min;
2460 
2461 		twh_cnt = calc_cycl(twh + if_skew, clk_period);
2462 	}
2463 
2464 	reg = FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TRH, trh_cnt);
2465 	reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TRP, trp_cnt);
2466 	reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TWH, twh_cnt);
2467 	reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TWP, twp_cnt);
2468 	t->async_toggle_timings = reg;
2469 	dev_dbg(cdns_ctrl->dev, "ASYNC_TOGGLE_TIMINGS_SDR\t%x\n", reg);
2470 
2471 	tadl_cnt = calc_cycl((sdr->tADL_min + if_skew), clk_period);
2472 	tccs_cnt = calc_cycl((sdr->tCCS_min + if_skew), clk_period);
2473 	twhr_cnt = calc_cycl((sdr->tWHR_min + if_skew), clk_period);
2474 	trhw_cnt = calc_cycl((sdr->tRHW_min + if_skew), clk_period);
2475 	reg = FIELD_PREP(TIMINGS0_TADL, tadl_cnt);
2476 
2477 	/*
2478 	 * If timing exceeds delay field in timing register
2479 	 * then use maximum value.
2480 	 */
2481 	if (FIELD_FIT(TIMINGS0_TCCS, tccs_cnt))
2482 		reg |= FIELD_PREP(TIMINGS0_TCCS, tccs_cnt);
2483 	else
2484 		reg |= TIMINGS0_TCCS;
2485 
2486 	reg |= FIELD_PREP(TIMINGS0_TWHR, twhr_cnt);
2487 	reg |= FIELD_PREP(TIMINGS0_TRHW, trhw_cnt);
2488 	t->timings0 = reg;
2489 	dev_dbg(cdns_ctrl->dev, "TIMINGS0_SDR\t%x\n", reg);
2490 
2491 	/* The following is related to single signal so skew is not needed. */
2492 	trhz_cnt = calc_cycl(sdr->tRHZ_max, clk_period);
2493 	trhz_cnt = trhz_cnt + 1;
2494 	twb_cnt = calc_cycl((sdr->tWB_max + board_delay), clk_period);
2495 	/*
2496 	 * Because of the two stage syncflop the value must be increased by 3
2497 	 * first value is related with sync, second value is related
2498 	 * with output if delay.
2499 	 */
2500 	twb_cnt = twb_cnt + 3 + 5;
2501 	/*
2502 	 * The following is related to the we edge of the random data input
2503 	 * sequence so skew is not needed.
2504 	 */
2505 	tvdly_cnt = calc_cycl(500000 + if_skew, clk_period);
2506 	reg = FIELD_PREP(TIMINGS1_TRHZ, trhz_cnt);
2507 	reg |= FIELD_PREP(TIMINGS1_TWB, twb_cnt);
2508 	reg |= FIELD_PREP(TIMINGS1_TVDLY, tvdly_cnt);
2509 	t->timings1 = reg;
2510 	dev_dbg(cdns_ctrl->dev, "TIMINGS1_SDR\t%x\n", reg);
2511 
2512 	tfeat_cnt = calc_cycl(sdr->tFEAT_max, clk_period);
2513 	if (tfeat_cnt < twb_cnt)
2514 		tfeat_cnt = twb_cnt;
2515 
2516 	tceh_cnt = calc_cycl(sdr->tCEH_min, clk_period);
2517 	tcs_cnt = calc_cycl((sdr->tCS_min + if_skew), clk_period);
2518 
2519 	reg = FIELD_PREP(TIMINGS2_TFEAT, tfeat_cnt);
2520 	reg |= FIELD_PREP(TIMINGS2_CS_HOLD_TIME, tceh_cnt);
2521 	reg |= FIELD_PREP(TIMINGS2_CS_SETUP_TIME, tcs_cnt);
2522 	t->timings2 = reg;
2523 	dev_dbg(cdns_ctrl->dev, "TIMINGS2_SDR\t%x\n", reg);
2524 
2525 	if (cdns_ctrl->caps2.is_phy_type_dll) {
2526 		reg = DLL_PHY_CTRL_DLL_RST_N;
2527 		if (ext_wr_mode)
2528 			reg |= DLL_PHY_CTRL_EXTENDED_WR_MODE;
2529 		if (ext_rd_mode)
2530 			reg |= DLL_PHY_CTRL_EXTENDED_RD_MODE;
2531 
2532 		reg |= FIELD_PREP(DLL_PHY_CTRL_RS_HIGH_WAIT_CNT, 7);
2533 		reg |= FIELD_PREP(DLL_PHY_CTRL_RS_IDLE_CNT, 7);
2534 		t->dll_phy_ctrl = reg;
2535 		dev_dbg(cdns_ctrl->dev, "DLL_PHY_CTRL_SDR\t%x\n", reg);
2536 	}
2537 
2538 	/* Sampling point calculation. */
2539 	if ((tdvw_max % dqs_sampl_res) > 0)
2540 		sampling_point = tdvw_max / dqs_sampl_res;
2541 	else
2542 		sampling_point = (tdvw_max / dqs_sampl_res - 1);
2543 
2544 	if (sampling_point * dqs_sampl_res > tdvw_min) {
2545 		dll_phy_dqs_timing =
2546 			FIELD_PREP(PHY_DQS_TIMING_DQS_SEL_OE_END, 4);
2547 		dll_phy_dqs_timing |= PHY_DQS_TIMING_USE_PHONY_DQS;
2548 		phony_dqs_timing = sampling_point / phony_dqs_mod;
2549 
2550 		if ((sampling_point % 2) > 0) {
2551 			dll_phy_dqs_timing |= PHY_DQS_TIMING_PHONY_DQS_SEL;
2552 			if ((tdvw_max % dqs_sampl_res) == 0)
2553 				/*
2554 				 * Calculation for sampling point at the edge
2555 				 * of data and being odd number.
2556 				 */
2557 				phony_dqs_timing = (tdvw_max / dqs_sampl_res)
2558 					/ phony_dqs_mod - 1;
2559 
2560 			if (!cdns_ctrl->caps2.is_phy_type_dll)
2561 				phony_dqs_timing--;
2562 
2563 		} else {
2564 			phony_dqs_timing--;
2565 		}
2566 		rd_del_sel = phony_dqs_timing + 3;
2567 	} else {
2568 		dev_warn(cdns_ctrl->dev,
2569 			 "ERROR : cannot find valid sampling point\n");
2570 	}
2571 
2572 	reg = FIELD_PREP(PHY_CTRL_PHONY_DQS, phony_dqs_timing);
2573 	if (cdns_ctrl->caps2.is_phy_type_dll)
2574 		reg  |= PHY_CTRL_SDR_DQS;
2575 	t->phy_ctrl = reg;
2576 	dev_dbg(cdns_ctrl->dev, "PHY_CTRL_REG_SDR\t%x\n", reg);
2577 
2578 	if (cdns_ctrl->caps2.is_phy_type_dll) {
2579 		dev_dbg(cdns_ctrl->dev, "PHY_TSEL_REG_SDR\t%x\n", 0);
2580 		dev_dbg(cdns_ctrl->dev, "PHY_DQ_TIMING_REG_SDR\t%x\n", 2);
2581 		dev_dbg(cdns_ctrl->dev, "PHY_DQS_TIMING_REG_SDR\t%x\n",
2582 			dll_phy_dqs_timing);
2583 		t->phy_dqs_timing = dll_phy_dqs_timing;
2584 
2585 		reg = FIELD_PREP(PHY_GATE_LPBK_CTRL_RDS, rd_del_sel);
2586 		dev_dbg(cdns_ctrl->dev, "PHY_GATE_LPBK_CTRL_REG_SDR\t%x\n",
2587 			reg);
2588 		t->phy_gate_lpbk_ctrl = reg;
2589 
2590 		dev_dbg(cdns_ctrl->dev, "PHY_DLL_MASTER_CTRL_REG_SDR\t%lx\n",
2591 			PHY_DLL_MASTER_CTRL_BYPASS_MODE);
2592 		dev_dbg(cdns_ctrl->dev, "PHY_DLL_SLAVE_CTRL_REG_SDR\t%x\n", 0);
2593 	}
2594 
2595 	return 0;
2596 }
2597 
2598 static int cadence_nand_attach_chip(struct nand_chip *chip)
2599 {
2600 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2601 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2602 	u32 ecc_size;
2603 	struct mtd_info *mtd = nand_to_mtd(chip);
2604 	int ret;
2605 
2606 	if (chip->options & NAND_BUSWIDTH_16) {
2607 		ret = cadence_nand_set_access_width16(cdns_ctrl, true);
2608 		if (ret)
2609 			return ret;
2610 	}
2611 
2612 	chip->bbt_options |= NAND_BBT_USE_FLASH;
2613 	chip->bbt_options |= NAND_BBT_NO_OOB;
2614 	chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2615 
2616 	chip->options |= NAND_NO_SUBPAGE_WRITE;
2617 
2618 	cdns_chip->bbm_offs = chip->badblockpos;
2619 	cdns_chip->bbm_offs &= ~0x01;
2620 	/* this value should be even number */
2621 	cdns_chip->bbm_len = 2;
2622 
2623 	ret = nand_ecc_choose_conf(chip,
2624 				   &cdns_ctrl->ecc_caps,
2625 				   mtd->oobsize - cdns_chip->bbm_len);
2626 	if (ret) {
2627 		dev_err(cdns_ctrl->dev, "ECC configuration failed\n");
2628 		return ret;
2629 	}
2630 
2631 	dev_dbg(cdns_ctrl->dev,
2632 		"chosen ECC settings: step=%d, strength=%d, bytes=%d\n",
2633 		chip->ecc.size, chip->ecc.strength, chip->ecc.bytes);
2634 
2635 	/* Error correction configuration. */
2636 	cdns_chip->sector_size = chip->ecc.size;
2637 	cdns_chip->sector_count = mtd->writesize / cdns_chip->sector_size;
2638 	ecc_size = cdns_chip->sector_count * chip->ecc.bytes;
2639 
2640 	cdns_chip->avail_oob_size = mtd->oobsize - ecc_size;
2641 
2642 	if (cdns_chip->avail_oob_size > cdns_ctrl->bch_metadata_size)
2643 		cdns_chip->avail_oob_size = cdns_ctrl->bch_metadata_size;
2644 
2645 	if ((cdns_chip->avail_oob_size + cdns_chip->bbm_len + ecc_size)
2646 	    > mtd->oobsize)
2647 		cdns_chip->avail_oob_size -= 4;
2648 
2649 	ret = cadence_nand_get_ecc_strength_idx(cdns_ctrl, chip->ecc.strength);
2650 	if (ret < 0)
2651 		return -EINVAL;
2652 
2653 	cdns_chip->corr_str_idx = (u8)ret;
2654 
2655 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
2656 					1000000,
2657 					CTRL_STATUS_CTRL_BUSY, true))
2658 		return -ETIMEDOUT;
2659 
2660 	cadence_nand_set_ecc_strength(cdns_ctrl,
2661 				      cdns_chip->corr_str_idx);
2662 
2663 	cadence_nand_set_erase_detection(cdns_ctrl, true,
2664 					 chip->ecc.strength);
2665 
2666 	/* Override the default read operations. */
2667 	chip->ecc.read_page = cadence_nand_read_page;
2668 	chip->ecc.read_page_raw = cadence_nand_read_page_raw;
2669 	chip->ecc.write_page = cadence_nand_write_page;
2670 	chip->ecc.write_page_raw = cadence_nand_write_page_raw;
2671 	chip->ecc.read_oob = cadence_nand_read_oob;
2672 	chip->ecc.write_oob = cadence_nand_write_oob;
2673 	chip->ecc.read_oob_raw = cadence_nand_read_oob_raw;
2674 	chip->ecc.write_oob_raw = cadence_nand_write_oob_raw;
2675 
2676 	if ((mtd->writesize + mtd->oobsize) > cdns_ctrl->buf_size)
2677 		cdns_ctrl->buf_size = mtd->writesize + mtd->oobsize;
2678 
2679 	/* Is 32-bit DMA supported? */
2680 	ret = dma_set_mask(cdns_ctrl->dev, DMA_BIT_MASK(32));
2681 	if (ret) {
2682 		dev_err(cdns_ctrl->dev, "no usable DMA configuration\n");
2683 		return ret;
2684 	}
2685 
2686 	mtd_set_ooblayout(mtd, &cadence_nand_ooblayout_ops);
2687 
2688 	return 0;
2689 }
2690 
2691 static const struct nand_controller_ops cadence_nand_controller_ops = {
2692 	.attach_chip = cadence_nand_attach_chip,
2693 	.exec_op = cadence_nand_exec_op,
2694 	.setup_interface = cadence_nand_setup_interface,
2695 };
2696 
2697 static int cadence_nand_chip_init(struct cdns_nand_ctrl *cdns_ctrl,
2698 				  struct device_node *np)
2699 {
2700 	struct cdns_nand_chip *cdns_chip;
2701 	struct mtd_info *mtd;
2702 	struct nand_chip *chip;
2703 	int nsels, ret, i;
2704 	u32 cs;
2705 
2706 	nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
2707 	if (nsels <= 0) {
2708 		dev_err(cdns_ctrl->dev, "missing/invalid reg property\n");
2709 		return -EINVAL;
2710 	}
2711 
2712 	/* Allocate the nand chip structure. */
2713 	cdns_chip = devm_kzalloc(cdns_ctrl->dev, sizeof(*cdns_chip) +
2714 				 (nsels * sizeof(u8)),
2715 				 GFP_KERNEL);
2716 	if (!cdns_chip) {
2717 		dev_err(cdns_ctrl->dev, "could not allocate chip structure\n");
2718 		return -ENOMEM;
2719 	}
2720 
2721 	cdns_chip->nsels = nsels;
2722 
2723 	for (i = 0; i < nsels; i++) {
2724 		/* Retrieve CS id. */
2725 		ret = of_property_read_u32_index(np, "reg", i, &cs);
2726 		if (ret) {
2727 			dev_err(cdns_ctrl->dev,
2728 				"could not retrieve reg property: %d\n",
2729 				ret);
2730 			return ret;
2731 		}
2732 
2733 		if (cs >= cdns_ctrl->caps2.max_banks) {
2734 			dev_err(cdns_ctrl->dev,
2735 				"invalid reg value: %u (max CS = %d)\n",
2736 				cs, cdns_ctrl->caps2.max_banks);
2737 			return -EINVAL;
2738 		}
2739 
2740 		if (test_and_set_bit(cs, &cdns_ctrl->assigned_cs)) {
2741 			dev_err(cdns_ctrl->dev,
2742 				"CS %d already assigned\n", cs);
2743 			return -EINVAL;
2744 		}
2745 
2746 		cdns_chip->cs[i] = cs;
2747 	}
2748 
2749 	chip = &cdns_chip->chip;
2750 	chip->controller = &cdns_ctrl->controller;
2751 	nand_set_flash_node(chip, np);
2752 
2753 	mtd = nand_to_mtd(chip);
2754 	mtd->dev.parent = cdns_ctrl->dev;
2755 
2756 	/*
2757 	 * Default to HW ECC engine mode. If the nand-ecc-mode property is given
2758 	 * in the DT node, this entry will be overwritten in nand_scan_ident().
2759 	 */
2760 	chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2761 
2762 	ret = nand_scan(chip, cdns_chip->nsels);
2763 	if (ret) {
2764 		dev_err(cdns_ctrl->dev, "could not scan the nand chip\n");
2765 		return ret;
2766 	}
2767 
2768 	ret = mtd_device_register(mtd, NULL, 0);
2769 	if (ret) {
2770 		dev_err(cdns_ctrl->dev,
2771 			"failed to register mtd device: %d\n", ret);
2772 		nand_cleanup(chip);
2773 		return ret;
2774 	}
2775 
2776 	list_add_tail(&cdns_chip->node, &cdns_ctrl->chips);
2777 
2778 	return 0;
2779 }
2780 
2781 static void cadence_nand_chips_cleanup(struct cdns_nand_ctrl *cdns_ctrl)
2782 {
2783 	struct cdns_nand_chip *entry, *temp;
2784 	struct nand_chip *chip;
2785 	int ret;
2786 
2787 	list_for_each_entry_safe(entry, temp, &cdns_ctrl->chips, node) {
2788 		chip = &entry->chip;
2789 		ret = mtd_device_unregister(nand_to_mtd(chip));
2790 		WARN_ON(ret);
2791 		nand_cleanup(chip);
2792 		list_del(&entry->node);
2793 	}
2794 }
2795 
2796 static int cadence_nand_chips_init(struct cdns_nand_ctrl *cdns_ctrl)
2797 {
2798 	struct device_node *np = cdns_ctrl->dev->of_node;
2799 	struct device_node *nand_np;
2800 	int max_cs = cdns_ctrl->caps2.max_banks;
2801 	int nchips, ret;
2802 
2803 	nchips = of_get_child_count(np);
2804 
2805 	if (nchips > max_cs) {
2806 		dev_err(cdns_ctrl->dev,
2807 			"too many NAND chips: %d (max = %d CS)\n",
2808 			nchips, max_cs);
2809 		return -EINVAL;
2810 	}
2811 
2812 	for_each_child_of_node(np, nand_np) {
2813 		ret = cadence_nand_chip_init(cdns_ctrl, nand_np);
2814 		if (ret) {
2815 			of_node_put(nand_np);
2816 			cadence_nand_chips_cleanup(cdns_ctrl);
2817 			return ret;
2818 		}
2819 	}
2820 
2821 	return 0;
2822 }
2823 
2824 static void
2825 cadence_nand_irq_cleanup(int irqnum, struct cdns_nand_ctrl *cdns_ctrl)
2826 {
2827 	/* Disable interrupts. */
2828 	writel_relaxed(INTR_ENABLE_INTR_EN, cdns_ctrl->reg + INTR_ENABLE);
2829 }
2830 
2831 static int cadence_nand_init(struct cdns_nand_ctrl *cdns_ctrl)
2832 {
2833 	dma_cap_mask_t mask;
2834 	int ret;
2835 
2836 	cdns_ctrl->cdma_desc = dma_alloc_coherent(cdns_ctrl->dev,
2837 						  sizeof(*cdns_ctrl->cdma_desc),
2838 						  &cdns_ctrl->dma_cdma_desc,
2839 						  GFP_KERNEL);
2840 	if (!cdns_ctrl->dma_cdma_desc)
2841 		return -ENOMEM;
2842 
2843 	cdns_ctrl->buf_size = SZ_16K;
2844 	cdns_ctrl->buf = kmalloc(cdns_ctrl->buf_size, GFP_KERNEL);
2845 	if (!cdns_ctrl->buf) {
2846 		ret = -ENOMEM;
2847 		goto free_buf_desc;
2848 	}
2849 
2850 	if (devm_request_irq(cdns_ctrl->dev, cdns_ctrl->irq, cadence_nand_isr,
2851 			     IRQF_SHARED, "cadence-nand-controller",
2852 			     cdns_ctrl)) {
2853 		dev_err(cdns_ctrl->dev, "Unable to allocate IRQ\n");
2854 		ret = -ENODEV;
2855 		goto free_buf;
2856 	}
2857 
2858 	spin_lock_init(&cdns_ctrl->irq_lock);
2859 	init_completion(&cdns_ctrl->complete);
2860 
2861 	ret = cadence_nand_hw_init(cdns_ctrl);
2862 	if (ret)
2863 		goto disable_irq;
2864 
2865 	dma_cap_zero(mask);
2866 	dma_cap_set(DMA_MEMCPY, mask);
2867 
2868 	if (cdns_ctrl->caps1->has_dma) {
2869 		cdns_ctrl->dmac = dma_request_channel(mask, NULL, NULL);
2870 		if (!cdns_ctrl->dmac) {
2871 			dev_err(cdns_ctrl->dev,
2872 				"Unable to get a DMA channel\n");
2873 			ret = -EBUSY;
2874 			goto disable_irq;
2875 		}
2876 	}
2877 
2878 	nand_controller_init(&cdns_ctrl->controller);
2879 	INIT_LIST_HEAD(&cdns_ctrl->chips);
2880 
2881 	cdns_ctrl->controller.ops = &cadence_nand_controller_ops;
2882 	cdns_ctrl->curr_corr_str_idx = 0xFF;
2883 
2884 	ret = cadence_nand_chips_init(cdns_ctrl);
2885 	if (ret) {
2886 		dev_err(cdns_ctrl->dev, "Failed to register MTD: %d\n",
2887 			ret);
2888 		goto dma_release_chnl;
2889 	}
2890 
2891 	kfree(cdns_ctrl->buf);
2892 	cdns_ctrl->buf = kzalloc(cdns_ctrl->buf_size, GFP_KERNEL);
2893 	if (!cdns_ctrl->buf) {
2894 		ret = -ENOMEM;
2895 		goto dma_release_chnl;
2896 	}
2897 
2898 	return 0;
2899 
2900 dma_release_chnl:
2901 	if (cdns_ctrl->dmac)
2902 		dma_release_channel(cdns_ctrl->dmac);
2903 
2904 disable_irq:
2905 	cadence_nand_irq_cleanup(cdns_ctrl->irq, cdns_ctrl);
2906 
2907 free_buf:
2908 	kfree(cdns_ctrl->buf);
2909 
2910 free_buf_desc:
2911 	dma_free_coherent(cdns_ctrl->dev, sizeof(struct cadence_nand_cdma_desc),
2912 			  cdns_ctrl->cdma_desc, cdns_ctrl->dma_cdma_desc);
2913 
2914 	return ret;
2915 }
2916 
2917 /* Driver exit point. */
2918 static void cadence_nand_remove(struct cdns_nand_ctrl *cdns_ctrl)
2919 {
2920 	cadence_nand_chips_cleanup(cdns_ctrl);
2921 	cadence_nand_irq_cleanup(cdns_ctrl->irq, cdns_ctrl);
2922 	kfree(cdns_ctrl->buf);
2923 	dma_free_coherent(cdns_ctrl->dev, sizeof(struct cadence_nand_cdma_desc),
2924 			  cdns_ctrl->cdma_desc, cdns_ctrl->dma_cdma_desc);
2925 
2926 	if (cdns_ctrl->dmac)
2927 		dma_release_channel(cdns_ctrl->dmac);
2928 }
2929 
2930 struct cadence_nand_dt {
2931 	struct cdns_nand_ctrl cdns_ctrl;
2932 	struct clk *clk;
2933 };
2934 
2935 static const struct cadence_nand_dt_devdata cadence_nand_default = {
2936 	.if_skew = 0,
2937 	.has_dma = 1,
2938 };
2939 
2940 static const struct of_device_id cadence_nand_dt_ids[] = {
2941 	{
2942 		.compatible = "cdns,hp-nfc",
2943 		.data = &cadence_nand_default
2944 	}, {}
2945 };
2946 
2947 MODULE_DEVICE_TABLE(of, cadence_nand_dt_ids);
2948 
2949 static int cadence_nand_dt_probe(struct platform_device *ofdev)
2950 {
2951 	struct resource *res;
2952 	struct cadence_nand_dt *dt;
2953 	struct cdns_nand_ctrl *cdns_ctrl;
2954 	int ret;
2955 	const struct of_device_id *of_id;
2956 	const struct cadence_nand_dt_devdata *devdata;
2957 	u32 val;
2958 
2959 	of_id = of_match_device(cadence_nand_dt_ids, &ofdev->dev);
2960 	if (of_id) {
2961 		ofdev->id_entry = of_id->data;
2962 		devdata = of_id->data;
2963 	} else {
2964 		pr_err("Failed to find the right device id.\n");
2965 		return -ENOMEM;
2966 	}
2967 
2968 	dt = devm_kzalloc(&ofdev->dev, sizeof(*dt), GFP_KERNEL);
2969 	if (!dt)
2970 		return -ENOMEM;
2971 
2972 	cdns_ctrl = &dt->cdns_ctrl;
2973 	cdns_ctrl->caps1 = devdata;
2974 
2975 	cdns_ctrl->dev = &ofdev->dev;
2976 	cdns_ctrl->irq = platform_get_irq(ofdev, 0);
2977 	if (cdns_ctrl->irq < 0)
2978 		return cdns_ctrl->irq;
2979 
2980 	dev_info(cdns_ctrl->dev, "IRQ: nr %d\n", cdns_ctrl->irq);
2981 
2982 	cdns_ctrl->reg = devm_platform_ioremap_resource(ofdev, 0);
2983 	if (IS_ERR(cdns_ctrl->reg))
2984 		return PTR_ERR(cdns_ctrl->reg);
2985 
2986 	cdns_ctrl->io.virt = devm_platform_get_and_ioremap_resource(ofdev, 1, &res);
2987 	if (IS_ERR(cdns_ctrl->io.virt))
2988 		return PTR_ERR(cdns_ctrl->io.virt);
2989 	cdns_ctrl->io.dma = res->start;
2990 
2991 	dt->clk = devm_clk_get(cdns_ctrl->dev, "nf_clk");
2992 	if (IS_ERR(dt->clk))
2993 		return PTR_ERR(dt->clk);
2994 
2995 	cdns_ctrl->nf_clk_rate = clk_get_rate(dt->clk);
2996 
2997 	ret = of_property_read_u32(ofdev->dev.of_node,
2998 				   "cdns,board-delay-ps", &val);
2999 	if (ret) {
3000 		val = 4830;
3001 		dev_info(cdns_ctrl->dev,
3002 			 "missing cdns,board-delay-ps property, %d was set\n",
3003 			 val);
3004 	}
3005 	cdns_ctrl->board_delay = val;
3006 
3007 	ret = cadence_nand_init(cdns_ctrl);
3008 	if (ret)
3009 		return ret;
3010 
3011 	platform_set_drvdata(ofdev, dt);
3012 	return 0;
3013 }
3014 
3015 static int cadence_nand_dt_remove(struct platform_device *ofdev)
3016 {
3017 	struct cadence_nand_dt *dt = platform_get_drvdata(ofdev);
3018 
3019 	cadence_nand_remove(&dt->cdns_ctrl);
3020 
3021 	return 0;
3022 }
3023 
3024 static struct platform_driver cadence_nand_dt_driver = {
3025 	.probe		= cadence_nand_dt_probe,
3026 	.remove		= cadence_nand_dt_remove,
3027 	.driver		= {
3028 		.name	= "cadence-nand-controller",
3029 		.of_match_table = cadence_nand_dt_ids,
3030 	},
3031 };
3032 
3033 module_platform_driver(cadence_nand_dt_driver);
3034 
3035 MODULE_AUTHOR("Piotr Sroka <piotrs@cadence.com>");
3036 MODULE_LICENSE("GPL v2");
3037 MODULE_DESCRIPTION("Driver for Cadence NAND flash controller");
3038 
3039