1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * Cadence NAND flash controller driver
4  *
5  * Copyright (C) 2019 Cadence
6  *
7  * Author: Piotr Sroka <piotrs@cadence.com>
8  */
9 
10 #include <linux/bitfield.h>
11 #include <linux/clk.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/dmaengine.h>
14 #include <linux/interrupt.h>
15 #include <linux/module.h>
16 #include <linux/mtd/mtd.h>
17 #include <linux/mtd/rawnand.h>
18 #include <linux/of_device.h>
19 #include <linux/iopoll.h>
20 #include <linux/slab.h>
21 
22 /*
23  * HPNFC can work in 3 modes:
24  * -  PIO - can work in master or slave DMA
25  * -  CDMA - needs Master DMA for accessing command descriptors.
26  * -  Generic mode - can use only slave DMA.
27  * CDMA and PIO modes can be used to execute only base commands.
28  * Generic mode can be used to execute any command
29  * on NAND flash memory. Driver uses CDMA mode for
30  * block erasing, page reading, page programing.
31  * Generic mode is used for executing rest of commands.
32  */
33 
34 #define MAX_ADDRESS_CYC		6
35 #define MAX_ERASE_ADDRESS_CYC	3
36 #define MAX_DATA_SIZE		0xFFFC
37 #define DMA_DATA_SIZE_ALIGN	8
38 
39 /* Register definition. */
40 /*
41  * Command register 0.
42  * Writing data to this register will initiate a new transaction
43  * of the NF controller.
44  */
45 #define CMD_REG0			0x0000
46 /* Command type field mask. */
47 #define		CMD_REG0_CT		GENMASK(31, 30)
48 /* Command type CDMA. */
49 #define		CMD_REG0_CT_CDMA	0uL
50 /* Command type generic. */
51 #define		CMD_REG0_CT_GEN		3uL
52 /* Command thread number field mask. */
53 #define		CMD_REG0_TN		GENMASK(27, 24)
54 
55 /* Command register 2. */
56 #define CMD_REG2			0x0008
57 /* Command register 3. */
58 #define CMD_REG3			0x000C
59 /* Pointer register to select which thread status will be selected. */
60 #define CMD_STATUS_PTR			0x0010
61 /* Command status register for selected thread. */
62 #define CMD_STATUS			0x0014
63 
64 /* Interrupt status register. */
65 #define INTR_STATUS			0x0110
66 #define		INTR_STATUS_SDMA_ERR	BIT(22)
67 #define		INTR_STATUS_SDMA_TRIGG	BIT(21)
68 #define		INTR_STATUS_UNSUPP_CMD	BIT(19)
69 #define		INTR_STATUS_DDMA_TERR	BIT(18)
70 #define		INTR_STATUS_CDMA_TERR	BIT(17)
71 #define		INTR_STATUS_CDMA_IDL	BIT(16)
72 
73 /* Interrupt enable register. */
74 #define INTR_ENABLE				0x0114
75 #define		INTR_ENABLE_INTR_EN		BIT(31)
76 #define		INTR_ENABLE_SDMA_ERR_EN		BIT(22)
77 #define		INTR_ENABLE_SDMA_TRIGG_EN	BIT(21)
78 #define		INTR_ENABLE_UNSUPP_CMD_EN	BIT(19)
79 #define		INTR_ENABLE_DDMA_TERR_EN	BIT(18)
80 #define		INTR_ENABLE_CDMA_TERR_EN	BIT(17)
81 #define		INTR_ENABLE_CDMA_IDLE_EN	BIT(16)
82 
83 /* Controller internal state. */
84 #define CTRL_STATUS				0x0118
85 #define		CTRL_STATUS_INIT_COMP		BIT(9)
86 #define		CTRL_STATUS_CTRL_BUSY		BIT(8)
87 
88 /* Command Engine threads state. */
89 #define TRD_STATUS				0x0120
90 
91 /* Command Engine interrupt thread error status. */
92 #define TRD_ERR_INT_STATUS			0x0128
93 /* Command Engine interrupt thread error enable. */
94 #define TRD_ERR_INT_STATUS_EN			0x0130
95 /* Command Engine interrupt thread complete status. */
96 #define TRD_COMP_INT_STATUS			0x0138
97 
98 /*
99  * Transfer config 0 register.
100  * Configures data transfer parameters.
101  */
102 #define TRAN_CFG_0				0x0400
103 /* Offset value from the beginning of the page. */
104 #define		TRAN_CFG_0_OFFSET		GENMASK(31, 16)
105 /* Numbers of sectors to transfer within singlNF device's page. */
106 #define		TRAN_CFG_0_SEC_CNT		GENMASK(7, 0)
107 
108 /*
109  * Transfer config 1 register.
110  * Configures data transfer parameters.
111  */
112 #define TRAN_CFG_1				0x0404
113 /* Size of last data sector. */
114 #define		TRAN_CFG_1_LAST_SEC_SIZE	GENMASK(31, 16)
115 /* Size of not-last data sector. */
116 #define		TRAN_CFG_1_SECTOR_SIZE		GENMASK(15, 0)
117 
118 /* ECC engine configuration register 0. */
119 #define ECC_CONFIG_0				0x0428
120 /* Correction strength. */
121 #define		ECC_CONFIG_0_CORR_STR		GENMASK(10, 8)
122 /* Enable erased pages detection mechanism. */
123 #define		ECC_CONFIG_0_ERASE_DET_EN	BIT(1)
124 /* Enable controller ECC check bits generation and correction. */
125 #define		ECC_CONFIG_0_ECC_EN		BIT(0)
126 
127 /* ECC engine configuration register 1. */
128 #define ECC_CONFIG_1				0x042C
129 
130 /* Multiplane settings register. */
131 #define MULTIPLANE_CFG				0x0434
132 /* Cache operation settings. */
133 #define CACHE_CFG				0x0438
134 
135 /* DMA settings register. */
136 #define DMA_SETINGS				0x043C
137 /* Enable SDMA error report on access unprepared slave DMA interface. */
138 #define		DMA_SETINGS_SDMA_ERR_RSP	BIT(17)
139 
140 /* Transferred data block size for the slave DMA module. */
141 #define SDMA_SIZE				0x0440
142 
143 /* Thread number associated with transferred data block
144  * for the slave DMA module.
145  */
146 #define SDMA_TRD_NUM				0x0444
147 /* Thread number mask. */
148 #define		SDMA_TRD_NUM_SDMA_TRD		GENMASK(2, 0)
149 
150 #define CONTROL_DATA_CTRL			0x0494
151 /* Thread number mask. */
152 #define		CONTROL_DATA_CTRL_SIZE		GENMASK(15, 0)
153 
154 #define CTRL_VERSION				0x800
155 #define		CTRL_VERSION_REV		GENMASK(7, 0)
156 
157 /* Available hardware features of the controller. */
158 #define CTRL_FEATURES				0x804
159 /* Support for NV-DDR2/3 work mode. */
160 #define		CTRL_FEATURES_NVDDR_2_3		BIT(28)
161 /* Support for NV-DDR work mode. */
162 #define		CTRL_FEATURES_NVDDR		BIT(27)
163 /* Support for asynchronous work mode. */
164 #define		CTRL_FEATURES_ASYNC		BIT(26)
165 /* Support for asynchronous work mode. */
166 #define		CTRL_FEATURES_N_BANKS		GENMASK(25, 24)
167 /* Slave and Master DMA data width. */
168 #define		CTRL_FEATURES_DMA_DWITH64	BIT(21)
169 /* Availability of Control Data feature.*/
170 #define		CTRL_FEATURES_CONTROL_DATA	BIT(10)
171 
172 /* BCH Engine identification register 0 - correction strengths. */
173 #define BCH_CFG_0				0x838
174 #define		BCH_CFG_0_CORR_CAP_0		GENMASK(7, 0)
175 #define		BCH_CFG_0_CORR_CAP_1		GENMASK(15, 8)
176 #define		BCH_CFG_0_CORR_CAP_2		GENMASK(23, 16)
177 #define		BCH_CFG_0_CORR_CAP_3		GENMASK(31, 24)
178 
179 /* BCH Engine identification register 1 - correction strengths. */
180 #define BCH_CFG_1				0x83C
181 #define		BCH_CFG_1_CORR_CAP_4		GENMASK(7, 0)
182 #define		BCH_CFG_1_CORR_CAP_5		GENMASK(15, 8)
183 #define		BCH_CFG_1_CORR_CAP_6		GENMASK(23, 16)
184 #define		BCH_CFG_1_CORR_CAP_7		GENMASK(31, 24)
185 
186 /* BCH Engine identification register 2 - sector sizes. */
187 #define BCH_CFG_2				0x840
188 #define		BCH_CFG_2_SECT_0		GENMASK(15, 0)
189 #define		BCH_CFG_2_SECT_1		GENMASK(31, 16)
190 
191 /* BCH Engine identification register 3. */
192 #define BCH_CFG_3				0x844
193 #define		BCH_CFG_3_METADATA_SIZE		GENMASK(23, 16)
194 
195 /* Ready/Busy# line status. */
196 #define RBN_SETINGS				0x1004
197 
198 /* Common settings. */
199 #define COMMON_SET				0x1008
200 /* 16 bit device connected to the NAND Flash interface. */
201 #define		COMMON_SET_DEVICE_16BIT		BIT(8)
202 
203 /* Skip_bytes registers. */
204 #define SKIP_BYTES_CONF				0x100C
205 #define		SKIP_BYTES_MARKER_VALUE		GENMASK(31, 16)
206 #define		SKIP_BYTES_NUM_OF_BYTES		GENMASK(7, 0)
207 
208 #define SKIP_BYTES_OFFSET			0x1010
209 #define		 SKIP_BYTES_OFFSET_VALUE	GENMASK(23, 0)
210 
211 /* Timings configuration. */
212 #define ASYNC_TOGGLE_TIMINGS			0x101c
213 #define		ASYNC_TOGGLE_TIMINGS_TRH	GENMASK(28, 24)
214 #define		ASYNC_TOGGLE_TIMINGS_TRP	GENMASK(20, 16)
215 #define		ASYNC_TOGGLE_TIMINGS_TWH	GENMASK(12, 8)
216 #define		ASYNC_TOGGLE_TIMINGS_TWP	GENMASK(4, 0)
217 
218 #define	TIMINGS0				0x1024
219 #define		TIMINGS0_TADL			GENMASK(31, 24)
220 #define		TIMINGS0_TCCS			GENMASK(23, 16)
221 #define		TIMINGS0_TWHR			GENMASK(15, 8)
222 #define		TIMINGS0_TRHW			GENMASK(7, 0)
223 
224 #define	TIMINGS1				0x1028
225 #define		TIMINGS1_TRHZ			GENMASK(31, 24)
226 #define		TIMINGS1_TWB			GENMASK(23, 16)
227 #define		TIMINGS1_TVDLY			GENMASK(7, 0)
228 
229 #define	TIMINGS2				0x102c
230 #define		TIMINGS2_TFEAT			GENMASK(25, 16)
231 #define		TIMINGS2_CS_HOLD_TIME		GENMASK(13, 8)
232 #define		TIMINGS2_CS_SETUP_TIME		GENMASK(5, 0)
233 
234 /* Configuration of the resynchronization of slave DLL of PHY. */
235 #define DLL_PHY_CTRL				0x1034
236 #define		DLL_PHY_CTRL_DLL_RST_N		BIT(24)
237 #define		DLL_PHY_CTRL_EXTENDED_WR_MODE	BIT(17)
238 #define		DLL_PHY_CTRL_EXTENDED_RD_MODE	BIT(16)
239 #define		DLL_PHY_CTRL_RS_HIGH_WAIT_CNT	GENMASK(11, 8)
240 #define		DLL_PHY_CTRL_RS_IDLE_CNT	GENMASK(7, 0)
241 
242 /* Register controlling DQ related timing. */
243 #define PHY_DQ_TIMING				0x2000
244 /* Register controlling DSQ related timing.  */
245 #define PHY_DQS_TIMING				0x2004
246 #define		PHY_DQS_TIMING_DQS_SEL_OE_END	GENMASK(3, 0)
247 #define		PHY_DQS_TIMING_PHONY_DQS_SEL	BIT(16)
248 #define		PHY_DQS_TIMING_USE_PHONY_DQS	BIT(20)
249 
250 /* Register controlling the gate and loopback control related timing. */
251 #define PHY_GATE_LPBK_CTRL			0x2008
252 #define		PHY_GATE_LPBK_CTRL_RDS		GENMASK(24, 19)
253 
254 /* Register holds the control for the master DLL logic. */
255 #define PHY_DLL_MASTER_CTRL			0x200C
256 #define		PHY_DLL_MASTER_CTRL_BYPASS_MODE	BIT(23)
257 
258 /* Register holds the control for the slave DLL logic. */
259 #define PHY_DLL_SLAVE_CTRL			0x2010
260 
261 /* This register handles the global control settings for the PHY. */
262 #define PHY_CTRL				0x2080
263 #define		PHY_CTRL_SDR_DQS		BIT(14)
264 #define		PHY_CTRL_PHONY_DQS		GENMASK(9, 4)
265 
266 /*
267  * This register handles the global control settings
268  * for the termination selects for reads.
269  */
270 #define PHY_TSEL				0x2084
271 
272 /* Generic command layout. */
273 #define GCMD_LAY_CS			GENMASK_ULL(11, 8)
274 /*
275  * This bit informs the minicotroller if it has to wait for tWB
276  * after sending the last CMD/ADDR/DATA in the sequence.
277  */
278 #define GCMD_LAY_TWB			BIT_ULL(6)
279 /* Type of generic instruction. */
280 #define GCMD_LAY_INSTR			GENMASK_ULL(5, 0)
281 
282 /* Generic CMD sequence type. */
283 #define		GCMD_LAY_INSTR_CMD	0
284 /* Generic ADDR sequence type. */
285 #define		GCMD_LAY_INSTR_ADDR	1
286 /* Generic data transfer sequence type. */
287 #define		GCMD_LAY_INSTR_DATA	2
288 
289 /* Input part of generic command type of input is command. */
290 #define GCMD_LAY_INPUT_CMD		GENMASK_ULL(23, 16)
291 
292 /* Generic command address sequence - address fields. */
293 #define GCMD_LAY_INPUT_ADDR		GENMASK_ULL(63, 16)
294 /* Generic command address sequence - address size. */
295 #define GCMD_LAY_INPUT_ADDR_SIZE	GENMASK_ULL(13, 11)
296 
297 /* Transfer direction field of generic command data sequence. */
298 #define GCMD_DIR			BIT_ULL(11)
299 /* Read transfer direction of generic command data sequence. */
300 #define		GCMD_DIR_READ		0
301 /* Write transfer direction of generic command data sequence. */
302 #define		GCMD_DIR_WRITE		1
303 
304 /* ECC enabled flag of generic command data sequence - ECC enabled. */
305 #define GCMD_ECC_EN			BIT_ULL(12)
306 /* Generic command data sequence - sector size. */
307 #define GCMD_SECT_SIZE			GENMASK_ULL(31, 16)
308 /* Generic command data sequence - sector count. */
309 #define GCMD_SECT_CNT			GENMASK_ULL(39, 32)
310 /* Generic command data sequence - last sector size. */
311 #define GCMD_LAST_SIZE			GENMASK_ULL(55, 40)
312 
313 /* CDMA descriptor fields. */
314 /* Erase command type of CDMA descriptor. */
315 #define CDMA_CT_ERASE		0x1000
316 /* Program page command type of CDMA descriptor. */
317 #define CDMA_CT_WR		0x2100
318 /* Read page command type of CDMA descriptor. */
319 #define CDMA_CT_RD		0x2200
320 
321 /* Flash pointer memory shift. */
322 #define CDMA_CFPTR_MEM_SHIFT	24
323 /* Flash pointer memory mask. */
324 #define CDMA_CFPTR_MEM		GENMASK(26, 24)
325 
326 /*
327  * Command DMA descriptor flags. If set causes issue interrupt after
328  * the completion of descriptor processing.
329  */
330 #define CDMA_CF_INT		BIT(8)
331 /*
332  * Command DMA descriptor flags - the next descriptor
333  * address field is valid and descriptor processing should continue.
334  */
335 #define CDMA_CF_CONT		BIT(9)
336 /* DMA master flag of command DMA descriptor. */
337 #define CDMA_CF_DMA_MASTER	BIT(10)
338 
339 /* Operation complete status of command descriptor. */
340 #define CDMA_CS_COMP		BIT(15)
341 /* Operation complete status of command descriptor. */
342 /* Command descriptor status - operation fail. */
343 #define CDMA_CS_FAIL		BIT(14)
344 /* Command descriptor status - page erased. */
345 #define CDMA_CS_ERP		BIT(11)
346 /* Command descriptor status - timeout occurred. */
347 #define CDMA_CS_TOUT		BIT(10)
348 /*
349  * Maximum amount of correction applied to one ECC sector.
350  * It is part of command descriptor status.
351  */
352 #define CDMA_CS_MAXERR		GENMASK(9, 2)
353 /* Command descriptor status - uncorrectable ECC error. */
354 #define CDMA_CS_UNCE		BIT(1)
355 /* Command descriptor status - descriptor error. */
356 #define CDMA_CS_ERR		BIT(0)
357 
358 /* Status of operation - OK. */
359 #define STAT_OK			0
360 /* Status of operation - FAIL. */
361 #define STAT_FAIL		2
362 /* Status of operation - uncorrectable ECC error. */
363 #define STAT_ECC_UNCORR		3
364 /* Status of operation - page erased. */
365 #define STAT_ERASED		5
366 /* Status of operation - correctable ECC error. */
367 #define STAT_ECC_CORR		6
368 /* Status of operation - unsuspected state. */
369 #define STAT_UNKNOWN		7
370 /* Status of operation - operation is not completed yet. */
371 #define STAT_BUSY		0xFF
372 
373 #define BCH_MAX_NUM_CORR_CAPS		8
374 #define BCH_MAX_NUM_SECTOR_SIZES	2
375 
376 struct cadence_nand_timings {
377 	u32 async_toggle_timings;
378 	u32 timings0;
379 	u32 timings1;
380 	u32 timings2;
381 	u32 dll_phy_ctrl;
382 	u32 phy_ctrl;
383 	u32 phy_dqs_timing;
384 	u32 phy_gate_lpbk_ctrl;
385 };
386 
387 /* Command DMA descriptor. */
388 struct cadence_nand_cdma_desc {
389 	/* Next descriptor address. */
390 	u64 next_pointer;
391 
392 	/* Flash address is a 32-bit address comprising of BANK and ROW ADDR. */
393 	u32 flash_pointer;
394 	/*field appears in HPNFC version 13*/
395 	u16 bank;
396 	u16 rsvd0;
397 
398 	/* Operation the controller needs to perform. */
399 	u16 command_type;
400 	u16 rsvd1;
401 	/* Flags for operation of this command. */
402 	u16 command_flags;
403 	u16 rsvd2;
404 
405 	/* System/host memory address required for data DMA commands. */
406 	u64 memory_pointer;
407 
408 	/* Status of operation. */
409 	u32 status;
410 	u32 rsvd3;
411 
412 	/* Address pointer to sync buffer location. */
413 	u64 sync_flag_pointer;
414 
415 	/* Controls the buffer sync mechanism. */
416 	u32 sync_arguments;
417 	u32 rsvd4;
418 
419 	/* Control data pointer. */
420 	u64 ctrl_data_ptr;
421 };
422 
423 /* Interrupt status. */
424 struct cadence_nand_irq_status {
425 	/* Thread operation complete status. */
426 	u32 trd_status;
427 	/* Thread operation error. */
428 	u32 trd_error;
429 	/* Controller status. */
430 	u32 status;
431 };
432 
433 /* Cadence NAND flash controller capabilities get from driver data. */
434 struct cadence_nand_dt_devdata {
435 	/* Skew value of the output signals of the NAND Flash interface. */
436 	u32 if_skew;
437 	/* It informs if slave DMA interface is connected to DMA engine. */
438 	unsigned int has_dma:1;
439 };
440 
441 /* Cadence NAND flash controller capabilities read from registers. */
442 struct cdns_nand_caps {
443 	/* Maximum number of banks supported by hardware. */
444 	u8 max_banks;
445 	/* Slave and Master DMA data width in bytes (4 or 8). */
446 	u8 data_dma_width;
447 	/* Control Data feature supported. */
448 	bool data_control_supp;
449 	/* Is PHY type DLL. */
450 	bool is_phy_type_dll;
451 };
452 
453 struct cdns_nand_ctrl {
454 	struct device *dev;
455 	struct nand_controller controller;
456 	struct cadence_nand_cdma_desc *cdma_desc;
457 	/* IP capability. */
458 	const struct cadence_nand_dt_devdata *caps1;
459 	struct cdns_nand_caps caps2;
460 	u8 ctrl_rev;
461 	dma_addr_t dma_cdma_desc;
462 	u8 *buf;
463 	u32 buf_size;
464 	u8 curr_corr_str_idx;
465 
466 	/* Register interface. */
467 	void __iomem *reg;
468 
469 	struct {
470 		void __iomem *virt;
471 		dma_addr_t dma;
472 	} io;
473 
474 	int irq;
475 	/* Interrupts that have happened. */
476 	struct cadence_nand_irq_status irq_status;
477 	/* Interrupts we are waiting for. */
478 	struct cadence_nand_irq_status irq_mask;
479 	struct completion complete;
480 	/* Protect irq_mask and irq_status. */
481 	spinlock_t irq_lock;
482 
483 	int ecc_strengths[BCH_MAX_NUM_CORR_CAPS];
484 	struct nand_ecc_step_info ecc_stepinfos[BCH_MAX_NUM_SECTOR_SIZES];
485 	struct nand_ecc_caps ecc_caps;
486 
487 	int curr_trans_type;
488 
489 	struct dma_chan *dmac;
490 
491 	u32 nf_clk_rate;
492 	/*
493 	 * Estimated Board delay. The value includes the total
494 	 * round trip delay for the signals and is used for deciding on values
495 	 * associated with data read capture.
496 	 */
497 	u32 board_delay;
498 
499 	struct nand_chip *selected_chip;
500 
501 	unsigned long assigned_cs;
502 	struct list_head chips;
503 	u8 bch_metadata_size;
504 };
505 
506 struct cdns_nand_chip {
507 	struct cadence_nand_timings timings;
508 	struct nand_chip chip;
509 	u8 nsels;
510 	struct list_head node;
511 
512 	/*
513 	 * part of oob area of NAND flash memory page.
514 	 * This part is available for user to read or write.
515 	 */
516 	u32 avail_oob_size;
517 
518 	/* Sector size. There are few sectors per mtd->writesize */
519 	u32 sector_size;
520 	u32 sector_count;
521 
522 	/* Offset of BBM. */
523 	u8 bbm_offs;
524 	/* Number of bytes reserved for BBM. */
525 	u8 bbm_len;
526 	/* ECC strength index. */
527 	u8 corr_str_idx;
528 
529 	u8 cs[];
530 };
531 
532 struct ecc_info {
533 	int (*calc_ecc_bytes)(int step_size, int strength);
534 	int max_step_size;
535 };
536 
537 static inline struct
538 cdns_nand_chip *to_cdns_nand_chip(struct nand_chip *chip)
539 {
540 	return container_of(chip, struct cdns_nand_chip, chip);
541 }
542 
543 static inline struct
544 cdns_nand_ctrl *to_cdns_nand_ctrl(struct nand_controller *controller)
545 {
546 	return container_of(controller, struct cdns_nand_ctrl, controller);
547 }
548 
549 static bool
550 cadence_nand_dma_buf_ok(struct cdns_nand_ctrl *cdns_ctrl, const void *buf,
551 			u32 buf_len)
552 {
553 	u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;
554 
555 	return buf && virt_addr_valid(buf) &&
556 		likely(IS_ALIGNED((uintptr_t)buf, data_dma_width)) &&
557 		likely(IS_ALIGNED(buf_len, DMA_DATA_SIZE_ALIGN));
558 }
559 
560 static int cadence_nand_wait_for_value(struct cdns_nand_ctrl *cdns_ctrl,
561 				       u32 reg_offset, u32 timeout_us,
562 				       u32 mask, bool is_clear)
563 {
564 	u32 val;
565 	int ret;
566 
567 	ret = readl_relaxed_poll_timeout(cdns_ctrl->reg + reg_offset,
568 					 val, !(val & mask) == is_clear,
569 					 10, timeout_us);
570 
571 	if (ret < 0) {
572 		dev_err(cdns_ctrl->dev,
573 			"Timeout while waiting for reg %x with mask %x is clear %d\n",
574 			reg_offset, mask, is_clear);
575 	}
576 
577 	return ret;
578 }
579 
580 static int cadence_nand_set_ecc_enable(struct cdns_nand_ctrl *cdns_ctrl,
581 				       bool enable)
582 {
583 	u32 reg;
584 
585 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
586 					1000000,
587 					CTRL_STATUS_CTRL_BUSY, true))
588 		return -ETIMEDOUT;
589 
590 	reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
591 
592 	if (enable)
593 		reg |= ECC_CONFIG_0_ECC_EN;
594 	else
595 		reg &= ~ECC_CONFIG_0_ECC_EN;
596 
597 	writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
598 
599 	return 0;
600 }
601 
602 static void cadence_nand_set_ecc_strength(struct cdns_nand_ctrl *cdns_ctrl,
603 					  u8 corr_str_idx)
604 {
605 	u32 reg;
606 
607 	if (cdns_ctrl->curr_corr_str_idx == corr_str_idx)
608 		return;
609 
610 	reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
611 	reg &= ~ECC_CONFIG_0_CORR_STR;
612 	reg |= FIELD_PREP(ECC_CONFIG_0_CORR_STR, corr_str_idx);
613 	writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
614 
615 	cdns_ctrl->curr_corr_str_idx = corr_str_idx;
616 }
617 
618 static int cadence_nand_get_ecc_strength_idx(struct cdns_nand_ctrl *cdns_ctrl,
619 					     u8 strength)
620 {
621 	int i, corr_str_idx = -1;
622 
623 	for (i = 0; i < BCH_MAX_NUM_CORR_CAPS; i++) {
624 		if (cdns_ctrl->ecc_strengths[i] == strength) {
625 			corr_str_idx = i;
626 			break;
627 		}
628 	}
629 
630 	return corr_str_idx;
631 }
632 
633 static int cadence_nand_set_skip_marker_val(struct cdns_nand_ctrl *cdns_ctrl,
634 					    u16 marker_value)
635 {
636 	u32 reg;
637 
638 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
639 					1000000,
640 					CTRL_STATUS_CTRL_BUSY, true))
641 		return -ETIMEDOUT;
642 
643 	reg = readl_relaxed(cdns_ctrl->reg + SKIP_BYTES_CONF);
644 	reg &= ~SKIP_BYTES_MARKER_VALUE;
645 	reg |= FIELD_PREP(SKIP_BYTES_MARKER_VALUE,
646 			  marker_value);
647 
648 	writel_relaxed(reg, cdns_ctrl->reg + SKIP_BYTES_CONF);
649 
650 	return 0;
651 }
652 
653 static int cadence_nand_set_skip_bytes_conf(struct cdns_nand_ctrl *cdns_ctrl,
654 					    u8 num_of_bytes,
655 					    u32 offset_value,
656 					    int enable)
657 {
658 	u32 reg, skip_bytes_offset;
659 
660 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
661 					1000000,
662 					CTRL_STATUS_CTRL_BUSY, true))
663 		return -ETIMEDOUT;
664 
665 	if (!enable) {
666 		num_of_bytes = 0;
667 		offset_value = 0;
668 	}
669 
670 	reg = readl_relaxed(cdns_ctrl->reg + SKIP_BYTES_CONF);
671 	reg &= ~SKIP_BYTES_NUM_OF_BYTES;
672 	reg |= FIELD_PREP(SKIP_BYTES_NUM_OF_BYTES,
673 			  num_of_bytes);
674 	skip_bytes_offset = FIELD_PREP(SKIP_BYTES_OFFSET_VALUE,
675 				       offset_value);
676 
677 	writel_relaxed(reg, cdns_ctrl->reg + SKIP_BYTES_CONF);
678 	writel_relaxed(skip_bytes_offset, cdns_ctrl->reg + SKIP_BYTES_OFFSET);
679 
680 	return 0;
681 }
682 
683 /* Functions enables/disables hardware detection of erased data */
684 static void cadence_nand_set_erase_detection(struct cdns_nand_ctrl *cdns_ctrl,
685 					     bool enable,
686 					     u8 bitflips_threshold)
687 {
688 	u32 reg;
689 
690 	reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
691 
692 	if (enable)
693 		reg |= ECC_CONFIG_0_ERASE_DET_EN;
694 	else
695 		reg &= ~ECC_CONFIG_0_ERASE_DET_EN;
696 
697 	writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
698 	writel_relaxed(bitflips_threshold, cdns_ctrl->reg + ECC_CONFIG_1);
699 }
700 
701 static int cadence_nand_set_access_width16(struct cdns_nand_ctrl *cdns_ctrl,
702 					   bool bit_bus16)
703 {
704 	u32 reg;
705 
706 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
707 					1000000,
708 					CTRL_STATUS_CTRL_BUSY, true))
709 		return -ETIMEDOUT;
710 
711 	reg = readl_relaxed(cdns_ctrl->reg + COMMON_SET);
712 
713 	if (!bit_bus16)
714 		reg &= ~COMMON_SET_DEVICE_16BIT;
715 	else
716 		reg |= COMMON_SET_DEVICE_16BIT;
717 	writel_relaxed(reg, cdns_ctrl->reg + COMMON_SET);
718 
719 	return 0;
720 }
721 
722 static void
723 cadence_nand_clear_interrupt(struct cdns_nand_ctrl *cdns_ctrl,
724 			     struct cadence_nand_irq_status *irq_status)
725 {
726 	writel_relaxed(irq_status->status, cdns_ctrl->reg + INTR_STATUS);
727 	writel_relaxed(irq_status->trd_status,
728 		       cdns_ctrl->reg + TRD_COMP_INT_STATUS);
729 	writel_relaxed(irq_status->trd_error,
730 		       cdns_ctrl->reg + TRD_ERR_INT_STATUS);
731 }
732 
733 static void
734 cadence_nand_read_int_status(struct cdns_nand_ctrl *cdns_ctrl,
735 			     struct cadence_nand_irq_status *irq_status)
736 {
737 	irq_status->status = readl_relaxed(cdns_ctrl->reg + INTR_STATUS);
738 	irq_status->trd_status = readl_relaxed(cdns_ctrl->reg
739 					       + TRD_COMP_INT_STATUS);
740 	irq_status->trd_error = readl_relaxed(cdns_ctrl->reg
741 					      + TRD_ERR_INT_STATUS);
742 }
743 
744 static u32 irq_detected(struct cdns_nand_ctrl *cdns_ctrl,
745 			struct cadence_nand_irq_status *irq_status)
746 {
747 	cadence_nand_read_int_status(cdns_ctrl, irq_status);
748 
749 	return irq_status->status || irq_status->trd_status ||
750 		irq_status->trd_error;
751 }
752 
753 static void cadence_nand_reset_irq(struct cdns_nand_ctrl *cdns_ctrl)
754 {
755 	unsigned long flags;
756 
757 	spin_lock_irqsave(&cdns_ctrl->irq_lock, flags);
758 	memset(&cdns_ctrl->irq_status, 0, sizeof(cdns_ctrl->irq_status));
759 	memset(&cdns_ctrl->irq_mask, 0, sizeof(cdns_ctrl->irq_mask));
760 	spin_unlock_irqrestore(&cdns_ctrl->irq_lock, flags);
761 }
762 
763 /*
764  * This is the interrupt service routine. It handles all interrupts
765  * sent to this device.
766  */
767 static irqreturn_t cadence_nand_isr(int irq, void *dev_id)
768 {
769 	struct cdns_nand_ctrl *cdns_ctrl = dev_id;
770 	struct cadence_nand_irq_status irq_status;
771 	irqreturn_t result = IRQ_NONE;
772 
773 	spin_lock(&cdns_ctrl->irq_lock);
774 
775 	if (irq_detected(cdns_ctrl, &irq_status)) {
776 		/* Handle interrupt. */
777 		/* First acknowledge it. */
778 		cadence_nand_clear_interrupt(cdns_ctrl, &irq_status);
779 		/* Status in the device context for someone to read. */
780 		cdns_ctrl->irq_status.status |= irq_status.status;
781 		cdns_ctrl->irq_status.trd_status |= irq_status.trd_status;
782 		cdns_ctrl->irq_status.trd_error |= irq_status.trd_error;
783 		/* Notify anyone who cares that it happened. */
784 		complete(&cdns_ctrl->complete);
785 		/* Tell the OS that we've handled this. */
786 		result = IRQ_HANDLED;
787 	}
788 	spin_unlock(&cdns_ctrl->irq_lock);
789 
790 	return result;
791 }
792 
793 static void cadence_nand_set_irq_mask(struct cdns_nand_ctrl *cdns_ctrl,
794 				      struct cadence_nand_irq_status *irq_mask)
795 {
796 	writel_relaxed(INTR_ENABLE_INTR_EN | irq_mask->status,
797 		       cdns_ctrl->reg + INTR_ENABLE);
798 
799 	writel_relaxed(irq_mask->trd_error,
800 		       cdns_ctrl->reg + TRD_ERR_INT_STATUS_EN);
801 }
802 
803 static void
804 cadence_nand_wait_for_irq(struct cdns_nand_ctrl *cdns_ctrl,
805 			  struct cadence_nand_irq_status *irq_mask,
806 			  struct cadence_nand_irq_status *irq_status)
807 {
808 	unsigned long timeout = msecs_to_jiffies(10000);
809 	unsigned long time_left;
810 
811 	time_left = wait_for_completion_timeout(&cdns_ctrl->complete,
812 						timeout);
813 
814 	*irq_status = cdns_ctrl->irq_status;
815 	if (time_left == 0) {
816 		/* Timeout error. */
817 		dev_err(cdns_ctrl->dev, "timeout occurred:\n");
818 		dev_err(cdns_ctrl->dev, "\tstatus = 0x%x, mask = 0x%x\n",
819 			irq_status->status, irq_mask->status);
820 		dev_err(cdns_ctrl->dev,
821 			"\ttrd_status = 0x%x, trd_status mask = 0x%x\n",
822 			irq_status->trd_status, irq_mask->trd_status);
823 		dev_err(cdns_ctrl->dev,
824 			"\t trd_error = 0x%x, trd_error mask = 0x%x\n",
825 			irq_status->trd_error, irq_mask->trd_error);
826 	}
827 }
828 
829 /* Execute generic command on NAND controller. */
830 static int cadence_nand_generic_cmd_send(struct cdns_nand_ctrl *cdns_ctrl,
831 					 u8 chip_nr,
832 					 u64 mini_ctrl_cmd)
833 {
834 	u32 mini_ctrl_cmd_l, mini_ctrl_cmd_h, reg;
835 
836 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_CS, chip_nr);
837 	mini_ctrl_cmd_l = mini_ctrl_cmd & 0xFFFFFFFF;
838 	mini_ctrl_cmd_h = mini_ctrl_cmd >> 32;
839 
840 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
841 					1000000,
842 					CTRL_STATUS_CTRL_BUSY, true))
843 		return -ETIMEDOUT;
844 
845 	cadence_nand_reset_irq(cdns_ctrl);
846 
847 	writel_relaxed(mini_ctrl_cmd_l, cdns_ctrl->reg + CMD_REG2);
848 	writel_relaxed(mini_ctrl_cmd_h, cdns_ctrl->reg + CMD_REG3);
849 
850 	/* Select generic command. */
851 	reg = FIELD_PREP(CMD_REG0_CT, CMD_REG0_CT_GEN);
852 	/* Thread number. */
853 	reg |= FIELD_PREP(CMD_REG0_TN, 0);
854 
855 	/* Issue command. */
856 	writel_relaxed(reg, cdns_ctrl->reg + CMD_REG0);
857 
858 	return 0;
859 }
860 
861 /* Wait for data on slave DMA interface. */
862 static int cadence_nand_wait_on_sdma(struct cdns_nand_ctrl *cdns_ctrl,
863 				     u8 *out_sdma_trd,
864 				     u32 *out_sdma_size)
865 {
866 	struct cadence_nand_irq_status irq_mask, irq_status;
867 
868 	irq_mask.trd_status = 0;
869 	irq_mask.trd_error = 0;
870 	irq_mask.status = INTR_STATUS_SDMA_TRIGG
871 		| INTR_STATUS_SDMA_ERR
872 		| INTR_STATUS_UNSUPP_CMD;
873 
874 	cadence_nand_set_irq_mask(cdns_ctrl, &irq_mask);
875 	cadence_nand_wait_for_irq(cdns_ctrl, &irq_mask, &irq_status);
876 	if (irq_status.status == 0) {
877 		dev_err(cdns_ctrl->dev, "Timeout while waiting for SDMA\n");
878 		return -ETIMEDOUT;
879 	}
880 
881 	if (irq_status.status & INTR_STATUS_SDMA_TRIGG) {
882 		*out_sdma_size = readl_relaxed(cdns_ctrl->reg + SDMA_SIZE);
883 		*out_sdma_trd  = readl_relaxed(cdns_ctrl->reg + SDMA_TRD_NUM);
884 		*out_sdma_trd =
885 			FIELD_GET(SDMA_TRD_NUM_SDMA_TRD, *out_sdma_trd);
886 	} else {
887 		dev_err(cdns_ctrl->dev, "SDMA error - irq_status %x\n",
888 			irq_status.status);
889 		return -EIO;
890 	}
891 
892 	return 0;
893 }
894 
895 static void cadence_nand_get_caps(struct cdns_nand_ctrl *cdns_ctrl)
896 {
897 	u32  reg;
898 
899 	reg = readl_relaxed(cdns_ctrl->reg + CTRL_FEATURES);
900 
901 	cdns_ctrl->caps2.max_banks = 1 << FIELD_GET(CTRL_FEATURES_N_BANKS, reg);
902 
903 	if (FIELD_GET(CTRL_FEATURES_DMA_DWITH64, reg))
904 		cdns_ctrl->caps2.data_dma_width = 8;
905 	else
906 		cdns_ctrl->caps2.data_dma_width = 4;
907 
908 	if (reg & CTRL_FEATURES_CONTROL_DATA)
909 		cdns_ctrl->caps2.data_control_supp = true;
910 
911 	if (reg & (CTRL_FEATURES_NVDDR_2_3
912 		   | CTRL_FEATURES_NVDDR))
913 		cdns_ctrl->caps2.is_phy_type_dll = true;
914 }
915 
916 /* Prepare CDMA descriptor. */
917 static void
918 cadence_nand_cdma_desc_prepare(struct cdns_nand_ctrl *cdns_ctrl,
919 			       char nf_mem, u32 flash_ptr, dma_addr_t mem_ptr,
920 				   dma_addr_t ctrl_data_ptr, u16 ctype)
921 {
922 	struct cadence_nand_cdma_desc *cdma_desc = cdns_ctrl->cdma_desc;
923 
924 	memset(cdma_desc, 0, sizeof(struct cadence_nand_cdma_desc));
925 
926 	/* Set fields for one descriptor. */
927 	cdma_desc->flash_pointer = flash_ptr;
928 	if (cdns_ctrl->ctrl_rev >= 13)
929 		cdma_desc->bank = nf_mem;
930 	else
931 		cdma_desc->flash_pointer |= (nf_mem << CDMA_CFPTR_MEM_SHIFT);
932 
933 	cdma_desc->command_flags |= CDMA_CF_DMA_MASTER;
934 	cdma_desc->command_flags  |= CDMA_CF_INT;
935 
936 	cdma_desc->memory_pointer = mem_ptr;
937 	cdma_desc->status = 0;
938 	cdma_desc->sync_flag_pointer = 0;
939 	cdma_desc->sync_arguments = 0;
940 
941 	cdma_desc->command_type = ctype;
942 	cdma_desc->ctrl_data_ptr = ctrl_data_ptr;
943 }
944 
945 static u8 cadence_nand_check_desc_error(struct cdns_nand_ctrl *cdns_ctrl,
946 					u32 desc_status)
947 {
948 	if (desc_status & CDMA_CS_ERP)
949 		return STAT_ERASED;
950 
951 	if (desc_status & CDMA_CS_UNCE)
952 		return STAT_ECC_UNCORR;
953 
954 	if (desc_status & CDMA_CS_ERR) {
955 		dev_err(cdns_ctrl->dev, ":CDMA desc error flag detected.\n");
956 		return STAT_FAIL;
957 	}
958 
959 	if (FIELD_GET(CDMA_CS_MAXERR, desc_status))
960 		return STAT_ECC_CORR;
961 
962 	return STAT_FAIL;
963 }
964 
965 static int cadence_nand_cdma_finish(struct cdns_nand_ctrl *cdns_ctrl)
966 {
967 	struct cadence_nand_cdma_desc *desc_ptr = cdns_ctrl->cdma_desc;
968 	u8 status = STAT_BUSY;
969 
970 	if (desc_ptr->status & CDMA_CS_FAIL) {
971 		status = cadence_nand_check_desc_error(cdns_ctrl,
972 						       desc_ptr->status);
973 		dev_err(cdns_ctrl->dev, ":CDMA error %x\n", desc_ptr->status);
974 	} else if (desc_ptr->status & CDMA_CS_COMP) {
975 		/* Descriptor finished with no errors. */
976 		if (desc_ptr->command_flags & CDMA_CF_CONT) {
977 			dev_info(cdns_ctrl->dev, "DMA unsupported flag is set");
978 			status = STAT_UNKNOWN;
979 		} else {
980 			/* Last descriptor.  */
981 			status = STAT_OK;
982 		}
983 	}
984 
985 	return status;
986 }
987 
988 static int cadence_nand_cdma_send(struct cdns_nand_ctrl *cdns_ctrl,
989 				  u8 thread)
990 {
991 	u32 reg;
992 	int status;
993 
994 	/* Wait for thread ready. */
995 	status = cadence_nand_wait_for_value(cdns_ctrl, TRD_STATUS,
996 					     1000000,
997 					     BIT(thread), true);
998 	if (status)
999 		return status;
1000 
1001 	cadence_nand_reset_irq(cdns_ctrl);
1002 	reinit_completion(&cdns_ctrl->complete);
1003 
1004 	writel_relaxed((u32)cdns_ctrl->dma_cdma_desc,
1005 		       cdns_ctrl->reg + CMD_REG2);
1006 	writel_relaxed(0, cdns_ctrl->reg + CMD_REG3);
1007 
1008 	/* Select CDMA mode. */
1009 	reg = FIELD_PREP(CMD_REG0_CT, CMD_REG0_CT_CDMA);
1010 	/* Thread number. */
1011 	reg |= FIELD_PREP(CMD_REG0_TN, thread);
1012 	/* Issue command. */
1013 	writel_relaxed(reg, cdns_ctrl->reg + CMD_REG0);
1014 
1015 	return 0;
1016 }
1017 
1018 /* Send SDMA command and wait for finish. */
1019 static u32
1020 cadence_nand_cdma_send_and_wait(struct cdns_nand_ctrl *cdns_ctrl,
1021 				u8 thread)
1022 {
1023 	struct cadence_nand_irq_status irq_mask, irq_status = {0};
1024 	int status;
1025 
1026 	irq_mask.trd_status = BIT(thread);
1027 	irq_mask.trd_error = BIT(thread);
1028 	irq_mask.status = INTR_STATUS_CDMA_TERR;
1029 
1030 	cadence_nand_set_irq_mask(cdns_ctrl, &irq_mask);
1031 
1032 	status = cadence_nand_cdma_send(cdns_ctrl, thread);
1033 	if (status)
1034 		return status;
1035 
1036 	cadence_nand_wait_for_irq(cdns_ctrl, &irq_mask, &irq_status);
1037 
1038 	if (irq_status.status == 0 && irq_status.trd_status == 0 &&
1039 	    irq_status.trd_error == 0) {
1040 		dev_err(cdns_ctrl->dev, "CDMA command timeout\n");
1041 		return -ETIMEDOUT;
1042 	}
1043 	if (irq_status.status & irq_mask.status) {
1044 		dev_err(cdns_ctrl->dev, "CDMA command failed\n");
1045 		return -EIO;
1046 	}
1047 
1048 	return 0;
1049 }
1050 
1051 /*
1052  * ECC size depends on configured ECC strength and on maximum supported
1053  * ECC step size.
1054  */
1055 static int cadence_nand_calc_ecc_bytes(int max_step_size, int strength)
1056 {
1057 	int nbytes = DIV_ROUND_UP(fls(8 * max_step_size) * strength, 8);
1058 
1059 	return ALIGN(nbytes, 2);
1060 }
1061 
1062 #define CADENCE_NAND_CALC_ECC_BYTES(max_step_size) \
1063 	static int \
1064 	cadence_nand_calc_ecc_bytes_##max_step_size(int step_size, \
1065 						    int strength)\
1066 	{\
1067 		return cadence_nand_calc_ecc_bytes(max_step_size, strength);\
1068 	}
1069 
1070 CADENCE_NAND_CALC_ECC_BYTES(256)
1071 CADENCE_NAND_CALC_ECC_BYTES(512)
1072 CADENCE_NAND_CALC_ECC_BYTES(1024)
1073 CADENCE_NAND_CALC_ECC_BYTES(2048)
1074 CADENCE_NAND_CALC_ECC_BYTES(4096)
1075 
1076 /* Function reads BCH capabilities. */
1077 static int cadence_nand_read_bch_caps(struct cdns_nand_ctrl *cdns_ctrl)
1078 {
1079 	struct nand_ecc_caps *ecc_caps = &cdns_ctrl->ecc_caps;
1080 	int max_step_size = 0, nstrengths, i;
1081 	u32 reg;
1082 
1083 	reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_3);
1084 	cdns_ctrl->bch_metadata_size = FIELD_GET(BCH_CFG_3_METADATA_SIZE, reg);
1085 	if (cdns_ctrl->bch_metadata_size < 4) {
1086 		dev_err(cdns_ctrl->dev,
1087 			"Driver needs at least 4 bytes of BCH meta data\n");
1088 		return -EIO;
1089 	}
1090 
1091 	reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_0);
1092 	cdns_ctrl->ecc_strengths[0] = FIELD_GET(BCH_CFG_0_CORR_CAP_0, reg);
1093 	cdns_ctrl->ecc_strengths[1] = FIELD_GET(BCH_CFG_0_CORR_CAP_1, reg);
1094 	cdns_ctrl->ecc_strengths[2] = FIELD_GET(BCH_CFG_0_CORR_CAP_2, reg);
1095 	cdns_ctrl->ecc_strengths[3] = FIELD_GET(BCH_CFG_0_CORR_CAP_3, reg);
1096 
1097 	reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_1);
1098 	cdns_ctrl->ecc_strengths[4] = FIELD_GET(BCH_CFG_1_CORR_CAP_4, reg);
1099 	cdns_ctrl->ecc_strengths[5] = FIELD_GET(BCH_CFG_1_CORR_CAP_5, reg);
1100 	cdns_ctrl->ecc_strengths[6] = FIELD_GET(BCH_CFG_1_CORR_CAP_6, reg);
1101 	cdns_ctrl->ecc_strengths[7] = FIELD_GET(BCH_CFG_1_CORR_CAP_7, reg);
1102 
1103 	reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_2);
1104 	cdns_ctrl->ecc_stepinfos[0].stepsize =
1105 		FIELD_GET(BCH_CFG_2_SECT_0, reg);
1106 
1107 	cdns_ctrl->ecc_stepinfos[1].stepsize =
1108 		FIELD_GET(BCH_CFG_2_SECT_1, reg);
1109 
1110 	nstrengths = 0;
1111 	for (i = 0; i < BCH_MAX_NUM_CORR_CAPS; i++) {
1112 		if (cdns_ctrl->ecc_strengths[i] != 0)
1113 			nstrengths++;
1114 	}
1115 
1116 	ecc_caps->nstepinfos = 0;
1117 	for (i = 0; i < BCH_MAX_NUM_SECTOR_SIZES; i++) {
1118 		/* ECC strengths are common for all step infos. */
1119 		cdns_ctrl->ecc_stepinfos[i].nstrengths = nstrengths;
1120 		cdns_ctrl->ecc_stepinfos[i].strengths =
1121 			cdns_ctrl->ecc_strengths;
1122 
1123 		if (cdns_ctrl->ecc_stepinfos[i].stepsize != 0)
1124 			ecc_caps->nstepinfos++;
1125 
1126 		if (cdns_ctrl->ecc_stepinfos[i].stepsize > max_step_size)
1127 			max_step_size = cdns_ctrl->ecc_stepinfos[i].stepsize;
1128 	}
1129 	ecc_caps->stepinfos = &cdns_ctrl->ecc_stepinfos[0];
1130 
1131 	switch (max_step_size) {
1132 	case 256:
1133 		ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_256;
1134 		break;
1135 	case 512:
1136 		ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_512;
1137 		break;
1138 	case 1024:
1139 		ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_1024;
1140 		break;
1141 	case 2048:
1142 		ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_2048;
1143 		break;
1144 	case 4096:
1145 		ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_4096;
1146 		break;
1147 	default:
1148 		dev_err(cdns_ctrl->dev,
1149 			"Unsupported sector size(ecc step size) %d\n",
1150 			max_step_size);
1151 		return -EIO;
1152 	}
1153 
1154 	return 0;
1155 }
1156 
1157 /* Hardware initialization. */
1158 static int cadence_nand_hw_init(struct cdns_nand_ctrl *cdns_ctrl)
1159 {
1160 	int status;
1161 	u32 reg;
1162 
1163 	status = cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
1164 					     1000000,
1165 					     CTRL_STATUS_INIT_COMP, false);
1166 	if (status)
1167 		return status;
1168 
1169 	reg = readl_relaxed(cdns_ctrl->reg + CTRL_VERSION);
1170 	cdns_ctrl->ctrl_rev = FIELD_GET(CTRL_VERSION_REV, reg);
1171 
1172 	dev_info(cdns_ctrl->dev,
1173 		 "%s: cadence nand controller version reg %x\n",
1174 		 __func__, reg);
1175 
1176 	/* Disable cache and multiplane. */
1177 	writel_relaxed(0, cdns_ctrl->reg + MULTIPLANE_CFG);
1178 	writel_relaxed(0, cdns_ctrl->reg + CACHE_CFG);
1179 
1180 	/* Clear all interrupts. */
1181 	writel_relaxed(0xFFFFFFFF, cdns_ctrl->reg + INTR_STATUS);
1182 
1183 	cadence_nand_get_caps(cdns_ctrl);
1184 	if (cadence_nand_read_bch_caps(cdns_ctrl))
1185 		return -EIO;
1186 
1187 	/*
1188 	 * Set IO width access to 8.
1189 	 * It is because during SW device discovering width access
1190 	 * is expected to be 8.
1191 	 */
1192 	status = cadence_nand_set_access_width16(cdns_ctrl, false);
1193 
1194 	return status;
1195 }
1196 
1197 #define TT_MAIN_OOB_AREAS	2
1198 #define TT_RAW_PAGE		3
1199 #define TT_BBM			4
1200 #define TT_MAIN_OOB_AREA_EXT	5
1201 
1202 /* Prepare size of data to transfer. */
1203 static void
1204 cadence_nand_prepare_data_size(struct nand_chip *chip,
1205 			       int transfer_type)
1206 {
1207 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1208 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1209 	struct mtd_info *mtd = nand_to_mtd(chip);
1210 	u32 sec_size = 0, offset = 0, sec_cnt = 1;
1211 	u32 last_sec_size = cdns_chip->sector_size;
1212 	u32 data_ctrl_size = 0;
1213 	u32 reg = 0;
1214 
1215 	if (cdns_ctrl->curr_trans_type == transfer_type)
1216 		return;
1217 
1218 	switch (transfer_type) {
1219 	case TT_MAIN_OOB_AREA_EXT:
1220 		sec_cnt = cdns_chip->sector_count;
1221 		sec_size = cdns_chip->sector_size;
1222 		data_ctrl_size = cdns_chip->avail_oob_size;
1223 		break;
1224 	case TT_MAIN_OOB_AREAS:
1225 		sec_cnt = cdns_chip->sector_count;
1226 		last_sec_size = cdns_chip->sector_size
1227 			+ cdns_chip->avail_oob_size;
1228 		sec_size = cdns_chip->sector_size;
1229 		break;
1230 	case TT_RAW_PAGE:
1231 		last_sec_size = mtd->writesize + mtd->oobsize;
1232 		break;
1233 	case TT_BBM:
1234 		offset = mtd->writesize + cdns_chip->bbm_offs;
1235 		last_sec_size = 8;
1236 		break;
1237 	}
1238 
1239 	reg = 0;
1240 	reg |= FIELD_PREP(TRAN_CFG_0_OFFSET, offset);
1241 	reg |= FIELD_PREP(TRAN_CFG_0_SEC_CNT, sec_cnt);
1242 	writel_relaxed(reg, cdns_ctrl->reg + TRAN_CFG_0);
1243 
1244 	reg = 0;
1245 	reg |= FIELD_PREP(TRAN_CFG_1_LAST_SEC_SIZE, last_sec_size);
1246 	reg |= FIELD_PREP(TRAN_CFG_1_SECTOR_SIZE, sec_size);
1247 	writel_relaxed(reg, cdns_ctrl->reg + TRAN_CFG_1);
1248 
1249 	if (cdns_ctrl->caps2.data_control_supp) {
1250 		reg = readl_relaxed(cdns_ctrl->reg + CONTROL_DATA_CTRL);
1251 		reg &= ~CONTROL_DATA_CTRL_SIZE;
1252 		reg |= FIELD_PREP(CONTROL_DATA_CTRL_SIZE, data_ctrl_size);
1253 		writel_relaxed(reg, cdns_ctrl->reg + CONTROL_DATA_CTRL);
1254 	}
1255 
1256 	cdns_ctrl->curr_trans_type = transfer_type;
1257 }
1258 
1259 static int
1260 cadence_nand_cdma_transfer(struct cdns_nand_ctrl *cdns_ctrl, u8 chip_nr,
1261 			   int page, void *buf, void *ctrl_dat, u32 buf_size,
1262 			   u32 ctrl_dat_size, enum dma_data_direction dir,
1263 			   bool with_ecc)
1264 {
1265 	dma_addr_t dma_buf, dma_ctrl_dat = 0;
1266 	u8 thread_nr = chip_nr;
1267 	int status;
1268 	u16 ctype;
1269 
1270 	if (dir == DMA_FROM_DEVICE)
1271 		ctype = CDMA_CT_RD;
1272 	else
1273 		ctype = CDMA_CT_WR;
1274 
1275 	cadence_nand_set_ecc_enable(cdns_ctrl, with_ecc);
1276 
1277 	dma_buf = dma_map_single(cdns_ctrl->dev, buf, buf_size, dir);
1278 	if (dma_mapping_error(cdns_ctrl->dev, dma_buf)) {
1279 		dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1280 		return -EIO;
1281 	}
1282 
1283 	if (ctrl_dat && ctrl_dat_size) {
1284 		dma_ctrl_dat = dma_map_single(cdns_ctrl->dev, ctrl_dat,
1285 					      ctrl_dat_size, dir);
1286 		if (dma_mapping_error(cdns_ctrl->dev, dma_ctrl_dat)) {
1287 			dma_unmap_single(cdns_ctrl->dev, dma_buf,
1288 					 buf_size, dir);
1289 			dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1290 			return -EIO;
1291 		}
1292 	}
1293 
1294 	cadence_nand_cdma_desc_prepare(cdns_ctrl, chip_nr, page,
1295 				       dma_buf, dma_ctrl_dat, ctype);
1296 
1297 	status = cadence_nand_cdma_send_and_wait(cdns_ctrl, thread_nr);
1298 
1299 	dma_unmap_single(cdns_ctrl->dev, dma_buf,
1300 			 buf_size, dir);
1301 
1302 	if (ctrl_dat && ctrl_dat_size)
1303 		dma_unmap_single(cdns_ctrl->dev, dma_ctrl_dat,
1304 				 ctrl_dat_size, dir);
1305 	if (status)
1306 		return status;
1307 
1308 	return cadence_nand_cdma_finish(cdns_ctrl);
1309 }
1310 
1311 static void cadence_nand_set_timings(struct cdns_nand_ctrl *cdns_ctrl,
1312 				     struct cadence_nand_timings *t)
1313 {
1314 	writel_relaxed(t->async_toggle_timings,
1315 		       cdns_ctrl->reg + ASYNC_TOGGLE_TIMINGS);
1316 	writel_relaxed(t->timings0, cdns_ctrl->reg + TIMINGS0);
1317 	writel_relaxed(t->timings1, cdns_ctrl->reg + TIMINGS1);
1318 	writel_relaxed(t->timings2, cdns_ctrl->reg + TIMINGS2);
1319 
1320 	if (cdns_ctrl->caps2.is_phy_type_dll)
1321 		writel_relaxed(t->dll_phy_ctrl, cdns_ctrl->reg + DLL_PHY_CTRL);
1322 
1323 	writel_relaxed(t->phy_ctrl, cdns_ctrl->reg + PHY_CTRL);
1324 
1325 	if (cdns_ctrl->caps2.is_phy_type_dll) {
1326 		writel_relaxed(0, cdns_ctrl->reg + PHY_TSEL);
1327 		writel_relaxed(2, cdns_ctrl->reg + PHY_DQ_TIMING);
1328 		writel_relaxed(t->phy_dqs_timing,
1329 			       cdns_ctrl->reg + PHY_DQS_TIMING);
1330 		writel_relaxed(t->phy_gate_lpbk_ctrl,
1331 			       cdns_ctrl->reg + PHY_GATE_LPBK_CTRL);
1332 		writel_relaxed(PHY_DLL_MASTER_CTRL_BYPASS_MODE,
1333 			       cdns_ctrl->reg + PHY_DLL_MASTER_CTRL);
1334 		writel_relaxed(0, cdns_ctrl->reg + PHY_DLL_SLAVE_CTRL);
1335 	}
1336 }
1337 
1338 static int cadence_nand_select_target(struct nand_chip *chip)
1339 {
1340 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1341 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1342 
1343 	if (chip == cdns_ctrl->selected_chip)
1344 		return 0;
1345 
1346 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
1347 					1000000,
1348 					CTRL_STATUS_CTRL_BUSY, true))
1349 		return -ETIMEDOUT;
1350 
1351 	cadence_nand_set_timings(cdns_ctrl, &cdns_chip->timings);
1352 
1353 	cadence_nand_set_ecc_strength(cdns_ctrl,
1354 				      cdns_chip->corr_str_idx);
1355 
1356 	cadence_nand_set_erase_detection(cdns_ctrl, true,
1357 					 chip->ecc.strength);
1358 
1359 	cdns_ctrl->curr_trans_type = -1;
1360 	cdns_ctrl->selected_chip = chip;
1361 
1362 	return 0;
1363 }
1364 
1365 static int cadence_nand_erase(struct nand_chip *chip, u32 page)
1366 {
1367 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1368 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1369 	int status;
1370 	u8 thread_nr = cdns_chip->cs[chip->cur_cs];
1371 
1372 	cadence_nand_cdma_desc_prepare(cdns_ctrl,
1373 				       cdns_chip->cs[chip->cur_cs],
1374 				       page, 0, 0,
1375 				       CDMA_CT_ERASE);
1376 	status = cadence_nand_cdma_send_and_wait(cdns_ctrl, thread_nr);
1377 	if (status) {
1378 		dev_err(cdns_ctrl->dev, "erase operation failed\n");
1379 		return -EIO;
1380 	}
1381 
1382 	status = cadence_nand_cdma_finish(cdns_ctrl);
1383 	if (status)
1384 		return status;
1385 
1386 	return 0;
1387 }
1388 
1389 static int cadence_nand_read_bbm(struct nand_chip *chip, int page, u8 *buf)
1390 {
1391 	int status;
1392 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1393 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1394 	struct mtd_info *mtd = nand_to_mtd(chip);
1395 
1396 	cadence_nand_prepare_data_size(chip, TT_BBM);
1397 
1398 	cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1399 
1400 	/*
1401 	 * Read only bad block marker from offset
1402 	 * defined by a memory manufacturer.
1403 	 */
1404 	status = cadence_nand_cdma_transfer(cdns_ctrl,
1405 					    cdns_chip->cs[chip->cur_cs],
1406 					    page, cdns_ctrl->buf, NULL,
1407 					    mtd->oobsize,
1408 					    0, DMA_FROM_DEVICE, false);
1409 	if (status) {
1410 		dev_err(cdns_ctrl->dev, "read BBM failed\n");
1411 		return -EIO;
1412 	}
1413 
1414 	memcpy(buf + cdns_chip->bbm_offs, cdns_ctrl->buf, cdns_chip->bbm_len);
1415 
1416 	return 0;
1417 }
1418 
1419 static int cadence_nand_write_page(struct nand_chip *chip,
1420 				   const u8 *buf, int oob_required,
1421 				   int page)
1422 {
1423 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1424 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1425 	struct mtd_info *mtd = nand_to_mtd(chip);
1426 	int status;
1427 	u16 marker_val = 0xFFFF;
1428 
1429 	status = cadence_nand_select_target(chip);
1430 	if (status)
1431 		return status;
1432 
1433 	cadence_nand_set_skip_bytes_conf(cdns_ctrl, cdns_chip->bbm_len,
1434 					 mtd->writesize
1435 					 + cdns_chip->bbm_offs,
1436 					 1);
1437 
1438 	if (oob_required) {
1439 		marker_val = *(u16 *)(chip->oob_poi
1440 				      + cdns_chip->bbm_offs);
1441 	} else {
1442 		/* Set oob data to 0xFF. */
1443 		memset(cdns_ctrl->buf + mtd->writesize, 0xFF,
1444 		       cdns_chip->avail_oob_size);
1445 	}
1446 
1447 	cadence_nand_set_skip_marker_val(cdns_ctrl, marker_val);
1448 
1449 	cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREA_EXT);
1450 
1451 	if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, mtd->writesize) &&
1452 	    cdns_ctrl->caps2.data_control_supp) {
1453 		u8 *oob;
1454 
1455 		if (oob_required)
1456 			oob = chip->oob_poi;
1457 		else
1458 			oob = cdns_ctrl->buf + mtd->writesize;
1459 
1460 		status = cadence_nand_cdma_transfer(cdns_ctrl,
1461 						    cdns_chip->cs[chip->cur_cs],
1462 						    page, (void *)buf, oob,
1463 						    mtd->writesize,
1464 						    cdns_chip->avail_oob_size,
1465 						    DMA_TO_DEVICE, true);
1466 		if (status) {
1467 			dev_err(cdns_ctrl->dev, "write page failed\n");
1468 			return -EIO;
1469 		}
1470 
1471 		return 0;
1472 	}
1473 
1474 	if (oob_required) {
1475 		/* Transfer the data to the oob area. */
1476 		memcpy(cdns_ctrl->buf + mtd->writesize, chip->oob_poi,
1477 		       cdns_chip->avail_oob_size);
1478 	}
1479 
1480 	memcpy(cdns_ctrl->buf, buf, mtd->writesize);
1481 
1482 	cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREAS);
1483 
1484 	return cadence_nand_cdma_transfer(cdns_ctrl,
1485 					  cdns_chip->cs[chip->cur_cs],
1486 					  page, cdns_ctrl->buf, NULL,
1487 					  mtd->writesize
1488 					  + cdns_chip->avail_oob_size,
1489 					  0, DMA_TO_DEVICE, true);
1490 }
1491 
1492 static int cadence_nand_write_oob(struct nand_chip *chip, int page)
1493 {
1494 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1495 	struct mtd_info *mtd = nand_to_mtd(chip);
1496 
1497 	memset(cdns_ctrl->buf, 0xFF, mtd->writesize);
1498 
1499 	return cadence_nand_write_page(chip, cdns_ctrl->buf, 1, page);
1500 }
1501 
1502 static int cadence_nand_write_page_raw(struct nand_chip *chip,
1503 				       const u8 *buf, int oob_required,
1504 				       int page)
1505 {
1506 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1507 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1508 	struct mtd_info *mtd = nand_to_mtd(chip);
1509 	int writesize = mtd->writesize;
1510 	int oobsize = mtd->oobsize;
1511 	int ecc_steps = chip->ecc.steps;
1512 	int ecc_size = chip->ecc.size;
1513 	int ecc_bytes = chip->ecc.bytes;
1514 	void *tmp_buf = cdns_ctrl->buf;
1515 	int oob_skip = cdns_chip->bbm_len;
1516 	size_t size = writesize + oobsize;
1517 	int i, pos, len;
1518 	int status = 0;
1519 
1520 	status = cadence_nand_select_target(chip);
1521 	if (status)
1522 		return status;
1523 
1524 	/*
1525 	 * Fill the buffer with 0xff first except the full page transfer.
1526 	 * This simplifies the logic.
1527 	 */
1528 	if (!buf || !oob_required)
1529 		memset(tmp_buf, 0xff, size);
1530 
1531 	cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1532 
1533 	/* Arrange the buffer for syndrome payload/ecc layout. */
1534 	if (buf) {
1535 		for (i = 0; i < ecc_steps; i++) {
1536 			pos = i * (ecc_size + ecc_bytes);
1537 			len = ecc_size;
1538 
1539 			if (pos >= writesize)
1540 				pos += oob_skip;
1541 			else if (pos + len > writesize)
1542 				len = writesize - pos;
1543 
1544 			memcpy(tmp_buf + pos, buf, len);
1545 			buf += len;
1546 			if (len < ecc_size) {
1547 				len = ecc_size - len;
1548 				memcpy(tmp_buf + writesize + oob_skip, buf,
1549 				       len);
1550 				buf += len;
1551 			}
1552 		}
1553 	}
1554 
1555 	if (oob_required) {
1556 		const u8 *oob = chip->oob_poi;
1557 		u32 oob_data_offset = (cdns_chip->sector_count - 1) *
1558 			(cdns_chip->sector_size + chip->ecc.bytes)
1559 			+ cdns_chip->sector_size + oob_skip;
1560 
1561 		/* BBM at the beginning of the OOB area. */
1562 		memcpy(tmp_buf + writesize, oob, oob_skip);
1563 
1564 		/* OOB free. */
1565 		memcpy(tmp_buf + oob_data_offset, oob,
1566 		       cdns_chip->avail_oob_size);
1567 		oob += cdns_chip->avail_oob_size;
1568 
1569 		/* OOB ECC. */
1570 		for (i = 0; i < ecc_steps; i++) {
1571 			pos = ecc_size + i * (ecc_size + ecc_bytes);
1572 			if (i == (ecc_steps - 1))
1573 				pos += cdns_chip->avail_oob_size;
1574 
1575 			len = ecc_bytes;
1576 
1577 			if (pos >= writesize)
1578 				pos += oob_skip;
1579 			else if (pos + len > writesize)
1580 				len = writesize - pos;
1581 
1582 			memcpy(tmp_buf + pos, oob, len);
1583 			oob += len;
1584 			if (len < ecc_bytes) {
1585 				len = ecc_bytes - len;
1586 				memcpy(tmp_buf + writesize + oob_skip, oob,
1587 				       len);
1588 				oob += len;
1589 			}
1590 		}
1591 	}
1592 
1593 	cadence_nand_prepare_data_size(chip, TT_RAW_PAGE);
1594 
1595 	return cadence_nand_cdma_transfer(cdns_ctrl,
1596 					  cdns_chip->cs[chip->cur_cs],
1597 					  page, cdns_ctrl->buf, NULL,
1598 					  mtd->writesize +
1599 					  mtd->oobsize,
1600 					  0, DMA_TO_DEVICE, false);
1601 }
1602 
1603 static int cadence_nand_write_oob_raw(struct nand_chip *chip,
1604 				      int page)
1605 {
1606 	return cadence_nand_write_page_raw(chip, NULL, true, page);
1607 }
1608 
1609 static int cadence_nand_read_page(struct nand_chip *chip,
1610 				  u8 *buf, int oob_required, int page)
1611 {
1612 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1613 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1614 	struct mtd_info *mtd = nand_to_mtd(chip);
1615 	int status = 0;
1616 	int ecc_err_count = 0;
1617 
1618 	status = cadence_nand_select_target(chip);
1619 	if (status)
1620 		return status;
1621 
1622 	cadence_nand_set_skip_bytes_conf(cdns_ctrl, cdns_chip->bbm_len,
1623 					 mtd->writesize
1624 					 + cdns_chip->bbm_offs, 1);
1625 
1626 	/*
1627 	 * If data buffer can be accessed by DMA and data_control feature
1628 	 * is supported then transfer data and oob directly.
1629 	 */
1630 	if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, mtd->writesize) &&
1631 	    cdns_ctrl->caps2.data_control_supp) {
1632 		u8 *oob;
1633 
1634 		if (oob_required)
1635 			oob = chip->oob_poi;
1636 		else
1637 			oob = cdns_ctrl->buf + mtd->writesize;
1638 
1639 		cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREA_EXT);
1640 		status = cadence_nand_cdma_transfer(cdns_ctrl,
1641 						    cdns_chip->cs[chip->cur_cs],
1642 						    page, buf, oob,
1643 						    mtd->writesize,
1644 						    cdns_chip->avail_oob_size,
1645 						    DMA_FROM_DEVICE, true);
1646 	/* Otherwise use bounce buffer. */
1647 	} else {
1648 		cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREAS);
1649 		status = cadence_nand_cdma_transfer(cdns_ctrl,
1650 						    cdns_chip->cs[chip->cur_cs],
1651 						    page, cdns_ctrl->buf,
1652 						    NULL, mtd->writesize
1653 						    + cdns_chip->avail_oob_size,
1654 						    0, DMA_FROM_DEVICE, true);
1655 
1656 		memcpy(buf, cdns_ctrl->buf, mtd->writesize);
1657 		if (oob_required)
1658 			memcpy(chip->oob_poi,
1659 			       cdns_ctrl->buf + mtd->writesize,
1660 			       mtd->oobsize);
1661 	}
1662 
1663 	switch (status) {
1664 	case STAT_ECC_UNCORR:
1665 		mtd->ecc_stats.failed++;
1666 		ecc_err_count++;
1667 		break;
1668 	case STAT_ECC_CORR:
1669 		ecc_err_count = FIELD_GET(CDMA_CS_MAXERR,
1670 					  cdns_ctrl->cdma_desc->status);
1671 		mtd->ecc_stats.corrected += ecc_err_count;
1672 		break;
1673 	case STAT_ERASED:
1674 	case STAT_OK:
1675 		break;
1676 	default:
1677 		dev_err(cdns_ctrl->dev, "read page failed\n");
1678 		return -EIO;
1679 	}
1680 
1681 	if (oob_required)
1682 		if (cadence_nand_read_bbm(chip, page, chip->oob_poi))
1683 			return -EIO;
1684 
1685 	return ecc_err_count;
1686 }
1687 
1688 /* Reads OOB data from the device. */
1689 static int cadence_nand_read_oob(struct nand_chip *chip, int page)
1690 {
1691 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1692 
1693 	return cadence_nand_read_page(chip, cdns_ctrl->buf, 1, page);
1694 }
1695 
1696 static int cadence_nand_read_page_raw(struct nand_chip *chip,
1697 				      u8 *buf, int oob_required, int page)
1698 {
1699 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1700 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1701 	struct mtd_info *mtd = nand_to_mtd(chip);
1702 	int oob_skip = cdns_chip->bbm_len;
1703 	int writesize = mtd->writesize;
1704 	int ecc_steps = chip->ecc.steps;
1705 	int ecc_size = chip->ecc.size;
1706 	int ecc_bytes = chip->ecc.bytes;
1707 	void *tmp_buf = cdns_ctrl->buf;
1708 	int i, pos, len;
1709 	int status = 0;
1710 
1711 	status = cadence_nand_select_target(chip);
1712 	if (status)
1713 		return status;
1714 
1715 	cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1716 
1717 	cadence_nand_prepare_data_size(chip, TT_RAW_PAGE);
1718 	status = cadence_nand_cdma_transfer(cdns_ctrl,
1719 					    cdns_chip->cs[chip->cur_cs],
1720 					    page, cdns_ctrl->buf, NULL,
1721 					    mtd->writesize
1722 					    + mtd->oobsize,
1723 					    0, DMA_FROM_DEVICE, false);
1724 
1725 	switch (status) {
1726 	case STAT_ERASED:
1727 	case STAT_OK:
1728 		break;
1729 	default:
1730 		dev_err(cdns_ctrl->dev, "read raw page failed\n");
1731 		return -EIO;
1732 	}
1733 
1734 	/* Arrange the buffer for syndrome payload/ecc layout. */
1735 	if (buf) {
1736 		for (i = 0; i < ecc_steps; i++) {
1737 			pos = i * (ecc_size + ecc_bytes);
1738 			len = ecc_size;
1739 
1740 			if (pos >= writesize)
1741 				pos += oob_skip;
1742 			else if (pos + len > writesize)
1743 				len = writesize - pos;
1744 
1745 			memcpy(buf, tmp_buf + pos, len);
1746 			buf += len;
1747 			if (len < ecc_size) {
1748 				len = ecc_size - len;
1749 				memcpy(buf, tmp_buf + writesize + oob_skip,
1750 				       len);
1751 				buf += len;
1752 			}
1753 		}
1754 	}
1755 
1756 	if (oob_required) {
1757 		u8 *oob = chip->oob_poi;
1758 		u32 oob_data_offset = (cdns_chip->sector_count - 1) *
1759 			(cdns_chip->sector_size + chip->ecc.bytes)
1760 			+ cdns_chip->sector_size + oob_skip;
1761 
1762 		/* OOB free. */
1763 		memcpy(oob, tmp_buf + oob_data_offset,
1764 		       cdns_chip->avail_oob_size);
1765 
1766 		/* BBM at the beginning of the OOB area. */
1767 		memcpy(oob, tmp_buf + writesize, oob_skip);
1768 
1769 		oob += cdns_chip->avail_oob_size;
1770 
1771 		/* OOB ECC */
1772 		for (i = 0; i < ecc_steps; i++) {
1773 			pos = ecc_size + i * (ecc_size + ecc_bytes);
1774 			len = ecc_bytes;
1775 
1776 			if (i == (ecc_steps - 1))
1777 				pos += cdns_chip->avail_oob_size;
1778 
1779 			if (pos >= writesize)
1780 				pos += oob_skip;
1781 			else if (pos + len > writesize)
1782 				len = writesize - pos;
1783 
1784 			memcpy(oob, tmp_buf + pos, len);
1785 			oob += len;
1786 			if (len < ecc_bytes) {
1787 				len = ecc_bytes - len;
1788 				memcpy(oob, tmp_buf + writesize + oob_skip,
1789 				       len);
1790 				oob += len;
1791 			}
1792 		}
1793 	}
1794 
1795 	return 0;
1796 }
1797 
1798 static int cadence_nand_read_oob_raw(struct nand_chip *chip,
1799 				     int page)
1800 {
1801 	return cadence_nand_read_page_raw(chip, NULL, true, page);
1802 }
1803 
1804 static void cadence_nand_slave_dma_transfer_finished(void *data)
1805 {
1806 	struct completion *finished = data;
1807 
1808 	complete(finished);
1809 }
1810 
1811 static int cadence_nand_slave_dma_transfer(struct cdns_nand_ctrl *cdns_ctrl,
1812 					   void *buf,
1813 					   dma_addr_t dev_dma, size_t len,
1814 					   enum dma_data_direction dir)
1815 {
1816 	DECLARE_COMPLETION_ONSTACK(finished);
1817 	struct dma_chan *chan;
1818 	struct dma_device *dma_dev;
1819 	dma_addr_t src_dma, dst_dma, buf_dma;
1820 	struct dma_async_tx_descriptor *tx;
1821 	dma_cookie_t cookie;
1822 
1823 	chan = cdns_ctrl->dmac;
1824 	dma_dev = chan->device;
1825 
1826 	buf_dma = dma_map_single(dma_dev->dev, buf, len, dir);
1827 	if (dma_mapping_error(dma_dev->dev, buf_dma)) {
1828 		dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1829 		goto err;
1830 	}
1831 
1832 	if (dir == DMA_FROM_DEVICE) {
1833 		src_dma = cdns_ctrl->io.dma;
1834 		dst_dma = buf_dma;
1835 	} else {
1836 		src_dma = buf_dma;
1837 		dst_dma = cdns_ctrl->io.dma;
1838 	}
1839 
1840 	tx = dmaengine_prep_dma_memcpy(cdns_ctrl->dmac, dst_dma, src_dma, len,
1841 				       DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
1842 	if (!tx) {
1843 		dev_err(cdns_ctrl->dev, "Failed to prepare DMA memcpy\n");
1844 		goto err_unmap;
1845 	}
1846 
1847 	tx->callback = cadence_nand_slave_dma_transfer_finished;
1848 	tx->callback_param = &finished;
1849 
1850 	cookie = dmaengine_submit(tx);
1851 	if (dma_submit_error(cookie)) {
1852 		dev_err(cdns_ctrl->dev, "Failed to do DMA tx_submit\n");
1853 		goto err_unmap;
1854 	}
1855 
1856 	dma_async_issue_pending(cdns_ctrl->dmac);
1857 	wait_for_completion(&finished);
1858 
1859 	dma_unmap_single(cdns_ctrl->dev, buf_dma, len, dir);
1860 
1861 	return 0;
1862 
1863 err_unmap:
1864 	dma_unmap_single(cdns_ctrl->dev, buf_dma, len, dir);
1865 
1866 err:
1867 	dev_dbg(cdns_ctrl->dev, "Fall back to CPU I/O\n");
1868 
1869 	return -EIO;
1870 }
1871 
1872 static int cadence_nand_read_buf(struct cdns_nand_ctrl *cdns_ctrl,
1873 				 u8 *buf, int len)
1874 {
1875 	u8 thread_nr = 0;
1876 	u32 sdma_size;
1877 	int status;
1878 
1879 	/* Wait until slave DMA interface is ready to data transfer. */
1880 	status = cadence_nand_wait_on_sdma(cdns_ctrl, &thread_nr, &sdma_size);
1881 	if (status)
1882 		return status;
1883 
1884 	if (!cdns_ctrl->caps1->has_dma) {
1885 		int len_in_words = len >> 2;
1886 
1887 		/* read alingment data */
1888 		ioread32_rep(cdns_ctrl->io.virt, buf, len_in_words);
1889 		if (sdma_size > len) {
1890 			/* read rest data from slave DMA interface if any */
1891 			ioread32_rep(cdns_ctrl->io.virt, cdns_ctrl->buf,
1892 				     sdma_size / 4 - len_in_words);
1893 			/* copy rest of data */
1894 			memcpy(buf + (len_in_words << 2), cdns_ctrl->buf,
1895 			       len - (len_in_words << 2));
1896 		}
1897 		return 0;
1898 	}
1899 
1900 	if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, len)) {
1901 		status = cadence_nand_slave_dma_transfer(cdns_ctrl, buf,
1902 							 cdns_ctrl->io.dma,
1903 							 len, DMA_FROM_DEVICE);
1904 		if (status == 0)
1905 			return 0;
1906 
1907 		dev_warn(cdns_ctrl->dev,
1908 			 "Slave DMA transfer failed. Try again using bounce buffer.");
1909 	}
1910 
1911 	/* If DMA transfer is not possible or failed then use bounce buffer. */
1912 	status = cadence_nand_slave_dma_transfer(cdns_ctrl, cdns_ctrl->buf,
1913 						 cdns_ctrl->io.dma,
1914 						 sdma_size, DMA_FROM_DEVICE);
1915 
1916 	if (status) {
1917 		dev_err(cdns_ctrl->dev, "Slave DMA transfer failed");
1918 		return status;
1919 	}
1920 
1921 	memcpy(buf, cdns_ctrl->buf, len);
1922 
1923 	return 0;
1924 }
1925 
1926 static int cadence_nand_write_buf(struct cdns_nand_ctrl *cdns_ctrl,
1927 				  const u8 *buf, int len)
1928 {
1929 	u8 thread_nr = 0;
1930 	u32 sdma_size;
1931 	int status;
1932 
1933 	/* Wait until slave DMA interface is ready to data transfer. */
1934 	status = cadence_nand_wait_on_sdma(cdns_ctrl, &thread_nr, &sdma_size);
1935 	if (status)
1936 		return status;
1937 
1938 	if (!cdns_ctrl->caps1->has_dma) {
1939 		int len_in_words = len >> 2;
1940 
1941 		iowrite32_rep(cdns_ctrl->io.virt, buf, len_in_words);
1942 		if (sdma_size > len) {
1943 			/* copy rest of data */
1944 			memcpy(cdns_ctrl->buf, buf + (len_in_words << 2),
1945 			       len - (len_in_words << 2));
1946 			/* write all expected by nand controller data */
1947 			iowrite32_rep(cdns_ctrl->io.virt, cdns_ctrl->buf,
1948 				      sdma_size / 4 - len_in_words);
1949 		}
1950 
1951 		return 0;
1952 	}
1953 
1954 	if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, len)) {
1955 		status = cadence_nand_slave_dma_transfer(cdns_ctrl, (void *)buf,
1956 							 cdns_ctrl->io.dma,
1957 							 len, DMA_TO_DEVICE);
1958 		if (status == 0)
1959 			return 0;
1960 
1961 		dev_warn(cdns_ctrl->dev,
1962 			 "Slave DMA transfer failed. Try again using bounce buffer.");
1963 	}
1964 
1965 	/* If DMA transfer is not possible or failed then use bounce buffer. */
1966 	memcpy(cdns_ctrl->buf, buf, len);
1967 
1968 	status = cadence_nand_slave_dma_transfer(cdns_ctrl, cdns_ctrl->buf,
1969 						 cdns_ctrl->io.dma,
1970 						 sdma_size, DMA_TO_DEVICE);
1971 
1972 	if (status)
1973 		dev_err(cdns_ctrl->dev, "Slave DMA transfer failed");
1974 
1975 	return status;
1976 }
1977 
1978 static int cadence_nand_force_byte_access(struct nand_chip *chip,
1979 					  bool force_8bit)
1980 {
1981 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1982 
1983 	/*
1984 	 * Callers of this function do not verify if the NAND is using a 16-bit
1985 	 * an 8-bit bus for normal operations, so we need to take care of that
1986 	 * here by leaving the configuration unchanged if the NAND does not have
1987 	 * the NAND_BUSWIDTH_16 flag set.
1988 	 */
1989 	if (!(chip->options & NAND_BUSWIDTH_16))
1990 		return 0;
1991 
1992 	return cadence_nand_set_access_width16(cdns_ctrl, !force_8bit);
1993 }
1994 
1995 static int cadence_nand_cmd_opcode(struct nand_chip *chip,
1996 				   const struct nand_subop *subop)
1997 {
1998 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1999 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2000 	const struct nand_op_instr *instr;
2001 	unsigned int op_id = 0;
2002 	u64 mini_ctrl_cmd = 0;
2003 	int ret;
2004 
2005 	instr = &subop->instrs[op_id];
2006 
2007 	if (instr->delay_ns > 0)
2008 		mini_ctrl_cmd |= GCMD_LAY_TWB;
2009 
2010 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2011 				    GCMD_LAY_INSTR_CMD);
2012 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_CMD,
2013 				    instr->ctx.cmd.opcode);
2014 
2015 	ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2016 					    cdns_chip->cs[chip->cur_cs],
2017 					    mini_ctrl_cmd);
2018 	if (ret)
2019 		dev_err(cdns_ctrl->dev, "send cmd %x failed\n",
2020 			instr->ctx.cmd.opcode);
2021 
2022 	return ret;
2023 }
2024 
2025 static int cadence_nand_cmd_address(struct nand_chip *chip,
2026 				    const struct nand_subop *subop)
2027 {
2028 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2029 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2030 	const struct nand_op_instr *instr;
2031 	unsigned int op_id = 0;
2032 	u64 mini_ctrl_cmd = 0;
2033 	unsigned int offset, naddrs;
2034 	u64 address = 0;
2035 	const u8 *addrs;
2036 	int ret;
2037 	int i;
2038 
2039 	instr = &subop->instrs[op_id];
2040 
2041 	if (instr->delay_ns > 0)
2042 		mini_ctrl_cmd |= GCMD_LAY_TWB;
2043 
2044 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2045 				    GCMD_LAY_INSTR_ADDR);
2046 
2047 	offset = nand_subop_get_addr_start_off(subop, op_id);
2048 	naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
2049 	addrs = &instr->ctx.addr.addrs[offset];
2050 
2051 	for (i = 0; i < naddrs; i++)
2052 		address |= (u64)addrs[i] << (8 * i);
2053 
2054 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_ADDR,
2055 				    address);
2056 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_ADDR_SIZE,
2057 				    naddrs - 1);
2058 
2059 	ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2060 					    cdns_chip->cs[chip->cur_cs],
2061 					    mini_ctrl_cmd);
2062 	if (ret)
2063 		dev_err(cdns_ctrl->dev, "send address %llx failed\n", address);
2064 
2065 	return ret;
2066 }
2067 
2068 static int cadence_nand_cmd_erase(struct nand_chip *chip,
2069 				  const struct nand_subop *subop)
2070 {
2071 	unsigned int op_id;
2072 
2073 	if (subop->instrs[0].ctx.cmd.opcode == NAND_CMD_ERASE1) {
2074 		int i;
2075 		const struct nand_op_instr *instr = NULL;
2076 		unsigned int offset, naddrs;
2077 		const u8 *addrs;
2078 		u32 page = 0;
2079 
2080 		instr = &subop->instrs[1];
2081 		offset = nand_subop_get_addr_start_off(subop, 1);
2082 		naddrs = nand_subop_get_num_addr_cyc(subop, 1);
2083 		addrs = &instr->ctx.addr.addrs[offset];
2084 
2085 		for (i = 0; i < naddrs; i++)
2086 			page |= (u32)addrs[i] << (8 * i);
2087 
2088 		return cadence_nand_erase(chip, page);
2089 	}
2090 
2091 	/*
2092 	 * If it is not an erase operation then handle operation
2093 	 * by calling exec_op function.
2094 	 */
2095 	for (op_id = 0; op_id < subop->ninstrs; op_id++) {
2096 		int ret;
2097 		const struct nand_operation nand_op = {
2098 			.cs = chip->cur_cs,
2099 			.instrs =  &subop->instrs[op_id],
2100 			.ninstrs = 1};
2101 		ret = chip->controller->ops->exec_op(chip, &nand_op, false);
2102 		if (ret)
2103 			return ret;
2104 	}
2105 
2106 	return 0;
2107 }
2108 
2109 static int cadence_nand_cmd_data(struct nand_chip *chip,
2110 				 const struct nand_subop *subop)
2111 {
2112 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2113 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2114 	const struct nand_op_instr *instr;
2115 	unsigned int offset, op_id = 0;
2116 	u64 mini_ctrl_cmd = 0;
2117 	int len = 0;
2118 	int ret;
2119 
2120 	instr = &subop->instrs[op_id];
2121 
2122 	if (instr->delay_ns > 0)
2123 		mini_ctrl_cmd |= GCMD_LAY_TWB;
2124 
2125 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2126 				    GCMD_LAY_INSTR_DATA);
2127 
2128 	if (instr->type == NAND_OP_DATA_OUT_INSTR)
2129 		mini_ctrl_cmd |= FIELD_PREP(GCMD_DIR,
2130 					    GCMD_DIR_WRITE);
2131 
2132 	len = nand_subop_get_data_len(subop, op_id);
2133 	offset = nand_subop_get_data_start_off(subop, op_id);
2134 	mini_ctrl_cmd |= FIELD_PREP(GCMD_SECT_CNT, 1);
2135 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAST_SIZE, len);
2136 	if (instr->ctx.data.force_8bit) {
2137 		ret = cadence_nand_force_byte_access(chip, true);
2138 		if (ret) {
2139 			dev_err(cdns_ctrl->dev,
2140 				"cannot change byte access generic data cmd failed\n");
2141 			return ret;
2142 		}
2143 	}
2144 
2145 	ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2146 					    cdns_chip->cs[chip->cur_cs],
2147 					    mini_ctrl_cmd);
2148 	if (ret) {
2149 		dev_err(cdns_ctrl->dev, "send generic data cmd failed\n");
2150 		return ret;
2151 	}
2152 
2153 	if (instr->type == NAND_OP_DATA_IN_INSTR) {
2154 		void *buf = instr->ctx.data.buf.in + offset;
2155 
2156 		ret = cadence_nand_read_buf(cdns_ctrl, buf, len);
2157 	} else {
2158 		const void *buf = instr->ctx.data.buf.out + offset;
2159 
2160 		ret = cadence_nand_write_buf(cdns_ctrl, buf, len);
2161 	}
2162 
2163 	if (ret) {
2164 		dev_err(cdns_ctrl->dev, "data transfer failed for generic command\n");
2165 		return ret;
2166 	}
2167 
2168 	if (instr->ctx.data.force_8bit) {
2169 		ret = cadence_nand_force_byte_access(chip, false);
2170 		if (ret) {
2171 			dev_err(cdns_ctrl->dev,
2172 				"cannot change byte access generic data cmd failed\n");
2173 		}
2174 	}
2175 
2176 	return ret;
2177 }
2178 
2179 static int cadence_nand_cmd_waitrdy(struct nand_chip *chip,
2180 				    const struct nand_subop *subop)
2181 {
2182 	int status;
2183 	unsigned int op_id = 0;
2184 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2185 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2186 	const struct nand_op_instr *instr = &subop->instrs[op_id];
2187 	u32 timeout_us = instr->ctx.waitrdy.timeout_ms * 1000;
2188 
2189 	status = cadence_nand_wait_for_value(cdns_ctrl, RBN_SETINGS,
2190 					     timeout_us,
2191 					     BIT(cdns_chip->cs[chip->cur_cs]),
2192 					     false);
2193 	return status;
2194 }
2195 
2196 static const struct nand_op_parser cadence_nand_op_parser = NAND_OP_PARSER(
2197 	NAND_OP_PARSER_PATTERN(
2198 		cadence_nand_cmd_erase,
2199 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2200 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ERASE_ADDRESS_CYC),
2201 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2202 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2203 	NAND_OP_PARSER_PATTERN(
2204 		cadence_nand_cmd_opcode,
2205 		NAND_OP_PARSER_PAT_CMD_ELEM(false)),
2206 	NAND_OP_PARSER_PATTERN(
2207 		cadence_nand_cmd_address,
2208 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC)),
2209 	NAND_OP_PARSER_PATTERN(
2210 		cadence_nand_cmd_data,
2211 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_DATA_SIZE)),
2212 	NAND_OP_PARSER_PATTERN(
2213 		cadence_nand_cmd_data,
2214 		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_DATA_SIZE)),
2215 	NAND_OP_PARSER_PATTERN(
2216 		cadence_nand_cmd_waitrdy,
2217 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false))
2218 	);
2219 
2220 static int cadence_nand_exec_op(struct nand_chip *chip,
2221 				const struct nand_operation *op,
2222 				bool check_only)
2223 {
2224 	if (!check_only) {
2225 		int status = cadence_nand_select_target(chip);
2226 
2227 		if (status)
2228 			return status;
2229 	}
2230 
2231 	return nand_op_parser_exec_op(chip, &cadence_nand_op_parser, op,
2232 				      check_only);
2233 }
2234 
2235 static int cadence_nand_ooblayout_free(struct mtd_info *mtd, int section,
2236 				       struct mtd_oob_region *oobregion)
2237 {
2238 	struct nand_chip *chip = mtd_to_nand(mtd);
2239 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2240 
2241 	if (section)
2242 		return -ERANGE;
2243 
2244 	oobregion->offset = cdns_chip->bbm_len;
2245 	oobregion->length = cdns_chip->avail_oob_size
2246 		- cdns_chip->bbm_len;
2247 
2248 	return 0;
2249 }
2250 
2251 static int cadence_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2252 				      struct mtd_oob_region *oobregion)
2253 {
2254 	struct nand_chip *chip = mtd_to_nand(mtd);
2255 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2256 
2257 	if (section)
2258 		return -ERANGE;
2259 
2260 	oobregion->offset = cdns_chip->avail_oob_size;
2261 	oobregion->length = chip->ecc.total;
2262 
2263 	return 0;
2264 }
2265 
2266 static const struct mtd_ooblayout_ops cadence_nand_ooblayout_ops = {
2267 	.free = cadence_nand_ooblayout_free,
2268 	.ecc = cadence_nand_ooblayout_ecc,
2269 };
2270 
2271 static int calc_cycl(u32 timing, u32 clock)
2272 {
2273 	if (timing == 0 || clock == 0)
2274 		return 0;
2275 
2276 	if ((timing % clock) > 0)
2277 		return timing / clock;
2278 	else
2279 		return timing / clock - 1;
2280 }
2281 
2282 /* Calculate max data valid window. */
2283 static inline u32 calc_tdvw_max(u32 trp_cnt, u32 clk_period, u32 trhoh_min,
2284 				u32 board_delay_skew_min, u32 ext_mode)
2285 {
2286 	if (ext_mode == 0)
2287 		clk_period /= 2;
2288 
2289 	return (trp_cnt + 1) * clk_period + trhoh_min +
2290 		board_delay_skew_min;
2291 }
2292 
2293 /* Calculate data valid window. */
2294 static inline u32 calc_tdvw(u32 trp_cnt, u32 clk_period, u32 trhoh_min,
2295 			    u32 trea_max, u32 ext_mode)
2296 {
2297 	if (ext_mode == 0)
2298 		clk_period /= 2;
2299 
2300 	return (trp_cnt + 1) * clk_period + trhoh_min - trea_max;
2301 }
2302 
2303 static int
2304 cadence_nand_setup_interface(struct nand_chip *chip, int chipnr,
2305 			     const struct nand_interface_config *conf)
2306 {
2307 	const struct nand_sdr_timings *sdr;
2308 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2309 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2310 	struct cadence_nand_timings *t = &cdns_chip->timings;
2311 	u32 reg;
2312 	u32 board_delay = cdns_ctrl->board_delay;
2313 	u32 clk_period = DIV_ROUND_DOWN_ULL(1000000000000ULL,
2314 					    cdns_ctrl->nf_clk_rate);
2315 	u32 tceh_cnt, tcs_cnt, tadl_cnt, tccs_cnt;
2316 	u32 tfeat_cnt, trhz_cnt, tvdly_cnt;
2317 	u32 trhw_cnt, twb_cnt, twh_cnt = 0, twhr_cnt;
2318 	u32 twp_cnt = 0, trp_cnt = 0, trh_cnt = 0;
2319 	u32 if_skew = cdns_ctrl->caps1->if_skew;
2320 	u32 board_delay_skew_min = board_delay - if_skew;
2321 	u32 board_delay_skew_max = board_delay + if_skew;
2322 	u32 dqs_sampl_res, phony_dqs_mod;
2323 	u32 tdvw, tdvw_min, tdvw_max;
2324 	u32 ext_rd_mode, ext_wr_mode;
2325 	u32 dll_phy_dqs_timing = 0, phony_dqs_timing = 0, rd_del_sel = 0;
2326 	u32 sampling_point;
2327 
2328 	sdr = nand_get_sdr_timings(conf);
2329 	if (IS_ERR(sdr))
2330 		return PTR_ERR(sdr);
2331 
2332 	memset(t, 0, sizeof(*t));
2333 	/* Sampling point calculation. */
2334 
2335 	if (cdns_ctrl->caps2.is_phy_type_dll)
2336 		phony_dqs_mod = 2;
2337 	else
2338 		phony_dqs_mod = 1;
2339 
2340 	dqs_sampl_res = clk_period / phony_dqs_mod;
2341 
2342 	tdvw_min = sdr->tREA_max + board_delay_skew_max;
2343 	/*
2344 	 * The idea of those calculation is to get the optimum value
2345 	 * for tRP and tRH timings. If it is NOT possible to sample data
2346 	 * with optimal tRP/tRH settings, the parameters will be extended.
2347 	 * If clk_period is 50ns (the lowest value) this condition is met
2348 	 * for SDR timing modes 1, 2, 3, 4 and 5.
2349 	 * If clk_period is 20ns the condition is met only for SDR timing
2350 	 * mode 5.
2351 	 */
2352 	if (sdr->tRC_min <= clk_period &&
2353 	    sdr->tRP_min <= (clk_period / 2) &&
2354 	    sdr->tREH_min <= (clk_period / 2)) {
2355 		/* Performance mode. */
2356 		ext_rd_mode = 0;
2357 		tdvw = calc_tdvw(trp_cnt, clk_period, sdr->tRHOH_min,
2358 				 sdr->tREA_max, ext_rd_mode);
2359 		tdvw_max = calc_tdvw_max(trp_cnt, clk_period, sdr->tRHOH_min,
2360 					 board_delay_skew_min,
2361 					 ext_rd_mode);
2362 		/*
2363 		 * Check if data valid window and sampling point can be found
2364 		 * and is not on the edge (ie. we have hold margin).
2365 		 * If not extend the tRP timings.
2366 		 */
2367 		if (tdvw > 0) {
2368 			if (tdvw_max <= tdvw_min ||
2369 			    (tdvw_max % dqs_sampl_res) == 0) {
2370 				/*
2371 				 * No valid sampling point so the RE pulse need
2372 				 * to be widen widening by half clock cycle.
2373 				 */
2374 				ext_rd_mode = 1;
2375 			}
2376 		} else {
2377 			/*
2378 			 * There is no valid window
2379 			 * to be able to sample data the tRP need to be widen.
2380 			 * Very safe calculations are performed here.
2381 			 */
2382 			trp_cnt = (sdr->tREA_max + board_delay_skew_max
2383 				   + dqs_sampl_res) / clk_period;
2384 			ext_rd_mode = 1;
2385 		}
2386 
2387 	} else {
2388 		/* Extended read mode. */
2389 		u32 trh;
2390 
2391 		ext_rd_mode = 1;
2392 		trp_cnt = calc_cycl(sdr->tRP_min, clk_period);
2393 		trh = sdr->tRC_min - ((trp_cnt + 1) * clk_period);
2394 		if (sdr->tREH_min >= trh)
2395 			trh_cnt = calc_cycl(sdr->tREH_min, clk_period);
2396 		else
2397 			trh_cnt = calc_cycl(trh, clk_period);
2398 
2399 		tdvw = calc_tdvw(trp_cnt, clk_period, sdr->tRHOH_min,
2400 				 sdr->tREA_max, ext_rd_mode);
2401 		/*
2402 		 * Check if data valid window and sampling point can be found
2403 		 * or if it is at the edge check if previous is valid
2404 		 * - if not extend the tRP timings.
2405 		 */
2406 		if (tdvw > 0) {
2407 			tdvw_max = calc_tdvw_max(trp_cnt, clk_period,
2408 						 sdr->tRHOH_min,
2409 						 board_delay_skew_min,
2410 						 ext_rd_mode);
2411 
2412 			if ((((tdvw_max / dqs_sampl_res)
2413 			      * dqs_sampl_res) <= tdvw_min) ||
2414 			    (((tdvw_max % dqs_sampl_res) == 0) &&
2415 			     (((tdvw_max / dqs_sampl_res - 1)
2416 			       * dqs_sampl_res) <= tdvw_min))) {
2417 				/*
2418 				 * Data valid window width is lower than
2419 				 * sampling resolution and do not hit any
2420 				 * sampling point to be sure the sampling point
2421 				 * will be found the RE low pulse width will be
2422 				 *  extended by one clock cycle.
2423 				 */
2424 				trp_cnt = trp_cnt + 1;
2425 			}
2426 		} else {
2427 			/*
2428 			 * There is no valid window to be able to sample data.
2429 			 * The tRP need to be widen.
2430 			 * Very safe calculations are performed here.
2431 			 */
2432 			trp_cnt = (sdr->tREA_max + board_delay_skew_max
2433 				   + dqs_sampl_res) / clk_period;
2434 		}
2435 	}
2436 
2437 	tdvw_max = calc_tdvw_max(trp_cnt, clk_period,
2438 				 sdr->tRHOH_min,
2439 				 board_delay_skew_min, ext_rd_mode);
2440 
2441 	if (sdr->tWC_min <= clk_period &&
2442 	    (sdr->tWP_min + if_skew) <= (clk_period / 2) &&
2443 	    (sdr->tWH_min + if_skew) <= (clk_period / 2)) {
2444 		ext_wr_mode = 0;
2445 	} else {
2446 		u32 twh;
2447 
2448 		ext_wr_mode = 1;
2449 		twp_cnt = calc_cycl(sdr->tWP_min + if_skew, clk_period);
2450 		if ((twp_cnt + 1) * clk_period < (sdr->tALS_min + if_skew))
2451 			twp_cnt = calc_cycl(sdr->tALS_min + if_skew,
2452 					    clk_period);
2453 
2454 		twh = (sdr->tWC_min - (twp_cnt + 1) * clk_period);
2455 		if (sdr->tWH_min >= twh)
2456 			twh = sdr->tWH_min;
2457 
2458 		twh_cnt = calc_cycl(twh + if_skew, clk_period);
2459 	}
2460 
2461 	reg = FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TRH, trh_cnt);
2462 	reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TRP, trp_cnt);
2463 	reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TWH, twh_cnt);
2464 	reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TWP, twp_cnt);
2465 	t->async_toggle_timings = reg;
2466 	dev_dbg(cdns_ctrl->dev, "ASYNC_TOGGLE_TIMINGS_SDR\t%x\n", reg);
2467 
2468 	tadl_cnt = calc_cycl((sdr->tADL_min + if_skew), clk_period);
2469 	tccs_cnt = calc_cycl((sdr->tCCS_min + if_skew), clk_period);
2470 	twhr_cnt = calc_cycl((sdr->tWHR_min + if_skew), clk_period);
2471 	trhw_cnt = calc_cycl((sdr->tRHW_min + if_skew), clk_period);
2472 	reg = FIELD_PREP(TIMINGS0_TADL, tadl_cnt);
2473 
2474 	/*
2475 	 * If timing exceeds delay field in timing register
2476 	 * then use maximum value.
2477 	 */
2478 	if (FIELD_FIT(TIMINGS0_TCCS, tccs_cnt))
2479 		reg |= FIELD_PREP(TIMINGS0_TCCS, tccs_cnt);
2480 	else
2481 		reg |= TIMINGS0_TCCS;
2482 
2483 	reg |= FIELD_PREP(TIMINGS0_TWHR, twhr_cnt);
2484 	reg |= FIELD_PREP(TIMINGS0_TRHW, trhw_cnt);
2485 	t->timings0 = reg;
2486 	dev_dbg(cdns_ctrl->dev, "TIMINGS0_SDR\t%x\n", reg);
2487 
2488 	/* The following is related to single signal so skew is not needed. */
2489 	trhz_cnt = calc_cycl(sdr->tRHZ_max, clk_period);
2490 	trhz_cnt = trhz_cnt + 1;
2491 	twb_cnt = calc_cycl((sdr->tWB_max + board_delay), clk_period);
2492 	/*
2493 	 * Because of the two stage syncflop the value must be increased by 3
2494 	 * first value is related with sync, second value is related
2495 	 * with output if delay.
2496 	 */
2497 	twb_cnt = twb_cnt + 3 + 5;
2498 	/*
2499 	 * The following is related to the we edge of the random data input
2500 	 * sequence so skew is not needed.
2501 	 */
2502 	tvdly_cnt = calc_cycl(500000 + if_skew, clk_period);
2503 	reg = FIELD_PREP(TIMINGS1_TRHZ, trhz_cnt);
2504 	reg |= FIELD_PREP(TIMINGS1_TWB, twb_cnt);
2505 	reg |= FIELD_PREP(TIMINGS1_TVDLY, tvdly_cnt);
2506 	t->timings1 = reg;
2507 	dev_dbg(cdns_ctrl->dev, "TIMINGS1_SDR\t%x\n", reg);
2508 
2509 	tfeat_cnt = calc_cycl(sdr->tFEAT_max, clk_period);
2510 	if (tfeat_cnt < twb_cnt)
2511 		tfeat_cnt = twb_cnt;
2512 
2513 	tceh_cnt = calc_cycl(sdr->tCEH_min, clk_period);
2514 	tcs_cnt = calc_cycl((sdr->tCS_min + if_skew), clk_period);
2515 
2516 	reg = FIELD_PREP(TIMINGS2_TFEAT, tfeat_cnt);
2517 	reg |= FIELD_PREP(TIMINGS2_CS_HOLD_TIME, tceh_cnt);
2518 	reg |= FIELD_PREP(TIMINGS2_CS_SETUP_TIME, tcs_cnt);
2519 	t->timings2 = reg;
2520 	dev_dbg(cdns_ctrl->dev, "TIMINGS2_SDR\t%x\n", reg);
2521 
2522 	if (cdns_ctrl->caps2.is_phy_type_dll) {
2523 		reg = DLL_PHY_CTRL_DLL_RST_N;
2524 		if (ext_wr_mode)
2525 			reg |= DLL_PHY_CTRL_EXTENDED_WR_MODE;
2526 		if (ext_rd_mode)
2527 			reg |= DLL_PHY_CTRL_EXTENDED_RD_MODE;
2528 
2529 		reg |= FIELD_PREP(DLL_PHY_CTRL_RS_HIGH_WAIT_CNT, 7);
2530 		reg |= FIELD_PREP(DLL_PHY_CTRL_RS_IDLE_CNT, 7);
2531 		t->dll_phy_ctrl = reg;
2532 		dev_dbg(cdns_ctrl->dev, "DLL_PHY_CTRL_SDR\t%x\n", reg);
2533 	}
2534 
2535 	/* Sampling point calculation. */
2536 	if ((tdvw_max % dqs_sampl_res) > 0)
2537 		sampling_point = tdvw_max / dqs_sampl_res;
2538 	else
2539 		sampling_point = (tdvw_max / dqs_sampl_res - 1);
2540 
2541 	if (sampling_point * dqs_sampl_res > tdvw_min) {
2542 		dll_phy_dqs_timing =
2543 			FIELD_PREP(PHY_DQS_TIMING_DQS_SEL_OE_END, 4);
2544 		dll_phy_dqs_timing |= PHY_DQS_TIMING_USE_PHONY_DQS;
2545 		phony_dqs_timing = sampling_point / phony_dqs_mod;
2546 
2547 		if ((sampling_point % 2) > 0) {
2548 			dll_phy_dqs_timing |= PHY_DQS_TIMING_PHONY_DQS_SEL;
2549 			if ((tdvw_max % dqs_sampl_res) == 0)
2550 				/*
2551 				 * Calculation for sampling point at the edge
2552 				 * of data and being odd number.
2553 				 */
2554 				phony_dqs_timing = (tdvw_max / dqs_sampl_res)
2555 					/ phony_dqs_mod - 1;
2556 
2557 			if (!cdns_ctrl->caps2.is_phy_type_dll)
2558 				phony_dqs_timing--;
2559 
2560 		} else {
2561 			phony_dqs_timing--;
2562 		}
2563 		rd_del_sel = phony_dqs_timing + 3;
2564 	} else {
2565 		dev_warn(cdns_ctrl->dev,
2566 			 "ERROR : cannot find valid sampling point\n");
2567 	}
2568 
2569 	reg = FIELD_PREP(PHY_CTRL_PHONY_DQS, phony_dqs_timing);
2570 	if (cdns_ctrl->caps2.is_phy_type_dll)
2571 		reg  |= PHY_CTRL_SDR_DQS;
2572 	t->phy_ctrl = reg;
2573 	dev_dbg(cdns_ctrl->dev, "PHY_CTRL_REG_SDR\t%x\n", reg);
2574 
2575 	if (cdns_ctrl->caps2.is_phy_type_dll) {
2576 		dev_dbg(cdns_ctrl->dev, "PHY_TSEL_REG_SDR\t%x\n", 0);
2577 		dev_dbg(cdns_ctrl->dev, "PHY_DQ_TIMING_REG_SDR\t%x\n", 2);
2578 		dev_dbg(cdns_ctrl->dev, "PHY_DQS_TIMING_REG_SDR\t%x\n",
2579 			dll_phy_dqs_timing);
2580 		t->phy_dqs_timing = dll_phy_dqs_timing;
2581 
2582 		reg = FIELD_PREP(PHY_GATE_LPBK_CTRL_RDS, rd_del_sel);
2583 		dev_dbg(cdns_ctrl->dev, "PHY_GATE_LPBK_CTRL_REG_SDR\t%x\n",
2584 			reg);
2585 		t->phy_gate_lpbk_ctrl = reg;
2586 
2587 		dev_dbg(cdns_ctrl->dev, "PHY_DLL_MASTER_CTRL_REG_SDR\t%lx\n",
2588 			PHY_DLL_MASTER_CTRL_BYPASS_MODE);
2589 		dev_dbg(cdns_ctrl->dev, "PHY_DLL_SLAVE_CTRL_REG_SDR\t%x\n", 0);
2590 	}
2591 
2592 	return 0;
2593 }
2594 
2595 static int cadence_nand_attach_chip(struct nand_chip *chip)
2596 {
2597 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2598 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2599 	u32 ecc_size;
2600 	struct mtd_info *mtd = nand_to_mtd(chip);
2601 	int ret;
2602 
2603 	if (chip->options & NAND_BUSWIDTH_16) {
2604 		ret = cadence_nand_set_access_width16(cdns_ctrl, true);
2605 		if (ret)
2606 			return ret;
2607 	}
2608 
2609 	chip->bbt_options |= NAND_BBT_USE_FLASH;
2610 	chip->bbt_options |= NAND_BBT_NO_OOB;
2611 	chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2612 
2613 	chip->options |= NAND_NO_SUBPAGE_WRITE;
2614 
2615 	cdns_chip->bbm_offs = chip->badblockpos;
2616 	cdns_chip->bbm_offs &= ~0x01;
2617 	/* this value should be even number */
2618 	cdns_chip->bbm_len = 2;
2619 
2620 	ret = nand_ecc_choose_conf(chip,
2621 				   &cdns_ctrl->ecc_caps,
2622 				   mtd->oobsize - cdns_chip->bbm_len);
2623 	if (ret) {
2624 		dev_err(cdns_ctrl->dev, "ECC configuration failed\n");
2625 		return ret;
2626 	}
2627 
2628 	dev_dbg(cdns_ctrl->dev,
2629 		"chosen ECC settings: step=%d, strength=%d, bytes=%d\n",
2630 		chip->ecc.size, chip->ecc.strength, chip->ecc.bytes);
2631 
2632 	/* Error correction configuration. */
2633 	cdns_chip->sector_size = chip->ecc.size;
2634 	cdns_chip->sector_count = mtd->writesize / cdns_chip->sector_size;
2635 	ecc_size = cdns_chip->sector_count * chip->ecc.bytes;
2636 
2637 	cdns_chip->avail_oob_size = mtd->oobsize - ecc_size;
2638 
2639 	if (cdns_chip->avail_oob_size > cdns_ctrl->bch_metadata_size)
2640 		cdns_chip->avail_oob_size = cdns_ctrl->bch_metadata_size;
2641 
2642 	if ((cdns_chip->avail_oob_size + cdns_chip->bbm_len + ecc_size)
2643 	    > mtd->oobsize)
2644 		cdns_chip->avail_oob_size -= 4;
2645 
2646 	ret = cadence_nand_get_ecc_strength_idx(cdns_ctrl, chip->ecc.strength);
2647 	if (ret < 0)
2648 		return -EINVAL;
2649 
2650 	cdns_chip->corr_str_idx = (u8)ret;
2651 
2652 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
2653 					1000000,
2654 					CTRL_STATUS_CTRL_BUSY, true))
2655 		return -ETIMEDOUT;
2656 
2657 	cadence_nand_set_ecc_strength(cdns_ctrl,
2658 				      cdns_chip->corr_str_idx);
2659 
2660 	cadence_nand_set_erase_detection(cdns_ctrl, true,
2661 					 chip->ecc.strength);
2662 
2663 	/* Override the default read operations. */
2664 	chip->ecc.read_page = cadence_nand_read_page;
2665 	chip->ecc.read_page_raw = cadence_nand_read_page_raw;
2666 	chip->ecc.write_page = cadence_nand_write_page;
2667 	chip->ecc.write_page_raw = cadence_nand_write_page_raw;
2668 	chip->ecc.read_oob = cadence_nand_read_oob;
2669 	chip->ecc.write_oob = cadence_nand_write_oob;
2670 	chip->ecc.read_oob_raw = cadence_nand_read_oob_raw;
2671 	chip->ecc.write_oob_raw = cadence_nand_write_oob_raw;
2672 
2673 	if ((mtd->writesize + mtd->oobsize) > cdns_ctrl->buf_size)
2674 		cdns_ctrl->buf_size = mtd->writesize + mtd->oobsize;
2675 
2676 	/* Is 32-bit DMA supported? */
2677 	ret = dma_set_mask(cdns_ctrl->dev, DMA_BIT_MASK(32));
2678 	if (ret) {
2679 		dev_err(cdns_ctrl->dev, "no usable DMA configuration\n");
2680 		return ret;
2681 	}
2682 
2683 	mtd_set_ooblayout(mtd, &cadence_nand_ooblayout_ops);
2684 
2685 	return 0;
2686 }
2687 
2688 static const struct nand_controller_ops cadence_nand_controller_ops = {
2689 	.attach_chip = cadence_nand_attach_chip,
2690 	.exec_op = cadence_nand_exec_op,
2691 	.setup_interface = cadence_nand_setup_interface,
2692 };
2693 
2694 static int cadence_nand_chip_init(struct cdns_nand_ctrl *cdns_ctrl,
2695 				  struct device_node *np)
2696 {
2697 	struct cdns_nand_chip *cdns_chip;
2698 	struct mtd_info *mtd;
2699 	struct nand_chip *chip;
2700 	int nsels, ret, i;
2701 	u32 cs;
2702 
2703 	nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
2704 	if (nsels <= 0) {
2705 		dev_err(cdns_ctrl->dev, "missing/invalid reg property\n");
2706 		return -EINVAL;
2707 	}
2708 
2709 	/* Allocate the nand chip structure. */
2710 	cdns_chip = devm_kzalloc(cdns_ctrl->dev, sizeof(*cdns_chip) +
2711 				 (nsels * sizeof(u8)),
2712 				 GFP_KERNEL);
2713 	if (!cdns_chip) {
2714 		dev_err(cdns_ctrl->dev, "could not allocate chip structure\n");
2715 		return -ENOMEM;
2716 	}
2717 
2718 	cdns_chip->nsels = nsels;
2719 
2720 	for (i = 0; i < nsels; i++) {
2721 		/* Retrieve CS id. */
2722 		ret = of_property_read_u32_index(np, "reg", i, &cs);
2723 		if (ret) {
2724 			dev_err(cdns_ctrl->dev,
2725 				"could not retrieve reg property: %d\n",
2726 				ret);
2727 			return ret;
2728 		}
2729 
2730 		if (cs >= cdns_ctrl->caps2.max_banks) {
2731 			dev_err(cdns_ctrl->dev,
2732 				"invalid reg value: %u (max CS = %d)\n",
2733 				cs, cdns_ctrl->caps2.max_banks);
2734 			return -EINVAL;
2735 		}
2736 
2737 		if (test_and_set_bit(cs, &cdns_ctrl->assigned_cs)) {
2738 			dev_err(cdns_ctrl->dev,
2739 				"CS %d already assigned\n", cs);
2740 			return -EINVAL;
2741 		}
2742 
2743 		cdns_chip->cs[i] = cs;
2744 	}
2745 
2746 	chip = &cdns_chip->chip;
2747 	chip->controller = &cdns_ctrl->controller;
2748 	nand_set_flash_node(chip, np);
2749 
2750 	mtd = nand_to_mtd(chip);
2751 	mtd->dev.parent = cdns_ctrl->dev;
2752 
2753 	/*
2754 	 * Default to HW ECC engine mode. If the nand-ecc-mode property is given
2755 	 * in the DT node, this entry will be overwritten in nand_scan_ident().
2756 	 */
2757 	chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2758 
2759 	ret = nand_scan(chip, cdns_chip->nsels);
2760 	if (ret) {
2761 		dev_err(cdns_ctrl->dev, "could not scan the nand chip\n");
2762 		return ret;
2763 	}
2764 
2765 	ret = mtd_device_register(mtd, NULL, 0);
2766 	if (ret) {
2767 		dev_err(cdns_ctrl->dev,
2768 			"failed to register mtd device: %d\n", ret);
2769 		nand_cleanup(chip);
2770 		return ret;
2771 	}
2772 
2773 	list_add_tail(&cdns_chip->node, &cdns_ctrl->chips);
2774 
2775 	return 0;
2776 }
2777 
2778 static void cadence_nand_chips_cleanup(struct cdns_nand_ctrl *cdns_ctrl)
2779 {
2780 	struct cdns_nand_chip *entry, *temp;
2781 	struct nand_chip *chip;
2782 	int ret;
2783 
2784 	list_for_each_entry_safe(entry, temp, &cdns_ctrl->chips, node) {
2785 		chip = &entry->chip;
2786 		ret = mtd_device_unregister(nand_to_mtd(chip));
2787 		WARN_ON(ret);
2788 		nand_cleanup(chip);
2789 		list_del(&entry->node);
2790 	}
2791 }
2792 
2793 static int cadence_nand_chips_init(struct cdns_nand_ctrl *cdns_ctrl)
2794 {
2795 	struct device_node *np = cdns_ctrl->dev->of_node;
2796 	struct device_node *nand_np;
2797 	int max_cs = cdns_ctrl->caps2.max_banks;
2798 	int nchips, ret;
2799 
2800 	nchips = of_get_child_count(np);
2801 
2802 	if (nchips > max_cs) {
2803 		dev_err(cdns_ctrl->dev,
2804 			"too many NAND chips: %d (max = %d CS)\n",
2805 			nchips, max_cs);
2806 		return -EINVAL;
2807 	}
2808 
2809 	for_each_child_of_node(np, nand_np) {
2810 		ret = cadence_nand_chip_init(cdns_ctrl, nand_np);
2811 		if (ret) {
2812 			of_node_put(nand_np);
2813 			cadence_nand_chips_cleanup(cdns_ctrl);
2814 			return ret;
2815 		}
2816 	}
2817 
2818 	return 0;
2819 }
2820 
2821 static void
2822 cadence_nand_irq_cleanup(int irqnum, struct cdns_nand_ctrl *cdns_ctrl)
2823 {
2824 	/* Disable interrupts. */
2825 	writel_relaxed(INTR_ENABLE_INTR_EN, cdns_ctrl->reg + INTR_ENABLE);
2826 }
2827 
2828 static int cadence_nand_init(struct cdns_nand_ctrl *cdns_ctrl)
2829 {
2830 	dma_cap_mask_t mask;
2831 	int ret;
2832 
2833 	cdns_ctrl->cdma_desc = dma_alloc_coherent(cdns_ctrl->dev,
2834 						  sizeof(*cdns_ctrl->cdma_desc),
2835 						  &cdns_ctrl->dma_cdma_desc,
2836 						  GFP_KERNEL);
2837 	if (!cdns_ctrl->dma_cdma_desc)
2838 		return -ENOMEM;
2839 
2840 	cdns_ctrl->buf_size = SZ_16K;
2841 	cdns_ctrl->buf = kmalloc(cdns_ctrl->buf_size, GFP_KERNEL);
2842 	if (!cdns_ctrl->buf) {
2843 		ret = -ENOMEM;
2844 		goto free_buf_desc;
2845 	}
2846 
2847 	if (devm_request_irq(cdns_ctrl->dev, cdns_ctrl->irq, cadence_nand_isr,
2848 			     IRQF_SHARED, "cadence-nand-controller",
2849 			     cdns_ctrl)) {
2850 		dev_err(cdns_ctrl->dev, "Unable to allocate IRQ\n");
2851 		ret = -ENODEV;
2852 		goto free_buf;
2853 	}
2854 
2855 	spin_lock_init(&cdns_ctrl->irq_lock);
2856 	init_completion(&cdns_ctrl->complete);
2857 
2858 	ret = cadence_nand_hw_init(cdns_ctrl);
2859 	if (ret)
2860 		goto disable_irq;
2861 
2862 	dma_cap_zero(mask);
2863 	dma_cap_set(DMA_MEMCPY, mask);
2864 
2865 	if (cdns_ctrl->caps1->has_dma) {
2866 		cdns_ctrl->dmac = dma_request_channel(mask, NULL, NULL);
2867 		if (!cdns_ctrl->dmac) {
2868 			dev_err(cdns_ctrl->dev,
2869 				"Unable to get a DMA channel\n");
2870 			ret = -EBUSY;
2871 			goto disable_irq;
2872 		}
2873 	}
2874 
2875 	nand_controller_init(&cdns_ctrl->controller);
2876 	INIT_LIST_HEAD(&cdns_ctrl->chips);
2877 
2878 	cdns_ctrl->controller.ops = &cadence_nand_controller_ops;
2879 	cdns_ctrl->curr_corr_str_idx = 0xFF;
2880 
2881 	ret = cadence_nand_chips_init(cdns_ctrl);
2882 	if (ret) {
2883 		dev_err(cdns_ctrl->dev, "Failed to register MTD: %d\n",
2884 			ret);
2885 		goto dma_release_chnl;
2886 	}
2887 
2888 	kfree(cdns_ctrl->buf);
2889 	cdns_ctrl->buf = kzalloc(cdns_ctrl->buf_size, GFP_KERNEL);
2890 	if (!cdns_ctrl->buf) {
2891 		ret = -ENOMEM;
2892 		goto dma_release_chnl;
2893 	}
2894 
2895 	return 0;
2896 
2897 dma_release_chnl:
2898 	if (cdns_ctrl->dmac)
2899 		dma_release_channel(cdns_ctrl->dmac);
2900 
2901 disable_irq:
2902 	cadence_nand_irq_cleanup(cdns_ctrl->irq, cdns_ctrl);
2903 
2904 free_buf:
2905 	kfree(cdns_ctrl->buf);
2906 
2907 free_buf_desc:
2908 	dma_free_coherent(cdns_ctrl->dev, sizeof(struct cadence_nand_cdma_desc),
2909 			  cdns_ctrl->cdma_desc, cdns_ctrl->dma_cdma_desc);
2910 
2911 	return ret;
2912 }
2913 
2914 /* Driver exit point. */
2915 static void cadence_nand_remove(struct cdns_nand_ctrl *cdns_ctrl)
2916 {
2917 	cadence_nand_chips_cleanup(cdns_ctrl);
2918 	cadence_nand_irq_cleanup(cdns_ctrl->irq, cdns_ctrl);
2919 	kfree(cdns_ctrl->buf);
2920 	dma_free_coherent(cdns_ctrl->dev, sizeof(struct cadence_nand_cdma_desc),
2921 			  cdns_ctrl->cdma_desc, cdns_ctrl->dma_cdma_desc);
2922 
2923 	if (cdns_ctrl->dmac)
2924 		dma_release_channel(cdns_ctrl->dmac);
2925 }
2926 
2927 struct cadence_nand_dt {
2928 	struct cdns_nand_ctrl cdns_ctrl;
2929 	struct clk *clk;
2930 };
2931 
2932 static const struct cadence_nand_dt_devdata cadence_nand_default = {
2933 	.if_skew = 0,
2934 	.has_dma = 1,
2935 };
2936 
2937 static const struct of_device_id cadence_nand_dt_ids[] = {
2938 	{
2939 		.compatible = "cdns,hp-nfc",
2940 		.data = &cadence_nand_default
2941 	}, {}
2942 };
2943 
2944 MODULE_DEVICE_TABLE(of, cadence_nand_dt_ids);
2945 
2946 static int cadence_nand_dt_probe(struct platform_device *ofdev)
2947 {
2948 	struct resource *res;
2949 	struct cadence_nand_dt *dt;
2950 	struct cdns_nand_ctrl *cdns_ctrl;
2951 	int ret;
2952 	const struct of_device_id *of_id;
2953 	const struct cadence_nand_dt_devdata *devdata;
2954 	u32 val;
2955 
2956 	of_id = of_match_device(cadence_nand_dt_ids, &ofdev->dev);
2957 	if (of_id) {
2958 		ofdev->id_entry = of_id->data;
2959 		devdata = of_id->data;
2960 	} else {
2961 		pr_err("Failed to find the right device id.\n");
2962 		return -ENOMEM;
2963 	}
2964 
2965 	dt = devm_kzalloc(&ofdev->dev, sizeof(*dt), GFP_KERNEL);
2966 	if (!dt)
2967 		return -ENOMEM;
2968 
2969 	cdns_ctrl = &dt->cdns_ctrl;
2970 	cdns_ctrl->caps1 = devdata;
2971 
2972 	cdns_ctrl->dev = &ofdev->dev;
2973 	cdns_ctrl->irq = platform_get_irq(ofdev, 0);
2974 	if (cdns_ctrl->irq < 0)
2975 		return cdns_ctrl->irq;
2976 
2977 	dev_info(cdns_ctrl->dev, "IRQ: nr %d\n", cdns_ctrl->irq);
2978 
2979 	cdns_ctrl->reg = devm_platform_ioremap_resource(ofdev, 0);
2980 	if (IS_ERR(cdns_ctrl->reg))
2981 		return PTR_ERR(cdns_ctrl->reg);
2982 
2983 	cdns_ctrl->io.virt = devm_platform_get_and_ioremap_resource(ofdev, 1, &res);
2984 	if (IS_ERR(cdns_ctrl->io.virt))
2985 		return PTR_ERR(cdns_ctrl->io.virt);
2986 	cdns_ctrl->io.dma = res->start;
2987 
2988 	dt->clk = devm_clk_get(cdns_ctrl->dev, "nf_clk");
2989 	if (IS_ERR(dt->clk))
2990 		return PTR_ERR(dt->clk);
2991 
2992 	cdns_ctrl->nf_clk_rate = clk_get_rate(dt->clk);
2993 
2994 	ret = of_property_read_u32(ofdev->dev.of_node,
2995 				   "cdns,board-delay-ps", &val);
2996 	if (ret) {
2997 		val = 4830;
2998 		dev_info(cdns_ctrl->dev,
2999 			 "missing cdns,board-delay-ps property, %d was set\n",
3000 			 val);
3001 	}
3002 	cdns_ctrl->board_delay = val;
3003 
3004 	ret = cadence_nand_init(cdns_ctrl);
3005 	if (ret)
3006 		return ret;
3007 
3008 	platform_set_drvdata(ofdev, dt);
3009 	return 0;
3010 }
3011 
3012 static int cadence_nand_dt_remove(struct platform_device *ofdev)
3013 {
3014 	struct cadence_nand_dt *dt = platform_get_drvdata(ofdev);
3015 
3016 	cadence_nand_remove(&dt->cdns_ctrl);
3017 
3018 	return 0;
3019 }
3020 
3021 static struct platform_driver cadence_nand_dt_driver = {
3022 	.probe		= cadence_nand_dt_probe,
3023 	.remove		= cadence_nand_dt_remove,
3024 	.driver		= {
3025 		.name	= "cadence-nand-controller",
3026 		.of_match_table = cadence_nand_dt_ids,
3027 	},
3028 };
3029 
3030 module_platform_driver(cadence_nand_dt_driver);
3031 
3032 MODULE_AUTHOR("Piotr Sroka <piotrs@cadence.com>");
3033 MODULE_LICENSE("GPL v2");
3034 MODULE_DESCRIPTION("Driver for Cadence NAND flash controller");
3035 
3036