1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Evatronix/Renesas R-Car Gen3, RZ/N1D, RZ/N1S, RZ/N1L NAND controller driver
4  *
5  * Copyright (C) 2021 Schneider Electric
6  * Author: Miquel RAYNAL <miquel.raynal@bootlin.com>
7  */
8 
9 #include <linux/bitfield.h>
10 #include <linux/clk.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/interrupt.h>
13 #include <linux/iopoll.h>
14 #include <linux/module.h>
15 #include <linux/mtd/mtd.h>
16 #include <linux/mtd/rawnand.h>
17 #include <linux/of.h>
18 #include <linux/platform_device.h>
19 #include <linux/slab.h>
20 
21 #define COMMAND_REG 0x00
22 #define   COMMAND_SEQ(x) FIELD_PREP(GENMASK(5, 0), (x))
23 #define     COMMAND_SEQ_10 COMMAND_SEQ(0x2A)
24 #define     COMMAND_SEQ_12 COMMAND_SEQ(0x0C)
25 #define     COMMAND_SEQ_18 COMMAND_SEQ(0x32)
26 #define     COMMAND_SEQ_19 COMMAND_SEQ(0x13)
27 #define     COMMAND_SEQ_GEN_IN COMMAND_SEQ_18
28 #define     COMMAND_SEQ_GEN_OUT COMMAND_SEQ_19
29 #define     COMMAND_SEQ_READ_PAGE COMMAND_SEQ_10
30 #define     COMMAND_SEQ_WRITE_PAGE COMMAND_SEQ_12
31 #define   COMMAND_INPUT_SEL_AHBS 0
32 #define   COMMAND_INPUT_SEL_DMA BIT(6)
33 #define   COMMAND_FIFO_SEL 0
34 #define   COMMAND_DATA_SEL BIT(7)
35 #define   COMMAND_0(x) FIELD_PREP(GENMASK(15, 8), (x))
36 #define   COMMAND_1(x) FIELD_PREP(GENMASK(23, 16), (x))
37 #define   COMMAND_2(x) FIELD_PREP(GENMASK(31, 24), (x))
38 
39 #define CONTROL_REG 0x04
40 #define   CONTROL_CHECK_RB_LINE 0
41 #define   CONTROL_ECC_BLOCK_SIZE(x) FIELD_PREP(GENMASK(2, 1), (x))
42 #define     CONTROL_ECC_BLOCK_SIZE_256 CONTROL_ECC_BLOCK_SIZE(0)
43 #define     CONTROL_ECC_BLOCK_SIZE_512 CONTROL_ECC_BLOCK_SIZE(1)
44 #define     CONTROL_ECC_BLOCK_SIZE_1024 CONTROL_ECC_BLOCK_SIZE(2)
45 #define   CONTROL_INT_EN BIT(4)
46 #define   CONTROL_ECC_EN BIT(5)
47 #define   CONTROL_BLOCK_SIZE(x) FIELD_PREP(GENMASK(7, 6), (x))
48 #define     CONTROL_BLOCK_SIZE_32P CONTROL_BLOCK_SIZE(0)
49 #define     CONTROL_BLOCK_SIZE_64P CONTROL_BLOCK_SIZE(1)
50 #define     CONTROL_BLOCK_SIZE_128P CONTROL_BLOCK_SIZE(2)
51 #define     CONTROL_BLOCK_SIZE_256P CONTROL_BLOCK_SIZE(3)
52 
53 #define STATUS_REG 0x8
54 #define   MEM_RDY(cs, reg) (FIELD_GET(GENMASK(3, 0), (reg)) & BIT(cs))
55 #define   CTRL_RDY(reg) (FIELD_GET(BIT(8), (reg)) == 0)
56 
57 #define ECC_CTRL_REG 0x18
58 #define   ECC_CTRL_CAP(x) FIELD_PREP(GENMASK(2, 0), (x))
59 #define     ECC_CTRL_CAP_2B ECC_CTRL_CAP(0)
60 #define     ECC_CTRL_CAP_4B ECC_CTRL_CAP(1)
61 #define     ECC_CTRL_CAP_8B ECC_CTRL_CAP(2)
62 #define     ECC_CTRL_CAP_16B ECC_CTRL_CAP(3)
63 #define     ECC_CTRL_CAP_24B ECC_CTRL_CAP(4)
64 #define     ECC_CTRL_CAP_32B ECC_CTRL_CAP(5)
65 #define   ECC_CTRL_ERR_THRESHOLD(x) FIELD_PREP(GENMASK(13, 8), (x))
66 
67 #define INT_MASK_REG 0x10
68 #define INT_STATUS_REG 0x14
69 #define   INT_CMD_END BIT(1)
70 #define   INT_DMA_END BIT(3)
71 #define   INT_MEM_RDY(cs) FIELD_PREP(GENMASK(11, 8), BIT(cs))
72 #define   INT_DMA_ENDED BIT(3)
73 #define   MEM_IS_RDY(cs, reg) (FIELD_GET(GENMASK(11, 8), (reg)) & BIT(cs))
74 #define   DMA_HAS_ENDED(reg) FIELD_GET(BIT(3), (reg))
75 
76 #define ECC_OFFSET_REG 0x1C
77 #define   ECC_OFFSET(x) FIELD_PREP(GENMASK(15, 0), (x))
78 
79 #define ECC_STAT_REG 0x20
80 #define   ECC_STAT_CORRECTABLE(cs, reg) (FIELD_GET(GENMASK(3, 0), (reg)) & BIT(cs))
81 #define   ECC_STAT_UNCORRECTABLE(cs, reg) (FIELD_GET(GENMASK(11, 8), (reg)) & BIT(cs))
82 
83 #define ADDR0_COL_REG 0x24
84 #define   ADDR0_COL(x) FIELD_PREP(GENMASK(15, 0), (x))
85 
86 #define ADDR0_ROW_REG 0x28
87 #define   ADDR0_ROW(x) FIELD_PREP(GENMASK(23, 0), (x))
88 
89 #define ADDR1_COL_REG 0x2C
90 #define   ADDR1_COL(x) FIELD_PREP(GENMASK(15, 0), (x))
91 
92 #define ADDR1_ROW_REG 0x30
93 #define   ADDR1_ROW(x) FIELD_PREP(GENMASK(23, 0), (x))
94 
95 #define FIFO_DATA_REG 0x38
96 
97 #define DATA_REG 0x3C
98 
99 #define DATA_REG_SIZE_REG 0x40
100 
101 #define DMA_ADDR_LOW_REG 0x64
102 
103 #define DMA_ADDR_HIGH_REG 0x68
104 
105 #define DMA_CNT_REG 0x6C
106 
107 #define DMA_CTRL_REG 0x70
108 #define   DMA_CTRL_INCREMENT_BURST_4 0
109 #define   DMA_CTRL_REGISTER_MANAGED_MODE 0
110 #define   DMA_CTRL_START BIT(7)
111 
112 #define MEM_CTRL_REG 0x80
113 #define   MEM_CTRL_CS(cs) FIELD_PREP(GENMASK(1, 0), (cs))
114 #define   MEM_CTRL_DIS_WP(cs) FIELD_PREP(GENMASK(11, 8), BIT((cs)))
115 
116 #define DATA_SIZE_REG 0x84
117 #define   DATA_SIZE(x) FIELD_PREP(GENMASK(14, 0), (x))
118 
119 #define TIMINGS_ASYN_REG 0x88
120 #define   TIMINGS_ASYN_TRWP(x) FIELD_PREP(GENMASK(3, 0), max((x), 1U) - 1)
121 #define   TIMINGS_ASYN_TRWH(x) FIELD_PREP(GENMASK(7, 4), max((x), 1U) - 1)
122 
123 #define TIM_SEQ0_REG 0x90
124 #define   TIM_SEQ0_TCCS(x) FIELD_PREP(GENMASK(5, 0), max((x), 1U) - 1)
125 #define   TIM_SEQ0_TADL(x) FIELD_PREP(GENMASK(13, 8), max((x), 1U) - 1)
126 #define   TIM_SEQ0_TRHW(x) FIELD_PREP(GENMASK(21, 16), max((x), 1U) - 1)
127 #define   TIM_SEQ0_TWHR(x) FIELD_PREP(GENMASK(29, 24), max((x), 1U) - 1)
128 
129 #define TIM_SEQ1_REG 0x94
130 #define   TIM_SEQ1_TWB(x) FIELD_PREP(GENMASK(5, 0), max((x), 1U) - 1)
131 #define   TIM_SEQ1_TRR(x) FIELD_PREP(GENMASK(13, 8), max((x), 1U) - 1)
132 #define   TIM_SEQ1_TWW(x) FIELD_PREP(GENMASK(21, 16), max((x), 1U) - 1)
133 
134 #define TIM_GEN_SEQ0_REG 0x98
135 #define   TIM_GEN_SEQ0_D0(x) FIELD_PREP(GENMASK(5, 0), max((x), 1U) - 1)
136 #define   TIM_GEN_SEQ0_D1(x) FIELD_PREP(GENMASK(13, 8), max((x), 1U) - 1)
137 #define   TIM_GEN_SEQ0_D2(x) FIELD_PREP(GENMASK(21, 16), max((x), 1U) - 1)
138 #define   TIM_GEN_SEQ0_D3(x) FIELD_PREP(GENMASK(29, 24), max((x), 1U) - 1)
139 
140 #define TIM_GEN_SEQ1_REG 0x9c
141 #define   TIM_GEN_SEQ1_D4(x) FIELD_PREP(GENMASK(5, 0), max((x), 1U) - 1)
142 #define   TIM_GEN_SEQ1_D5(x) FIELD_PREP(GENMASK(13, 8), max((x), 1U) - 1)
143 #define   TIM_GEN_SEQ1_D6(x) FIELD_PREP(GENMASK(21, 16), max((x), 1U) - 1)
144 #define   TIM_GEN_SEQ1_D7(x) FIELD_PREP(GENMASK(29, 24), max((x), 1U) - 1)
145 
146 #define TIM_GEN_SEQ2_REG 0xA0
147 #define   TIM_GEN_SEQ2_D8(x) FIELD_PREP(GENMASK(5, 0), max((x), 1U) - 1)
148 #define   TIM_GEN_SEQ2_D9(x) FIELD_PREP(GENMASK(13, 8), max((x), 1U) - 1)
149 #define   TIM_GEN_SEQ2_D10(x) FIELD_PREP(GENMASK(21, 16), max((x), 1U) - 1)
150 #define   TIM_GEN_SEQ2_D11(x) FIELD_PREP(GENMASK(29, 24), max((x), 1U) - 1)
151 
152 #define FIFO_INIT_REG 0xB4
153 #define   FIFO_INIT BIT(0)
154 
155 #define FIFO_STATE_REG 0xB4
156 #define   FIFO_STATE_R_EMPTY(reg) FIELD_GET(BIT(0), (reg))
157 #define   FIFO_STATE_W_FULL(reg) FIELD_GET(BIT(1), (reg))
158 #define   FIFO_STATE_C_EMPTY(reg) FIELD_GET(BIT(2), (reg))
159 #define   FIFO_STATE_R_FULL(reg) FIELD_GET(BIT(6), (reg))
160 #define   FIFO_STATE_W_EMPTY(reg) FIELD_GET(BIT(7), (reg))
161 
162 #define GEN_SEQ_CTRL_REG 0xB8
163 #define   GEN_SEQ_CMD0_EN BIT(0)
164 #define   GEN_SEQ_CMD1_EN BIT(1)
165 #define   GEN_SEQ_CMD2_EN BIT(2)
166 #define   GEN_SEQ_CMD3_EN BIT(3)
167 #define   GEN_SEQ_COL_A0(x) FIELD_PREP(GENMASK(5, 4), min((x), 2U))
168 #define   GEN_SEQ_COL_A1(x) FIELD_PREP(GENMASK(7, 6), min((x), 2U))
169 #define   GEN_SEQ_ROW_A0(x) FIELD_PREP(GENMASK(9, 8), min((x), 3U))
170 #define   GEN_SEQ_ROW_A1(x) FIELD_PREP(GENMASK(11, 10), min((x), 3U))
171 #define   GEN_SEQ_DATA_EN BIT(12)
172 #define   GEN_SEQ_DELAY_EN(x) FIELD_PREP(GENMASK(14, 13), (x))
173 #define     GEN_SEQ_DELAY0_EN GEN_SEQ_DELAY_EN(1)
174 #define     GEN_SEQ_DELAY1_EN GEN_SEQ_DELAY_EN(2)
175 #define   GEN_SEQ_IMD_SEQ BIT(15)
176 #define   GEN_SEQ_COMMAND_3(x) FIELD_PREP(GENMASK(26, 16), (x))
177 
178 #define DMA_TLVL_REG 0x114
179 #define   DMA_TLVL(x) FIELD_PREP(GENMASK(7, 0), (x))
180 #define   DMA_TLVL_MAX DMA_TLVL(0xFF)
181 
182 #define TIM_GEN_SEQ3_REG 0x134
183 #define   TIM_GEN_SEQ3_D12(x) FIELD_PREP(GENMASK(5, 0), max((x), 1U) - 1)
184 
185 #define ECC_CNT_REG 0x14C
186 #define   ECC_CNT(cs, reg) FIELD_GET(GENMASK(5, 0), (reg) >> ((cs) * 8))
187 
188 #define RNANDC_CS_NUM 4
189 
190 #define TO_CYCLES64(ps, period_ns) ((unsigned int)DIV_ROUND_UP_ULL(div_u64(ps, 1000), \
191 								   period_ns))
192 
193 struct rnand_chip_sel {
194 	unsigned int cs;
195 };
196 
197 struct rnand_chip {
198 	struct nand_chip chip;
199 	struct list_head node;
200 	int selected_die;
201 	u32 ctrl;
202 	unsigned int nsels;
203 	u32 control;
204 	u32 ecc_ctrl;
205 	u32 timings_asyn;
206 	u32 tim_seq0;
207 	u32 tim_seq1;
208 	u32 tim_gen_seq0;
209 	u32 tim_gen_seq1;
210 	u32 tim_gen_seq2;
211 	u32 tim_gen_seq3;
212 	struct rnand_chip_sel sels[];
213 };
214 
215 struct rnandc {
216 	struct nand_controller controller;
217 	struct device *dev;
218 	void __iomem *regs;
219 	struct clk *hclk;
220 	struct clk *eclk;
221 	unsigned long assigned_cs;
222 	struct list_head chips;
223 	struct nand_chip *selected_chip;
224 	struct completion complete;
225 	bool use_polling;
226 	u8 *buf;
227 	unsigned int buf_sz;
228 };
229 
230 struct rnandc_op {
231 	u32 command;
232 	u32 addr0_col;
233 	u32 addr0_row;
234 	u32 addr1_col;
235 	u32 addr1_row;
236 	u32 data_size;
237 	u32 ecc_offset;
238 	u32 gen_seq_ctrl;
239 	u8 *buf;
240 	bool read;
241 	unsigned int len;
242 };
243 
244 static inline struct rnandc *to_rnandc(struct nand_controller *ctrl)
245 {
246 	return container_of(ctrl, struct rnandc, controller);
247 }
248 
249 static inline struct rnand_chip *to_rnand(struct nand_chip *chip)
250 {
251 	return container_of(chip, struct rnand_chip, chip);
252 }
253 
254 static inline unsigned int to_rnandc_cs(struct rnand_chip *nand)
255 {
256 	return nand->sels[nand->selected_die].cs;
257 }
258 
259 static void rnandc_dis_correction(struct rnandc *rnandc)
260 {
261 	u32 control;
262 
263 	control = readl_relaxed(rnandc->regs + CONTROL_REG);
264 	control &= ~CONTROL_ECC_EN;
265 	writel_relaxed(control, rnandc->regs + CONTROL_REG);
266 }
267 
268 static void rnandc_en_correction(struct rnandc *rnandc)
269 {
270 	u32 control;
271 
272 	control = readl_relaxed(rnandc->regs + CONTROL_REG);
273 	control |= CONTROL_ECC_EN;
274 	writel_relaxed(control, rnandc->regs + CONTROL_REG);
275 }
276 
277 static void rnandc_clear_status(struct rnandc *rnandc)
278 {
279 	writel_relaxed(0, rnandc->regs + INT_STATUS_REG);
280 	writel_relaxed(0, rnandc->regs + ECC_STAT_REG);
281 	writel_relaxed(0, rnandc->regs + ECC_CNT_REG);
282 }
283 
284 static void rnandc_dis_interrupts(struct rnandc *rnandc)
285 {
286 	writel_relaxed(0, rnandc->regs + INT_MASK_REG);
287 }
288 
289 static void rnandc_en_interrupts(struct rnandc *rnandc, u32 val)
290 {
291 	if (!rnandc->use_polling)
292 		writel_relaxed(val, rnandc->regs + INT_MASK_REG);
293 }
294 
295 static void rnandc_clear_fifo(struct rnandc *rnandc)
296 {
297 	writel_relaxed(FIFO_INIT, rnandc->regs + FIFO_INIT_REG);
298 }
299 
300 static void rnandc_select_target(struct nand_chip *chip, int die_nr)
301 {
302 	struct rnand_chip *rnand = to_rnand(chip);
303 	struct rnandc *rnandc = to_rnandc(chip->controller);
304 	unsigned int cs = rnand->sels[die_nr].cs;
305 
306 	if (chip == rnandc->selected_chip && die_nr == rnand->selected_die)
307 		return;
308 
309 	rnandc_clear_status(rnandc);
310 	writel_relaxed(MEM_CTRL_CS(cs) | MEM_CTRL_DIS_WP(cs), rnandc->regs + MEM_CTRL_REG);
311 	writel_relaxed(rnand->control, rnandc->regs + CONTROL_REG);
312 	writel_relaxed(rnand->ecc_ctrl, rnandc->regs + ECC_CTRL_REG);
313 	writel_relaxed(rnand->timings_asyn, rnandc->regs + TIMINGS_ASYN_REG);
314 	writel_relaxed(rnand->tim_seq0, rnandc->regs + TIM_SEQ0_REG);
315 	writel_relaxed(rnand->tim_seq1, rnandc->regs + TIM_SEQ1_REG);
316 	writel_relaxed(rnand->tim_gen_seq0, rnandc->regs + TIM_GEN_SEQ0_REG);
317 	writel_relaxed(rnand->tim_gen_seq1, rnandc->regs + TIM_GEN_SEQ1_REG);
318 	writel_relaxed(rnand->tim_gen_seq2, rnandc->regs + TIM_GEN_SEQ2_REG);
319 	writel_relaxed(rnand->tim_gen_seq3, rnandc->regs + TIM_GEN_SEQ3_REG);
320 
321 	rnandc->selected_chip = chip;
322 	rnand->selected_die = die_nr;
323 }
324 
325 static void rnandc_trigger_op(struct rnandc *rnandc, struct rnandc_op *rop)
326 {
327 	writel_relaxed(rop->addr0_col, rnandc->regs + ADDR0_COL_REG);
328 	writel_relaxed(rop->addr0_row, rnandc->regs + ADDR0_ROW_REG);
329 	writel_relaxed(rop->addr1_col, rnandc->regs + ADDR1_COL_REG);
330 	writel_relaxed(rop->addr1_row, rnandc->regs + ADDR1_ROW_REG);
331 	writel_relaxed(rop->ecc_offset, rnandc->regs + ECC_OFFSET_REG);
332 	writel_relaxed(rop->gen_seq_ctrl, rnandc->regs + GEN_SEQ_CTRL_REG);
333 	writel_relaxed(DATA_SIZE(rop->len), rnandc->regs + DATA_SIZE_REG);
334 	writel_relaxed(rop->command, rnandc->regs + COMMAND_REG);
335 }
336 
337 static void rnandc_trigger_dma(struct rnandc *rnandc)
338 {
339 	writel_relaxed(DMA_CTRL_INCREMENT_BURST_4 |
340 		       DMA_CTRL_REGISTER_MANAGED_MODE |
341 		       DMA_CTRL_START, rnandc->regs + DMA_CTRL_REG);
342 }
343 
344 static irqreturn_t rnandc_irq_handler(int irq, void *private)
345 {
346 	struct rnandc *rnandc = private;
347 
348 	rnandc_dis_interrupts(rnandc);
349 	complete(&rnandc->complete);
350 
351 	return IRQ_HANDLED;
352 }
353 
354 static int rnandc_wait_end_of_op(struct rnandc *rnandc,
355 				 struct nand_chip *chip)
356 {
357 	struct rnand_chip *rnand = to_rnand(chip);
358 	unsigned int cs = to_rnandc_cs(rnand);
359 	u32 status;
360 	int ret;
361 
362 	ret = readl_poll_timeout(rnandc->regs + STATUS_REG, status,
363 				 MEM_RDY(cs, status) && CTRL_RDY(status),
364 				 1, 100000);
365 	if (ret)
366 		dev_err(rnandc->dev, "Operation timed out, status: 0x%08x\n",
367 			status);
368 
369 	return ret;
370 }
371 
372 static int rnandc_wait_end_of_io(struct rnandc *rnandc,
373 				 struct nand_chip *chip)
374 {
375 	int timeout_ms = 1000;
376 	int ret;
377 
378 	if (rnandc->use_polling) {
379 		struct rnand_chip *rnand = to_rnand(chip);
380 		unsigned int cs = to_rnandc_cs(rnand);
381 		u32 status;
382 
383 		ret = readl_poll_timeout(rnandc->regs + INT_STATUS_REG, status,
384 					 MEM_IS_RDY(cs, status) &
385 					 DMA_HAS_ENDED(status),
386 					 0, timeout_ms * 1000);
387 	} else {
388 		ret = wait_for_completion_timeout(&rnandc->complete,
389 						  msecs_to_jiffies(timeout_ms));
390 		if (!ret)
391 			ret = -ETIMEDOUT;
392 		else
393 			ret = 0;
394 	}
395 
396 	return ret;
397 }
398 
399 static int rnandc_read_page_hw_ecc(struct nand_chip *chip, u8 *buf,
400 				   int oob_required, int page)
401 {
402 	struct rnandc *rnandc = to_rnandc(chip->controller);
403 	struct mtd_info *mtd = nand_to_mtd(chip);
404 	struct rnand_chip *rnand = to_rnand(chip);
405 	unsigned int cs = to_rnandc_cs(rnand);
406 	struct rnandc_op rop = {
407 		.command = COMMAND_INPUT_SEL_DMA | COMMAND_0(NAND_CMD_READ0) |
408 			   COMMAND_2(NAND_CMD_READSTART) | COMMAND_FIFO_SEL |
409 			   COMMAND_SEQ_READ_PAGE,
410 		.addr0_row = page,
411 		.len = mtd->writesize,
412 		.ecc_offset = ECC_OFFSET(mtd->writesize + 2),
413 	};
414 	unsigned int max_bitflips = 0;
415 	dma_addr_t dma_addr;
416 	u32 ecc_stat;
417 	int bf, ret, i;
418 
419 	/* Prepare controller */
420 	rnandc_select_target(chip, chip->cur_cs);
421 	rnandc_clear_status(rnandc);
422 	reinit_completion(&rnandc->complete);
423 	rnandc_en_interrupts(rnandc, INT_DMA_ENDED);
424 	rnandc_en_correction(rnandc);
425 
426 	/* Configure DMA */
427 	dma_addr = dma_map_single(rnandc->dev, rnandc->buf, mtd->writesize,
428 				  DMA_FROM_DEVICE);
429 	writel(dma_addr, rnandc->regs + DMA_ADDR_LOW_REG);
430 	writel(mtd->writesize, rnandc->regs + DMA_CNT_REG);
431 	writel(DMA_TLVL_MAX, rnandc->regs + DMA_TLVL_REG);
432 
433 	rnandc_trigger_op(rnandc, &rop);
434 	rnandc_trigger_dma(rnandc);
435 
436 	ret = rnandc_wait_end_of_io(rnandc, chip);
437 	dma_unmap_single(rnandc->dev, dma_addr, mtd->writesize, DMA_FROM_DEVICE);
438 	rnandc_dis_correction(rnandc);
439 	if (ret) {
440 		dev_err(rnandc->dev, "Read page operation never ending\n");
441 		return ret;
442 	}
443 
444 	ecc_stat = readl_relaxed(rnandc->regs + ECC_STAT_REG);
445 
446 	if (oob_required || ECC_STAT_UNCORRECTABLE(cs, ecc_stat)) {
447 		ret = nand_change_read_column_op(chip, mtd->writesize,
448 						 chip->oob_poi, mtd->oobsize,
449 						 false);
450 		if (ret)
451 			return ret;
452 	}
453 
454 	if (ECC_STAT_UNCORRECTABLE(cs, ecc_stat)) {
455 		for (i = 0; i < chip->ecc.steps; i++) {
456 			unsigned int off = i * chip->ecc.size;
457 			unsigned int eccoff = i * chip->ecc.bytes;
458 
459 			bf = nand_check_erased_ecc_chunk(rnandc->buf + off,
460 							 chip->ecc.size,
461 							 chip->oob_poi + 2 + eccoff,
462 							 chip->ecc.bytes,
463 							 NULL, 0,
464 							 chip->ecc.strength);
465 			if (bf < 0) {
466 				mtd->ecc_stats.failed++;
467 			} else {
468 				mtd->ecc_stats.corrected += bf;
469 				max_bitflips = max_t(unsigned int, max_bitflips, bf);
470 			}
471 		}
472 	} else if (ECC_STAT_CORRECTABLE(cs, ecc_stat)) {
473 		bf = ECC_CNT(cs, readl_relaxed(rnandc->regs + ECC_CNT_REG));
474 		/*
475 		 * The number of bitflips is an approximation given the fact
476 		 * that this controller does not provide per-chunk details but
477 		 * only gives statistics on the entire page.
478 		 */
479 		mtd->ecc_stats.corrected += bf;
480 	}
481 
482 	memcpy(buf, rnandc->buf, mtd->writesize);
483 
484 	return 0;
485 }
486 
487 static int rnandc_read_subpage_hw_ecc(struct nand_chip *chip, u32 req_offset,
488 				      u32 req_len, u8 *bufpoi, int page)
489 {
490 	struct rnandc *rnandc = to_rnandc(chip->controller);
491 	struct mtd_info *mtd = nand_to_mtd(chip);
492 	struct rnand_chip *rnand = to_rnand(chip);
493 	unsigned int cs = to_rnandc_cs(rnand);
494 	unsigned int page_off = round_down(req_offset, chip->ecc.size);
495 	unsigned int real_len = round_up(req_offset + req_len - page_off,
496 					 chip->ecc.size);
497 	unsigned int start_chunk = page_off / chip->ecc.size;
498 	unsigned int nchunks = real_len / chip->ecc.size;
499 	unsigned int ecc_off = 2 + (start_chunk * chip->ecc.bytes);
500 	struct rnandc_op rop = {
501 		.command = COMMAND_INPUT_SEL_AHBS | COMMAND_0(NAND_CMD_READ0) |
502 			   COMMAND_2(NAND_CMD_READSTART) | COMMAND_FIFO_SEL |
503 			   COMMAND_SEQ_READ_PAGE,
504 		.addr0_row = page,
505 		.addr0_col = page_off,
506 		.len = real_len,
507 		.ecc_offset = ECC_OFFSET(mtd->writesize + ecc_off),
508 	};
509 	unsigned int max_bitflips = 0, i;
510 	u32 ecc_stat;
511 	int bf, ret;
512 
513 	/* Prepare controller */
514 	rnandc_select_target(chip, chip->cur_cs);
515 	rnandc_clear_status(rnandc);
516 	rnandc_en_correction(rnandc);
517 	rnandc_trigger_op(rnandc, &rop);
518 
519 	while (!FIFO_STATE_C_EMPTY(readl(rnandc->regs + FIFO_STATE_REG)))
520 		cpu_relax();
521 
522 	while (FIFO_STATE_R_EMPTY(readl(rnandc->regs + FIFO_STATE_REG)))
523 		cpu_relax();
524 
525 	ioread32_rep(rnandc->regs + FIFO_DATA_REG, bufpoi + page_off,
526 		     real_len / 4);
527 
528 	if (!FIFO_STATE_R_EMPTY(readl(rnandc->regs + FIFO_STATE_REG))) {
529 		dev_err(rnandc->dev, "Clearing residual data in the read FIFO\n");
530 		rnandc_clear_fifo(rnandc);
531 	}
532 
533 	ret = rnandc_wait_end_of_op(rnandc, chip);
534 	rnandc_dis_correction(rnandc);
535 	if (ret) {
536 		dev_err(rnandc->dev, "Read subpage operation never ending\n");
537 		return ret;
538 	}
539 
540 	ecc_stat = readl_relaxed(rnandc->regs + ECC_STAT_REG);
541 
542 	if (ECC_STAT_UNCORRECTABLE(cs, ecc_stat)) {
543 		ret = nand_change_read_column_op(chip, mtd->writesize,
544 						 chip->oob_poi, mtd->oobsize,
545 						 false);
546 		if (ret)
547 			return ret;
548 
549 		for (i = start_chunk; i < nchunks; i++) {
550 			unsigned int dataoff = i * chip->ecc.size;
551 			unsigned int eccoff = 2 + (i * chip->ecc.bytes);
552 
553 			bf = nand_check_erased_ecc_chunk(bufpoi + dataoff,
554 							 chip->ecc.size,
555 							 chip->oob_poi + eccoff,
556 							 chip->ecc.bytes,
557 							 NULL, 0,
558 							 chip->ecc.strength);
559 			if (bf < 0) {
560 				mtd->ecc_stats.failed++;
561 			} else {
562 				mtd->ecc_stats.corrected += bf;
563 				max_bitflips = max_t(unsigned int, max_bitflips, bf);
564 			}
565 		}
566 	} else if (ECC_STAT_CORRECTABLE(cs, ecc_stat)) {
567 		bf = ECC_CNT(cs, readl_relaxed(rnandc->regs + ECC_CNT_REG));
568 		/*
569 		 * The number of bitflips is an approximation given the fact
570 		 * that this controller does not provide per-chunk details but
571 		 * only gives statistics on the entire page.
572 		 */
573 		mtd->ecc_stats.corrected += bf;
574 	}
575 
576 	return 0;
577 }
578 
579 static int rnandc_write_page_hw_ecc(struct nand_chip *chip, const u8 *buf,
580 				    int oob_required, int page)
581 {
582 	struct rnandc *rnandc = to_rnandc(chip->controller);
583 	struct mtd_info *mtd = nand_to_mtd(chip);
584 	struct rnand_chip *rnand = to_rnand(chip);
585 	unsigned int cs = to_rnandc_cs(rnand);
586 	struct rnandc_op rop = {
587 		.command = COMMAND_INPUT_SEL_DMA | COMMAND_0(NAND_CMD_SEQIN) |
588 			   COMMAND_1(NAND_CMD_PAGEPROG) | COMMAND_FIFO_SEL |
589 			   COMMAND_SEQ_WRITE_PAGE,
590 		.addr0_row = page,
591 		.len = mtd->writesize,
592 		.ecc_offset = ECC_OFFSET(mtd->writesize + 2),
593 	};
594 	dma_addr_t dma_addr;
595 	int ret;
596 
597 	memcpy(rnandc->buf, buf, mtd->writesize);
598 
599 	/* Prepare controller */
600 	rnandc_select_target(chip, chip->cur_cs);
601 	rnandc_clear_status(rnandc);
602 	reinit_completion(&rnandc->complete);
603 	rnandc_en_interrupts(rnandc, INT_MEM_RDY(cs));
604 	rnandc_en_correction(rnandc);
605 
606 	/* Configure DMA */
607 	dma_addr = dma_map_single(rnandc->dev, (void *)rnandc->buf, mtd->writesize,
608 				  DMA_TO_DEVICE);
609 	writel(dma_addr, rnandc->regs + DMA_ADDR_LOW_REG);
610 	writel(mtd->writesize, rnandc->regs + DMA_CNT_REG);
611 	writel(DMA_TLVL_MAX, rnandc->regs + DMA_TLVL_REG);
612 
613 	rnandc_trigger_op(rnandc, &rop);
614 	rnandc_trigger_dma(rnandc);
615 
616 	ret = rnandc_wait_end_of_io(rnandc, chip);
617 	dma_unmap_single(rnandc->dev, dma_addr, mtd->writesize, DMA_TO_DEVICE);
618 	rnandc_dis_correction(rnandc);
619 	if (ret) {
620 		dev_err(rnandc->dev, "Write page operation never ending\n");
621 		return ret;
622 	}
623 
624 	if (!oob_required)
625 		return 0;
626 
627 	return nand_change_write_column_op(chip, mtd->writesize, chip->oob_poi,
628 					   mtd->oobsize, false);
629 }
630 
631 static int rnandc_write_subpage_hw_ecc(struct nand_chip *chip, u32 req_offset,
632 				       u32 req_len, const u8 *bufpoi,
633 				       int oob_required, int page)
634 {
635 	struct rnandc *rnandc = to_rnandc(chip->controller);
636 	struct mtd_info *mtd = nand_to_mtd(chip);
637 	unsigned int page_off = round_down(req_offset, chip->ecc.size);
638 	unsigned int real_len = round_up(req_offset + req_len - page_off,
639 					 chip->ecc.size);
640 	unsigned int start_chunk = page_off / chip->ecc.size;
641 	unsigned int ecc_off = 2 + (start_chunk * chip->ecc.bytes);
642 	struct rnandc_op rop = {
643 		.command = COMMAND_INPUT_SEL_AHBS | COMMAND_0(NAND_CMD_SEQIN) |
644 			   COMMAND_1(NAND_CMD_PAGEPROG) | COMMAND_FIFO_SEL |
645 			   COMMAND_SEQ_WRITE_PAGE,
646 		.addr0_row = page,
647 		.addr0_col = page_off,
648 		.len = real_len,
649 		.ecc_offset = ECC_OFFSET(mtd->writesize + ecc_off),
650 	};
651 	int ret;
652 
653 	/* Prepare controller */
654 	rnandc_select_target(chip, chip->cur_cs);
655 	rnandc_clear_status(rnandc);
656 	rnandc_en_correction(rnandc);
657 	rnandc_trigger_op(rnandc, &rop);
658 
659 	while (FIFO_STATE_W_FULL(readl(rnandc->regs + FIFO_STATE_REG)))
660 		cpu_relax();
661 
662 	iowrite32_rep(rnandc->regs + FIFO_DATA_REG, bufpoi + page_off,
663 		      real_len / 4);
664 
665 	while (!FIFO_STATE_W_EMPTY(readl(rnandc->regs + FIFO_STATE_REG)))
666 		cpu_relax();
667 
668 	ret = rnandc_wait_end_of_op(rnandc, chip);
669 	rnandc_dis_correction(rnandc);
670 	if (ret) {
671 		dev_err(rnandc->dev, "Write subpage operation never ending\n");
672 		return ret;
673 	}
674 
675 	return 0;
676 }
677 
678 /*
679  * This controller is simple enough and thus does not need to use the parser
680  * provided by the core, instead, handle every situation here.
681  */
682 static int rnandc_exec_op(struct nand_chip *chip,
683 			  const struct nand_operation *op, bool check_only)
684 {
685 	struct rnandc *rnandc = to_rnandc(chip->controller);
686 	const struct nand_op_instr *instr = NULL;
687 	struct rnandc_op rop = {
688 		.command = COMMAND_INPUT_SEL_AHBS,
689 		.gen_seq_ctrl = GEN_SEQ_IMD_SEQ,
690 	};
691 	unsigned int cmd_phase = 0, addr_phase = 0, data_phase = 0,
692 		delay_phase = 0, delays = 0;
693 	unsigned int op_id, col_addrs, row_addrs, naddrs, remainder, words, i;
694 	const u8 *addrs;
695 	u32 last_bytes;
696 	int ret;
697 
698 	if (!check_only)
699 		rnandc_select_target(chip, op->cs);
700 
701 	for (op_id = 0; op_id < op->ninstrs; op_id++) {
702 		instr = &op->instrs[op_id];
703 
704 		nand_op_trace("  ", instr);
705 
706 		switch (instr->type) {
707 		case NAND_OP_CMD_INSTR:
708 			switch (cmd_phase++) {
709 			case 0:
710 				rop.command |= COMMAND_0(instr->ctx.cmd.opcode);
711 				rop.gen_seq_ctrl |= GEN_SEQ_CMD0_EN;
712 				break;
713 			case 1:
714 				rop.gen_seq_ctrl |= GEN_SEQ_COMMAND_3(instr->ctx.cmd.opcode);
715 				rop.gen_seq_ctrl |= GEN_SEQ_CMD3_EN;
716 				if (addr_phase == 0)
717 					addr_phase = 1;
718 				break;
719 			case 2:
720 				rop.command |= COMMAND_2(instr->ctx.cmd.opcode);
721 				rop.gen_seq_ctrl |= GEN_SEQ_CMD2_EN;
722 				if (addr_phase <= 1)
723 					addr_phase = 2;
724 				break;
725 			case 3:
726 				rop.command |= COMMAND_1(instr->ctx.cmd.opcode);
727 				rop.gen_seq_ctrl |= GEN_SEQ_CMD1_EN;
728 				if (addr_phase <= 1)
729 					addr_phase = 2;
730 				if (delay_phase == 0)
731 					delay_phase = 1;
732 				if (data_phase == 0)
733 					data_phase = 1;
734 				break;
735 			default:
736 				return -EOPNOTSUPP;
737 			}
738 			break;
739 
740 		case NAND_OP_ADDR_INSTR:
741 			addrs = instr->ctx.addr.addrs;
742 			naddrs = instr->ctx.addr.naddrs;
743 			if (naddrs > 5)
744 				return -EOPNOTSUPP;
745 
746 			col_addrs = min(2U, naddrs);
747 			row_addrs = naddrs > 2 ? naddrs - col_addrs : 0;
748 
749 			switch (addr_phase++) {
750 			case 0:
751 				for (i = 0; i < col_addrs; i++)
752 					rop.addr0_col |= addrs[i] << (i * 8);
753 				rop.gen_seq_ctrl |= GEN_SEQ_COL_A0(col_addrs);
754 
755 				for (i = 0; i < row_addrs; i++)
756 					rop.addr0_row |= addrs[2 + i] << (i * 8);
757 				rop.gen_seq_ctrl |= GEN_SEQ_ROW_A0(row_addrs);
758 
759 				if (cmd_phase == 0)
760 					cmd_phase = 1;
761 				break;
762 			case 1:
763 				for (i = 0; i < col_addrs; i++)
764 					rop.addr1_col |= addrs[i] << (i * 8);
765 				rop.gen_seq_ctrl |= GEN_SEQ_COL_A1(col_addrs);
766 
767 				for (i = 0; i < row_addrs; i++)
768 					rop.addr1_row |= addrs[2 + i] << (i * 8);
769 				rop.gen_seq_ctrl |= GEN_SEQ_ROW_A1(row_addrs);
770 
771 				if (cmd_phase <= 1)
772 					cmd_phase = 2;
773 				break;
774 			default:
775 				return -EOPNOTSUPP;
776 			}
777 			break;
778 
779 		case NAND_OP_DATA_IN_INSTR:
780 			rop.read = true;
781 			fallthrough;
782 		case NAND_OP_DATA_OUT_INSTR:
783 			rop.gen_seq_ctrl |= GEN_SEQ_DATA_EN;
784 			rop.buf = instr->ctx.data.buf.in;
785 			rop.len = instr->ctx.data.len;
786 			rop.command |= COMMAND_FIFO_SEL;
787 
788 			switch (data_phase++) {
789 			case 0:
790 				if (cmd_phase <= 2)
791 					cmd_phase = 3;
792 				if (addr_phase <= 1)
793 					addr_phase = 2;
794 				if (delay_phase == 0)
795 					delay_phase = 1;
796 				break;
797 			default:
798 				return -EOPNOTSUPP;
799 			}
800 			break;
801 
802 		case NAND_OP_WAITRDY_INSTR:
803 			switch (delay_phase++) {
804 			case 0:
805 				rop.gen_seq_ctrl |= GEN_SEQ_DELAY0_EN;
806 
807 				if (cmd_phase <= 2)
808 					cmd_phase = 3;
809 				break;
810 			case 1:
811 				rop.gen_seq_ctrl |= GEN_SEQ_DELAY1_EN;
812 
813 				if (cmd_phase <= 3)
814 					cmd_phase = 4;
815 				if (data_phase == 0)
816 					data_phase = 1;
817 				break;
818 			default:
819 				return -EOPNOTSUPP;
820 			}
821 			break;
822 		}
823 	}
824 
825 	/*
826 	 * Sequence 19 is generic and dedicated to write operations.
827 	 * Sequence 18 is also generic and works for all other operations.
828 	 */
829 	if (rop.buf && !rop.read)
830 		rop.command |= COMMAND_SEQ_GEN_OUT;
831 	else
832 		rop.command |= COMMAND_SEQ_GEN_IN;
833 
834 	if (delays > 1) {
835 		dev_err(rnandc->dev, "Cannot handle more than one wait delay\n");
836 		return -EOPNOTSUPP;
837 	}
838 
839 	if (check_only)
840 		return 0;
841 
842 	rnandc_trigger_op(rnandc, &rop);
843 
844 	words = rop.len / sizeof(u32);
845 	remainder = rop.len % sizeof(u32);
846 	if (rop.buf && rop.read) {
847 		while (!FIFO_STATE_C_EMPTY(readl(rnandc->regs + FIFO_STATE_REG)))
848 			cpu_relax();
849 
850 		while (FIFO_STATE_R_EMPTY(readl(rnandc->regs + FIFO_STATE_REG)))
851 			cpu_relax();
852 
853 		ioread32_rep(rnandc->regs + FIFO_DATA_REG, rop.buf, words);
854 		if (remainder) {
855 			last_bytes = readl_relaxed(rnandc->regs + FIFO_DATA_REG);
856 			memcpy(rop.buf + (words * sizeof(u32)), &last_bytes,
857 			       remainder);
858 		}
859 
860 		if (!FIFO_STATE_R_EMPTY(readl(rnandc->regs + FIFO_STATE_REG))) {
861 			dev_warn(rnandc->dev,
862 				 "Clearing residual data in the read FIFO\n");
863 			rnandc_clear_fifo(rnandc);
864 		}
865 	} else if (rop.len && !rop.read) {
866 		while (FIFO_STATE_W_FULL(readl(rnandc->regs + FIFO_STATE_REG)))
867 			cpu_relax();
868 
869 		iowrite32_rep(rnandc->regs + FIFO_DATA_REG, rop.buf,
870 			      DIV_ROUND_UP(rop.len, 4));
871 
872 		if (remainder) {
873 			last_bytes = 0;
874 			memcpy(&last_bytes, rop.buf + (words * sizeof(u32)), remainder);
875 			writel_relaxed(last_bytes, rnandc->regs + FIFO_DATA_REG);
876 		}
877 
878 		while (!FIFO_STATE_W_EMPTY(readl(rnandc->regs + FIFO_STATE_REG)))
879 			cpu_relax();
880 	}
881 
882 	ret = rnandc_wait_end_of_op(rnandc, chip);
883 	if (ret)
884 		return ret;
885 
886 	return 0;
887 }
888 
889 static int rnandc_setup_interface(struct nand_chip *chip, int chipnr,
890 				  const struct nand_interface_config *conf)
891 {
892 	struct rnand_chip *rnand = to_rnand(chip);
893 	struct rnandc *rnandc = to_rnandc(chip->controller);
894 	unsigned int period_ns = 1000000000 / clk_get_rate(rnandc->eclk);
895 	const struct nand_sdr_timings *sdr;
896 	unsigned int cyc, cle, ale, bef_dly, ca_to_data;
897 
898 	sdr = nand_get_sdr_timings(conf);
899 	if (IS_ERR(sdr))
900 		return PTR_ERR(sdr);
901 
902 	if (sdr->tRP_min != sdr->tWP_min || sdr->tREH_min != sdr->tWH_min) {
903 		dev_err(rnandc->dev, "Read and write hold times must be identical\n");
904 		return -EINVAL;
905 	}
906 
907 	if (chipnr < 0)
908 		return 0;
909 
910 	rnand->timings_asyn =
911 		TIMINGS_ASYN_TRWP(TO_CYCLES64(sdr->tRP_min, period_ns)) |
912 		TIMINGS_ASYN_TRWH(TO_CYCLES64(sdr->tREH_min, period_ns));
913 	rnand->tim_seq0 =
914 		TIM_SEQ0_TCCS(TO_CYCLES64(sdr->tCCS_min, period_ns)) |
915 		TIM_SEQ0_TADL(TO_CYCLES64(sdr->tADL_min, period_ns)) |
916 		TIM_SEQ0_TRHW(TO_CYCLES64(sdr->tRHW_min, period_ns)) |
917 		TIM_SEQ0_TWHR(TO_CYCLES64(sdr->tWHR_min, period_ns));
918 	rnand->tim_seq1 =
919 		TIM_SEQ1_TWB(TO_CYCLES64(sdr->tWB_max, period_ns)) |
920 		TIM_SEQ1_TRR(TO_CYCLES64(sdr->tRR_min, period_ns)) |
921 		TIM_SEQ1_TWW(TO_CYCLES64(sdr->tWW_min, period_ns));
922 
923 	cyc = sdr->tDS_min + sdr->tDH_min;
924 	cle = sdr->tCLH_min + sdr->tCLS_min;
925 	ale = sdr->tALH_min + sdr->tALS_min;
926 	bef_dly = sdr->tWB_max - sdr->tDH_min;
927 	ca_to_data = sdr->tWHR_min + sdr->tREA_max - sdr->tDH_min;
928 
929 	/*
930 	 * D0 = CMD -> ADDR = tCLH + tCLS - 1 cycle
931 	 * D1 = CMD -> CMD = tCLH + tCLS - 1 cycle
932 	 * D2 = CMD -> DLY = tWB - tDH
933 	 * D3 = CMD -> DATA = tWHR + tREA - tDH
934 	 */
935 	rnand->tim_gen_seq0 =
936 		TIM_GEN_SEQ0_D0(TO_CYCLES64(cle - cyc, period_ns)) |
937 		TIM_GEN_SEQ0_D1(TO_CYCLES64(cle - cyc, period_ns)) |
938 		TIM_GEN_SEQ0_D2(TO_CYCLES64(bef_dly, period_ns)) |
939 		TIM_GEN_SEQ0_D3(TO_CYCLES64(ca_to_data, period_ns));
940 
941 	/*
942 	 * D4 = ADDR -> CMD = tALH + tALS - 1 cyle
943 	 * D5 = ADDR -> ADDR = tALH + tALS - 1 cyle
944 	 * D6 = ADDR -> DLY = tWB - tDH
945 	 * D7 = ADDR -> DATA = tWHR + tREA - tDH
946 	 */
947 	rnand->tim_gen_seq1 =
948 		TIM_GEN_SEQ1_D4(TO_CYCLES64(ale - cyc, period_ns)) |
949 		TIM_GEN_SEQ1_D5(TO_CYCLES64(ale - cyc, period_ns)) |
950 		TIM_GEN_SEQ1_D6(TO_CYCLES64(bef_dly, period_ns)) |
951 		TIM_GEN_SEQ1_D7(TO_CYCLES64(ca_to_data, period_ns));
952 
953 	/*
954 	 * D8 = DLY -> DATA = tRR + tREA
955 	 * D9 = DLY -> CMD = tRR
956 	 * D10 = DATA -> CMD = tCLH + tCLS - 1 cycle
957 	 * D11 = DATA -> DLY = tWB - tDH
958 	 */
959 	rnand->tim_gen_seq2 =
960 		TIM_GEN_SEQ2_D8(TO_CYCLES64(sdr->tRR_min + sdr->tREA_max, period_ns)) |
961 		TIM_GEN_SEQ2_D9(TO_CYCLES64(sdr->tRR_min, period_ns)) |
962 		TIM_GEN_SEQ2_D10(TO_CYCLES64(cle - cyc, period_ns)) |
963 		TIM_GEN_SEQ2_D11(TO_CYCLES64(bef_dly, period_ns));
964 
965 	/* D12 = DATA -> END = tCLH - tDH */
966 	rnand->tim_gen_seq3 =
967 		TIM_GEN_SEQ3_D12(TO_CYCLES64(sdr->tCLH_min - sdr->tDH_min, period_ns));
968 
969 	return 0;
970 }
971 
972 static int rnandc_ooblayout_ecc(struct mtd_info *mtd, int section,
973 				struct mtd_oob_region *oobregion)
974 {
975 	struct nand_chip *chip = mtd_to_nand(mtd);
976 	unsigned int eccbytes = round_up(chip->ecc.bytes, 4) * chip->ecc.steps;
977 
978 	if (section)
979 		return -ERANGE;
980 
981 	oobregion->offset = 2;
982 	oobregion->length = eccbytes;
983 
984 	return 0;
985 }
986 
987 static int rnandc_ooblayout_free(struct mtd_info *mtd, int section,
988 				 struct mtd_oob_region *oobregion)
989 {
990 	struct nand_chip *chip = mtd_to_nand(mtd);
991 	unsigned int eccbytes = round_up(chip->ecc.bytes, 4) * chip->ecc.steps;
992 
993 	if (section)
994 		return -ERANGE;
995 
996 	oobregion->offset = 2 + eccbytes;
997 	oobregion->length = mtd->oobsize - oobregion->offset;
998 
999 	return 0;
1000 }
1001 
1002 static const struct mtd_ooblayout_ops rnandc_ooblayout_ops = {
1003 	.ecc = rnandc_ooblayout_ecc,
1004 	.free = rnandc_ooblayout_free,
1005 };
1006 
1007 static int rnandc_hw_ecc_controller_init(struct nand_chip *chip)
1008 {
1009 	struct rnand_chip *rnand = to_rnand(chip);
1010 	struct mtd_info *mtd = nand_to_mtd(chip);
1011 	struct rnandc *rnandc = to_rnandc(chip->controller);
1012 
1013 	if (mtd->writesize > SZ_16K) {
1014 		dev_err(rnandc->dev, "Unsupported page size\n");
1015 		return -EINVAL;
1016 	}
1017 
1018 	switch (chip->ecc.size) {
1019 	case SZ_256:
1020 		rnand->control |= CONTROL_ECC_BLOCK_SIZE_256;
1021 		break;
1022 	case SZ_512:
1023 		rnand->control |= CONTROL_ECC_BLOCK_SIZE_512;
1024 		break;
1025 	case SZ_1K:
1026 		rnand->control |= CONTROL_ECC_BLOCK_SIZE_1024;
1027 		break;
1028 	default:
1029 		dev_err(rnandc->dev, "Unsupported ECC chunk size\n");
1030 		return -EINVAL;
1031 	}
1032 
1033 	switch (chip->ecc.strength) {
1034 	case 2:
1035 		chip->ecc.bytes = 4;
1036 		rnand->ecc_ctrl |= ECC_CTRL_CAP_2B;
1037 		break;
1038 	case 4:
1039 		chip->ecc.bytes = 7;
1040 		rnand->ecc_ctrl |= ECC_CTRL_CAP_4B;
1041 		break;
1042 	case 8:
1043 		chip->ecc.bytes = 14;
1044 		rnand->ecc_ctrl |= ECC_CTRL_CAP_8B;
1045 		break;
1046 	case 16:
1047 		chip->ecc.bytes = 28;
1048 		rnand->ecc_ctrl |= ECC_CTRL_CAP_16B;
1049 		break;
1050 	case 24:
1051 		chip->ecc.bytes = 42;
1052 		rnand->ecc_ctrl |= ECC_CTRL_CAP_24B;
1053 		break;
1054 	case 32:
1055 		chip->ecc.bytes = 56;
1056 		rnand->ecc_ctrl |= ECC_CTRL_CAP_32B;
1057 		break;
1058 	default:
1059 		dev_err(rnandc->dev, "Unsupported ECC strength\n");
1060 		return -EINVAL;
1061 	}
1062 
1063 	rnand->ecc_ctrl |= ECC_CTRL_ERR_THRESHOLD(chip->ecc.strength);
1064 
1065 	mtd_set_ooblayout(mtd, &rnandc_ooblayout_ops);
1066 	chip->ecc.steps = mtd->writesize / chip->ecc.size;
1067 	chip->ecc.read_page = rnandc_read_page_hw_ecc;
1068 	chip->ecc.read_subpage = rnandc_read_subpage_hw_ecc;
1069 	chip->ecc.write_page = rnandc_write_page_hw_ecc;
1070 	chip->ecc.write_subpage = rnandc_write_subpage_hw_ecc;
1071 
1072 	return 0;
1073 }
1074 
1075 static int rnandc_ecc_init(struct nand_chip *chip)
1076 {
1077 	struct nand_ecc_ctrl *ecc = &chip->ecc;
1078 	const struct nand_ecc_props *requirements =
1079 		nanddev_get_ecc_requirements(&chip->base);
1080 	struct rnandc *rnandc = to_rnandc(chip->controller);
1081 	int ret;
1082 
1083 	if (ecc->engine_type != NAND_ECC_ENGINE_TYPE_NONE &&
1084 	    (!ecc->size || !ecc->strength)) {
1085 		if (requirements->step_size && requirements->strength) {
1086 			ecc->size = requirements->step_size;
1087 			ecc->strength = requirements->strength;
1088 		} else {
1089 			dev_err(rnandc->dev, "No minimum ECC strength\n");
1090 			return -EINVAL;
1091 		}
1092 	}
1093 
1094 	switch (ecc->engine_type) {
1095 	case NAND_ECC_ENGINE_TYPE_ON_HOST:
1096 		ret = rnandc_hw_ecc_controller_init(chip);
1097 		if (ret)
1098 			return ret;
1099 		break;
1100 	case NAND_ECC_ENGINE_TYPE_NONE:
1101 	case NAND_ECC_ENGINE_TYPE_SOFT:
1102 	case NAND_ECC_ENGINE_TYPE_ON_DIE:
1103 		break;
1104 	default:
1105 		return -EINVAL;
1106 	}
1107 
1108 	return 0;
1109 }
1110 
1111 static int rnandc_attach_chip(struct nand_chip *chip)
1112 {
1113 	struct rnand_chip *rnand = to_rnand(chip);
1114 	struct rnandc *rnandc = to_rnandc(chip->controller);
1115 	struct mtd_info *mtd = nand_to_mtd(chip);
1116 	struct nand_memory_organization *memorg = nanddev_get_memorg(&chip->base);
1117 	int ret;
1118 
1119 	/* Do not store BBT bits in the OOB section as it is not protected */
1120 	if (chip->bbt_options & NAND_BBT_USE_FLASH)
1121 		chip->bbt_options |= NAND_BBT_NO_OOB;
1122 
1123 	if (mtd->writesize <= 512) {
1124 		dev_err(rnandc->dev, "Small page devices not supported\n");
1125 		return -EINVAL;
1126 	}
1127 
1128 	rnand->control |= CONTROL_CHECK_RB_LINE | CONTROL_INT_EN;
1129 
1130 	switch (memorg->pages_per_eraseblock) {
1131 	case 32:
1132 		rnand->control |= CONTROL_BLOCK_SIZE_32P;
1133 		break;
1134 	case 64:
1135 		rnand->control |= CONTROL_BLOCK_SIZE_64P;
1136 		break;
1137 	case 128:
1138 		rnand->control |= CONTROL_BLOCK_SIZE_128P;
1139 		break;
1140 	case 256:
1141 		rnand->control |= CONTROL_BLOCK_SIZE_256P;
1142 		break;
1143 	default:
1144 		dev_err(rnandc->dev, "Unsupported memory organization\n");
1145 		return -EINVAL;
1146 	}
1147 
1148 	chip->options |= NAND_SUBPAGE_READ;
1149 
1150 	ret = rnandc_ecc_init(chip);
1151 	if (ret) {
1152 		dev_err(rnandc->dev, "ECC initialization failed (%d)\n", ret);
1153 		return ret;
1154 	}
1155 
1156 	/* Force an update of the configuration registers */
1157 	rnand->selected_die = -1;
1158 
1159 	return 0;
1160 }
1161 
1162 static const struct nand_controller_ops rnandc_ops = {
1163 	.attach_chip = rnandc_attach_chip,
1164 	.exec_op = rnandc_exec_op,
1165 	.setup_interface = rnandc_setup_interface,
1166 };
1167 
1168 static int rnandc_alloc_dma_buf(struct rnandc *rnandc,
1169 				struct mtd_info *new_mtd)
1170 {
1171 	unsigned int max_len = new_mtd->writesize + new_mtd->oobsize;
1172 	struct rnand_chip *entry, *temp;
1173 	struct nand_chip *chip;
1174 	struct mtd_info *mtd;
1175 
1176 	list_for_each_entry_safe(entry, temp, &rnandc->chips, node) {
1177 		chip = &entry->chip;
1178 		mtd = nand_to_mtd(chip);
1179 		max_len = max(max_len, mtd->writesize + mtd->oobsize);
1180 	}
1181 
1182 	if (rnandc->buf && rnandc->buf_sz < max_len) {
1183 		devm_kfree(rnandc->dev, rnandc->buf);
1184 		rnandc->buf = NULL;
1185 	}
1186 
1187 	if (!rnandc->buf) {
1188 		rnandc->buf_sz = max_len;
1189 		rnandc->buf = devm_kmalloc(rnandc->dev, max_len,
1190 					   GFP_KERNEL | GFP_DMA);
1191 		if (!rnandc->buf)
1192 			return -ENOMEM;
1193 	}
1194 
1195 	return 0;
1196 }
1197 
1198 static int rnandc_chip_init(struct rnandc *rnandc, struct device_node *np)
1199 {
1200 	struct rnand_chip *rnand;
1201 	struct mtd_info *mtd;
1202 	struct nand_chip *chip;
1203 	int nsels, ret, i;
1204 	u32 cs;
1205 
1206 	nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
1207 	if (nsels <= 0) {
1208 		ret = (nsels < 0) ? nsels : -EINVAL;
1209 		dev_err(rnandc->dev, "Invalid reg property (%d)\n", ret);
1210 		return ret;
1211 	}
1212 
1213 	/* Alloc the driver's NAND chip structure */
1214 	rnand = devm_kzalloc(rnandc->dev, struct_size(rnand, sels, nsels),
1215 			     GFP_KERNEL);
1216 	if (!rnand)
1217 		return -ENOMEM;
1218 
1219 	rnand->nsels = nsels;
1220 	rnand->selected_die = -1;
1221 
1222 	for (i = 0; i < nsels; i++) {
1223 		ret = of_property_read_u32_index(np, "reg", i, &cs);
1224 		if (ret) {
1225 			dev_err(rnandc->dev, "Incomplete reg property (%d)\n", ret);
1226 			return ret;
1227 		}
1228 
1229 		if (cs >= RNANDC_CS_NUM) {
1230 			dev_err(rnandc->dev, "Invalid reg property (%d)\n", cs);
1231 			return -EINVAL;
1232 		}
1233 
1234 		if (test_and_set_bit(cs, &rnandc->assigned_cs)) {
1235 			dev_err(rnandc->dev, "CS %d already assigned\n", cs);
1236 			return -EINVAL;
1237 		}
1238 
1239 		/*
1240 		 * No need to check for RB or WP properties, there is a 1:1
1241 		 * mandatory mapping with the CS.
1242 		 */
1243 		rnand->sels[i].cs = cs;
1244 	}
1245 
1246 	chip = &rnand->chip;
1247 	chip->controller = &rnandc->controller;
1248 	nand_set_flash_node(chip, np);
1249 
1250 	mtd = nand_to_mtd(chip);
1251 	mtd->dev.parent = rnandc->dev;
1252 	if (!mtd->name) {
1253 		dev_err(rnandc->dev, "Missing MTD label\n");
1254 		return -EINVAL;
1255 	}
1256 
1257 	ret = nand_scan(chip, rnand->nsels);
1258 	if (ret) {
1259 		dev_err(rnandc->dev, "Failed to scan the NAND chip (%d)\n", ret);
1260 		return ret;
1261 	}
1262 
1263 	ret = rnandc_alloc_dma_buf(rnandc, mtd);
1264 	if (ret)
1265 		goto cleanup_nand;
1266 
1267 	ret = mtd_device_register(mtd, NULL, 0);
1268 	if (ret) {
1269 		dev_err(rnandc->dev, "Failed to register MTD device (%d)\n", ret);
1270 		goto cleanup_nand;
1271 	}
1272 
1273 	list_add_tail(&rnand->node, &rnandc->chips);
1274 
1275 	return 0;
1276 
1277 cleanup_nand:
1278 	nand_cleanup(chip);
1279 
1280 	return ret;
1281 }
1282 
1283 static void rnandc_chips_cleanup(struct rnandc *rnandc)
1284 {
1285 	struct rnand_chip *entry, *temp;
1286 	struct nand_chip *chip;
1287 	int ret;
1288 
1289 	list_for_each_entry_safe(entry, temp, &rnandc->chips, node) {
1290 		chip = &entry->chip;
1291 		ret = mtd_device_unregister(nand_to_mtd(chip));
1292 		WARN_ON(ret);
1293 		nand_cleanup(chip);
1294 		list_del(&entry->node);
1295 	}
1296 }
1297 
1298 static int rnandc_chips_init(struct rnandc *rnandc)
1299 {
1300 	struct device_node *np;
1301 	int ret;
1302 
1303 	for_each_child_of_node(rnandc->dev->of_node, np) {
1304 		ret = rnandc_chip_init(rnandc, np);
1305 		if (ret) {
1306 			of_node_put(np);
1307 			goto cleanup_chips;
1308 		}
1309 	}
1310 
1311 	return 0;
1312 
1313 cleanup_chips:
1314 	rnandc_chips_cleanup(rnandc);
1315 
1316 	return ret;
1317 }
1318 
1319 static int rnandc_probe(struct platform_device *pdev)
1320 {
1321 	struct rnandc *rnandc;
1322 	int irq, ret;
1323 
1324 	rnandc = devm_kzalloc(&pdev->dev, sizeof(*rnandc), GFP_KERNEL);
1325 	if (!rnandc)
1326 		return -ENOMEM;
1327 
1328 	rnandc->dev = &pdev->dev;
1329 	nand_controller_init(&rnandc->controller);
1330 	rnandc->controller.ops = &rnandc_ops;
1331 	INIT_LIST_HEAD(&rnandc->chips);
1332 	init_completion(&rnandc->complete);
1333 
1334 	rnandc->regs = devm_platform_ioremap_resource(pdev, 0);
1335 	if (IS_ERR(rnandc->regs))
1336 		return PTR_ERR(rnandc->regs);
1337 
1338 	/* APB clock */
1339 	rnandc->hclk = devm_clk_get(&pdev->dev, "hclk");
1340 	if (IS_ERR(rnandc->hclk))
1341 		return PTR_ERR(rnandc->hclk);
1342 
1343 	/* External NAND bus clock */
1344 	rnandc->eclk = devm_clk_get(&pdev->dev, "eclk");
1345 	if (IS_ERR(rnandc->eclk))
1346 		return PTR_ERR(rnandc->eclk);
1347 
1348 	ret = clk_prepare_enable(rnandc->hclk);
1349 	if (ret)
1350 		return ret;
1351 
1352 	ret = clk_prepare_enable(rnandc->eclk);
1353 	if (ret)
1354 		goto disable_hclk;
1355 
1356 	rnandc_dis_interrupts(rnandc);
1357 	irq = platform_get_irq_optional(pdev, 0);
1358 	if (irq == -EPROBE_DEFER) {
1359 		ret = irq;
1360 		goto disable_eclk;
1361 	} else if (irq < 0) {
1362 		dev_info(&pdev->dev, "No IRQ found, fallback to polling\n");
1363 		rnandc->use_polling = true;
1364 	} else {
1365 		ret = devm_request_irq(&pdev->dev, irq, rnandc_irq_handler, 0,
1366 				       "renesas-nand-controller", rnandc);
1367 		if (ret < 0)
1368 			goto disable_eclk;
1369 	}
1370 
1371 	ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
1372 	if (ret)
1373 		goto disable_eclk;
1374 
1375 	rnandc_clear_fifo(rnandc);
1376 
1377 	platform_set_drvdata(pdev, rnandc);
1378 
1379 	ret = rnandc_chips_init(rnandc);
1380 	if (ret)
1381 		goto disable_eclk;
1382 
1383 	return 0;
1384 
1385 disable_eclk:
1386 	clk_disable_unprepare(rnandc->eclk);
1387 disable_hclk:
1388 	clk_disable_unprepare(rnandc->hclk);
1389 
1390 	return ret;
1391 }
1392 
1393 static int rnandc_remove(struct platform_device *pdev)
1394 {
1395 	struct rnandc *rnandc = platform_get_drvdata(pdev);
1396 
1397 	rnandc_chips_cleanup(rnandc);
1398 
1399 	clk_disable_unprepare(rnandc->eclk);
1400 	clk_disable_unprepare(rnandc->hclk);
1401 
1402 	return 0;
1403 }
1404 
1405 static const struct of_device_id rnandc_id_table[] = {
1406 	{ .compatible = "renesas,rcar-gen3-nandc" },
1407 	{ .compatible = "renesas,rzn1-nandc" },
1408 	{} /* sentinel */
1409 };
1410 MODULE_DEVICE_TABLE(of, rnandc_id_table);
1411 
1412 static struct platform_driver rnandc_driver = {
1413 	.driver = {
1414 		.name = "renesas-nandc",
1415 		.of_match_table = of_match_ptr(rnandc_id_table),
1416 	},
1417 	.probe = rnandc_probe,
1418 	.remove = rnandc_remove,
1419 };
1420 module_platform_driver(rnandc_driver);
1421 
1422 MODULE_AUTHOR("Miquel Raynal <miquel.raynal@bootlin.com>");
1423 MODULE_DESCRIPTION("Renesas R-Car Gen3 & RZ/N1 NAND controller driver");
1424 MODULE_LICENSE("GPL v2");
1425