1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Arasan NAND Flash Controller Driver
4  *
5  * Copyright (C) 2014 - 2020 Xilinx, Inc.
6  * Author:
7  *   Miquel Raynal <miquel.raynal@bootlin.com>
8  * Original work (fully rewritten):
9  *   Punnaiah Choudary Kalluri <punnaia@xilinx.com>
10  *   Naga Sureshkumar Relli <nagasure@xilinx.com>
11  */
12 
13 #include <linux/bch.h>
14 #include <linux/bitfield.h>
15 #include <linux/clk.h>
16 #include <linux/delay.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/gpio/consumer.h>
19 #include <linux/interrupt.h>
20 #include <linux/iopoll.h>
21 #include <linux/module.h>
22 #include <linux/mtd/mtd.h>
23 #include <linux/mtd/partitions.h>
24 #include <linux/mtd/rawnand.h>
25 #include <linux/of.h>
26 #include <linux/platform_device.h>
27 #include <linux/slab.h>
28 
29 #define PKT_REG				0x00
30 #define   PKT_SIZE(x)			FIELD_PREP(GENMASK(10, 0), (x))
31 #define   PKT_STEPS(x)			FIELD_PREP(GENMASK(23, 12), (x))
32 
33 #define MEM_ADDR1_REG			0x04
34 
35 #define MEM_ADDR2_REG			0x08
36 #define   ADDR2_STRENGTH(x)		FIELD_PREP(GENMASK(27, 25), (x))
37 #define   ADDR2_CS(x)			FIELD_PREP(GENMASK(31, 30), (x))
38 
39 #define CMD_REG				0x0C
40 #define   CMD_1(x)			FIELD_PREP(GENMASK(7, 0), (x))
41 #define   CMD_2(x)			FIELD_PREP(GENMASK(15, 8), (x))
42 #define   CMD_PAGE_SIZE(x)		FIELD_PREP(GENMASK(25, 23), (x))
43 #define   CMD_DMA_ENABLE		BIT(27)
44 #define   CMD_NADDRS(x)			FIELD_PREP(GENMASK(30, 28), (x))
45 #define   CMD_ECC_ENABLE		BIT(31)
46 
47 #define PROG_REG			0x10
48 #define   PROG_PGRD			BIT(0)
49 #define   PROG_ERASE			BIT(2)
50 #define   PROG_STATUS			BIT(3)
51 #define   PROG_PGPROG			BIT(4)
52 #define   PROG_RDID			BIT(6)
53 #define   PROG_RDPARAM			BIT(7)
54 #define   PROG_RST			BIT(8)
55 #define   PROG_GET_FEATURE		BIT(9)
56 #define   PROG_SET_FEATURE		BIT(10)
57 #define   PROG_CHG_RD_COL_ENH		BIT(14)
58 
59 #define INTR_STS_EN_REG			0x14
60 #define INTR_SIG_EN_REG			0x18
61 #define INTR_STS_REG			0x1C
62 #define   WRITE_READY			BIT(0)
63 #define   READ_READY			BIT(1)
64 #define   XFER_COMPLETE			BIT(2)
65 #define   DMA_BOUNDARY			BIT(6)
66 #define   EVENT_MASK			GENMASK(7, 0)
67 
68 #define READY_STS_REG			0x20
69 
70 #define DMA_ADDR0_REG			0x50
71 #define DMA_ADDR1_REG			0x24
72 
73 #define FLASH_STS_REG			0x28
74 
75 #define TIMING_REG			0x2C
76 #define   TCCS_TIME_500NS		0
77 #define   TCCS_TIME_300NS		3
78 #define   TCCS_TIME_200NS		2
79 #define   TCCS_TIME_100NS		1
80 #define   FAST_TCAD			BIT(2)
81 #define   DQS_BUFF_SEL_IN(x)		FIELD_PREP(GENMASK(6, 3), (x))
82 #define   DQS_BUFF_SEL_OUT(x)		FIELD_PREP(GENMASK(18, 15), (x))
83 
84 #define DATA_PORT_REG			0x30
85 
86 #define ECC_CONF_REG			0x34
87 #define   ECC_CONF_COL(x)		FIELD_PREP(GENMASK(15, 0), (x))
88 #define   ECC_CONF_LEN(x)		FIELD_PREP(GENMASK(26, 16), (x))
89 #define   ECC_CONF_BCH_EN		BIT(27)
90 
91 #define ECC_ERR_CNT_REG			0x38
92 #define   GET_PKT_ERR_CNT(x)		FIELD_GET(GENMASK(7, 0), (x))
93 #define   GET_PAGE_ERR_CNT(x)		FIELD_GET(GENMASK(16, 8), (x))
94 
95 #define ECC_SP_REG			0x3C
96 #define   ECC_SP_CMD1(x)		FIELD_PREP(GENMASK(7, 0), (x))
97 #define   ECC_SP_CMD2(x)		FIELD_PREP(GENMASK(15, 8), (x))
98 #define   ECC_SP_ADDRS(x)		FIELD_PREP(GENMASK(30, 28), (x))
99 
100 #define ECC_1ERR_CNT_REG		0x40
101 #define ECC_2ERR_CNT_REG		0x44
102 
103 #define DATA_INTERFACE_REG		0x6C
104 #define   DIFACE_SDR_MODE(x)		FIELD_PREP(GENMASK(2, 0), (x))
105 #define   DIFACE_DDR_MODE(x)		FIELD_PREP(GENMASK(5, 3), (x))
106 #define   DIFACE_SDR			0
107 #define   DIFACE_NVDDR			BIT(9)
108 
109 #define ANFC_MAX_CS			2
110 #define ANFC_DFLT_TIMEOUT_US		1000000
111 #define ANFC_MAX_CHUNK_SIZE		SZ_1M
112 #define ANFC_MAX_PARAM_SIZE		SZ_4K
113 #define ANFC_MAX_STEPS			SZ_2K
114 #define ANFC_MAX_PKT_SIZE		(SZ_2K - 1)
115 #define ANFC_MAX_ADDR_CYC		5U
116 #define ANFC_RSVD_ECC_BYTES		21
117 
118 #define ANFC_XLNX_SDR_DFLT_CORE_CLK	100000000
119 #define ANFC_XLNX_SDR_HS_CORE_CLK	80000000
120 
121 static struct gpio_desc *anfc_default_cs_array[2] = {NULL, NULL};
122 
123 /**
124  * struct anfc_op - Defines how to execute an operation
125  * @pkt_reg: Packet register
126  * @addr1_reg: Memory address 1 register
127  * @addr2_reg: Memory address 2 register
128  * @cmd_reg: Command register
129  * @prog_reg: Program register
130  * @steps: Number of "packets" to read/write
131  * @rdy_timeout_ms: Timeout for waits on Ready/Busy pin
132  * @len: Data transfer length
133  * @read: Data transfer direction from the controller point of view
134  * @buf: Data buffer
135  */
136 struct anfc_op {
137 	u32 pkt_reg;
138 	u32 addr1_reg;
139 	u32 addr2_reg;
140 	u32 cmd_reg;
141 	u32 prog_reg;
142 	int steps;
143 	unsigned int rdy_timeout_ms;
144 	unsigned int len;
145 	bool read;
146 	u8 *buf;
147 };
148 
149 /**
150  * struct anand - Defines the NAND chip related information
151  * @node:		Used to store NAND chips into a list
152  * @chip:		NAND chip information structure
153  * @rb:			Ready-busy line
154  * @page_sz:		Register value of the page_sz field to use
155  * @clk:		Expected clock frequency to use
156  * @data_iface:		Data interface timing mode to use
157  * @timings:		NV-DDR specific timings to use
158  * @ecc_conf:		Hardware ECC configuration value
159  * @strength:		Register value of the ECC strength
160  * @raddr_cycles:	Row address cycle information
161  * @caddr_cycles:	Column address cycle information
162  * @ecc_bits:		Exact number of ECC bits per syndrome
163  * @ecc_total:		Total number of ECC bytes
164  * @errloc:		Array of errors located with soft BCH
165  * @hw_ecc:		Buffer to store syndromes computed by hardware
166  * @bch:		BCH structure
167  * @cs_idx:		Array of chip-select for this device, values are indexes
168  *			of the controller structure @gpio_cs array
169  * @ncs_idx:		Size of the @cs_idx array
170  */
171 struct anand {
172 	struct list_head node;
173 	struct nand_chip chip;
174 	unsigned int rb;
175 	unsigned int page_sz;
176 	unsigned long clk;
177 	u32 data_iface;
178 	u32 timings;
179 	u32 ecc_conf;
180 	u32 strength;
181 	u16 raddr_cycles;
182 	u16 caddr_cycles;
183 	unsigned int ecc_bits;
184 	unsigned int ecc_total;
185 	unsigned int *errloc;
186 	u8 *hw_ecc;
187 	struct bch_control *bch;
188 	int *cs_idx;
189 	int ncs_idx;
190 };
191 
192 /**
193  * struct arasan_nfc - Defines the Arasan NAND flash controller driver instance
194  * @dev:		Pointer to the device structure
195  * @base:		Remapped register area
196  * @controller_clk:		Pointer to the system clock
197  * @bus_clk:		Pointer to the flash clock
198  * @controller:		Base controller structure
199  * @chips:		List of all NAND chips attached to the controller
200  * @cur_clk:		Current clock rate
201  * @cs_array:		CS array. Native CS are left empty, the other cells are
202  *			populated with their corresponding GPIO descriptor.
203  * @ncs:		Size of @cs_array
204  * @cur_cs:		Index in @cs_array of the currently in use CS
205  * @native_cs:		Currently selected native CS
206  * @spare_cs:		Native CS that is not wired (may be selected when a GPIO
207  *			CS is in use)
208  */
209 struct arasan_nfc {
210 	struct device *dev;
211 	void __iomem *base;
212 	struct clk *controller_clk;
213 	struct clk *bus_clk;
214 	struct nand_controller controller;
215 	struct list_head chips;
216 	unsigned int cur_clk;
217 	struct gpio_desc **cs_array;
218 	unsigned int ncs;
219 	int cur_cs;
220 	unsigned int native_cs;
221 	unsigned int spare_cs;
222 };
223 
224 static struct anand *to_anand(struct nand_chip *nand)
225 {
226 	return container_of(nand, struct anand, chip);
227 }
228 
229 static struct arasan_nfc *to_anfc(struct nand_controller *ctrl)
230 {
231 	return container_of(ctrl, struct arasan_nfc, controller);
232 }
233 
234 static int anfc_wait_for_event(struct arasan_nfc *nfc, unsigned int event)
235 {
236 	u32 val;
237 	int ret;
238 
239 	ret = readl_relaxed_poll_timeout(nfc->base + INTR_STS_REG, val,
240 					 val & event, 0,
241 					 ANFC_DFLT_TIMEOUT_US);
242 	if (ret) {
243 		dev_err(nfc->dev, "Timeout waiting for event 0x%x\n", event);
244 		return -ETIMEDOUT;
245 	}
246 
247 	writel_relaxed(event, nfc->base + INTR_STS_REG);
248 
249 	return 0;
250 }
251 
252 static int anfc_wait_for_rb(struct arasan_nfc *nfc, struct nand_chip *chip,
253 			    unsigned int timeout_ms)
254 {
255 	struct anand *anand = to_anand(chip);
256 	u32 val;
257 	int ret;
258 
259 	/* There is no R/B interrupt, we must poll a register */
260 	ret = readl_relaxed_poll_timeout(nfc->base + READY_STS_REG, val,
261 					 val & BIT(anand->rb),
262 					 1, timeout_ms * 1000);
263 	if (ret) {
264 		dev_err(nfc->dev, "Timeout waiting for R/B 0x%x\n",
265 			readl_relaxed(nfc->base + READY_STS_REG));
266 		return -ETIMEDOUT;
267 	}
268 
269 	return 0;
270 }
271 
272 static void anfc_trigger_op(struct arasan_nfc *nfc, struct anfc_op *nfc_op)
273 {
274 	writel_relaxed(nfc_op->pkt_reg, nfc->base + PKT_REG);
275 	writel_relaxed(nfc_op->addr1_reg, nfc->base + MEM_ADDR1_REG);
276 	writel_relaxed(nfc_op->addr2_reg, nfc->base + MEM_ADDR2_REG);
277 	writel_relaxed(nfc_op->cmd_reg, nfc->base + CMD_REG);
278 	writel_relaxed(nfc_op->prog_reg, nfc->base + PROG_REG);
279 }
280 
281 static int anfc_pkt_len_config(unsigned int len, unsigned int *steps,
282 			       unsigned int *pktsize)
283 {
284 	unsigned int nb, sz;
285 
286 	for (nb = 1; nb < ANFC_MAX_STEPS; nb *= 2) {
287 		sz = len / nb;
288 		if (sz <= ANFC_MAX_PKT_SIZE)
289 			break;
290 	}
291 
292 	if (sz * nb != len)
293 		return -ENOTSUPP;
294 
295 	if (steps)
296 		*steps = nb;
297 
298 	if (pktsize)
299 		*pktsize = sz;
300 
301 	return 0;
302 }
303 
304 static bool anfc_is_gpio_cs(struct arasan_nfc *nfc, int nfc_cs)
305 {
306 	return nfc_cs >= 0 && nfc->cs_array[nfc_cs];
307 }
308 
309 static int anfc_relative_to_absolute_cs(struct anand *anand, int num)
310 {
311 	return anand->cs_idx[num];
312 }
313 
314 static void anfc_assert_cs(struct arasan_nfc *nfc, unsigned int nfc_cs_idx)
315 {
316 	/* CS did not change: do nothing */
317 	if (nfc->cur_cs == nfc_cs_idx)
318 		return;
319 
320 	/* Deassert the previous CS if it was a GPIO */
321 	if (anfc_is_gpio_cs(nfc, nfc->cur_cs))
322 		gpiod_set_value_cansleep(nfc->cs_array[nfc->cur_cs], 1);
323 
324 	/* Assert the new one */
325 	if (anfc_is_gpio_cs(nfc, nfc_cs_idx)) {
326 		nfc->native_cs = nfc->spare_cs;
327 		gpiod_set_value_cansleep(nfc->cs_array[nfc_cs_idx], 0);
328 	} else {
329 		nfc->native_cs = nfc_cs_idx;
330 	}
331 
332 	nfc->cur_cs = nfc_cs_idx;
333 }
334 
335 static int anfc_select_target(struct nand_chip *chip, int target)
336 {
337 	struct anand *anand = to_anand(chip);
338 	struct arasan_nfc *nfc = to_anfc(chip->controller);
339 	unsigned int nfc_cs_idx = anfc_relative_to_absolute_cs(anand, target);
340 	int ret;
341 
342 	anfc_assert_cs(nfc, nfc_cs_idx);
343 
344 	/* Update the controller timings and the potential ECC configuration */
345 	writel_relaxed(anand->data_iface, nfc->base + DATA_INTERFACE_REG);
346 	writel_relaxed(anand->timings, nfc->base + TIMING_REG);
347 
348 	/* Update clock frequency */
349 	if (nfc->cur_clk != anand->clk) {
350 		clk_disable_unprepare(nfc->controller_clk);
351 		ret = clk_set_rate(nfc->controller_clk, anand->clk);
352 		if (ret) {
353 			dev_err(nfc->dev, "Failed to change clock rate\n");
354 			return ret;
355 		}
356 
357 		ret = clk_prepare_enable(nfc->controller_clk);
358 		if (ret) {
359 			dev_err(nfc->dev,
360 				"Failed to re-enable the controller clock\n");
361 			return ret;
362 		}
363 
364 		nfc->cur_clk = anand->clk;
365 	}
366 
367 	return 0;
368 }
369 
370 /*
371  * When using the embedded hardware ECC engine, the controller is in charge of
372  * feeding the engine with, first, the ECC residue present in the data array.
373  * A typical read operation is:
374  * 1/ Assert the read operation by sending the relevant command/address cycles
375  *    but targeting the column of the first ECC bytes in the OOB area instead of
376  *    the main data directly.
377  * 2/ After having read the relevant number of ECC bytes, the controller uses
378  *    the RNDOUT/RNDSTART commands which are set into the "ECC Spare Command
379  *    Register" to move the pointer back at the beginning of the main data.
380  * 3/ It will read the content of the main area for a given size (pktsize) and
381  *    will feed the ECC engine with this buffer again.
382  * 4/ The ECC engine derives the ECC bytes for the given data and compare them
383  *    with the ones already received. It eventually trigger status flags and
384  *    then set the "Buffer Read Ready" flag.
385  * 5/ The corrected data is then available for reading from the data port
386  *    register.
387  *
388  * The hardware BCH ECC engine is known to be inconstent in BCH mode and never
389  * reports uncorrectable errors. Because of this bug, we have to use the
390  * software BCH implementation in the read path.
391  */
392 static int anfc_read_page_hw_ecc(struct nand_chip *chip, u8 *buf,
393 				 int oob_required, int page)
394 {
395 	struct arasan_nfc *nfc = to_anfc(chip->controller);
396 	struct mtd_info *mtd = nand_to_mtd(chip);
397 	struct anand *anand = to_anand(chip);
398 	unsigned int len = mtd->writesize + (oob_required ? mtd->oobsize : 0);
399 	unsigned int max_bitflips = 0;
400 	dma_addr_t dma_addr;
401 	int step, ret;
402 	struct anfc_op nfc_op = {
403 		.pkt_reg =
404 			PKT_SIZE(chip->ecc.size) |
405 			PKT_STEPS(chip->ecc.steps),
406 		.addr1_reg =
407 			(page & 0xFF) << (8 * (anand->caddr_cycles)) |
408 			(((page >> 8) & 0xFF) << (8 * (1 + anand->caddr_cycles))),
409 		.addr2_reg =
410 			((page >> 16) & 0xFF) |
411 			ADDR2_STRENGTH(anand->strength) |
412 			ADDR2_CS(nfc->native_cs),
413 		.cmd_reg =
414 			CMD_1(NAND_CMD_READ0) |
415 			CMD_2(NAND_CMD_READSTART) |
416 			CMD_PAGE_SIZE(anand->page_sz) |
417 			CMD_DMA_ENABLE |
418 			CMD_NADDRS(anand->caddr_cycles +
419 				   anand->raddr_cycles),
420 		.prog_reg = PROG_PGRD,
421 	};
422 
423 	dma_addr = dma_map_single(nfc->dev, (void *)buf, len, DMA_FROM_DEVICE);
424 	if (dma_mapping_error(nfc->dev, dma_addr)) {
425 		dev_err(nfc->dev, "Buffer mapping error");
426 		return -EIO;
427 	}
428 
429 	writel_relaxed(lower_32_bits(dma_addr), nfc->base + DMA_ADDR0_REG);
430 	writel_relaxed(upper_32_bits(dma_addr), nfc->base + DMA_ADDR1_REG);
431 
432 	anfc_trigger_op(nfc, &nfc_op);
433 
434 	ret = anfc_wait_for_event(nfc, XFER_COMPLETE);
435 	dma_unmap_single(nfc->dev, dma_addr, len, DMA_FROM_DEVICE);
436 	if (ret) {
437 		dev_err(nfc->dev, "Error reading page %d\n", page);
438 		return ret;
439 	}
440 
441 	/* Store the raw OOB bytes as well */
442 	ret = nand_change_read_column_op(chip, mtd->writesize, chip->oob_poi,
443 					 mtd->oobsize, 0);
444 	if (ret)
445 		return ret;
446 
447 	/*
448 	 * For each step, compute by softare the BCH syndrome over the raw data.
449 	 * Compare the theoretical amount of errors and compare with the
450 	 * hardware engine feedback.
451 	 */
452 	for (step = 0; step < chip->ecc.steps; step++) {
453 		u8 *raw_buf = &buf[step * chip->ecc.size];
454 		unsigned int bit, byte;
455 		int bf, i;
456 
457 		/* Extract the syndrome, it is not necessarily aligned */
458 		memset(anand->hw_ecc, 0, chip->ecc.bytes);
459 		nand_extract_bits(anand->hw_ecc, 0,
460 				  &chip->oob_poi[mtd->oobsize - anand->ecc_total],
461 				  anand->ecc_bits * step, anand->ecc_bits);
462 
463 		bf = bch_decode(anand->bch, raw_buf, chip->ecc.size,
464 				anand->hw_ecc, NULL, NULL, anand->errloc);
465 		if (!bf) {
466 			continue;
467 		} else if (bf > 0) {
468 			for (i = 0; i < bf; i++) {
469 				/* Only correct the data, not the syndrome */
470 				if (anand->errloc[i] < (chip->ecc.size * 8)) {
471 					bit = BIT(anand->errloc[i] & 7);
472 					byte = anand->errloc[i] >> 3;
473 					raw_buf[byte] ^= bit;
474 				}
475 			}
476 
477 			mtd->ecc_stats.corrected += bf;
478 			max_bitflips = max_t(unsigned int, max_bitflips, bf);
479 
480 			continue;
481 		}
482 
483 		bf = nand_check_erased_ecc_chunk(raw_buf, chip->ecc.size,
484 						 NULL, 0, NULL, 0,
485 						 chip->ecc.strength);
486 		if (bf > 0) {
487 			mtd->ecc_stats.corrected += bf;
488 			max_bitflips = max_t(unsigned int, max_bitflips, bf);
489 			memset(raw_buf, 0xFF, chip->ecc.size);
490 		} else if (bf < 0) {
491 			mtd->ecc_stats.failed++;
492 		}
493 	}
494 
495 	return 0;
496 }
497 
498 static int anfc_sel_read_page_hw_ecc(struct nand_chip *chip, u8 *buf,
499 				     int oob_required, int page)
500 {
501 	int ret;
502 
503 	ret = anfc_select_target(chip, chip->cur_cs);
504 	if (ret)
505 		return ret;
506 
507 	return anfc_read_page_hw_ecc(chip, buf, oob_required, page);
508 };
509 
510 static int anfc_write_page_hw_ecc(struct nand_chip *chip, const u8 *buf,
511 				  int oob_required, int page)
512 {
513 	struct anand *anand = to_anand(chip);
514 	struct arasan_nfc *nfc = to_anfc(chip->controller);
515 	struct mtd_info *mtd = nand_to_mtd(chip);
516 	unsigned int len = mtd->writesize + (oob_required ? mtd->oobsize : 0);
517 	dma_addr_t dma_addr;
518 	int ret;
519 	struct anfc_op nfc_op = {
520 		.pkt_reg =
521 			PKT_SIZE(chip->ecc.size) |
522 			PKT_STEPS(chip->ecc.steps),
523 		.addr1_reg =
524 			(page & 0xFF) << (8 * (anand->caddr_cycles)) |
525 			(((page >> 8) & 0xFF) << (8 * (1 + anand->caddr_cycles))),
526 		.addr2_reg =
527 			((page >> 16) & 0xFF) |
528 			ADDR2_STRENGTH(anand->strength) |
529 			ADDR2_CS(nfc->native_cs),
530 		.cmd_reg =
531 			CMD_1(NAND_CMD_SEQIN) |
532 			CMD_2(NAND_CMD_PAGEPROG) |
533 			CMD_PAGE_SIZE(anand->page_sz) |
534 			CMD_DMA_ENABLE |
535 			CMD_NADDRS(anand->caddr_cycles +
536 				   anand->raddr_cycles) |
537 			CMD_ECC_ENABLE,
538 		.prog_reg = PROG_PGPROG,
539 	};
540 
541 	writel_relaxed(anand->ecc_conf, nfc->base + ECC_CONF_REG);
542 	writel_relaxed(ECC_SP_CMD1(NAND_CMD_RNDIN) |
543 		       ECC_SP_ADDRS(anand->caddr_cycles),
544 		       nfc->base + ECC_SP_REG);
545 
546 	dma_addr = dma_map_single(nfc->dev, (void *)buf, len, DMA_TO_DEVICE);
547 	if (dma_mapping_error(nfc->dev, dma_addr)) {
548 		dev_err(nfc->dev, "Buffer mapping error");
549 		return -EIO;
550 	}
551 
552 	writel_relaxed(lower_32_bits(dma_addr), nfc->base + DMA_ADDR0_REG);
553 	writel_relaxed(upper_32_bits(dma_addr), nfc->base + DMA_ADDR1_REG);
554 
555 	anfc_trigger_op(nfc, &nfc_op);
556 	ret = anfc_wait_for_event(nfc, XFER_COMPLETE);
557 	dma_unmap_single(nfc->dev, dma_addr, len, DMA_TO_DEVICE);
558 	if (ret) {
559 		dev_err(nfc->dev, "Error writing page %d\n", page);
560 		return ret;
561 	}
562 
563 	/* Spare data is not protected */
564 	if (oob_required)
565 		ret = nand_write_oob_std(chip, page);
566 
567 	return ret;
568 }
569 
570 static int anfc_sel_write_page_hw_ecc(struct nand_chip *chip, const u8 *buf,
571 				      int oob_required, int page)
572 {
573 	int ret;
574 
575 	ret = anfc_select_target(chip, chip->cur_cs);
576 	if (ret)
577 		return ret;
578 
579 	return anfc_write_page_hw_ecc(chip, buf, oob_required, page);
580 };
581 
582 /* NAND framework ->exec_op() hooks and related helpers */
583 static int anfc_parse_instructions(struct nand_chip *chip,
584 				   const struct nand_subop *subop,
585 				   struct anfc_op *nfc_op)
586 {
587 	struct arasan_nfc *nfc = to_anfc(chip->controller);
588 	struct anand *anand = to_anand(chip);
589 	const struct nand_op_instr *instr = NULL;
590 	bool first_cmd = true;
591 	unsigned int op_id;
592 	int ret, i;
593 
594 	memset(nfc_op, 0, sizeof(*nfc_op));
595 	nfc_op->addr2_reg = ADDR2_CS(nfc->native_cs);
596 	nfc_op->cmd_reg = CMD_PAGE_SIZE(anand->page_sz);
597 
598 	for (op_id = 0; op_id < subop->ninstrs; op_id++) {
599 		unsigned int offset, naddrs, pktsize;
600 		const u8 *addrs;
601 		u8 *buf;
602 
603 		instr = &subop->instrs[op_id];
604 
605 		switch (instr->type) {
606 		case NAND_OP_CMD_INSTR:
607 			if (first_cmd)
608 				nfc_op->cmd_reg |= CMD_1(instr->ctx.cmd.opcode);
609 			else
610 				nfc_op->cmd_reg |= CMD_2(instr->ctx.cmd.opcode);
611 
612 			first_cmd = false;
613 			break;
614 
615 		case NAND_OP_ADDR_INSTR:
616 			offset = nand_subop_get_addr_start_off(subop, op_id);
617 			naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
618 			addrs = &instr->ctx.addr.addrs[offset];
619 			nfc_op->cmd_reg |= CMD_NADDRS(naddrs);
620 
621 			for (i = 0; i < min(ANFC_MAX_ADDR_CYC, naddrs); i++) {
622 				if (i < 4)
623 					nfc_op->addr1_reg |= (u32)addrs[i] << i * 8;
624 				else
625 					nfc_op->addr2_reg |= addrs[i];
626 			}
627 
628 			break;
629 		case NAND_OP_DATA_IN_INSTR:
630 			nfc_op->read = true;
631 			fallthrough;
632 		case NAND_OP_DATA_OUT_INSTR:
633 			offset = nand_subop_get_data_start_off(subop, op_id);
634 			buf = instr->ctx.data.buf.in;
635 			nfc_op->buf = &buf[offset];
636 			nfc_op->len = nand_subop_get_data_len(subop, op_id);
637 			ret = anfc_pkt_len_config(nfc_op->len, &nfc_op->steps,
638 						  &pktsize);
639 			if (ret)
640 				return ret;
641 
642 			/*
643 			 * Number of DATA cycles must be aligned on 4, this
644 			 * means the controller might read/write more than
645 			 * requested. This is harmless most of the time as extra
646 			 * DATA are discarded in the write path and read pointer
647 			 * adjusted in the read path.
648 			 *
649 			 * FIXME: The core should mark operations where
650 			 * reading/writing more is allowed so the exec_op()
651 			 * implementation can take the right decision when the
652 			 * alignment constraint is not met: adjust the number of
653 			 * DATA cycles when it's allowed, reject the operation
654 			 * otherwise.
655 			 */
656 			nfc_op->pkt_reg |= PKT_SIZE(round_up(pktsize, 4)) |
657 					   PKT_STEPS(nfc_op->steps);
658 			break;
659 		case NAND_OP_WAITRDY_INSTR:
660 			nfc_op->rdy_timeout_ms = instr->ctx.waitrdy.timeout_ms;
661 			break;
662 		}
663 	}
664 
665 	return 0;
666 }
667 
668 static int anfc_rw_pio_op(struct arasan_nfc *nfc, struct anfc_op *nfc_op)
669 {
670 	unsigned int dwords = (nfc_op->len / 4) / nfc_op->steps;
671 	unsigned int last_len = nfc_op->len % 4;
672 	unsigned int offset, dir;
673 	u8 *buf = nfc_op->buf;
674 	int ret, i;
675 
676 	for (i = 0; i < nfc_op->steps; i++) {
677 		dir = nfc_op->read ? READ_READY : WRITE_READY;
678 		ret = anfc_wait_for_event(nfc, dir);
679 		if (ret) {
680 			dev_err(nfc->dev, "PIO %s ready signal not received\n",
681 				nfc_op->read ? "Read" : "Write");
682 			return ret;
683 		}
684 
685 		offset = i * (dwords * 4);
686 		if (nfc_op->read)
687 			ioread32_rep(nfc->base + DATA_PORT_REG, &buf[offset],
688 				     dwords);
689 		else
690 			iowrite32_rep(nfc->base + DATA_PORT_REG, &buf[offset],
691 				      dwords);
692 	}
693 
694 	if (last_len) {
695 		u32 remainder;
696 
697 		offset = nfc_op->len - last_len;
698 
699 		if (nfc_op->read) {
700 			remainder = readl_relaxed(nfc->base + DATA_PORT_REG);
701 			memcpy(&buf[offset], &remainder, last_len);
702 		} else {
703 			memcpy(&remainder, &buf[offset], last_len);
704 			writel_relaxed(remainder, nfc->base + DATA_PORT_REG);
705 		}
706 	}
707 
708 	return anfc_wait_for_event(nfc, XFER_COMPLETE);
709 }
710 
711 static int anfc_misc_data_type_exec(struct nand_chip *chip,
712 				    const struct nand_subop *subop,
713 				    u32 prog_reg)
714 {
715 	struct arasan_nfc *nfc = to_anfc(chip->controller);
716 	struct anfc_op nfc_op = {};
717 	int ret;
718 
719 	ret = anfc_parse_instructions(chip, subop, &nfc_op);
720 	if (ret)
721 		return ret;
722 
723 	nfc_op.prog_reg = prog_reg;
724 	anfc_trigger_op(nfc, &nfc_op);
725 
726 	if (nfc_op.rdy_timeout_ms) {
727 		ret = anfc_wait_for_rb(nfc, chip, nfc_op.rdy_timeout_ms);
728 		if (ret)
729 			return ret;
730 	}
731 
732 	return anfc_rw_pio_op(nfc, &nfc_op);
733 }
734 
735 static int anfc_param_read_type_exec(struct nand_chip *chip,
736 				     const struct nand_subop *subop)
737 {
738 	return anfc_misc_data_type_exec(chip, subop, PROG_RDPARAM);
739 }
740 
741 static int anfc_data_read_type_exec(struct nand_chip *chip,
742 				    const struct nand_subop *subop)
743 {
744 	u32 prog_reg = PROG_PGRD;
745 
746 	/*
747 	 * Experience shows that while in SDR mode sending a CHANGE READ COLUMN
748 	 * command through the READ PAGE "type" always works fine, when in
749 	 * NV-DDR mode the same command simply fails. However, it was also
750 	 * spotted that any CHANGE READ COLUMN command sent through the CHANGE
751 	 * READ COLUMN ENHANCED "type" would correctly work in both cases (SDR
752 	 * and NV-DDR). So, for simplicity, let's program the controller with
753 	 * the CHANGE READ COLUMN ENHANCED "type" whenever we are requested to
754 	 * perform a CHANGE READ COLUMN operation.
755 	 */
756 	if (subop->instrs[0].ctx.cmd.opcode == NAND_CMD_RNDOUT &&
757 	    subop->instrs[2].ctx.cmd.opcode == NAND_CMD_RNDOUTSTART)
758 		prog_reg = PROG_CHG_RD_COL_ENH;
759 
760 	return anfc_misc_data_type_exec(chip, subop, prog_reg);
761 }
762 
763 static int anfc_param_write_type_exec(struct nand_chip *chip,
764 				      const struct nand_subop *subop)
765 {
766 	return anfc_misc_data_type_exec(chip, subop, PROG_SET_FEATURE);
767 }
768 
769 static int anfc_data_write_type_exec(struct nand_chip *chip,
770 				     const struct nand_subop *subop)
771 {
772 	return anfc_misc_data_type_exec(chip, subop, PROG_PGPROG);
773 }
774 
775 static int anfc_misc_zerolen_type_exec(struct nand_chip *chip,
776 				       const struct nand_subop *subop,
777 				       u32 prog_reg)
778 {
779 	struct arasan_nfc *nfc = to_anfc(chip->controller);
780 	struct anfc_op nfc_op = {};
781 	int ret;
782 
783 	ret = anfc_parse_instructions(chip, subop, &nfc_op);
784 	if (ret)
785 		return ret;
786 
787 	nfc_op.prog_reg = prog_reg;
788 	anfc_trigger_op(nfc, &nfc_op);
789 
790 	ret = anfc_wait_for_event(nfc, XFER_COMPLETE);
791 	if (ret)
792 		return ret;
793 
794 	if (nfc_op.rdy_timeout_ms)
795 		ret = anfc_wait_for_rb(nfc, chip, nfc_op.rdy_timeout_ms);
796 
797 	return ret;
798 }
799 
800 static int anfc_status_type_exec(struct nand_chip *chip,
801 				 const struct nand_subop *subop)
802 {
803 	struct arasan_nfc *nfc = to_anfc(chip->controller);
804 	u32 tmp;
805 	int ret;
806 
807 	/* See anfc_check_op() for details about this constraint */
808 	if (subop->instrs[0].ctx.cmd.opcode != NAND_CMD_STATUS)
809 		return -ENOTSUPP;
810 
811 	ret = anfc_misc_zerolen_type_exec(chip, subop, PROG_STATUS);
812 	if (ret)
813 		return ret;
814 
815 	tmp = readl_relaxed(nfc->base + FLASH_STS_REG);
816 	memcpy(subop->instrs[1].ctx.data.buf.in, &tmp, 1);
817 
818 	return 0;
819 }
820 
821 static int anfc_reset_type_exec(struct nand_chip *chip,
822 				const struct nand_subop *subop)
823 {
824 	return anfc_misc_zerolen_type_exec(chip, subop, PROG_RST);
825 }
826 
827 static int anfc_erase_type_exec(struct nand_chip *chip,
828 				const struct nand_subop *subop)
829 {
830 	return anfc_misc_zerolen_type_exec(chip, subop, PROG_ERASE);
831 }
832 
833 static int anfc_wait_type_exec(struct nand_chip *chip,
834 			       const struct nand_subop *subop)
835 {
836 	struct arasan_nfc *nfc = to_anfc(chip->controller);
837 	struct anfc_op nfc_op = {};
838 	int ret;
839 
840 	ret = anfc_parse_instructions(chip, subop, &nfc_op);
841 	if (ret)
842 		return ret;
843 
844 	return anfc_wait_for_rb(nfc, chip, nfc_op.rdy_timeout_ms);
845 }
846 
847 static const struct nand_op_parser anfc_op_parser = NAND_OP_PARSER(
848 	NAND_OP_PARSER_PATTERN(
849 		anfc_param_read_type_exec,
850 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
851 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, ANFC_MAX_ADDR_CYC),
852 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
853 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, ANFC_MAX_CHUNK_SIZE)),
854 	NAND_OP_PARSER_PATTERN(
855 		anfc_param_write_type_exec,
856 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
857 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, ANFC_MAX_ADDR_CYC),
858 		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, ANFC_MAX_PARAM_SIZE)),
859 	NAND_OP_PARSER_PATTERN(
860 		anfc_data_read_type_exec,
861 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
862 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, ANFC_MAX_ADDR_CYC),
863 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
864 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
865 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, ANFC_MAX_CHUNK_SIZE)),
866 	NAND_OP_PARSER_PATTERN(
867 		anfc_data_write_type_exec,
868 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
869 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, ANFC_MAX_ADDR_CYC),
870 		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, ANFC_MAX_CHUNK_SIZE),
871 		NAND_OP_PARSER_PAT_CMD_ELEM(false)),
872 	NAND_OP_PARSER_PATTERN(
873 		anfc_reset_type_exec,
874 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
875 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
876 	NAND_OP_PARSER_PATTERN(
877 		anfc_erase_type_exec,
878 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
879 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, ANFC_MAX_ADDR_CYC),
880 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
881 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
882 	NAND_OP_PARSER_PATTERN(
883 		anfc_status_type_exec,
884 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
885 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, ANFC_MAX_CHUNK_SIZE)),
886 	NAND_OP_PARSER_PATTERN(
887 		anfc_wait_type_exec,
888 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
889 	);
890 
891 static int anfc_check_op(struct nand_chip *chip,
892 			 const struct nand_operation *op)
893 {
894 	const struct nand_op_instr *instr;
895 	int op_id;
896 
897 	/*
898 	 * The controller abstracts all the NAND operations and do not support
899 	 * data only operations.
900 	 *
901 	 * TODO: The nand_op_parser framework should be extended to
902 	 * support custom checks on DATA instructions.
903 	 */
904 	for (op_id = 0; op_id < op->ninstrs; op_id++) {
905 		instr = &op->instrs[op_id];
906 
907 		switch (instr->type) {
908 		case NAND_OP_ADDR_INSTR:
909 			if (instr->ctx.addr.naddrs > ANFC_MAX_ADDR_CYC)
910 				return -ENOTSUPP;
911 
912 			break;
913 		case NAND_OP_DATA_IN_INSTR:
914 		case NAND_OP_DATA_OUT_INSTR:
915 			if (instr->ctx.data.len > ANFC_MAX_CHUNK_SIZE)
916 				return -ENOTSUPP;
917 
918 			if (anfc_pkt_len_config(instr->ctx.data.len, 0, 0))
919 				return -ENOTSUPP;
920 
921 			break;
922 		default:
923 			break;
924 		}
925 	}
926 
927 	/*
928 	 * The controller does not allow to proceed with a CMD+DATA_IN cycle
929 	 * manually on the bus by reading data from the data register. Instead,
930 	 * the controller abstract a status read operation with its own status
931 	 * register after ordering a read status operation. Hence, we cannot
932 	 * support any CMD+DATA_IN operation other than a READ STATUS.
933 	 *
934 	 * TODO: The nand_op_parser() framework should be extended to describe
935 	 * fixed patterns instead of open-coding this check here.
936 	 */
937 	if (op->ninstrs == 2 &&
938 	    op->instrs[0].type == NAND_OP_CMD_INSTR &&
939 	    op->instrs[0].ctx.cmd.opcode != NAND_CMD_STATUS &&
940 	    op->instrs[1].type == NAND_OP_DATA_IN_INSTR)
941 		return -ENOTSUPP;
942 
943 	return nand_op_parser_exec_op(chip, &anfc_op_parser, op, true);
944 }
945 
946 static int anfc_exec_op(struct nand_chip *chip,
947 			const struct nand_operation *op,
948 			bool check_only)
949 {
950 	int ret;
951 
952 	if (check_only)
953 		return anfc_check_op(chip, op);
954 
955 	ret = anfc_select_target(chip, op->cs);
956 	if (ret)
957 		return ret;
958 
959 	return nand_op_parser_exec_op(chip, &anfc_op_parser, op, check_only);
960 }
961 
962 static int anfc_setup_interface(struct nand_chip *chip, int target,
963 				const struct nand_interface_config *conf)
964 {
965 	struct anand *anand = to_anand(chip);
966 	struct arasan_nfc *nfc = to_anfc(chip->controller);
967 	struct device_node *np = nfc->dev->of_node;
968 	const struct nand_sdr_timings *sdr;
969 	const struct nand_nvddr_timings *nvddr;
970 	unsigned int tccs_min, dqs_mode, fast_tcad;
971 
972 	if (nand_interface_is_nvddr(conf)) {
973 		nvddr = nand_get_nvddr_timings(conf);
974 		if (IS_ERR(nvddr))
975 			return PTR_ERR(nvddr);
976 	} else {
977 		sdr = nand_get_sdr_timings(conf);
978 		if (IS_ERR(sdr))
979 			return PTR_ERR(sdr);
980 	}
981 
982 	if (target < 0)
983 		return 0;
984 
985 	if (nand_interface_is_sdr(conf)) {
986 		anand->data_iface = DIFACE_SDR |
987 				    DIFACE_SDR_MODE(conf->timings.mode);
988 		anand->timings = 0;
989 	} else {
990 		anand->data_iface = DIFACE_NVDDR |
991 				    DIFACE_DDR_MODE(conf->timings.mode);
992 
993 		if (conf->timings.nvddr.tCCS_min <= 100000)
994 			tccs_min = TCCS_TIME_100NS;
995 		else if (conf->timings.nvddr.tCCS_min <= 200000)
996 			tccs_min = TCCS_TIME_200NS;
997 		else if (conf->timings.nvddr.tCCS_min <= 300000)
998 			tccs_min = TCCS_TIME_300NS;
999 		else
1000 			tccs_min = TCCS_TIME_500NS;
1001 
1002 		fast_tcad = 0;
1003 		if (conf->timings.nvddr.tCAD_min < 45000)
1004 			fast_tcad = FAST_TCAD;
1005 
1006 		switch (conf->timings.mode) {
1007 		case 5:
1008 		case 4:
1009 			dqs_mode = 2;
1010 			break;
1011 		case 3:
1012 			dqs_mode = 3;
1013 			break;
1014 		case 2:
1015 			dqs_mode = 4;
1016 			break;
1017 		case 1:
1018 			dqs_mode = 5;
1019 			break;
1020 		case 0:
1021 		default:
1022 			dqs_mode = 6;
1023 			break;
1024 		}
1025 
1026 		anand->timings = tccs_min | fast_tcad |
1027 				 DQS_BUFF_SEL_IN(dqs_mode) |
1028 				 DQS_BUFF_SEL_OUT(dqs_mode);
1029 	}
1030 
1031 	anand->clk = ANFC_XLNX_SDR_DFLT_CORE_CLK;
1032 
1033 	/*
1034 	 * Due to a hardware bug in the ZynqMP SoC, SDR timing modes 0-1 work
1035 	 * with f > 90MHz (default clock is 100MHz) but signals are unstable
1036 	 * with higher modes. Hence we decrease a little bit the clock rate to
1037 	 * 80MHz when using SDR modes 2-5 with this SoC.
1038 	 */
1039 	if (of_device_is_compatible(np, "xlnx,zynqmp-nand-controller") &&
1040 	    nand_interface_is_sdr(conf) && conf->timings.mode >= 2)
1041 		anand->clk = ANFC_XLNX_SDR_HS_CORE_CLK;
1042 
1043 	return 0;
1044 }
1045 
1046 static int anfc_calc_hw_ecc_bytes(int step_size, int strength)
1047 {
1048 	unsigned int bch_gf_mag, ecc_bits;
1049 
1050 	switch (step_size) {
1051 	case SZ_512:
1052 		bch_gf_mag = 13;
1053 		break;
1054 	case SZ_1K:
1055 		bch_gf_mag = 14;
1056 		break;
1057 	default:
1058 		return -EINVAL;
1059 	}
1060 
1061 	ecc_bits = bch_gf_mag * strength;
1062 
1063 	return DIV_ROUND_UP(ecc_bits, 8);
1064 }
1065 
1066 static const int anfc_hw_ecc_512_strengths[] = {4, 8, 12};
1067 
1068 static const int anfc_hw_ecc_1024_strengths[] = {24};
1069 
1070 static const struct nand_ecc_step_info anfc_hw_ecc_step_infos[] = {
1071 	{
1072 		.stepsize = SZ_512,
1073 		.strengths = anfc_hw_ecc_512_strengths,
1074 		.nstrengths = ARRAY_SIZE(anfc_hw_ecc_512_strengths),
1075 	},
1076 	{
1077 		.stepsize = SZ_1K,
1078 		.strengths = anfc_hw_ecc_1024_strengths,
1079 		.nstrengths = ARRAY_SIZE(anfc_hw_ecc_1024_strengths),
1080 	},
1081 };
1082 
1083 static const struct nand_ecc_caps anfc_hw_ecc_caps = {
1084 	.stepinfos = anfc_hw_ecc_step_infos,
1085 	.nstepinfos = ARRAY_SIZE(anfc_hw_ecc_step_infos),
1086 	.calc_ecc_bytes = anfc_calc_hw_ecc_bytes,
1087 };
1088 
1089 static int anfc_init_hw_ecc_controller(struct arasan_nfc *nfc,
1090 				       struct nand_chip *chip)
1091 {
1092 	struct anand *anand = to_anand(chip);
1093 	struct mtd_info *mtd = nand_to_mtd(chip);
1094 	struct nand_ecc_ctrl *ecc = &chip->ecc;
1095 	unsigned int bch_prim_poly = 0, bch_gf_mag = 0, ecc_offset;
1096 	int ret;
1097 
1098 	switch (mtd->writesize) {
1099 	case SZ_512:
1100 	case SZ_2K:
1101 	case SZ_4K:
1102 	case SZ_8K:
1103 	case SZ_16K:
1104 		break;
1105 	default:
1106 		dev_err(nfc->dev, "Unsupported page size %d\n", mtd->writesize);
1107 		return -EINVAL;
1108 	}
1109 
1110 	ret = nand_ecc_choose_conf(chip, &anfc_hw_ecc_caps, mtd->oobsize);
1111 	if (ret)
1112 		return ret;
1113 
1114 	switch (ecc->strength) {
1115 	case 12:
1116 		anand->strength = 0x1;
1117 		break;
1118 	case 8:
1119 		anand->strength = 0x2;
1120 		break;
1121 	case 4:
1122 		anand->strength = 0x3;
1123 		break;
1124 	case 24:
1125 		anand->strength = 0x4;
1126 		break;
1127 	default:
1128 		dev_err(nfc->dev, "Unsupported strength %d\n", ecc->strength);
1129 		return -EINVAL;
1130 	}
1131 
1132 	switch (ecc->size) {
1133 	case SZ_512:
1134 		bch_gf_mag = 13;
1135 		bch_prim_poly = 0x201b;
1136 		break;
1137 	case SZ_1K:
1138 		bch_gf_mag = 14;
1139 		bch_prim_poly = 0x4443;
1140 		break;
1141 	default:
1142 		dev_err(nfc->dev, "Unsupported step size %d\n", ecc->strength);
1143 		return -EINVAL;
1144 	}
1145 
1146 	mtd_set_ooblayout(mtd, nand_get_large_page_ooblayout());
1147 
1148 	ecc->steps = mtd->writesize / ecc->size;
1149 	ecc->algo = NAND_ECC_ALGO_BCH;
1150 	anand->ecc_bits = bch_gf_mag * ecc->strength;
1151 	ecc->bytes = DIV_ROUND_UP(anand->ecc_bits, 8);
1152 	anand->ecc_total = DIV_ROUND_UP(anand->ecc_bits * ecc->steps, 8);
1153 	ecc_offset = mtd->writesize + mtd->oobsize - anand->ecc_total;
1154 	anand->ecc_conf = ECC_CONF_COL(ecc_offset) |
1155 			  ECC_CONF_LEN(anand->ecc_total) |
1156 			  ECC_CONF_BCH_EN;
1157 
1158 	anand->errloc = devm_kmalloc_array(nfc->dev, ecc->strength,
1159 					   sizeof(*anand->errloc), GFP_KERNEL);
1160 	if (!anand->errloc)
1161 		return -ENOMEM;
1162 
1163 	anand->hw_ecc = devm_kmalloc(nfc->dev, ecc->bytes, GFP_KERNEL);
1164 	if (!anand->hw_ecc)
1165 		return -ENOMEM;
1166 
1167 	/* Enforce bit swapping to fit the hardware */
1168 	anand->bch = bch_init(bch_gf_mag, ecc->strength, bch_prim_poly, true);
1169 	if (!anand->bch)
1170 		return -EINVAL;
1171 
1172 	ecc->read_page = anfc_sel_read_page_hw_ecc;
1173 	ecc->write_page = anfc_sel_write_page_hw_ecc;
1174 
1175 	return 0;
1176 }
1177 
1178 static int anfc_attach_chip(struct nand_chip *chip)
1179 {
1180 	struct anand *anand = to_anand(chip);
1181 	struct arasan_nfc *nfc = to_anfc(chip->controller);
1182 	struct mtd_info *mtd = nand_to_mtd(chip);
1183 	int ret = 0;
1184 
1185 	if (mtd->writesize <= SZ_512)
1186 		anand->caddr_cycles = 1;
1187 	else
1188 		anand->caddr_cycles = 2;
1189 
1190 	if (chip->options & NAND_ROW_ADDR_3)
1191 		anand->raddr_cycles = 3;
1192 	else
1193 		anand->raddr_cycles = 2;
1194 
1195 	switch (mtd->writesize) {
1196 	case 512:
1197 		anand->page_sz = 0;
1198 		break;
1199 	case 1024:
1200 		anand->page_sz = 5;
1201 		break;
1202 	case 2048:
1203 		anand->page_sz = 1;
1204 		break;
1205 	case 4096:
1206 		anand->page_sz = 2;
1207 		break;
1208 	case 8192:
1209 		anand->page_sz = 3;
1210 		break;
1211 	case 16384:
1212 		anand->page_sz = 4;
1213 		break;
1214 	default:
1215 		return -EINVAL;
1216 	}
1217 
1218 	/* These hooks are valid for all ECC providers */
1219 	chip->ecc.read_page_raw = nand_monolithic_read_page_raw;
1220 	chip->ecc.write_page_raw = nand_monolithic_write_page_raw;
1221 
1222 	switch (chip->ecc.engine_type) {
1223 	case NAND_ECC_ENGINE_TYPE_NONE:
1224 	case NAND_ECC_ENGINE_TYPE_SOFT:
1225 	case NAND_ECC_ENGINE_TYPE_ON_DIE:
1226 		break;
1227 	case NAND_ECC_ENGINE_TYPE_ON_HOST:
1228 		ret = anfc_init_hw_ecc_controller(nfc, chip);
1229 		break;
1230 	default:
1231 		dev_err(nfc->dev, "Unsupported ECC mode: %d\n",
1232 			chip->ecc.engine_type);
1233 		return -EINVAL;
1234 	}
1235 
1236 	return ret;
1237 }
1238 
1239 static void anfc_detach_chip(struct nand_chip *chip)
1240 {
1241 	struct anand *anand = to_anand(chip);
1242 
1243 	if (anand->bch)
1244 		bch_free(anand->bch);
1245 }
1246 
1247 static const struct nand_controller_ops anfc_ops = {
1248 	.exec_op = anfc_exec_op,
1249 	.setup_interface = anfc_setup_interface,
1250 	.attach_chip = anfc_attach_chip,
1251 	.detach_chip = anfc_detach_chip,
1252 };
1253 
1254 static int anfc_chip_init(struct arasan_nfc *nfc, struct device_node *np)
1255 {
1256 	struct anand *anand;
1257 	struct nand_chip *chip;
1258 	struct mtd_info *mtd;
1259 	int rb, ret, i;
1260 
1261 	anand = devm_kzalloc(nfc->dev, sizeof(*anand), GFP_KERNEL);
1262 	if (!anand)
1263 		return -ENOMEM;
1264 
1265 	/* Chip-select init */
1266 	anand->ncs_idx = of_property_count_elems_of_size(np, "reg", sizeof(u32));
1267 	if (anand->ncs_idx <= 0 || anand->ncs_idx > nfc->ncs) {
1268 		dev_err(nfc->dev, "Invalid reg property\n");
1269 		return -EINVAL;
1270 	}
1271 
1272 	anand->cs_idx = devm_kcalloc(nfc->dev, anand->ncs_idx,
1273 				     sizeof(*anand->cs_idx), GFP_KERNEL);
1274 	if (!anand->cs_idx)
1275 		return -ENOMEM;
1276 
1277 	for (i = 0; i < anand->ncs_idx; i++) {
1278 		ret = of_property_read_u32_index(np, "reg", i,
1279 						 &anand->cs_idx[i]);
1280 		if (ret) {
1281 			dev_err(nfc->dev, "invalid CS property: %d\n", ret);
1282 			return ret;
1283 		}
1284 	}
1285 
1286 	/* Ready-busy init */
1287 	ret = of_property_read_u32(np, "nand-rb", &rb);
1288 	if (ret)
1289 		return ret;
1290 
1291 	if (rb >= ANFC_MAX_CS) {
1292 		dev_err(nfc->dev, "Wrong RB %d\n", rb);
1293 		return -EINVAL;
1294 	}
1295 
1296 	anand->rb = rb;
1297 
1298 	chip = &anand->chip;
1299 	mtd = nand_to_mtd(chip);
1300 	mtd->dev.parent = nfc->dev;
1301 	chip->controller = &nfc->controller;
1302 	chip->options = NAND_BUSWIDTH_AUTO | NAND_NO_SUBPAGE_WRITE |
1303 			NAND_USES_DMA;
1304 
1305 	nand_set_flash_node(chip, np);
1306 	if (!mtd->name) {
1307 		dev_err(nfc->dev, "NAND label property is mandatory\n");
1308 		return -EINVAL;
1309 	}
1310 
1311 	ret = nand_scan(chip, anand->ncs_idx);
1312 	if (ret) {
1313 		dev_err(nfc->dev, "Scan operation failed\n");
1314 		return ret;
1315 	}
1316 
1317 	ret = mtd_device_register(mtd, NULL, 0);
1318 	if (ret) {
1319 		nand_cleanup(chip);
1320 		return ret;
1321 	}
1322 
1323 	list_add_tail(&anand->node, &nfc->chips);
1324 
1325 	return 0;
1326 }
1327 
1328 static void anfc_chips_cleanup(struct arasan_nfc *nfc)
1329 {
1330 	struct anand *anand, *tmp;
1331 	struct nand_chip *chip;
1332 	int ret;
1333 
1334 	list_for_each_entry_safe(anand, tmp, &nfc->chips, node) {
1335 		chip = &anand->chip;
1336 		ret = mtd_device_unregister(nand_to_mtd(chip));
1337 		WARN_ON(ret);
1338 		nand_cleanup(chip);
1339 		list_del(&anand->node);
1340 	}
1341 }
1342 
1343 static int anfc_chips_init(struct arasan_nfc *nfc)
1344 {
1345 	struct device_node *np = nfc->dev->of_node, *nand_np;
1346 	int nchips = of_get_child_count(np);
1347 	int ret;
1348 
1349 	if (!nchips) {
1350 		dev_err(nfc->dev, "Incorrect number of NAND chips (%d)\n",
1351 			nchips);
1352 		return -EINVAL;
1353 	}
1354 
1355 	for_each_child_of_node(np, nand_np) {
1356 		ret = anfc_chip_init(nfc, nand_np);
1357 		if (ret) {
1358 			of_node_put(nand_np);
1359 			anfc_chips_cleanup(nfc);
1360 			break;
1361 		}
1362 	}
1363 
1364 	return ret;
1365 }
1366 
1367 static void anfc_reset(struct arasan_nfc *nfc)
1368 {
1369 	/* Disable interrupt signals */
1370 	writel_relaxed(0, nfc->base + INTR_SIG_EN_REG);
1371 
1372 	/* Enable interrupt status */
1373 	writel_relaxed(EVENT_MASK, nfc->base + INTR_STS_EN_REG);
1374 
1375 	nfc->cur_cs = -1;
1376 }
1377 
1378 static int anfc_parse_cs(struct arasan_nfc *nfc)
1379 {
1380 	int ret;
1381 
1382 	/* Check the gpio-cs property */
1383 	ret = rawnand_dt_parse_gpio_cs(nfc->dev, &nfc->cs_array, &nfc->ncs);
1384 	if (ret)
1385 		return ret;
1386 
1387 	/*
1388 	 * The controller native CS cannot be both disabled at the same time.
1389 	 * Hence, only one native CS can be used if GPIO CS are needed, so that
1390 	 * the other is selected when a non-native CS must be asserted (not
1391 	 * wired physically or configured as GPIO instead of NAND CS). In this
1392 	 * case, the "not" chosen CS is assigned to nfc->spare_cs and selected
1393 	 * whenever a GPIO CS must be asserted.
1394 	 */
1395 	if (nfc->cs_array && nfc->ncs > 2) {
1396 		if (!nfc->cs_array[0] && !nfc->cs_array[1]) {
1397 			dev_err(nfc->dev,
1398 				"Assign a single native CS when using GPIOs\n");
1399 			return -EINVAL;
1400 		}
1401 
1402 		if (nfc->cs_array[0])
1403 			nfc->spare_cs = 0;
1404 		else
1405 			nfc->spare_cs = 1;
1406 	}
1407 
1408 	if (!nfc->cs_array) {
1409 		nfc->cs_array = anfc_default_cs_array;
1410 		nfc->ncs = ANFC_MAX_CS;
1411 		return 0;
1412 	}
1413 
1414 	return 0;
1415 }
1416 
1417 static int anfc_probe(struct platform_device *pdev)
1418 {
1419 	struct arasan_nfc *nfc;
1420 	int ret;
1421 
1422 	nfc = devm_kzalloc(&pdev->dev, sizeof(*nfc), GFP_KERNEL);
1423 	if (!nfc)
1424 		return -ENOMEM;
1425 
1426 	nfc->dev = &pdev->dev;
1427 	nand_controller_init(&nfc->controller);
1428 	nfc->controller.ops = &anfc_ops;
1429 	INIT_LIST_HEAD(&nfc->chips);
1430 
1431 	nfc->base = devm_platform_ioremap_resource(pdev, 0);
1432 	if (IS_ERR(nfc->base))
1433 		return PTR_ERR(nfc->base);
1434 
1435 	anfc_reset(nfc);
1436 
1437 	nfc->controller_clk = devm_clk_get(&pdev->dev, "controller");
1438 	if (IS_ERR(nfc->controller_clk))
1439 		return PTR_ERR(nfc->controller_clk);
1440 
1441 	nfc->bus_clk = devm_clk_get(&pdev->dev, "bus");
1442 	if (IS_ERR(nfc->bus_clk))
1443 		return PTR_ERR(nfc->bus_clk);
1444 
1445 	ret = clk_prepare_enable(nfc->controller_clk);
1446 	if (ret)
1447 		return ret;
1448 
1449 	ret = clk_prepare_enable(nfc->bus_clk);
1450 	if (ret)
1451 		goto disable_controller_clk;
1452 
1453 	ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
1454 	if (ret)
1455 		goto disable_bus_clk;
1456 
1457 	ret = anfc_parse_cs(nfc);
1458 	if (ret)
1459 		goto disable_bus_clk;
1460 
1461 	ret = anfc_chips_init(nfc);
1462 	if (ret)
1463 		goto disable_bus_clk;
1464 
1465 	platform_set_drvdata(pdev, nfc);
1466 
1467 	return 0;
1468 
1469 disable_bus_clk:
1470 	clk_disable_unprepare(nfc->bus_clk);
1471 
1472 disable_controller_clk:
1473 	clk_disable_unprepare(nfc->controller_clk);
1474 
1475 	return ret;
1476 }
1477 
1478 static int anfc_remove(struct platform_device *pdev)
1479 {
1480 	struct arasan_nfc *nfc = platform_get_drvdata(pdev);
1481 
1482 	anfc_chips_cleanup(nfc);
1483 
1484 	clk_disable_unprepare(nfc->bus_clk);
1485 	clk_disable_unprepare(nfc->controller_clk);
1486 
1487 	return 0;
1488 }
1489 
1490 static const struct of_device_id anfc_ids[] = {
1491 	{
1492 		.compatible = "xlnx,zynqmp-nand-controller",
1493 	},
1494 	{
1495 		.compatible = "arasan,nfc-v3p10",
1496 	},
1497 	{}
1498 };
1499 MODULE_DEVICE_TABLE(of, anfc_ids);
1500 
1501 static struct platform_driver anfc_driver = {
1502 	.driver = {
1503 		.name = "arasan-nand-controller",
1504 		.of_match_table = anfc_ids,
1505 	},
1506 	.probe = anfc_probe,
1507 	.remove = anfc_remove,
1508 };
1509 module_platform_driver(anfc_driver);
1510 
1511 MODULE_LICENSE("GPL v2");
1512 MODULE_AUTHOR("Punnaiah Choudary Kalluri <punnaia@xilinx.com>");
1513 MODULE_AUTHOR("Naga Sureshkumar Relli <nagasure@xilinx.com>");
1514 MODULE_AUTHOR("Miquel Raynal <miquel.raynal@bootlin.com>");
1515 MODULE_DESCRIPTION("Arasan NAND Flash Controller Driver");
1516