xref: /openbmc/linux/drivers/mtd/nand/raw/meson_nand.c (revision d8ea3e78)
1 // SPDX-License-Identifier: (GPL-2.0+ OR MIT)
2 /*
3  * Amlogic Meson Nand Flash Controller Driver
4  *
5  * Copyright (c) 2018 Amlogic, inc.
6  * Author: Liang Yang <liang.yang@amlogic.com>
7  */
8 
9 #include <linux/platform_device.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/interrupt.h>
12 #include <linux/clk.h>
13 #include <linux/clk-provider.h>
14 #include <linux/mtd/rawnand.h>
15 #include <linux/mtd/mtd.h>
16 #include <linux/mfd/syscon.h>
17 #include <linux/regmap.h>
18 #include <linux/slab.h>
19 #include <linux/module.h>
20 #include <linux/iopoll.h>
21 #include <linux/of.h>
22 #include <linux/sched/task_stack.h>
23 
24 #define NFC_REG_CMD		0x00
25 #define NFC_CMD_IDLE		(0xc << 14)
26 #define NFC_CMD_CLE		(0x5 << 14)
27 #define NFC_CMD_ALE		(0x6 << 14)
28 #define NFC_CMD_ADL		((0 << 16) | (3 << 20))
29 #define NFC_CMD_ADH		((1 << 16) | (3 << 20))
30 #define NFC_CMD_AIL		((2 << 16) | (3 << 20))
31 #define NFC_CMD_AIH		((3 << 16) | (3 << 20))
32 #define NFC_CMD_SEED		((8 << 16) | (3 << 20))
33 #define NFC_CMD_M2N		((0 << 17) | (2 << 20))
34 #define NFC_CMD_N2M		((1 << 17) | (2 << 20))
35 #define NFC_CMD_RB		BIT(20)
36 #define NFC_CMD_SCRAMBLER_ENABLE	BIT(19)
37 #define NFC_CMD_SCRAMBLER_DISABLE	0
38 #define NFC_CMD_SHORTMODE_DISABLE	0
39 #define NFC_CMD_RB_INT		BIT(14)
40 #define NFC_CMD_RB_INT_NO_PIN	((0xb << 10) | BIT(18) | BIT(16))
41 
42 #define NFC_CMD_GET_SIZE(x)	(((x) >> 22) & GENMASK(4, 0))
43 
44 #define NFC_REG_CFG		0x04
45 #define NFC_REG_DADR		0x08
46 #define NFC_REG_IADR		0x0c
47 #define NFC_REG_BUF		0x10
48 #define NFC_REG_INFO		0x14
49 #define NFC_REG_DC		0x18
50 #define NFC_REG_ADR		0x1c
51 #define NFC_REG_DL		0x20
52 #define NFC_REG_DH		0x24
53 #define NFC_REG_CADR		0x28
54 #define NFC_REG_SADR		0x2c
55 #define NFC_REG_PINS		0x30
56 #define NFC_REG_VER		0x38
57 
58 #define NFC_RB_IRQ_EN		BIT(21)
59 
60 #define CLK_DIV_SHIFT		0
61 #define CLK_DIV_WIDTH		6
62 
63 #define CMDRWGEN(cmd_dir, ran, bch, short_mode, page_size, pages)	\
64 	(								\
65 		(cmd_dir)			|			\
66 		(ran)				|			\
67 		((bch) << 14)			|			\
68 		((short_mode) << 13)		|			\
69 		(((page_size) & 0x7f) << 6)	|			\
70 		((pages) & 0x3f)					\
71 	)
72 
73 #define GENCMDDADDRL(adl, addr)		((adl) | ((addr) & 0xffff))
74 #define GENCMDDADDRH(adh, addr)		((adh) | (((addr) >> 16) & 0xffff))
75 #define GENCMDIADDRL(ail, addr)		((ail) | ((addr) & 0xffff))
76 #define GENCMDIADDRH(aih, addr)		((aih) | (((addr) >> 16) & 0xffff))
77 
78 #define DMA_DIR(dir)		((dir) ? NFC_CMD_N2M : NFC_CMD_M2N)
79 #define DMA_ADDR_ALIGN		8
80 
81 #define ECC_CHECK_RETURN_FF	(-1)
82 
83 #define NAND_CE0		(0xe << 10)
84 #define NAND_CE1		(0xd << 10)
85 
86 #define DMA_BUSY_TIMEOUT	0x100000
87 #define CMD_FIFO_EMPTY_TIMEOUT	1000
88 
89 #define MAX_CE_NUM		2
90 
91 /* eMMC clock register, misc control */
92 #define CLK_SELECT_NAND		BIT(31)
93 
94 #define NFC_CLK_CYCLE		6
95 
96 /* nand flash controller delay 3 ns */
97 #define NFC_DEFAULT_DELAY	3000
98 
99 #define ROW_ADDER(page, index)	(((page) >> (8 * (index))) & 0xff)
100 #define MAX_CYCLE_ADDRS		5
101 #define DIRREAD			1
102 #define DIRWRITE		0
103 
104 #define ECC_PARITY_BCH8_512B	14
105 #define ECC_COMPLETE            BIT(31)
106 #define ECC_ERR_CNT(x)		(((x) >> 24) & GENMASK(5, 0))
107 #define ECC_ZERO_CNT(x)		(((x) >> 16) & GENMASK(5, 0))
108 #define ECC_UNCORRECTABLE	0x3f
109 
110 #define PER_INFO_BYTE		8
111 
112 #define NFC_CMD_RAW_LEN	GENMASK(13, 0)
113 
114 #define NFC_COLUMN_ADDR_0	0
115 #define NFC_COLUMN_ADDR_1	0
116 
117 struct meson_nfc_nand_chip {
118 	struct list_head node;
119 	struct nand_chip nand;
120 	unsigned long clk_rate;
121 	unsigned long level1_divider;
122 	u32 bus_timing;
123 	u32 twb;
124 	u32 tadl;
125 	u32 tbers_max;
126 
127 	u32 bch_mode;
128 	u8 *data_buf;
129 	__le64 *info_buf;
130 	u32 nsels;
131 	u8 sels[];
132 };
133 
134 struct meson_nand_ecc {
135 	u32 bch;
136 	u32 strength;
137 	u32 size;
138 };
139 
140 struct meson_nfc_data {
141 	const struct nand_ecc_caps *ecc_caps;
142 };
143 
144 struct meson_nfc_param {
145 	u32 chip_select;
146 	u32 rb_select;
147 };
148 
149 struct nand_rw_cmd {
150 	u32 cmd0;
151 	u32 addrs[MAX_CYCLE_ADDRS];
152 	u32 cmd1;
153 };
154 
155 struct nand_timing {
156 	u32 twb;
157 	u32 tadl;
158 	u32 tbers_max;
159 };
160 
161 struct meson_nfc {
162 	struct nand_controller controller;
163 	struct clk *core_clk;
164 	struct clk *device_clk;
165 	struct clk *nand_clk;
166 	struct clk_divider nand_divider;
167 
168 	unsigned long clk_rate;
169 	u32 bus_timing;
170 
171 	struct device *dev;
172 	void __iomem *reg_base;
173 	void __iomem *reg_clk;
174 	struct completion completion;
175 	struct list_head chips;
176 	const struct meson_nfc_data *data;
177 	struct meson_nfc_param param;
178 	struct nand_timing timing;
179 	union {
180 		int cmd[32];
181 		struct nand_rw_cmd rw;
182 	} cmdfifo;
183 
184 	dma_addr_t daddr;
185 	dma_addr_t iaddr;
186 	u32 info_bytes;
187 
188 	unsigned long assigned_cs;
189 	bool no_rb_pin;
190 };
191 
192 enum {
193 	NFC_ECC_BCH8_512	= 1,
194 	NFC_ECC_BCH8_1K,
195 	NFC_ECC_BCH24_1K,
196 	NFC_ECC_BCH30_1K,
197 	NFC_ECC_BCH40_1K,
198 	NFC_ECC_BCH50_1K,
199 	NFC_ECC_BCH60_1K,
200 };
201 
202 #define MESON_ECC_DATA(b, s, sz)	{ .bch = (b), .strength = (s), .size = (sz) }
203 
204 static struct meson_nand_ecc meson_ecc[] = {
205 	MESON_ECC_DATA(NFC_ECC_BCH8_512, 8,  512),
206 	MESON_ECC_DATA(NFC_ECC_BCH8_1K,  8,  1024),
207 	MESON_ECC_DATA(NFC_ECC_BCH24_1K, 24, 1024),
208 	MESON_ECC_DATA(NFC_ECC_BCH30_1K, 30, 1024),
209 	MESON_ECC_DATA(NFC_ECC_BCH40_1K, 40, 1024),
210 	MESON_ECC_DATA(NFC_ECC_BCH50_1K, 50, 1024),
211 	MESON_ECC_DATA(NFC_ECC_BCH60_1K, 60, 1024),
212 };
213 
meson_nand_calc_ecc_bytes(int step_size,int strength)214 static int meson_nand_calc_ecc_bytes(int step_size, int strength)
215 {
216 	int ecc_bytes;
217 
218 	if (step_size == 512 && strength == 8)
219 		return ECC_PARITY_BCH8_512B;
220 
221 	ecc_bytes = DIV_ROUND_UP(strength * fls(step_size * 8), 8);
222 	ecc_bytes = ALIGN(ecc_bytes, 2);
223 
224 	return ecc_bytes;
225 }
226 
227 NAND_ECC_CAPS_SINGLE(meson_gxl_ecc_caps,
228 		     meson_nand_calc_ecc_bytes, 1024, 8, 24, 30, 40, 50, 60);
229 
230 static const int axg_stepinfo_strengths[] = { 8 };
231 
232 static const struct nand_ecc_step_info axg_stepinfo[] = {
233 	{
234 		.stepsize = 1024,
235 		.strengths = axg_stepinfo_strengths,
236 		.nstrengths = ARRAY_SIZE(axg_stepinfo_strengths)
237 	},
238 	{
239 		.stepsize = 512,
240 		.strengths = axg_stepinfo_strengths,
241 		.nstrengths = ARRAY_SIZE(axg_stepinfo_strengths)
242 	},
243 };
244 
245 static const struct nand_ecc_caps meson_axg_ecc_caps = {
246 	.stepinfos = axg_stepinfo,
247 	.nstepinfos = ARRAY_SIZE(axg_stepinfo),
248 	.calc_ecc_bytes = meson_nand_calc_ecc_bytes,
249 };
250 
to_meson_nand(struct nand_chip * nand)251 static struct meson_nfc_nand_chip *to_meson_nand(struct nand_chip *nand)
252 {
253 	return container_of(nand, struct meson_nfc_nand_chip, nand);
254 }
255 
meson_nfc_select_chip(struct nand_chip * nand,int chip)256 static void meson_nfc_select_chip(struct nand_chip *nand, int chip)
257 {
258 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
259 	struct meson_nfc *nfc = nand_get_controller_data(nand);
260 	int ret, value;
261 
262 	if (chip < 0 || WARN_ON_ONCE(chip >= meson_chip->nsels))
263 		return;
264 
265 	nfc->param.chip_select = meson_chip->sels[chip] ? NAND_CE1 : NAND_CE0;
266 	nfc->param.rb_select = nfc->param.chip_select;
267 	nfc->timing.twb = meson_chip->twb;
268 	nfc->timing.tadl = meson_chip->tadl;
269 	nfc->timing.tbers_max = meson_chip->tbers_max;
270 
271 	if (nfc->clk_rate != meson_chip->clk_rate) {
272 		ret = clk_set_rate(nfc->nand_clk, meson_chip->clk_rate);
273 		if (ret) {
274 			dev_err(nfc->dev, "failed to set clock rate\n");
275 			return;
276 		}
277 		nfc->clk_rate = meson_chip->clk_rate;
278 	}
279 	if (nfc->bus_timing != meson_chip->bus_timing) {
280 		value = (NFC_CLK_CYCLE - 1) | (meson_chip->bus_timing << 5);
281 		writel(value, nfc->reg_base + NFC_REG_CFG);
282 		writel((1 << 31), nfc->reg_base + NFC_REG_CMD);
283 		nfc->bus_timing =  meson_chip->bus_timing;
284 	}
285 }
286 
meson_nfc_cmd_idle(struct meson_nfc * nfc,u32 time)287 static void meson_nfc_cmd_idle(struct meson_nfc *nfc, u32 time)
288 {
289 	writel(nfc->param.chip_select | NFC_CMD_IDLE | (time & 0x3ff),
290 	       nfc->reg_base + NFC_REG_CMD);
291 }
292 
meson_nfc_cmd_seed(struct meson_nfc * nfc,u32 seed)293 static void meson_nfc_cmd_seed(struct meson_nfc *nfc, u32 seed)
294 {
295 	writel(NFC_CMD_SEED | (0xc2 + (seed & 0x7fff)),
296 	       nfc->reg_base + NFC_REG_CMD);
297 }
298 
meson_nfc_cmd_access(struct nand_chip * nand,int raw,bool dir,int scrambler)299 static void meson_nfc_cmd_access(struct nand_chip *nand, int raw, bool dir,
300 				 int scrambler)
301 {
302 	struct mtd_info *mtd = nand_to_mtd(nand);
303 	struct meson_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd));
304 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
305 	u32 bch = meson_chip->bch_mode, cmd;
306 	int len = mtd->writesize, pagesize, pages;
307 
308 	pagesize = nand->ecc.size;
309 
310 	if (raw) {
311 		len = mtd->writesize + mtd->oobsize;
312 		cmd = len | scrambler | DMA_DIR(dir);
313 		writel(cmd, nfc->reg_base + NFC_REG_CMD);
314 		return;
315 	}
316 
317 	pages = len / nand->ecc.size;
318 
319 	cmd = CMDRWGEN(DMA_DIR(dir), scrambler, bch,
320 		       NFC_CMD_SHORTMODE_DISABLE, pagesize, pages);
321 
322 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
323 }
324 
meson_nfc_drain_cmd(struct meson_nfc * nfc)325 static void meson_nfc_drain_cmd(struct meson_nfc *nfc)
326 {
327 	/*
328 	 * Insert two commands to make sure all valid commands are finished.
329 	 *
330 	 * The Nand flash controller is designed as two stages pipleline -
331 	 *  a) fetch and b) excute.
332 	 * There might be cases when the driver see command queue is empty,
333 	 * but the Nand flash controller still has two commands buffered,
334 	 * one is fetched into NFC request queue (ready to run), and another
335 	 * is actively executing. So pushing 2 "IDLE" commands guarantees that
336 	 * the pipeline is emptied.
337 	 */
338 	meson_nfc_cmd_idle(nfc, 0);
339 	meson_nfc_cmd_idle(nfc, 0);
340 }
341 
meson_nfc_wait_cmd_finish(struct meson_nfc * nfc,unsigned int timeout_ms)342 static int meson_nfc_wait_cmd_finish(struct meson_nfc *nfc,
343 				     unsigned int timeout_ms)
344 {
345 	u32 cmd_size = 0;
346 	int ret;
347 
348 	/* wait cmd fifo is empty */
349 	ret = readl_relaxed_poll_timeout(nfc->reg_base + NFC_REG_CMD, cmd_size,
350 					 !NFC_CMD_GET_SIZE(cmd_size),
351 					 10, timeout_ms * 1000);
352 	if (ret)
353 		dev_err(nfc->dev, "wait for empty CMD FIFO time out\n");
354 
355 	return ret;
356 }
357 
meson_nfc_wait_dma_finish(struct meson_nfc * nfc)358 static int meson_nfc_wait_dma_finish(struct meson_nfc *nfc)
359 {
360 	meson_nfc_drain_cmd(nfc);
361 
362 	return meson_nfc_wait_cmd_finish(nfc, DMA_BUSY_TIMEOUT);
363 }
364 
meson_nfc_oob_ptr(struct nand_chip * nand,int i)365 static u8 *meson_nfc_oob_ptr(struct nand_chip *nand, int i)
366 {
367 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
368 	int len;
369 
370 	len = nand->ecc.size * (i + 1) + (nand->ecc.bytes + 2) * i;
371 
372 	return meson_chip->data_buf + len;
373 }
374 
meson_nfc_data_ptr(struct nand_chip * nand,int i)375 static u8 *meson_nfc_data_ptr(struct nand_chip *nand, int i)
376 {
377 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
378 	int len, temp;
379 
380 	temp = nand->ecc.size + nand->ecc.bytes;
381 	len = (temp + 2) * i;
382 
383 	return meson_chip->data_buf + len;
384 }
385 
meson_nfc_get_data_oob(struct nand_chip * nand,u8 * buf,u8 * oobbuf)386 static void meson_nfc_get_data_oob(struct nand_chip *nand,
387 				   u8 *buf, u8 *oobbuf)
388 {
389 	int i, oob_len = 0;
390 	u8 *dsrc, *osrc;
391 
392 	oob_len = nand->ecc.bytes + 2;
393 	for (i = 0; i < nand->ecc.steps; i++) {
394 		if (buf) {
395 			dsrc = meson_nfc_data_ptr(nand, i);
396 			memcpy(buf, dsrc, nand->ecc.size);
397 			buf += nand->ecc.size;
398 		}
399 		osrc = meson_nfc_oob_ptr(nand, i);
400 		memcpy(oobbuf, osrc, oob_len);
401 		oobbuf += oob_len;
402 	}
403 }
404 
meson_nfc_set_data_oob(struct nand_chip * nand,const u8 * buf,u8 * oobbuf)405 static void meson_nfc_set_data_oob(struct nand_chip *nand,
406 				   const u8 *buf, u8 *oobbuf)
407 {
408 	int i, oob_len = 0;
409 	u8 *dsrc, *osrc;
410 
411 	oob_len = nand->ecc.bytes + 2;
412 	for (i = 0; i < nand->ecc.steps; i++) {
413 		if (buf) {
414 			dsrc = meson_nfc_data_ptr(nand, i);
415 			memcpy(dsrc, buf, nand->ecc.size);
416 			buf += nand->ecc.size;
417 		}
418 		osrc = meson_nfc_oob_ptr(nand, i);
419 		memcpy(osrc, oobbuf, oob_len);
420 		oobbuf += oob_len;
421 	}
422 }
423 
meson_nfc_wait_no_rb_pin(struct nand_chip * nand,int timeout_ms,bool need_cmd_read0)424 static int meson_nfc_wait_no_rb_pin(struct nand_chip *nand, int timeout_ms,
425 				    bool need_cmd_read0)
426 {
427 	struct meson_nfc *nfc = nand_get_controller_data(nand);
428 	u32 cmd, cfg;
429 
430 	meson_nfc_cmd_idle(nfc, nfc->timing.twb);
431 	meson_nfc_drain_cmd(nfc);
432 	meson_nfc_wait_cmd_finish(nfc, CMD_FIFO_EMPTY_TIMEOUT);
433 
434 	cfg = readl(nfc->reg_base + NFC_REG_CFG);
435 	cfg |= NFC_RB_IRQ_EN;
436 	writel(cfg, nfc->reg_base + NFC_REG_CFG);
437 
438 	reinit_completion(&nfc->completion);
439 	nand_status_op(nand, NULL);
440 
441 	/* use the max erase time as the maximum clock for waiting R/B */
442 	cmd = NFC_CMD_RB | NFC_CMD_RB_INT_NO_PIN | nfc->timing.tbers_max;
443 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
444 
445 	if (!wait_for_completion_timeout(&nfc->completion,
446 					 msecs_to_jiffies(timeout_ms)))
447 		return -ETIMEDOUT;
448 
449 	if (need_cmd_read0)
450 		nand_exit_status_op(nand);
451 
452 	return 0;
453 }
454 
meson_nfc_wait_rb_pin(struct meson_nfc * nfc,int timeout_ms)455 static int meson_nfc_wait_rb_pin(struct meson_nfc *nfc, int timeout_ms)
456 {
457 	u32 cmd, cfg;
458 	int ret = 0;
459 
460 	meson_nfc_cmd_idle(nfc, nfc->timing.twb);
461 	meson_nfc_drain_cmd(nfc);
462 	meson_nfc_wait_cmd_finish(nfc, CMD_FIFO_EMPTY_TIMEOUT);
463 
464 	cfg = readl(nfc->reg_base + NFC_REG_CFG);
465 	cfg |= NFC_RB_IRQ_EN;
466 	writel(cfg, nfc->reg_base + NFC_REG_CFG);
467 
468 	reinit_completion(&nfc->completion);
469 
470 	/* use the max erase time as the maximum clock for waiting R/B */
471 	cmd = NFC_CMD_RB | NFC_CMD_RB_INT
472 		| nfc->param.chip_select | nfc->timing.tbers_max;
473 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
474 
475 	ret = wait_for_completion_timeout(&nfc->completion,
476 					  msecs_to_jiffies(timeout_ms));
477 	if (ret == 0)
478 		ret = -1;
479 
480 	return ret;
481 }
482 
meson_nfc_queue_rb(struct nand_chip * nand,int timeout_ms,bool need_cmd_read0)483 static int meson_nfc_queue_rb(struct nand_chip *nand, int timeout_ms,
484 			      bool need_cmd_read0)
485 {
486 	struct meson_nfc *nfc = nand_get_controller_data(nand);
487 
488 	if (nfc->no_rb_pin) {
489 		/* This mode is used when there is no wired R/B pin.
490 		 * It works like 'nand_soft_waitrdy()', but instead of
491 		 * polling NAND_CMD_STATUS bit in the software loop,
492 		 * it will wait for interrupt - controllers checks IO
493 		 * bus and when it detects NAND_CMD_STATUS on it, it
494 		 * raises interrupt. After interrupt, NAND_CMD_READ0 is
495 		 * sent as terminator of the ready waiting procedure if
496 		 * needed (for all cases except page programming - this
497 		 * is reason of 'need_cmd_read0' flag).
498 		 */
499 		return meson_nfc_wait_no_rb_pin(nand, timeout_ms,
500 						need_cmd_read0);
501 	} else {
502 		return meson_nfc_wait_rb_pin(nfc, timeout_ms);
503 	}
504 }
505 
meson_nfc_set_user_byte(struct nand_chip * nand,u8 * oob_buf)506 static void meson_nfc_set_user_byte(struct nand_chip *nand, u8 *oob_buf)
507 {
508 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
509 	__le64 *info;
510 	int i, count;
511 
512 	for (i = 0, count = 0; i < nand->ecc.steps; i++, count += 2) {
513 		info = &meson_chip->info_buf[i];
514 		*info |= oob_buf[count];
515 		*info |= oob_buf[count + 1] << 8;
516 	}
517 }
518 
meson_nfc_get_user_byte(struct nand_chip * nand,u8 * oob_buf)519 static void meson_nfc_get_user_byte(struct nand_chip *nand, u8 *oob_buf)
520 {
521 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
522 	__le64 *info;
523 	int i, count;
524 
525 	for (i = 0, count = 0; i < nand->ecc.steps; i++, count += 2) {
526 		info = &meson_chip->info_buf[i];
527 		oob_buf[count] = *info;
528 		oob_buf[count + 1] = *info >> 8;
529 	}
530 }
531 
meson_nfc_ecc_correct(struct nand_chip * nand,u32 * bitflips,u64 * correct_bitmap)532 static int meson_nfc_ecc_correct(struct nand_chip *nand, u32 *bitflips,
533 				 u64 *correct_bitmap)
534 {
535 	struct mtd_info *mtd = nand_to_mtd(nand);
536 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
537 	__le64 *info;
538 	int ret = 0, i;
539 
540 	for (i = 0; i < nand->ecc.steps; i++) {
541 		info = &meson_chip->info_buf[i];
542 		if (ECC_ERR_CNT(*info) != ECC_UNCORRECTABLE) {
543 			mtd->ecc_stats.corrected += ECC_ERR_CNT(*info);
544 			*bitflips = max_t(u32, *bitflips, ECC_ERR_CNT(*info));
545 			*correct_bitmap |= BIT_ULL(i);
546 			continue;
547 		}
548 		if ((nand->options & NAND_NEED_SCRAMBLING) &&
549 		    ECC_ZERO_CNT(*info) < nand->ecc.strength) {
550 			mtd->ecc_stats.corrected += ECC_ZERO_CNT(*info);
551 			*bitflips = max_t(u32, *bitflips,
552 					  ECC_ZERO_CNT(*info));
553 			ret = ECC_CHECK_RETURN_FF;
554 		} else {
555 			ret = -EBADMSG;
556 		}
557 	}
558 	return ret;
559 }
560 
meson_nfc_dma_buffer_setup(struct nand_chip * nand,void * databuf,int datalen,void * infobuf,int infolen,enum dma_data_direction dir)561 static int meson_nfc_dma_buffer_setup(struct nand_chip *nand, void *databuf,
562 				      int datalen, void *infobuf, int infolen,
563 				      enum dma_data_direction dir)
564 {
565 	struct meson_nfc *nfc = nand_get_controller_data(nand);
566 	u32 cmd;
567 	int ret = 0;
568 
569 	nfc->daddr = dma_map_single(nfc->dev, databuf, datalen, dir);
570 	ret = dma_mapping_error(nfc->dev, nfc->daddr);
571 	if (ret) {
572 		dev_err(nfc->dev, "DMA mapping error\n");
573 		return ret;
574 	}
575 	cmd = GENCMDDADDRL(NFC_CMD_ADL, nfc->daddr);
576 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
577 
578 	cmd = GENCMDDADDRH(NFC_CMD_ADH, nfc->daddr);
579 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
580 
581 	if (infobuf) {
582 		nfc->iaddr = dma_map_single(nfc->dev, infobuf, infolen, dir);
583 		ret = dma_mapping_error(nfc->dev, nfc->iaddr);
584 		if (ret) {
585 			dev_err(nfc->dev, "DMA mapping error\n");
586 			dma_unmap_single(nfc->dev,
587 					 nfc->daddr, datalen, dir);
588 			return ret;
589 		}
590 		nfc->info_bytes = infolen;
591 		cmd = GENCMDIADDRL(NFC_CMD_AIL, nfc->iaddr);
592 		writel(cmd, nfc->reg_base + NFC_REG_CMD);
593 
594 		cmd = GENCMDIADDRH(NFC_CMD_AIH, nfc->iaddr);
595 		writel(cmd, nfc->reg_base + NFC_REG_CMD);
596 	}
597 
598 	return ret;
599 }
600 
meson_nfc_dma_buffer_release(struct nand_chip * nand,int datalen,int infolen,enum dma_data_direction dir)601 static void meson_nfc_dma_buffer_release(struct nand_chip *nand,
602 					 int datalen, int infolen,
603 					 enum dma_data_direction dir)
604 {
605 	struct meson_nfc *nfc = nand_get_controller_data(nand);
606 
607 	dma_unmap_single(nfc->dev, nfc->daddr, datalen, dir);
608 	if (infolen) {
609 		dma_unmap_single(nfc->dev, nfc->iaddr, infolen, dir);
610 		nfc->info_bytes = 0;
611 	}
612 }
613 
meson_nfc_read_buf(struct nand_chip * nand,u8 * buf,int len)614 static int meson_nfc_read_buf(struct nand_chip *nand, u8 *buf, int len)
615 {
616 	struct meson_nfc *nfc = nand_get_controller_data(nand);
617 	int ret = 0;
618 	u32 cmd;
619 	u8 *info;
620 
621 	info = kzalloc(PER_INFO_BYTE, GFP_KERNEL);
622 	if (!info)
623 		return -ENOMEM;
624 
625 	ret = meson_nfc_dma_buffer_setup(nand, buf, len, info,
626 					 PER_INFO_BYTE, DMA_FROM_DEVICE);
627 	if (ret)
628 		goto out;
629 
630 	cmd = NFC_CMD_N2M | len;
631 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
632 
633 	meson_nfc_drain_cmd(nfc);
634 	meson_nfc_wait_cmd_finish(nfc, 1000);
635 	meson_nfc_dma_buffer_release(nand, len, PER_INFO_BYTE, DMA_FROM_DEVICE);
636 
637 out:
638 	kfree(info);
639 
640 	return ret;
641 }
642 
meson_nfc_write_buf(struct nand_chip * nand,u8 * buf,int len)643 static int meson_nfc_write_buf(struct nand_chip *nand, u8 *buf, int len)
644 {
645 	struct meson_nfc *nfc = nand_get_controller_data(nand);
646 	int ret = 0;
647 	u32 cmd;
648 
649 	ret = meson_nfc_dma_buffer_setup(nand, buf, len, NULL,
650 					 0, DMA_TO_DEVICE);
651 	if (ret)
652 		return ret;
653 
654 	cmd = NFC_CMD_M2N | len;
655 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
656 
657 	meson_nfc_drain_cmd(nfc);
658 	meson_nfc_wait_cmd_finish(nfc, 1000);
659 	meson_nfc_dma_buffer_release(nand, len, 0, DMA_TO_DEVICE);
660 
661 	return ret;
662 }
663 
meson_nfc_rw_cmd_prepare_and_execute(struct nand_chip * nand,int page,bool in)664 static int meson_nfc_rw_cmd_prepare_and_execute(struct nand_chip *nand,
665 						int page, bool in)
666 {
667 	const struct nand_sdr_timings *sdr =
668 		nand_get_sdr_timings(nand_get_interface_config(nand));
669 	struct mtd_info *mtd = nand_to_mtd(nand);
670 	struct meson_nfc *nfc = nand_get_controller_data(nand);
671 	u32 *addrs = nfc->cmdfifo.rw.addrs;
672 	u32 cs = nfc->param.chip_select;
673 	u32 cmd0, cmd_num, row_start;
674 	int i;
675 
676 	cmd_num = sizeof(struct nand_rw_cmd) / sizeof(int);
677 
678 	cmd0 = in ? NAND_CMD_READ0 : NAND_CMD_SEQIN;
679 	nfc->cmdfifo.rw.cmd0 = cs | NFC_CMD_CLE | cmd0;
680 
681 	addrs[0] = cs | NFC_CMD_ALE | NFC_COLUMN_ADDR_0;
682 	if (mtd->writesize <= 512) {
683 		cmd_num--;
684 		row_start = 1;
685 	} else {
686 		addrs[1] = cs | NFC_CMD_ALE | NFC_COLUMN_ADDR_1;
687 		row_start = 2;
688 	}
689 
690 	addrs[row_start] = cs | NFC_CMD_ALE | ROW_ADDER(page, 0);
691 	addrs[row_start + 1] = cs | NFC_CMD_ALE | ROW_ADDER(page, 1);
692 
693 	if (nand->options & NAND_ROW_ADDR_3)
694 		addrs[row_start + 2] =
695 			cs | NFC_CMD_ALE | ROW_ADDER(page, 2);
696 	else
697 		cmd_num--;
698 
699 	/* subtract cmd1 */
700 	cmd_num--;
701 
702 	for (i = 0; i < cmd_num; i++)
703 		writel_relaxed(nfc->cmdfifo.cmd[i],
704 			       nfc->reg_base + NFC_REG_CMD);
705 
706 	if (in) {
707 		nfc->cmdfifo.rw.cmd1 = cs | NFC_CMD_CLE | NAND_CMD_READSTART;
708 		writel(nfc->cmdfifo.rw.cmd1, nfc->reg_base + NFC_REG_CMD);
709 		meson_nfc_queue_rb(nand, PSEC_TO_MSEC(sdr->tR_max), true);
710 	} else {
711 		meson_nfc_cmd_idle(nfc, nfc->timing.tadl);
712 	}
713 
714 	return 0;
715 }
716 
meson_nfc_write_page_sub(struct nand_chip * nand,int page,int raw)717 static int meson_nfc_write_page_sub(struct nand_chip *nand,
718 				    int page, int raw)
719 {
720 	const struct nand_sdr_timings *sdr =
721 		nand_get_sdr_timings(nand_get_interface_config(nand));
722 	struct mtd_info *mtd = nand_to_mtd(nand);
723 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
724 	struct meson_nfc *nfc = nand_get_controller_data(nand);
725 	int data_len, info_len;
726 	u32 cmd;
727 	int ret;
728 
729 	meson_nfc_select_chip(nand, nand->cur_cs);
730 
731 	data_len =  mtd->writesize + mtd->oobsize;
732 	info_len = nand->ecc.steps * PER_INFO_BYTE;
733 
734 	ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRWRITE);
735 	if (ret)
736 		return ret;
737 
738 	ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf,
739 					 data_len, meson_chip->info_buf,
740 					 info_len, DMA_TO_DEVICE);
741 	if (ret)
742 		return ret;
743 
744 	if (nand->options & NAND_NEED_SCRAMBLING) {
745 		meson_nfc_cmd_seed(nfc, page);
746 		meson_nfc_cmd_access(nand, raw, DIRWRITE,
747 				     NFC_CMD_SCRAMBLER_ENABLE);
748 	} else {
749 		meson_nfc_cmd_access(nand, raw, DIRWRITE,
750 				     NFC_CMD_SCRAMBLER_DISABLE);
751 	}
752 
753 	cmd = nfc->param.chip_select | NFC_CMD_CLE | NAND_CMD_PAGEPROG;
754 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
755 	meson_nfc_queue_rb(nand, PSEC_TO_MSEC(sdr->tPROG_max), false);
756 
757 	meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_TO_DEVICE);
758 
759 	return ret;
760 }
761 
meson_nfc_write_page_raw(struct nand_chip * nand,const u8 * buf,int oob_required,int page)762 static int meson_nfc_write_page_raw(struct nand_chip *nand, const u8 *buf,
763 				    int oob_required, int page)
764 {
765 	u8 *oob_buf = nand->oob_poi;
766 
767 	meson_nfc_set_data_oob(nand, buf, oob_buf);
768 
769 	return meson_nfc_write_page_sub(nand, page, 1);
770 }
771 
meson_nfc_write_page_hwecc(struct nand_chip * nand,const u8 * buf,int oob_required,int page)772 static int meson_nfc_write_page_hwecc(struct nand_chip *nand,
773 				      const u8 *buf, int oob_required, int page)
774 {
775 	struct mtd_info *mtd = nand_to_mtd(nand);
776 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
777 	u8 *oob_buf = nand->oob_poi;
778 
779 	memcpy(meson_chip->data_buf, buf, mtd->writesize);
780 	memset(meson_chip->info_buf, 0, nand->ecc.steps * PER_INFO_BYTE);
781 	meson_nfc_set_user_byte(nand, oob_buf);
782 
783 	return meson_nfc_write_page_sub(nand, page, 0);
784 }
785 
meson_nfc_check_ecc_pages_valid(struct meson_nfc * nfc,struct nand_chip * nand,int raw)786 static void meson_nfc_check_ecc_pages_valid(struct meson_nfc *nfc,
787 					    struct nand_chip *nand, int raw)
788 {
789 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
790 	__le64 *info;
791 	u32 neccpages;
792 	int ret;
793 
794 	neccpages = raw ? 1 : nand->ecc.steps;
795 	info = &meson_chip->info_buf[neccpages - 1];
796 	do {
797 		usleep_range(10, 15);
798 		/* info is updated by nfc dma engine*/
799 		smp_rmb();
800 		dma_sync_single_for_cpu(nfc->dev, nfc->iaddr, nfc->info_bytes,
801 					DMA_FROM_DEVICE);
802 		ret = *info & ECC_COMPLETE;
803 	} while (!ret);
804 }
805 
meson_nfc_read_page_sub(struct nand_chip * nand,int page,int raw)806 static int meson_nfc_read_page_sub(struct nand_chip *nand,
807 				   int page, int raw)
808 {
809 	struct mtd_info *mtd = nand_to_mtd(nand);
810 	struct meson_nfc *nfc = nand_get_controller_data(nand);
811 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
812 	int data_len, info_len;
813 	int ret;
814 
815 	meson_nfc_select_chip(nand, nand->cur_cs);
816 
817 	data_len =  mtd->writesize + mtd->oobsize;
818 	info_len = nand->ecc.steps * PER_INFO_BYTE;
819 
820 	ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRREAD);
821 	if (ret)
822 		return ret;
823 
824 	ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf,
825 					 data_len, meson_chip->info_buf,
826 					 info_len, DMA_FROM_DEVICE);
827 	if (ret)
828 		return ret;
829 
830 	if (nand->options & NAND_NEED_SCRAMBLING) {
831 		meson_nfc_cmd_seed(nfc, page);
832 		meson_nfc_cmd_access(nand, raw, DIRREAD,
833 				     NFC_CMD_SCRAMBLER_ENABLE);
834 	} else {
835 		meson_nfc_cmd_access(nand, raw, DIRREAD,
836 				     NFC_CMD_SCRAMBLER_DISABLE);
837 	}
838 
839 	ret = meson_nfc_wait_dma_finish(nfc);
840 	meson_nfc_check_ecc_pages_valid(nfc, nand, raw);
841 
842 	meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_FROM_DEVICE);
843 
844 	return ret;
845 }
846 
meson_nfc_read_page_raw(struct nand_chip * nand,u8 * buf,int oob_required,int page)847 static int meson_nfc_read_page_raw(struct nand_chip *nand, u8 *buf,
848 				   int oob_required, int page)
849 {
850 	u8 *oob_buf = nand->oob_poi;
851 	int ret;
852 
853 	ret = meson_nfc_read_page_sub(nand, page, 1);
854 	if (ret)
855 		return ret;
856 
857 	meson_nfc_get_data_oob(nand, buf, oob_buf);
858 
859 	return 0;
860 }
861 
meson_nfc_read_page_hwecc(struct nand_chip * nand,u8 * buf,int oob_required,int page)862 static int meson_nfc_read_page_hwecc(struct nand_chip *nand, u8 *buf,
863 				     int oob_required, int page)
864 {
865 	struct mtd_info *mtd = nand_to_mtd(nand);
866 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
867 	struct nand_ecc_ctrl *ecc = &nand->ecc;
868 	u64 correct_bitmap = 0;
869 	u32 bitflips = 0;
870 	u8 *oob_buf = nand->oob_poi;
871 	int ret, i;
872 
873 	ret = meson_nfc_read_page_sub(nand, page, 0);
874 	if (ret)
875 		return ret;
876 
877 	meson_nfc_get_user_byte(nand, oob_buf);
878 	ret = meson_nfc_ecc_correct(nand, &bitflips, &correct_bitmap);
879 	if (ret == ECC_CHECK_RETURN_FF) {
880 		if (buf)
881 			memset(buf, 0xff, mtd->writesize);
882 		memset(oob_buf, 0xff, mtd->oobsize);
883 	} else if (ret < 0) {
884 		if ((nand->options & NAND_NEED_SCRAMBLING) || !buf) {
885 			mtd->ecc_stats.failed++;
886 			return bitflips;
887 		}
888 		ret  = meson_nfc_read_page_raw(nand, buf, 0, page);
889 		if (ret)
890 			return ret;
891 
892 		for (i = 0; i < nand->ecc.steps ; i++) {
893 			u8 *data = buf + i * ecc->size;
894 			u8 *oob = nand->oob_poi + i * (ecc->bytes + 2);
895 
896 			if (correct_bitmap & BIT_ULL(i))
897 				continue;
898 			ret = nand_check_erased_ecc_chunk(data,	ecc->size,
899 							  oob, ecc->bytes + 2,
900 							  NULL, 0,
901 							  ecc->strength);
902 			if (ret < 0) {
903 				mtd->ecc_stats.failed++;
904 			} else {
905 				mtd->ecc_stats.corrected += ret;
906 				bitflips =  max_t(u32, bitflips, ret);
907 			}
908 		}
909 	} else if (buf && buf != meson_chip->data_buf) {
910 		memcpy(buf, meson_chip->data_buf, mtd->writesize);
911 	}
912 
913 	return bitflips;
914 }
915 
meson_nfc_read_oob_raw(struct nand_chip * nand,int page)916 static int meson_nfc_read_oob_raw(struct nand_chip *nand, int page)
917 {
918 	return meson_nfc_read_page_raw(nand, NULL, 1, page);
919 }
920 
meson_nfc_read_oob(struct nand_chip * nand,int page)921 static int meson_nfc_read_oob(struct nand_chip *nand, int page)
922 {
923 	return meson_nfc_read_page_hwecc(nand, NULL, 1, page);
924 }
925 
meson_nfc_is_buffer_dma_safe(const void * buffer)926 static bool meson_nfc_is_buffer_dma_safe(const void *buffer)
927 {
928 	if ((uintptr_t)buffer % DMA_ADDR_ALIGN)
929 		return false;
930 
931 	if (virt_addr_valid(buffer) && (!object_is_on_stack(buffer)))
932 		return true;
933 	return false;
934 }
935 
936 static void *
meson_nand_op_get_dma_safe_input_buf(const struct nand_op_instr * instr)937 meson_nand_op_get_dma_safe_input_buf(const struct nand_op_instr *instr)
938 {
939 	if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR))
940 		return NULL;
941 
942 	if (meson_nfc_is_buffer_dma_safe(instr->ctx.data.buf.in))
943 		return instr->ctx.data.buf.in;
944 
945 	return kzalloc(instr->ctx.data.len, GFP_KERNEL);
946 }
947 
948 static void
meson_nand_op_put_dma_safe_input_buf(const struct nand_op_instr * instr,void * buf)949 meson_nand_op_put_dma_safe_input_buf(const struct nand_op_instr *instr,
950 				     void *buf)
951 {
952 	if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR) ||
953 	    WARN_ON(!buf))
954 		return;
955 
956 	if (buf == instr->ctx.data.buf.in)
957 		return;
958 
959 	memcpy(instr->ctx.data.buf.in, buf, instr->ctx.data.len);
960 	kfree(buf);
961 }
962 
963 static void *
meson_nand_op_get_dma_safe_output_buf(const struct nand_op_instr * instr)964 meson_nand_op_get_dma_safe_output_buf(const struct nand_op_instr *instr)
965 {
966 	if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR))
967 		return NULL;
968 
969 	if (meson_nfc_is_buffer_dma_safe(instr->ctx.data.buf.out))
970 		return (void *)instr->ctx.data.buf.out;
971 
972 	return kmemdup(instr->ctx.data.buf.out,
973 		       instr->ctx.data.len, GFP_KERNEL);
974 }
975 
976 static void
meson_nand_op_put_dma_safe_output_buf(const struct nand_op_instr * instr,const void * buf)977 meson_nand_op_put_dma_safe_output_buf(const struct nand_op_instr *instr,
978 				      const void *buf)
979 {
980 	if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR) ||
981 	    WARN_ON(!buf))
982 		return;
983 
984 	if (buf != instr->ctx.data.buf.out)
985 		kfree(buf);
986 }
987 
meson_nfc_check_op(struct nand_chip * chip,const struct nand_operation * op)988 static int meson_nfc_check_op(struct nand_chip *chip,
989 			      const struct nand_operation *op)
990 {
991 	int op_id;
992 
993 	for (op_id = 0; op_id < op->ninstrs; op_id++) {
994 		const struct nand_op_instr *instr;
995 
996 		instr = &op->instrs[op_id];
997 
998 		switch (instr->type) {
999 		case NAND_OP_DATA_IN_INSTR:
1000 		case NAND_OP_DATA_OUT_INSTR:
1001 			if (instr->ctx.data.len > NFC_CMD_RAW_LEN)
1002 				return -ENOTSUPP;
1003 
1004 			break;
1005 		default:
1006 			break;
1007 		}
1008 	}
1009 
1010 	return 0;
1011 }
1012 
meson_nfc_exec_op(struct nand_chip * nand,const struct nand_operation * op,bool check_only)1013 static int meson_nfc_exec_op(struct nand_chip *nand,
1014 			     const struct nand_operation *op, bool check_only)
1015 {
1016 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1017 	struct meson_nfc *nfc = nand_get_controller_data(nand);
1018 	const struct nand_op_instr *instr = NULL;
1019 	void *buf;
1020 	u32 op_id, delay_idle, cmd;
1021 	int err;
1022 	int i;
1023 
1024 	err = meson_nfc_check_op(nand, op);
1025 	if (err)
1026 		return err;
1027 
1028 	if (check_only)
1029 		return 0;
1030 
1031 	meson_nfc_select_chip(nand, op->cs);
1032 	for (op_id = 0; op_id < op->ninstrs; op_id++) {
1033 		instr = &op->instrs[op_id];
1034 		delay_idle = DIV_ROUND_UP(PSEC_TO_NSEC(instr->delay_ns),
1035 					  meson_chip->level1_divider *
1036 					  NFC_CLK_CYCLE);
1037 		switch (instr->type) {
1038 		case NAND_OP_CMD_INSTR:
1039 			cmd = nfc->param.chip_select | NFC_CMD_CLE;
1040 			cmd |= instr->ctx.cmd.opcode & 0xff;
1041 			writel(cmd, nfc->reg_base + NFC_REG_CMD);
1042 			meson_nfc_cmd_idle(nfc, delay_idle);
1043 			break;
1044 
1045 		case NAND_OP_ADDR_INSTR:
1046 			for (i = 0; i < instr->ctx.addr.naddrs; i++) {
1047 				cmd = nfc->param.chip_select | NFC_CMD_ALE;
1048 				cmd |= instr->ctx.addr.addrs[i] & 0xff;
1049 				writel(cmd, nfc->reg_base + NFC_REG_CMD);
1050 			}
1051 			meson_nfc_cmd_idle(nfc, delay_idle);
1052 			break;
1053 
1054 		case NAND_OP_DATA_IN_INSTR:
1055 			buf = meson_nand_op_get_dma_safe_input_buf(instr);
1056 			if (!buf)
1057 				return -ENOMEM;
1058 			meson_nfc_read_buf(nand, buf, instr->ctx.data.len);
1059 			meson_nand_op_put_dma_safe_input_buf(instr, buf);
1060 			break;
1061 
1062 		case NAND_OP_DATA_OUT_INSTR:
1063 			buf = meson_nand_op_get_dma_safe_output_buf(instr);
1064 			if (!buf)
1065 				return -ENOMEM;
1066 			meson_nfc_write_buf(nand, buf, instr->ctx.data.len);
1067 			meson_nand_op_put_dma_safe_output_buf(instr, buf);
1068 			break;
1069 
1070 		case NAND_OP_WAITRDY_INSTR:
1071 			meson_nfc_queue_rb(nand, instr->ctx.waitrdy.timeout_ms,
1072 					   true);
1073 			if (instr->delay_ns)
1074 				meson_nfc_cmd_idle(nfc, delay_idle);
1075 			break;
1076 		}
1077 	}
1078 	meson_nfc_wait_cmd_finish(nfc, 1000);
1079 	return 0;
1080 }
1081 
meson_ooblayout_ecc(struct mtd_info * mtd,int section,struct mtd_oob_region * oobregion)1082 static int meson_ooblayout_ecc(struct mtd_info *mtd, int section,
1083 			       struct mtd_oob_region *oobregion)
1084 {
1085 	struct nand_chip *nand = mtd_to_nand(mtd);
1086 
1087 	if (section >= nand->ecc.steps)
1088 		return -ERANGE;
1089 
1090 	oobregion->offset =  2 + (section * (2 + nand->ecc.bytes));
1091 	oobregion->length = nand->ecc.bytes;
1092 
1093 	return 0;
1094 }
1095 
meson_ooblayout_free(struct mtd_info * mtd,int section,struct mtd_oob_region * oobregion)1096 static int meson_ooblayout_free(struct mtd_info *mtd, int section,
1097 				struct mtd_oob_region *oobregion)
1098 {
1099 	struct nand_chip *nand = mtd_to_nand(mtd);
1100 
1101 	if (section >= nand->ecc.steps)
1102 		return -ERANGE;
1103 
1104 	oobregion->offset = section * (2 + nand->ecc.bytes);
1105 	oobregion->length = 2;
1106 
1107 	return 0;
1108 }
1109 
1110 static const struct mtd_ooblayout_ops meson_ooblayout_ops = {
1111 	.ecc = meson_ooblayout_ecc,
1112 	.free = meson_ooblayout_free,
1113 };
1114 
meson_nfc_clk_init(struct meson_nfc * nfc)1115 static int meson_nfc_clk_init(struct meson_nfc *nfc)
1116 {
1117 	struct clk_parent_data nfc_divider_parent_data[1] = {0};
1118 	struct clk_init_data init = {0};
1119 	int ret;
1120 
1121 	/* request core clock */
1122 	nfc->core_clk = devm_clk_get(nfc->dev, "core");
1123 	if (IS_ERR(nfc->core_clk)) {
1124 		dev_err(nfc->dev, "failed to get core clock\n");
1125 		return PTR_ERR(nfc->core_clk);
1126 	}
1127 
1128 	nfc->device_clk = devm_clk_get(nfc->dev, "device");
1129 	if (IS_ERR(nfc->device_clk)) {
1130 		dev_err(nfc->dev, "failed to get device clock\n");
1131 		return PTR_ERR(nfc->device_clk);
1132 	}
1133 
1134 	init.name = devm_kasprintf(nfc->dev,
1135 				   GFP_KERNEL, "%s#div",
1136 				   dev_name(nfc->dev));
1137 	if (!init.name)
1138 		return -ENOMEM;
1139 
1140 	init.ops = &clk_divider_ops;
1141 	nfc_divider_parent_data[0].fw_name = "device";
1142 	init.parent_data = nfc_divider_parent_data;
1143 	init.num_parents = 1;
1144 	nfc->nand_divider.reg = nfc->reg_clk;
1145 	nfc->nand_divider.shift = CLK_DIV_SHIFT;
1146 	nfc->nand_divider.width = CLK_DIV_WIDTH;
1147 	nfc->nand_divider.hw.init = &init;
1148 	nfc->nand_divider.flags = CLK_DIVIDER_ONE_BASED |
1149 				  CLK_DIVIDER_ROUND_CLOSEST |
1150 				  CLK_DIVIDER_ALLOW_ZERO;
1151 
1152 	nfc->nand_clk = devm_clk_register(nfc->dev, &nfc->nand_divider.hw);
1153 	if (IS_ERR(nfc->nand_clk))
1154 		return PTR_ERR(nfc->nand_clk);
1155 
1156 	/* init SD_EMMC_CLOCK to sane defaults w/min clock rate */
1157 	writel(CLK_SELECT_NAND | readl(nfc->reg_clk),
1158 	       nfc->reg_clk);
1159 
1160 	ret = clk_prepare_enable(nfc->core_clk);
1161 	if (ret) {
1162 		dev_err(nfc->dev, "failed to enable core clock\n");
1163 		return ret;
1164 	}
1165 
1166 	ret = clk_prepare_enable(nfc->device_clk);
1167 	if (ret) {
1168 		dev_err(nfc->dev, "failed to enable device clock\n");
1169 		goto err_device_clk;
1170 	}
1171 
1172 	ret = clk_prepare_enable(nfc->nand_clk);
1173 	if (ret) {
1174 		dev_err(nfc->dev, "pre enable NFC divider fail\n");
1175 		goto err_nand_clk;
1176 	}
1177 
1178 	ret = clk_set_rate(nfc->nand_clk, 24000000);
1179 	if (ret)
1180 		goto err_disable_clk;
1181 
1182 	return 0;
1183 
1184 err_disable_clk:
1185 	clk_disable_unprepare(nfc->nand_clk);
1186 err_nand_clk:
1187 	clk_disable_unprepare(nfc->device_clk);
1188 err_device_clk:
1189 	clk_disable_unprepare(nfc->core_clk);
1190 	return ret;
1191 }
1192 
meson_nfc_disable_clk(struct meson_nfc * nfc)1193 static void meson_nfc_disable_clk(struct meson_nfc *nfc)
1194 {
1195 	clk_disable_unprepare(nfc->nand_clk);
1196 	clk_disable_unprepare(nfc->device_clk);
1197 	clk_disable_unprepare(nfc->core_clk);
1198 }
1199 
meson_nfc_free_buffer(struct nand_chip * nand)1200 static void meson_nfc_free_buffer(struct nand_chip *nand)
1201 {
1202 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1203 
1204 	kfree(meson_chip->info_buf);
1205 	kfree(meson_chip->data_buf);
1206 }
1207 
meson_chip_buffer_init(struct nand_chip * nand)1208 static int meson_chip_buffer_init(struct nand_chip *nand)
1209 {
1210 	struct mtd_info *mtd = nand_to_mtd(nand);
1211 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1212 	u32 page_bytes, info_bytes, nsectors;
1213 
1214 	nsectors = mtd->writesize / nand->ecc.size;
1215 
1216 	page_bytes =  mtd->writesize + mtd->oobsize;
1217 	info_bytes = nsectors * PER_INFO_BYTE;
1218 
1219 	meson_chip->data_buf = kmalloc(page_bytes, GFP_KERNEL);
1220 	if (!meson_chip->data_buf)
1221 		return -ENOMEM;
1222 
1223 	meson_chip->info_buf = kmalloc(info_bytes, GFP_KERNEL);
1224 	if (!meson_chip->info_buf) {
1225 		kfree(meson_chip->data_buf);
1226 		return -ENOMEM;
1227 	}
1228 
1229 	return 0;
1230 }
1231 
1232 static
meson_nfc_setup_interface(struct nand_chip * nand,int csline,const struct nand_interface_config * conf)1233 int meson_nfc_setup_interface(struct nand_chip *nand, int csline,
1234 			      const struct nand_interface_config *conf)
1235 {
1236 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1237 	const struct nand_sdr_timings *timings;
1238 	u32 div, bt_min, bt_max, tbers_clocks;
1239 
1240 	timings = nand_get_sdr_timings(conf);
1241 	if (IS_ERR(timings))
1242 		return -ENOTSUPP;
1243 
1244 	if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1245 		return 0;
1246 
1247 	div = DIV_ROUND_UP((timings->tRC_min / 1000), NFC_CLK_CYCLE);
1248 	bt_min = (timings->tREA_max + NFC_DEFAULT_DELAY) / div;
1249 	bt_max = (NFC_DEFAULT_DELAY + timings->tRHOH_min +
1250 		  timings->tRC_min / 2) / div;
1251 
1252 	meson_chip->twb = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tWB_max),
1253 				       div * NFC_CLK_CYCLE);
1254 	meson_chip->tadl = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tADL_min),
1255 					div * NFC_CLK_CYCLE);
1256 	tbers_clocks = DIV_ROUND_UP_ULL(PSEC_TO_NSEC(timings->tBERS_max),
1257 					div * NFC_CLK_CYCLE);
1258 	meson_chip->tbers_max = ilog2(tbers_clocks);
1259 	if (!is_power_of_2(tbers_clocks))
1260 		meson_chip->tbers_max++;
1261 
1262 	bt_min = DIV_ROUND_UP(bt_min, 1000);
1263 	bt_max = DIV_ROUND_UP(bt_max, 1000);
1264 
1265 	if (bt_max < bt_min)
1266 		return -EINVAL;
1267 
1268 	meson_chip->level1_divider = div;
1269 	meson_chip->clk_rate = 1000000000 / meson_chip->level1_divider;
1270 	meson_chip->bus_timing = (bt_min + bt_max) / 2 + 1;
1271 
1272 	return 0;
1273 }
1274 
meson_nand_bch_mode(struct nand_chip * nand)1275 static int meson_nand_bch_mode(struct nand_chip *nand)
1276 {
1277 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1278 	int i;
1279 
1280 	if (nand->ecc.strength > 60 || nand->ecc.strength < 8)
1281 		return -EINVAL;
1282 
1283 	for (i = 0; i < ARRAY_SIZE(meson_ecc); i++) {
1284 		if (meson_ecc[i].strength == nand->ecc.strength &&
1285 		    meson_ecc[i].size == nand->ecc.size) {
1286 			meson_chip->bch_mode = meson_ecc[i].bch;
1287 			return 0;
1288 		}
1289 	}
1290 
1291 	return -EINVAL;
1292 }
1293 
meson_nand_detach_chip(struct nand_chip * nand)1294 static void meson_nand_detach_chip(struct nand_chip *nand)
1295 {
1296 	meson_nfc_free_buffer(nand);
1297 }
1298 
meson_nand_attach_chip(struct nand_chip * nand)1299 static int meson_nand_attach_chip(struct nand_chip *nand)
1300 {
1301 	struct meson_nfc *nfc = nand_get_controller_data(nand);
1302 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1303 	struct mtd_info *mtd = nand_to_mtd(nand);
1304 	int raw_writesize;
1305 	int ret;
1306 
1307 	if (!mtd->name) {
1308 		mtd->name = devm_kasprintf(nfc->dev, GFP_KERNEL,
1309 					   "%s:nand%d",
1310 					   dev_name(nfc->dev),
1311 					   meson_chip->sels[0]);
1312 		if (!mtd->name)
1313 			return -ENOMEM;
1314 	}
1315 
1316 	raw_writesize = mtd->writesize + mtd->oobsize;
1317 	if (raw_writesize > NFC_CMD_RAW_LEN) {
1318 		dev_err(nfc->dev, "too big write size in raw mode: %d > %ld\n",
1319 			raw_writesize, NFC_CMD_RAW_LEN);
1320 		return -EINVAL;
1321 	}
1322 
1323 	if (nand->bbt_options & NAND_BBT_USE_FLASH)
1324 		nand->bbt_options |= NAND_BBT_NO_OOB;
1325 
1326 	nand->options |= NAND_NO_SUBPAGE_WRITE;
1327 
1328 	ret = nand_ecc_choose_conf(nand, nfc->data->ecc_caps,
1329 				   mtd->oobsize - 2);
1330 	if (ret) {
1331 		dev_err(nfc->dev, "failed to ECC init\n");
1332 		return -EINVAL;
1333 	}
1334 
1335 	mtd_set_ooblayout(mtd, &meson_ooblayout_ops);
1336 
1337 	ret = meson_nand_bch_mode(nand);
1338 	if (ret)
1339 		return -EINVAL;
1340 
1341 	nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
1342 	nand->ecc.write_page_raw = meson_nfc_write_page_raw;
1343 	nand->ecc.write_page = meson_nfc_write_page_hwecc;
1344 	nand->ecc.write_oob_raw = nand_write_oob_std;
1345 	nand->ecc.write_oob = nand_write_oob_std;
1346 
1347 	nand->ecc.read_page_raw = meson_nfc_read_page_raw;
1348 	nand->ecc.read_page = meson_nfc_read_page_hwecc;
1349 	nand->ecc.read_oob_raw = meson_nfc_read_oob_raw;
1350 	nand->ecc.read_oob = meson_nfc_read_oob;
1351 
1352 	if (nand->options & NAND_BUSWIDTH_16) {
1353 		dev_err(nfc->dev, "16bits bus width not supported");
1354 		return -EINVAL;
1355 	}
1356 	ret = meson_chip_buffer_init(nand);
1357 	if (ret)
1358 		return -ENOMEM;
1359 
1360 	return ret;
1361 }
1362 
1363 static const struct nand_controller_ops meson_nand_controller_ops = {
1364 	.attach_chip = meson_nand_attach_chip,
1365 	.detach_chip = meson_nand_detach_chip,
1366 	.setup_interface = meson_nfc_setup_interface,
1367 	.exec_op = meson_nfc_exec_op,
1368 };
1369 
1370 static int
meson_nfc_nand_chip_init(struct device * dev,struct meson_nfc * nfc,struct device_node * np)1371 meson_nfc_nand_chip_init(struct device *dev,
1372 			 struct meson_nfc *nfc, struct device_node *np)
1373 {
1374 	struct meson_nfc_nand_chip *meson_chip;
1375 	struct nand_chip *nand;
1376 	struct mtd_info *mtd;
1377 	int ret, i;
1378 	u32 tmp, nsels;
1379 	u32 nand_rb_val = 0;
1380 
1381 	nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
1382 	if (!nsels || nsels > MAX_CE_NUM) {
1383 		dev_err(dev, "invalid register property size\n");
1384 		return -EINVAL;
1385 	}
1386 
1387 	meson_chip = devm_kzalloc(dev, struct_size(meson_chip, sels, nsels),
1388 				  GFP_KERNEL);
1389 	if (!meson_chip)
1390 		return -ENOMEM;
1391 
1392 	meson_chip->nsels = nsels;
1393 
1394 	for (i = 0; i < nsels; i++) {
1395 		ret = of_property_read_u32_index(np, "reg", i, &tmp);
1396 		if (ret) {
1397 			dev_err(dev, "could not retrieve register property: %d\n",
1398 				ret);
1399 			return ret;
1400 		}
1401 
1402 		if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1403 			dev_err(dev, "CS %d already assigned\n", tmp);
1404 			return -EINVAL;
1405 		}
1406 	}
1407 
1408 	nand = &meson_chip->nand;
1409 	nand->controller = &nfc->controller;
1410 	nand->controller->ops = &meson_nand_controller_ops;
1411 	nand_set_flash_node(nand, np);
1412 	nand_set_controller_data(nand, nfc);
1413 
1414 	nand->options |= NAND_USES_DMA;
1415 	mtd = nand_to_mtd(nand);
1416 	mtd->owner = THIS_MODULE;
1417 	mtd->dev.parent = dev;
1418 
1419 	ret = of_property_read_u32(np, "nand-rb", &nand_rb_val);
1420 	if (ret == -EINVAL)
1421 		nfc->no_rb_pin = true;
1422 	else if (ret)
1423 		return ret;
1424 
1425 	if (nand_rb_val)
1426 		return -EINVAL;
1427 
1428 	ret = nand_scan(nand, nsels);
1429 	if (ret)
1430 		return ret;
1431 
1432 	ret = mtd_device_register(mtd, NULL, 0);
1433 	if (ret) {
1434 		dev_err(dev, "failed to register MTD device: %d\n", ret);
1435 		nand_cleanup(nand);
1436 		return ret;
1437 	}
1438 
1439 	list_add_tail(&meson_chip->node, &nfc->chips);
1440 
1441 	return 0;
1442 }
1443 
meson_nfc_nand_chip_cleanup(struct meson_nfc * nfc)1444 static void meson_nfc_nand_chip_cleanup(struct meson_nfc *nfc)
1445 {
1446 	struct meson_nfc_nand_chip *meson_chip;
1447 	struct mtd_info *mtd;
1448 
1449 	while (!list_empty(&nfc->chips)) {
1450 		meson_chip = list_first_entry(&nfc->chips,
1451 					      struct meson_nfc_nand_chip, node);
1452 		mtd = nand_to_mtd(&meson_chip->nand);
1453 		WARN_ON(mtd_device_unregister(mtd));
1454 
1455 		nand_cleanup(&meson_chip->nand);
1456 		list_del(&meson_chip->node);
1457 	}
1458 }
1459 
meson_nfc_nand_chips_init(struct device * dev,struct meson_nfc * nfc)1460 static int meson_nfc_nand_chips_init(struct device *dev,
1461 				     struct meson_nfc *nfc)
1462 {
1463 	struct device_node *np = dev->of_node;
1464 	struct device_node *nand_np;
1465 	int ret;
1466 
1467 	for_each_child_of_node(np, nand_np) {
1468 		ret = meson_nfc_nand_chip_init(dev, nfc, nand_np);
1469 		if (ret) {
1470 			meson_nfc_nand_chip_cleanup(nfc);
1471 			of_node_put(nand_np);
1472 			return ret;
1473 		}
1474 	}
1475 
1476 	return 0;
1477 }
1478 
meson_nfc_irq(int irq,void * id)1479 static irqreturn_t meson_nfc_irq(int irq, void *id)
1480 {
1481 	struct meson_nfc *nfc = id;
1482 	u32 cfg;
1483 
1484 	cfg = readl(nfc->reg_base + NFC_REG_CFG);
1485 	if (!(cfg & NFC_RB_IRQ_EN))
1486 		return IRQ_NONE;
1487 
1488 	cfg &= ~(NFC_RB_IRQ_EN);
1489 	writel(cfg, nfc->reg_base + NFC_REG_CFG);
1490 
1491 	complete(&nfc->completion);
1492 	return IRQ_HANDLED;
1493 }
1494 
1495 static const struct meson_nfc_data meson_gxl_data = {
1496 	.ecc_caps = &meson_gxl_ecc_caps,
1497 };
1498 
1499 static const struct meson_nfc_data meson_axg_data = {
1500 	.ecc_caps = &meson_axg_ecc_caps,
1501 };
1502 
1503 static const struct of_device_id meson_nfc_id_table[] = {
1504 	{
1505 		.compatible = "amlogic,meson-gxl-nfc",
1506 		.data = &meson_gxl_data,
1507 	}, {
1508 		.compatible = "amlogic,meson-axg-nfc",
1509 		.data = &meson_axg_data,
1510 	},
1511 	{}
1512 };
1513 MODULE_DEVICE_TABLE(of, meson_nfc_id_table);
1514 
meson_nfc_probe(struct platform_device * pdev)1515 static int meson_nfc_probe(struct platform_device *pdev)
1516 {
1517 	struct device *dev = &pdev->dev;
1518 	struct meson_nfc *nfc;
1519 	int ret, irq;
1520 
1521 	nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
1522 	if (!nfc)
1523 		return -ENOMEM;
1524 
1525 	nfc->data = of_device_get_match_data(&pdev->dev);
1526 	if (!nfc->data)
1527 		return -ENODEV;
1528 
1529 	nand_controller_init(&nfc->controller);
1530 	INIT_LIST_HEAD(&nfc->chips);
1531 	init_completion(&nfc->completion);
1532 
1533 	nfc->dev = dev;
1534 
1535 	nfc->reg_base = devm_platform_ioremap_resource_byname(pdev, "nfc");
1536 	if (IS_ERR(nfc->reg_base))
1537 		return PTR_ERR(nfc->reg_base);
1538 
1539 	nfc->reg_clk = devm_platform_ioremap_resource_byname(pdev, "emmc");
1540 	if (IS_ERR(nfc->reg_clk))
1541 		return PTR_ERR(nfc->reg_clk);
1542 
1543 	irq = platform_get_irq(pdev, 0);
1544 	if (irq < 0)
1545 		return -EINVAL;
1546 
1547 	ret = meson_nfc_clk_init(nfc);
1548 	if (ret) {
1549 		dev_err(dev, "failed to initialize NAND clock\n");
1550 		return ret;
1551 	}
1552 
1553 	writel(0, nfc->reg_base + NFC_REG_CFG);
1554 	ret = devm_request_irq(dev, irq, meson_nfc_irq, 0, dev_name(dev), nfc);
1555 	if (ret) {
1556 		dev_err(dev, "failed to request NFC IRQ\n");
1557 		ret = -EINVAL;
1558 		goto err_clk;
1559 	}
1560 
1561 	ret = dma_set_mask(dev, DMA_BIT_MASK(32));
1562 	if (ret) {
1563 		dev_err(dev, "failed to set DMA mask\n");
1564 		goto err_clk;
1565 	}
1566 
1567 	platform_set_drvdata(pdev, nfc);
1568 
1569 	ret = meson_nfc_nand_chips_init(dev, nfc);
1570 	if (ret) {
1571 		dev_err(dev, "failed to init NAND chips\n");
1572 		goto err_clk;
1573 	}
1574 
1575 	return 0;
1576 err_clk:
1577 	meson_nfc_disable_clk(nfc);
1578 	return ret;
1579 }
1580 
meson_nfc_remove(struct platform_device * pdev)1581 static void meson_nfc_remove(struct platform_device *pdev)
1582 {
1583 	struct meson_nfc *nfc = platform_get_drvdata(pdev);
1584 
1585 	meson_nfc_nand_chip_cleanup(nfc);
1586 
1587 	meson_nfc_disable_clk(nfc);
1588 }
1589 
1590 static struct platform_driver meson_nfc_driver = {
1591 	.probe  = meson_nfc_probe,
1592 	.remove_new = meson_nfc_remove,
1593 	.driver = {
1594 		.name  = "meson-nand",
1595 		.of_match_table = meson_nfc_id_table,
1596 	},
1597 };
1598 module_platform_driver(meson_nfc_driver);
1599 
1600 MODULE_LICENSE("Dual MIT/GPL");
1601 MODULE_AUTHOR("Liang Yang <liang.yang@amlogic.com>");
1602 MODULE_DESCRIPTION("Amlogic's Meson NAND Flash Controller driver");
1603