1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com> 4 * 5 * Derived from: 6 * https://github.com/yuq/sunxi-nfc-mtd 7 * Copyright (C) 2013 Qiang Yu <yuq825@gmail.com> 8 * 9 * https://github.com/hno/Allwinner-Info 10 * Copyright (C) 2013 Henrik Nordström <Henrik Nordström> 11 * 12 * Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com> 13 * Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org> 14 */ 15 16 #include <linux/dma-mapping.h> 17 #include <linux/slab.h> 18 #include <linux/module.h> 19 #include <linux/moduleparam.h> 20 #include <linux/platform_device.h> 21 #include <linux/of.h> 22 #include <linux/of_device.h> 23 #include <linux/mtd/mtd.h> 24 #include <linux/mtd/rawnand.h> 25 #include <linux/mtd/partitions.h> 26 #include <linux/clk.h> 27 #include <linux/delay.h> 28 #include <linux/dmaengine.h> 29 #include <linux/interrupt.h> 30 #include <linux/iopoll.h> 31 #include <linux/reset.h> 32 33 #define NFC_REG_CTL 0x0000 34 #define NFC_REG_ST 0x0004 35 #define NFC_REG_INT 0x0008 36 #define NFC_REG_TIMING_CTL 0x000C 37 #define NFC_REG_TIMING_CFG 0x0010 38 #define NFC_REG_ADDR_LOW 0x0014 39 #define NFC_REG_ADDR_HIGH 0x0018 40 #define NFC_REG_SECTOR_NUM 0x001C 41 #define NFC_REG_CNT 0x0020 42 #define NFC_REG_CMD 0x0024 43 #define NFC_REG_RCMD_SET 0x0028 44 #define NFC_REG_WCMD_SET 0x002C 45 #define NFC_REG_A10_IO_DATA 0x0030 46 #define NFC_REG_A23_IO_DATA 0x0300 47 #define NFC_REG_ECC_CTL 0x0034 48 #define NFC_REG_ECC_ST 0x0038 49 #define NFC_REG_DEBUG 0x003C 50 #define NFC_REG_ECC_ERR_CNT(x) ((0x0040 + (x)) & ~0x3) 51 #define NFC_REG_USER_DATA(x) (0x0050 + ((x) * 4)) 52 #define NFC_REG_SPARE_AREA 0x00A0 53 #define NFC_REG_PAT_ID 0x00A4 54 #define NFC_RAM0_BASE 0x0400 55 #define NFC_RAM1_BASE 0x0800 56 57 /* define bit use in NFC_CTL */ 58 #define NFC_EN BIT(0) 59 #define NFC_RESET BIT(1) 60 #define NFC_BUS_WIDTH_MSK BIT(2) 61 #define NFC_BUS_WIDTH_8 (0 << 2) 62 #define NFC_BUS_WIDTH_16 (1 << 2) 63 #define NFC_RB_SEL_MSK BIT(3) 64 #define NFC_RB_SEL(x) ((x) << 3) 65 #define NFC_CE_SEL_MSK GENMASK(26, 24) 66 #define NFC_CE_SEL(x) ((x) << 24) 67 #define NFC_CE_CTL BIT(6) 68 #define NFC_PAGE_SHIFT_MSK GENMASK(11, 8) 69 #define NFC_PAGE_SHIFT(x) (((x) < 10 ? 0 : (x) - 10) << 8) 70 #define NFC_SAM BIT(12) 71 #define NFC_RAM_METHOD BIT(14) 72 #define NFC_DEBUG_CTL BIT(31) 73 74 /* define bit use in NFC_ST */ 75 #define NFC_RB_B2R BIT(0) 76 #define NFC_CMD_INT_FLAG BIT(1) 77 #define NFC_DMA_INT_FLAG BIT(2) 78 #define NFC_CMD_FIFO_STATUS BIT(3) 79 #define NFC_STA BIT(4) 80 #define NFC_NATCH_INT_FLAG BIT(5) 81 #define NFC_RB_STATE(x) BIT(x + 8) 82 83 /* define bit use in NFC_INT */ 84 #define NFC_B2R_INT_ENABLE BIT(0) 85 #define NFC_CMD_INT_ENABLE BIT(1) 86 #define NFC_DMA_INT_ENABLE BIT(2) 87 #define NFC_INT_MASK (NFC_B2R_INT_ENABLE | \ 88 NFC_CMD_INT_ENABLE | \ 89 NFC_DMA_INT_ENABLE) 90 91 /* define bit use in NFC_TIMING_CTL */ 92 #define NFC_TIMING_CTL_EDO BIT(8) 93 94 /* define NFC_TIMING_CFG register layout */ 95 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD) \ 96 (((tWB) & 0x3) | (((tADL) & 0x3) << 2) | \ 97 (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) | \ 98 (((tCAD) & 0x7) << 8)) 99 100 /* define bit use in NFC_CMD */ 101 #define NFC_CMD_LOW_BYTE_MSK GENMASK(7, 0) 102 #define NFC_CMD_HIGH_BYTE_MSK GENMASK(15, 8) 103 #define NFC_CMD(x) (x) 104 #define NFC_ADR_NUM_MSK GENMASK(18, 16) 105 #define NFC_ADR_NUM(x) (((x) - 1) << 16) 106 #define NFC_SEND_ADR BIT(19) 107 #define NFC_ACCESS_DIR BIT(20) 108 #define NFC_DATA_TRANS BIT(21) 109 #define NFC_SEND_CMD1 BIT(22) 110 #define NFC_WAIT_FLAG BIT(23) 111 #define NFC_SEND_CMD2 BIT(24) 112 #define NFC_SEQ BIT(25) 113 #define NFC_DATA_SWAP_METHOD BIT(26) 114 #define NFC_ROW_AUTO_INC BIT(27) 115 #define NFC_SEND_CMD3 BIT(28) 116 #define NFC_SEND_CMD4 BIT(29) 117 #define NFC_CMD_TYPE_MSK GENMASK(31, 30) 118 #define NFC_NORMAL_OP (0 << 30) 119 #define NFC_ECC_OP (1 << 30) 120 #define NFC_PAGE_OP (2U << 30) 121 122 /* define bit use in NFC_RCMD_SET */ 123 #define NFC_READ_CMD_MSK GENMASK(7, 0) 124 #define NFC_RND_READ_CMD0_MSK GENMASK(15, 8) 125 #define NFC_RND_READ_CMD1_MSK GENMASK(23, 16) 126 127 /* define bit use in NFC_WCMD_SET */ 128 #define NFC_PROGRAM_CMD_MSK GENMASK(7, 0) 129 #define NFC_RND_WRITE_CMD_MSK GENMASK(15, 8) 130 #define NFC_READ_CMD0_MSK GENMASK(23, 16) 131 #define NFC_READ_CMD1_MSK GENMASK(31, 24) 132 133 /* define bit use in NFC_ECC_CTL */ 134 #define NFC_ECC_EN BIT(0) 135 #define NFC_ECC_PIPELINE BIT(3) 136 #define NFC_ECC_EXCEPTION BIT(4) 137 #define NFC_ECC_BLOCK_SIZE_MSK BIT(5) 138 #define NFC_ECC_BLOCK_512 BIT(5) 139 #define NFC_RANDOM_EN BIT(9) 140 #define NFC_RANDOM_DIRECTION BIT(10) 141 #define NFC_ECC_MODE_MSK GENMASK(15, 12) 142 #define NFC_ECC_MODE(x) ((x) << 12) 143 #define NFC_RANDOM_SEED_MSK GENMASK(30, 16) 144 #define NFC_RANDOM_SEED(x) ((x) << 16) 145 146 /* define bit use in NFC_ECC_ST */ 147 #define NFC_ECC_ERR(x) BIT(x) 148 #define NFC_ECC_ERR_MSK GENMASK(15, 0) 149 #define NFC_ECC_PAT_FOUND(x) BIT(x + 16) 150 #define NFC_ECC_ERR_CNT(b, x) (((x) >> (((b) % 4) * 8)) & 0xff) 151 152 #define NFC_DEFAULT_TIMEOUT_MS 1000 153 154 #define NFC_SRAM_SIZE 1024 155 156 #define NFC_MAX_CS 7 157 158 /** 159 * struct sunxi_nand_chip_sel - stores information related to NAND Chip Select 160 * 161 * @cs: the NAND CS id used to communicate with a NAND Chip 162 * @rb: the Ready/Busy pin ID. -1 means no R/B pin connected to the NFC 163 */ 164 struct sunxi_nand_chip_sel { 165 u8 cs; 166 s8 rb; 167 }; 168 169 /** 170 * struct sunxi_nand_hw_ecc - stores information related to HW ECC support 171 * 172 * @mode: the sunxi ECC mode field deduced from ECC requirements 173 */ 174 struct sunxi_nand_hw_ecc { 175 int mode; 176 }; 177 178 /** 179 * struct sunxi_nand_chip - stores NAND chip device related information 180 * 181 * @node: used to store NAND chips into a list 182 * @nand: base NAND chip structure 183 * @clk_rate: clk_rate required for this NAND chip 184 * @timing_cfg: TIMING_CFG register value for this NAND chip 185 * @timing_ctl: TIMING_CTL register value for this NAND chip 186 * @nsels: number of CS lines required by the NAND chip 187 * @sels: array of CS lines descriptions 188 */ 189 struct sunxi_nand_chip { 190 struct list_head node; 191 struct nand_chip nand; 192 unsigned long clk_rate; 193 u32 timing_cfg; 194 u32 timing_ctl; 195 int nsels; 196 struct sunxi_nand_chip_sel sels[0]; 197 }; 198 199 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand) 200 { 201 return container_of(nand, struct sunxi_nand_chip, nand); 202 } 203 204 /* 205 * NAND Controller capabilities structure: stores NAND controller capabilities 206 * for distinction between compatible strings. 207 * 208 * @sram_through_ahb: On A23, we choose to access the internal RAM through AHB 209 * instead of MBUS (less configuration). A10, A10s, A13 and 210 * A20 use the MBUS but no extra configuration is needed. 211 * @reg_io_data: I/O data register 212 * @dma_maxburst: DMA maxburst 213 */ 214 struct sunxi_nfc_caps { 215 bool sram_through_ahb; 216 unsigned int reg_io_data; 217 unsigned int dma_maxburst; 218 }; 219 220 /** 221 * struct sunxi_nfc - stores sunxi NAND controller information 222 * 223 * @controller: base controller structure 224 * @dev: parent device (used to print error messages) 225 * @regs: NAND controller registers 226 * @ahb_clk: NAND controller AHB clock 227 * @mod_clk: NAND controller mod clock 228 * @reset: NAND controller reset line 229 * @assigned_cs: bitmask describing already assigned CS lines 230 * @clk_rate: NAND controller current clock rate 231 * @chips: a list containing all the NAND chips attached to this NAND 232 * controller 233 * @complete: a completion object used to wait for NAND controller events 234 * @dmac: the DMA channel attached to the NAND controller 235 */ 236 struct sunxi_nfc { 237 struct nand_controller controller; 238 struct device *dev; 239 void __iomem *regs; 240 struct clk *ahb_clk; 241 struct clk *mod_clk; 242 struct reset_control *reset; 243 unsigned long assigned_cs; 244 unsigned long clk_rate; 245 struct list_head chips; 246 struct completion complete; 247 struct dma_chan *dmac; 248 const struct sunxi_nfc_caps *caps; 249 }; 250 251 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_controller *ctrl) 252 { 253 return container_of(ctrl, struct sunxi_nfc, controller); 254 } 255 256 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id) 257 { 258 struct sunxi_nfc *nfc = dev_id; 259 u32 st = readl(nfc->regs + NFC_REG_ST); 260 u32 ien = readl(nfc->regs + NFC_REG_INT); 261 262 if (!(ien & st)) 263 return IRQ_NONE; 264 265 if ((ien & st) == ien) 266 complete(&nfc->complete); 267 268 writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST); 269 writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT); 270 271 return IRQ_HANDLED; 272 } 273 274 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events, 275 bool use_polling, unsigned int timeout_ms) 276 { 277 int ret; 278 279 if (events & ~NFC_INT_MASK) 280 return -EINVAL; 281 282 if (!timeout_ms) 283 timeout_ms = NFC_DEFAULT_TIMEOUT_MS; 284 285 if (!use_polling) { 286 init_completion(&nfc->complete); 287 288 writel(events, nfc->regs + NFC_REG_INT); 289 290 ret = wait_for_completion_timeout(&nfc->complete, 291 msecs_to_jiffies(timeout_ms)); 292 if (!ret) 293 ret = -ETIMEDOUT; 294 else 295 ret = 0; 296 297 writel(0, nfc->regs + NFC_REG_INT); 298 } else { 299 u32 status; 300 301 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status, 302 (status & events) == events, 1, 303 timeout_ms * 1000); 304 } 305 306 writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST); 307 308 if (ret) 309 dev_err(nfc->dev, "wait interrupt timedout\n"); 310 311 return ret; 312 } 313 314 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc) 315 { 316 u32 status; 317 int ret; 318 319 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status, 320 !(status & NFC_CMD_FIFO_STATUS), 1, 321 NFC_DEFAULT_TIMEOUT_MS * 1000); 322 if (ret) 323 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n"); 324 325 return ret; 326 } 327 328 static int sunxi_nfc_rst(struct sunxi_nfc *nfc) 329 { 330 u32 ctl; 331 int ret; 332 333 writel(0, nfc->regs + NFC_REG_ECC_CTL); 334 writel(NFC_RESET, nfc->regs + NFC_REG_CTL); 335 336 ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl, 337 !(ctl & NFC_RESET), 1, 338 NFC_DEFAULT_TIMEOUT_MS * 1000); 339 if (ret) 340 dev_err(nfc->dev, "wait for NAND controller reset timedout\n"); 341 342 return ret; 343 } 344 345 static int sunxi_nfc_dma_op_prepare(struct sunxi_nfc *nfc, const void *buf, 346 int chunksize, int nchunks, 347 enum dma_data_direction ddir, 348 struct scatterlist *sg) 349 { 350 struct dma_async_tx_descriptor *dmad; 351 enum dma_transfer_direction tdir; 352 dma_cookie_t dmat; 353 int ret; 354 355 if (ddir == DMA_FROM_DEVICE) 356 tdir = DMA_DEV_TO_MEM; 357 else 358 tdir = DMA_MEM_TO_DEV; 359 360 sg_init_one(sg, buf, nchunks * chunksize); 361 ret = dma_map_sg(nfc->dev, sg, 1, ddir); 362 if (!ret) 363 return -ENOMEM; 364 365 dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK); 366 if (!dmad) { 367 ret = -EINVAL; 368 goto err_unmap_buf; 369 } 370 371 /* 372 * On A23, we suppose the "internal RAM" (p.12 of the NFC user manual) 373 * refers to the NAND controller's internal SRAM. This memory is mapped 374 * and so is accessible from the AHB. It seems that it can also be 375 * accessed by the MBUS. MBUS accesses are mandatory when using the 376 * internal DMA instead of the external DMA engine. 377 * 378 * During DMA I/O operation, either we access this memory from the AHB 379 * by clearing the NFC_RAM_METHOD bit, or we set the bit and use the 380 * MBUS. In this case, we should also configure the MBUS DMA length 381 * NFC_REG_MDMA_CNT(0xC4) to be chunksize * nchunks. NAND I/O over MBUS 382 * are also limited to 32kiB pages. 383 */ 384 if (nfc->caps->sram_through_ahb) 385 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD, 386 nfc->regs + NFC_REG_CTL); 387 else 388 writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD, 389 nfc->regs + NFC_REG_CTL); 390 391 writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM); 392 writel(chunksize, nfc->regs + NFC_REG_CNT); 393 394 dmat = dmaengine_submit(dmad); 395 396 ret = dma_submit_error(dmat); 397 if (ret) 398 goto err_clr_dma_flag; 399 400 return 0; 401 402 err_clr_dma_flag: 403 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD, 404 nfc->regs + NFC_REG_CTL); 405 406 err_unmap_buf: 407 dma_unmap_sg(nfc->dev, sg, 1, ddir); 408 return ret; 409 } 410 411 static void sunxi_nfc_dma_op_cleanup(struct sunxi_nfc *nfc, 412 enum dma_data_direction ddir, 413 struct scatterlist *sg) 414 { 415 dma_unmap_sg(nfc->dev, sg, 1, ddir); 416 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD, 417 nfc->regs + NFC_REG_CTL); 418 } 419 420 static void sunxi_nfc_select_chip(struct nand_chip *nand, unsigned int cs) 421 { 422 struct mtd_info *mtd = nand_to_mtd(nand); 423 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 424 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 425 struct sunxi_nand_chip_sel *sel; 426 u32 ctl; 427 428 if (cs > 0 && cs >= sunxi_nand->nsels) 429 return; 430 431 ctl = readl(nfc->regs + NFC_REG_CTL) & 432 ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN); 433 434 sel = &sunxi_nand->sels[cs]; 435 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN | NFC_PAGE_SHIFT(nand->page_shift); 436 if (sel->rb >= 0) 437 ctl |= NFC_RB_SEL(sel->rb); 438 439 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA); 440 441 if (nfc->clk_rate != sunxi_nand->clk_rate) { 442 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate); 443 nfc->clk_rate = sunxi_nand->clk_rate; 444 } 445 446 writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL); 447 writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG); 448 writel(ctl, nfc->regs + NFC_REG_CTL); 449 } 450 451 static void sunxi_nfc_read_buf(struct nand_chip *nand, uint8_t *buf, int len) 452 { 453 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 454 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 455 int ret; 456 int cnt; 457 int offs = 0; 458 u32 tmp; 459 460 while (len > offs) { 461 bool poll = false; 462 463 cnt = min(len - offs, NFC_SRAM_SIZE); 464 465 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 466 if (ret) 467 break; 468 469 writel(cnt, nfc->regs + NFC_REG_CNT); 470 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD; 471 writel(tmp, nfc->regs + NFC_REG_CMD); 472 473 /* Arbitrary limit for polling mode */ 474 if (cnt < 64) 475 poll = true; 476 477 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0); 478 if (ret) 479 break; 480 481 if (buf) 482 memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE, 483 cnt); 484 offs += cnt; 485 } 486 } 487 488 static void sunxi_nfc_write_buf(struct nand_chip *nand, const uint8_t *buf, 489 int len) 490 { 491 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 492 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 493 int ret; 494 int cnt; 495 int offs = 0; 496 u32 tmp; 497 498 while (len > offs) { 499 bool poll = false; 500 501 cnt = min(len - offs, NFC_SRAM_SIZE); 502 503 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 504 if (ret) 505 break; 506 507 writel(cnt, nfc->regs + NFC_REG_CNT); 508 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt); 509 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | 510 NFC_ACCESS_DIR; 511 writel(tmp, nfc->regs + NFC_REG_CMD); 512 513 /* Arbitrary limit for polling mode */ 514 if (cnt < 64) 515 poll = true; 516 517 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0); 518 if (ret) 519 break; 520 521 offs += cnt; 522 } 523 } 524 525 /* These seed values have been extracted from Allwinner's BSP */ 526 static const u16 sunxi_nfc_randomizer_page_seeds[] = { 527 0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72, 528 0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436, 529 0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d, 530 0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130, 531 0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56, 532 0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55, 533 0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb, 534 0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17, 535 0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62, 536 0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064, 537 0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126, 538 0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e, 539 0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3, 540 0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b, 541 0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d, 542 0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db, 543 }; 544 545 /* 546 * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds 547 * have been generated using 548 * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what 549 * the randomizer engine does internally before de/scrambling OOB data. 550 * 551 * Those tables are statically defined to avoid calculating randomizer state 552 * at runtime. 553 */ 554 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = { 555 0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64, 556 0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409, 557 0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617, 558 0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d, 559 0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91, 560 0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d, 561 0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab, 562 0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8, 563 0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8, 564 0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b, 565 0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5, 566 0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a, 567 0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891, 568 0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36, 569 0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd, 570 0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0, 571 }; 572 573 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = { 574 0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6, 575 0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982, 576 0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9, 577 0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07, 578 0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e, 579 0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2, 580 0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c, 581 0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f, 582 0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc, 583 0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e, 584 0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8, 585 0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68, 586 0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d, 587 0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179, 588 0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601, 589 0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd, 590 }; 591 592 static u16 sunxi_nfc_randomizer_step(u16 state, int count) 593 { 594 state &= 0x7fff; 595 596 /* 597 * This loop is just a simple implementation of a Fibonacci LFSR using 598 * the x16 + x15 + 1 polynomial. 599 */ 600 while (count--) 601 state = ((state >> 1) | 602 (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff; 603 604 return state; 605 } 606 607 static u16 sunxi_nfc_randomizer_state(struct nand_chip *nand, int page, 608 bool ecc) 609 { 610 struct mtd_info *mtd = nand_to_mtd(nand); 611 const u16 *seeds = sunxi_nfc_randomizer_page_seeds; 612 int mod = mtd_div_by_ws(mtd->erasesize, mtd); 613 614 if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds)) 615 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds); 616 617 if (ecc) { 618 if (mtd->ecc_step_size == 512) 619 seeds = sunxi_nfc_randomizer_ecc512_seeds; 620 else 621 seeds = sunxi_nfc_randomizer_ecc1024_seeds; 622 } 623 624 return seeds[page % mod]; 625 } 626 627 static void sunxi_nfc_randomizer_config(struct nand_chip *nand, int page, 628 bool ecc) 629 { 630 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 631 u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); 632 u16 state; 633 634 if (!(nand->options & NAND_NEED_SCRAMBLING)) 635 return; 636 637 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); 638 state = sunxi_nfc_randomizer_state(nand, page, ecc); 639 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK; 640 writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL); 641 } 642 643 static void sunxi_nfc_randomizer_enable(struct nand_chip *nand) 644 { 645 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 646 647 if (!(nand->options & NAND_NEED_SCRAMBLING)) 648 return; 649 650 writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN, 651 nfc->regs + NFC_REG_ECC_CTL); 652 } 653 654 static void sunxi_nfc_randomizer_disable(struct nand_chip *nand) 655 { 656 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 657 658 if (!(nand->options & NAND_NEED_SCRAMBLING)) 659 return; 660 661 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN, 662 nfc->regs + NFC_REG_ECC_CTL); 663 } 664 665 static void sunxi_nfc_randomize_bbm(struct nand_chip *nand, int page, u8 *bbm) 666 { 667 u16 state = sunxi_nfc_randomizer_state(nand, page, true); 668 669 bbm[0] ^= state; 670 bbm[1] ^= sunxi_nfc_randomizer_step(state, 8); 671 } 672 673 static void sunxi_nfc_randomizer_write_buf(struct nand_chip *nand, 674 const uint8_t *buf, int len, 675 bool ecc, int page) 676 { 677 sunxi_nfc_randomizer_config(nand, page, ecc); 678 sunxi_nfc_randomizer_enable(nand); 679 sunxi_nfc_write_buf(nand, buf, len); 680 sunxi_nfc_randomizer_disable(nand); 681 } 682 683 static void sunxi_nfc_randomizer_read_buf(struct nand_chip *nand, uint8_t *buf, 684 int len, bool ecc, int page) 685 { 686 sunxi_nfc_randomizer_config(nand, page, ecc); 687 sunxi_nfc_randomizer_enable(nand); 688 sunxi_nfc_read_buf(nand, buf, len); 689 sunxi_nfc_randomizer_disable(nand); 690 } 691 692 static void sunxi_nfc_hw_ecc_enable(struct nand_chip *nand) 693 { 694 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 695 struct sunxi_nand_hw_ecc *data = nand->ecc.priv; 696 u32 ecc_ctl; 697 698 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); 699 ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE | 700 NFC_ECC_BLOCK_SIZE_MSK); 701 ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION | 702 NFC_ECC_PIPELINE; 703 704 if (nand->ecc.size == 512) 705 ecc_ctl |= NFC_ECC_BLOCK_512; 706 707 writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL); 708 } 709 710 static void sunxi_nfc_hw_ecc_disable(struct nand_chip *nand) 711 { 712 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 713 714 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN, 715 nfc->regs + NFC_REG_ECC_CTL); 716 } 717 718 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf) 719 { 720 buf[0] = user_data; 721 buf[1] = user_data >> 8; 722 buf[2] = user_data >> 16; 723 buf[3] = user_data >> 24; 724 } 725 726 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf) 727 { 728 return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24); 729 } 730 731 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct nand_chip *nand, u8 *oob, 732 int step, bool bbm, int page) 733 { 734 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 735 736 sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)), 737 oob); 738 739 /* De-randomize the Bad Block Marker. */ 740 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) 741 sunxi_nfc_randomize_bbm(nand, page, oob); 742 } 743 744 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand, 745 const u8 *oob, int step, 746 bool bbm, int page) 747 { 748 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 749 u8 user_data[4]; 750 751 /* Randomize the Bad Block Marker. */ 752 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) { 753 memcpy(user_data, oob, sizeof(user_data)); 754 sunxi_nfc_randomize_bbm(nand, page, user_data); 755 oob = user_data; 756 } 757 758 writel(sunxi_nfc_buf_to_user_data(oob), 759 nfc->regs + NFC_REG_USER_DATA(step)); 760 } 761 762 static void sunxi_nfc_hw_ecc_update_stats(struct nand_chip *nand, 763 unsigned int *max_bitflips, int ret) 764 { 765 struct mtd_info *mtd = nand_to_mtd(nand); 766 767 if (ret < 0) { 768 mtd->ecc_stats.failed++; 769 } else { 770 mtd->ecc_stats.corrected += ret; 771 *max_bitflips = max_t(unsigned int, *max_bitflips, ret); 772 } 773 } 774 775 static int sunxi_nfc_hw_ecc_correct(struct nand_chip *nand, u8 *data, u8 *oob, 776 int step, u32 status, bool *erased) 777 { 778 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 779 struct nand_ecc_ctrl *ecc = &nand->ecc; 780 u32 tmp; 781 782 *erased = false; 783 784 if (status & NFC_ECC_ERR(step)) 785 return -EBADMSG; 786 787 if (status & NFC_ECC_PAT_FOUND(step)) { 788 u8 pattern; 789 790 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) { 791 pattern = 0x0; 792 } else { 793 pattern = 0xff; 794 *erased = true; 795 } 796 797 if (data) 798 memset(data, pattern, ecc->size); 799 800 if (oob) 801 memset(oob, pattern, ecc->bytes + 4); 802 803 return 0; 804 } 805 806 tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step)); 807 808 return NFC_ECC_ERR_CNT(step, tmp); 809 } 810 811 static int sunxi_nfc_hw_ecc_read_chunk(struct nand_chip *nand, 812 u8 *data, int data_off, 813 u8 *oob, int oob_off, 814 int *cur_off, 815 unsigned int *max_bitflips, 816 bool bbm, bool oob_required, int page) 817 { 818 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 819 struct nand_ecc_ctrl *ecc = &nand->ecc; 820 int raw_mode = 0; 821 bool erased; 822 int ret; 823 824 if (*cur_off != data_off) 825 nand_change_read_column_op(nand, data_off, NULL, 0, false); 826 827 sunxi_nfc_randomizer_read_buf(nand, NULL, ecc->size, false, page); 828 829 if (data_off + ecc->size != oob_off) 830 nand_change_read_column_op(nand, oob_off, NULL, 0, false); 831 832 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 833 if (ret) 834 return ret; 835 836 sunxi_nfc_randomizer_enable(nand); 837 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP, 838 nfc->regs + NFC_REG_CMD); 839 840 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 841 sunxi_nfc_randomizer_disable(nand); 842 if (ret) 843 return ret; 844 845 *cur_off = oob_off + ecc->bytes + 4; 846 847 ret = sunxi_nfc_hw_ecc_correct(nand, data, oob_required ? oob : NULL, 0, 848 readl(nfc->regs + NFC_REG_ECC_ST), 849 &erased); 850 if (erased) 851 return 1; 852 853 if (ret < 0) { 854 /* 855 * Re-read the data with the randomizer disabled to identify 856 * bitflips in erased pages. 857 */ 858 if (nand->options & NAND_NEED_SCRAMBLING) 859 nand_change_read_column_op(nand, data_off, data, 860 ecc->size, false); 861 else 862 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, 863 ecc->size); 864 865 nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4, 866 false); 867 868 ret = nand_check_erased_ecc_chunk(data, ecc->size, 869 oob, ecc->bytes + 4, 870 NULL, 0, ecc->strength); 871 if (ret >= 0) 872 raw_mode = 1; 873 } else { 874 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size); 875 876 if (oob_required) { 877 nand_change_read_column_op(nand, oob_off, NULL, 0, 878 false); 879 sunxi_nfc_randomizer_read_buf(nand, oob, ecc->bytes + 4, 880 true, page); 881 882 sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, 0, 883 bbm, page); 884 } 885 } 886 887 sunxi_nfc_hw_ecc_update_stats(nand, max_bitflips, ret); 888 889 return raw_mode; 890 } 891 892 static void sunxi_nfc_hw_ecc_read_extra_oob(struct nand_chip *nand, 893 u8 *oob, int *cur_off, 894 bool randomize, int page) 895 { 896 struct mtd_info *mtd = nand_to_mtd(nand); 897 struct nand_ecc_ctrl *ecc = &nand->ecc; 898 int offset = ((ecc->bytes + 4) * ecc->steps); 899 int len = mtd->oobsize - offset; 900 901 if (len <= 0) 902 return; 903 904 if (!cur_off || *cur_off != offset) 905 nand_change_read_column_op(nand, mtd->writesize, NULL, 0, 906 false); 907 908 if (!randomize) 909 sunxi_nfc_read_buf(nand, oob + offset, len); 910 else 911 sunxi_nfc_randomizer_read_buf(nand, oob + offset, len, 912 false, page); 913 914 if (cur_off) 915 *cur_off = mtd->oobsize + mtd->writesize; 916 } 917 918 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf, 919 int oob_required, int page, 920 int nchunks) 921 { 922 bool randomized = nand->options & NAND_NEED_SCRAMBLING; 923 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 924 struct mtd_info *mtd = nand_to_mtd(nand); 925 struct nand_ecc_ctrl *ecc = &nand->ecc; 926 unsigned int max_bitflips = 0; 927 int ret, i, raw_mode = 0; 928 struct scatterlist sg; 929 u32 status; 930 931 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 932 if (ret) 933 return ret; 934 935 ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, nchunks, 936 DMA_FROM_DEVICE, &sg); 937 if (ret) 938 return ret; 939 940 sunxi_nfc_hw_ecc_enable(nand); 941 sunxi_nfc_randomizer_config(nand, page, false); 942 sunxi_nfc_randomizer_enable(nand); 943 944 writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) | 945 NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET); 946 947 dma_async_issue_pending(nfc->dmac); 948 949 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS, 950 nfc->regs + NFC_REG_CMD); 951 952 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 953 if (ret) 954 dmaengine_terminate_all(nfc->dmac); 955 956 sunxi_nfc_randomizer_disable(nand); 957 sunxi_nfc_hw_ecc_disable(nand); 958 959 sunxi_nfc_dma_op_cleanup(nfc, DMA_FROM_DEVICE, &sg); 960 961 if (ret) 962 return ret; 963 964 status = readl(nfc->regs + NFC_REG_ECC_ST); 965 966 for (i = 0; i < nchunks; i++) { 967 int data_off = i * ecc->size; 968 int oob_off = i * (ecc->bytes + 4); 969 u8 *data = buf + data_off; 970 u8 *oob = nand->oob_poi + oob_off; 971 bool erased; 972 973 ret = sunxi_nfc_hw_ecc_correct(nand, randomized ? data : NULL, 974 oob_required ? oob : NULL, 975 i, status, &erased); 976 977 /* ECC errors are handled in the second loop. */ 978 if (ret < 0) 979 continue; 980 981 if (oob_required && !erased) { 982 /* TODO: use DMA to retrieve OOB */ 983 nand_change_read_column_op(nand, 984 mtd->writesize + oob_off, 985 oob, ecc->bytes + 4, false); 986 987 sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, i, 988 !i, page); 989 } 990 991 if (erased) 992 raw_mode = 1; 993 994 sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret); 995 } 996 997 if (status & NFC_ECC_ERR_MSK) { 998 for (i = 0; i < nchunks; i++) { 999 int data_off = i * ecc->size; 1000 int oob_off = i * (ecc->bytes + 4); 1001 u8 *data = buf + data_off; 1002 u8 *oob = nand->oob_poi + oob_off; 1003 1004 if (!(status & NFC_ECC_ERR(i))) 1005 continue; 1006 1007 /* 1008 * Re-read the data with the randomizer disabled to 1009 * identify bitflips in erased pages. 1010 * TODO: use DMA to read page in raw mode 1011 */ 1012 if (randomized) 1013 nand_change_read_column_op(nand, data_off, 1014 data, ecc->size, 1015 false); 1016 1017 /* TODO: use DMA to retrieve OOB */ 1018 nand_change_read_column_op(nand, 1019 mtd->writesize + oob_off, 1020 oob, ecc->bytes + 4, false); 1021 1022 ret = nand_check_erased_ecc_chunk(data, ecc->size, 1023 oob, ecc->bytes + 4, 1024 NULL, 0, 1025 ecc->strength); 1026 if (ret >= 0) 1027 raw_mode = 1; 1028 1029 sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret); 1030 } 1031 } 1032 1033 if (oob_required) 1034 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi, 1035 NULL, !raw_mode, 1036 page); 1037 1038 return max_bitflips; 1039 } 1040 1041 static int sunxi_nfc_hw_ecc_write_chunk(struct nand_chip *nand, 1042 const u8 *data, int data_off, 1043 const u8 *oob, int oob_off, 1044 int *cur_off, bool bbm, 1045 int page) 1046 { 1047 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1048 struct nand_ecc_ctrl *ecc = &nand->ecc; 1049 int ret; 1050 1051 if (data_off != *cur_off) 1052 nand_change_write_column_op(nand, data_off, NULL, 0, false); 1053 1054 sunxi_nfc_randomizer_write_buf(nand, data, ecc->size, false, page); 1055 1056 if (data_off + ecc->size != oob_off) 1057 nand_change_write_column_op(nand, oob_off, NULL, 0, false); 1058 1059 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1060 if (ret) 1061 return ret; 1062 1063 sunxi_nfc_randomizer_enable(nand); 1064 sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, 0, bbm, page); 1065 1066 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | 1067 NFC_ACCESS_DIR | NFC_ECC_OP, 1068 nfc->regs + NFC_REG_CMD); 1069 1070 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 1071 sunxi_nfc_randomizer_disable(nand); 1072 if (ret) 1073 return ret; 1074 1075 *cur_off = oob_off + ecc->bytes + 4; 1076 1077 return 0; 1078 } 1079 1080 static void sunxi_nfc_hw_ecc_write_extra_oob(struct nand_chip *nand, 1081 u8 *oob, int *cur_off, 1082 int page) 1083 { 1084 struct mtd_info *mtd = nand_to_mtd(nand); 1085 struct nand_ecc_ctrl *ecc = &nand->ecc; 1086 int offset = ((ecc->bytes + 4) * ecc->steps); 1087 int len = mtd->oobsize - offset; 1088 1089 if (len <= 0) 1090 return; 1091 1092 if (!cur_off || *cur_off != offset) 1093 nand_change_write_column_op(nand, offset + mtd->writesize, 1094 NULL, 0, false); 1095 1096 sunxi_nfc_randomizer_write_buf(nand, oob + offset, len, false, page); 1097 1098 if (cur_off) 1099 *cur_off = mtd->oobsize + mtd->writesize; 1100 } 1101 1102 static int sunxi_nfc_hw_ecc_read_page(struct nand_chip *nand, uint8_t *buf, 1103 int oob_required, int page) 1104 { 1105 struct mtd_info *mtd = nand_to_mtd(nand); 1106 struct nand_ecc_ctrl *ecc = &nand->ecc; 1107 unsigned int max_bitflips = 0; 1108 int ret, i, cur_off = 0; 1109 bool raw_mode = false; 1110 1111 sunxi_nfc_select_chip(nand, nand->cur_cs); 1112 1113 nand_read_page_op(nand, page, 0, NULL, 0); 1114 1115 sunxi_nfc_hw_ecc_enable(nand); 1116 1117 for (i = 0; i < ecc->steps; i++) { 1118 int data_off = i * ecc->size; 1119 int oob_off = i * (ecc->bytes + 4); 1120 u8 *data = buf + data_off; 1121 u8 *oob = nand->oob_poi + oob_off; 1122 1123 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off, oob, 1124 oob_off + mtd->writesize, 1125 &cur_off, &max_bitflips, 1126 !i, oob_required, page); 1127 if (ret < 0) 1128 return ret; 1129 else if (ret) 1130 raw_mode = true; 1131 } 1132 1133 if (oob_required) 1134 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi, &cur_off, 1135 !raw_mode, page); 1136 1137 sunxi_nfc_hw_ecc_disable(nand); 1138 1139 return max_bitflips; 1140 } 1141 1142 static int sunxi_nfc_hw_ecc_read_page_dma(struct nand_chip *nand, u8 *buf, 1143 int oob_required, int page) 1144 { 1145 int ret; 1146 1147 sunxi_nfc_select_chip(nand, nand->cur_cs); 1148 1149 nand_read_page_op(nand, page, 0, NULL, 0); 1150 1151 ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, oob_required, page, 1152 nand->ecc.steps); 1153 if (ret >= 0) 1154 return ret; 1155 1156 /* Fallback to PIO mode */ 1157 return sunxi_nfc_hw_ecc_read_page(nand, buf, oob_required, page); 1158 } 1159 1160 static int sunxi_nfc_hw_ecc_read_subpage(struct nand_chip *nand, 1161 u32 data_offs, u32 readlen, 1162 u8 *bufpoi, int page) 1163 { 1164 struct mtd_info *mtd = nand_to_mtd(nand); 1165 struct nand_ecc_ctrl *ecc = &nand->ecc; 1166 int ret, i, cur_off = 0; 1167 unsigned int max_bitflips = 0; 1168 1169 sunxi_nfc_select_chip(nand, nand->cur_cs); 1170 1171 nand_read_page_op(nand, page, 0, NULL, 0); 1172 1173 sunxi_nfc_hw_ecc_enable(nand); 1174 1175 for (i = data_offs / ecc->size; 1176 i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) { 1177 int data_off = i * ecc->size; 1178 int oob_off = i * (ecc->bytes + 4); 1179 u8 *data = bufpoi + data_off; 1180 u8 *oob = nand->oob_poi + oob_off; 1181 1182 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off, 1183 oob, 1184 oob_off + mtd->writesize, 1185 &cur_off, &max_bitflips, !i, 1186 false, page); 1187 if (ret < 0) 1188 return ret; 1189 } 1190 1191 sunxi_nfc_hw_ecc_disable(nand); 1192 1193 return max_bitflips; 1194 } 1195 1196 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct nand_chip *nand, 1197 u32 data_offs, u32 readlen, 1198 u8 *buf, int page) 1199 { 1200 int nchunks = DIV_ROUND_UP(data_offs + readlen, nand->ecc.size); 1201 int ret; 1202 1203 sunxi_nfc_select_chip(nand, nand->cur_cs); 1204 1205 nand_read_page_op(nand, page, 0, NULL, 0); 1206 1207 ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, false, page, nchunks); 1208 if (ret >= 0) 1209 return ret; 1210 1211 /* Fallback to PIO mode */ 1212 return sunxi_nfc_hw_ecc_read_subpage(nand, data_offs, readlen, 1213 buf, page); 1214 } 1215 1216 static int sunxi_nfc_hw_ecc_write_page(struct nand_chip *nand, 1217 const uint8_t *buf, int oob_required, 1218 int page) 1219 { 1220 struct mtd_info *mtd = nand_to_mtd(nand); 1221 struct nand_ecc_ctrl *ecc = &nand->ecc; 1222 int ret, i, cur_off = 0; 1223 1224 sunxi_nfc_select_chip(nand, nand->cur_cs); 1225 1226 nand_prog_page_begin_op(nand, page, 0, NULL, 0); 1227 1228 sunxi_nfc_hw_ecc_enable(nand); 1229 1230 for (i = 0; i < ecc->steps; i++) { 1231 int data_off = i * ecc->size; 1232 int oob_off = i * (ecc->bytes + 4); 1233 const u8 *data = buf + data_off; 1234 const u8 *oob = nand->oob_poi + oob_off; 1235 1236 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob, 1237 oob_off + mtd->writesize, 1238 &cur_off, !i, page); 1239 if (ret) 1240 return ret; 1241 } 1242 1243 if (oob_required || (nand->options & NAND_NEED_SCRAMBLING)) 1244 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi, 1245 &cur_off, page); 1246 1247 sunxi_nfc_hw_ecc_disable(nand); 1248 1249 return nand_prog_page_end_op(nand); 1250 } 1251 1252 static int sunxi_nfc_hw_ecc_write_subpage(struct nand_chip *nand, 1253 u32 data_offs, u32 data_len, 1254 const u8 *buf, int oob_required, 1255 int page) 1256 { 1257 struct mtd_info *mtd = nand_to_mtd(nand); 1258 struct nand_ecc_ctrl *ecc = &nand->ecc; 1259 int ret, i, cur_off = 0; 1260 1261 sunxi_nfc_select_chip(nand, nand->cur_cs); 1262 1263 nand_prog_page_begin_op(nand, page, 0, NULL, 0); 1264 1265 sunxi_nfc_hw_ecc_enable(nand); 1266 1267 for (i = data_offs / ecc->size; 1268 i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) { 1269 int data_off = i * ecc->size; 1270 int oob_off = i * (ecc->bytes + 4); 1271 const u8 *data = buf + data_off; 1272 const u8 *oob = nand->oob_poi + oob_off; 1273 1274 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob, 1275 oob_off + mtd->writesize, 1276 &cur_off, !i, page); 1277 if (ret) 1278 return ret; 1279 } 1280 1281 sunxi_nfc_hw_ecc_disable(nand); 1282 1283 return nand_prog_page_end_op(nand); 1284 } 1285 1286 static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *nand, 1287 const u8 *buf, 1288 int oob_required, 1289 int page) 1290 { 1291 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1292 struct nand_ecc_ctrl *ecc = &nand->ecc; 1293 struct scatterlist sg; 1294 int ret, i; 1295 1296 sunxi_nfc_select_chip(nand, nand->cur_cs); 1297 1298 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1299 if (ret) 1300 return ret; 1301 1302 ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, ecc->steps, 1303 DMA_TO_DEVICE, &sg); 1304 if (ret) 1305 goto pio_fallback; 1306 1307 for (i = 0; i < ecc->steps; i++) { 1308 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4)); 1309 1310 sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, i, !i, page); 1311 } 1312 1313 nand_prog_page_begin_op(nand, page, 0, NULL, 0); 1314 1315 sunxi_nfc_hw_ecc_enable(nand); 1316 sunxi_nfc_randomizer_config(nand, page, false); 1317 sunxi_nfc_randomizer_enable(nand); 1318 1319 writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG, 1320 nfc->regs + NFC_REG_WCMD_SET); 1321 1322 dma_async_issue_pending(nfc->dmac); 1323 1324 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | 1325 NFC_DATA_TRANS | NFC_ACCESS_DIR, 1326 nfc->regs + NFC_REG_CMD); 1327 1328 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 1329 if (ret) 1330 dmaengine_terminate_all(nfc->dmac); 1331 1332 sunxi_nfc_randomizer_disable(nand); 1333 sunxi_nfc_hw_ecc_disable(nand); 1334 1335 sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg); 1336 1337 if (ret) 1338 return ret; 1339 1340 if (oob_required || (nand->options & NAND_NEED_SCRAMBLING)) 1341 /* TODO: use DMA to transfer extra OOB bytes ? */ 1342 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi, 1343 NULL, page); 1344 1345 return nand_prog_page_end_op(nand); 1346 1347 pio_fallback: 1348 return sunxi_nfc_hw_ecc_write_page(nand, buf, oob_required, page); 1349 } 1350 1351 static int sunxi_nfc_hw_ecc_read_oob(struct nand_chip *nand, int page) 1352 { 1353 u8 *buf = nand_get_data_buf(nand); 1354 1355 return nand->ecc.read_page(nand, buf, 1, page); 1356 } 1357 1358 static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *nand, int page) 1359 { 1360 struct mtd_info *mtd = nand_to_mtd(nand); 1361 u8 *buf = nand_get_data_buf(nand); 1362 int ret; 1363 1364 memset(buf, 0xff, mtd->writesize); 1365 ret = nand->ecc.write_page(nand, buf, 1, page); 1366 if (ret) 1367 return ret; 1368 1369 /* Send command to program the OOB data */ 1370 return nand_prog_page_end_op(nand); 1371 } 1372 1373 static const s32 tWB_lut[] = {6, 12, 16, 20}; 1374 static const s32 tRHW_lut[] = {4, 8, 12, 20}; 1375 1376 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration, 1377 u32 clk_period) 1378 { 1379 u32 clk_cycles = DIV_ROUND_UP(duration, clk_period); 1380 int i; 1381 1382 for (i = 0; i < lut_size; i++) { 1383 if (clk_cycles <= lut[i]) 1384 return i; 1385 } 1386 1387 /* Doesn't fit */ 1388 return -EINVAL; 1389 } 1390 1391 #define sunxi_nand_lookup_timing(l, p, c) \ 1392 _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c) 1393 1394 static int sunxi_nfc_setup_data_interface(struct nand_chip *nand, int csline, 1395 const struct nand_data_interface *conf) 1396 { 1397 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 1398 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 1399 const struct nand_sdr_timings *timings; 1400 u32 min_clk_period = 0; 1401 s32 tWB, tADL, tWHR, tRHW, tCAD; 1402 long real_clk_rate; 1403 1404 timings = nand_get_sdr_timings(conf); 1405 if (IS_ERR(timings)) 1406 return -ENOTSUPP; 1407 1408 /* T1 <=> tCLS */ 1409 if (timings->tCLS_min > min_clk_period) 1410 min_clk_period = timings->tCLS_min; 1411 1412 /* T2 <=> tCLH */ 1413 if (timings->tCLH_min > min_clk_period) 1414 min_clk_period = timings->tCLH_min; 1415 1416 /* T3 <=> tCS */ 1417 if (timings->tCS_min > min_clk_period) 1418 min_clk_period = timings->tCS_min; 1419 1420 /* T4 <=> tCH */ 1421 if (timings->tCH_min > min_clk_period) 1422 min_clk_period = timings->tCH_min; 1423 1424 /* T5 <=> tWP */ 1425 if (timings->tWP_min > min_clk_period) 1426 min_clk_period = timings->tWP_min; 1427 1428 /* T6 <=> tWH */ 1429 if (timings->tWH_min > min_clk_period) 1430 min_clk_period = timings->tWH_min; 1431 1432 /* T7 <=> tALS */ 1433 if (timings->tALS_min > min_clk_period) 1434 min_clk_period = timings->tALS_min; 1435 1436 /* T8 <=> tDS */ 1437 if (timings->tDS_min > min_clk_period) 1438 min_clk_period = timings->tDS_min; 1439 1440 /* T9 <=> tDH */ 1441 if (timings->tDH_min > min_clk_period) 1442 min_clk_period = timings->tDH_min; 1443 1444 /* T10 <=> tRR */ 1445 if (timings->tRR_min > (min_clk_period * 3)) 1446 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3); 1447 1448 /* T11 <=> tALH */ 1449 if (timings->tALH_min > min_clk_period) 1450 min_clk_period = timings->tALH_min; 1451 1452 /* T12 <=> tRP */ 1453 if (timings->tRP_min > min_clk_period) 1454 min_clk_period = timings->tRP_min; 1455 1456 /* T13 <=> tREH */ 1457 if (timings->tREH_min > min_clk_period) 1458 min_clk_period = timings->tREH_min; 1459 1460 /* T14 <=> tRC */ 1461 if (timings->tRC_min > (min_clk_period * 2)) 1462 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2); 1463 1464 /* T15 <=> tWC */ 1465 if (timings->tWC_min > (min_clk_period * 2)) 1466 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2); 1467 1468 /* T16 - T19 + tCAD */ 1469 if (timings->tWB_max > (min_clk_period * 20)) 1470 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20); 1471 1472 if (timings->tADL_min > (min_clk_period * 32)) 1473 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32); 1474 1475 if (timings->tWHR_min > (min_clk_period * 32)) 1476 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32); 1477 1478 if (timings->tRHW_min > (min_clk_period * 20)) 1479 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20); 1480 1481 /* 1482 * In non-EDO, tREA should be less than tRP to guarantee that the 1483 * controller does not sample the IO lines too early. Unfortunately, 1484 * the sunxi NAND controller does not allow us to have different 1485 * values for tRP and tREH (tRP = tREH = tRW / 2). 1486 * 1487 * We have 2 options to overcome this limitation: 1488 * 1489 * 1/ Extend tRC to fulfil the tREA <= tRC / 2 constraint 1490 * 2/ Use EDO mode (only works if timings->tRLOH > 0) 1491 */ 1492 if (timings->tREA_max > min_clk_period && !timings->tRLOH_min) 1493 min_clk_period = timings->tREA_max; 1494 1495 tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max, 1496 min_clk_period); 1497 if (tWB < 0) { 1498 dev_err(nfc->dev, "unsupported tWB\n"); 1499 return tWB; 1500 } 1501 1502 tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3; 1503 if (tADL > 3) { 1504 dev_err(nfc->dev, "unsupported tADL\n"); 1505 return -EINVAL; 1506 } 1507 1508 tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3; 1509 if (tWHR > 3) { 1510 dev_err(nfc->dev, "unsupported tWHR\n"); 1511 return -EINVAL; 1512 } 1513 1514 tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min, 1515 min_clk_period); 1516 if (tRHW < 0) { 1517 dev_err(nfc->dev, "unsupported tRHW\n"); 1518 return tRHW; 1519 } 1520 1521 if (csline == NAND_DATA_IFACE_CHECK_ONLY) 1522 return 0; 1523 1524 /* 1525 * TODO: according to ONFI specs this value only applies for DDR NAND, 1526 * but Allwinner seems to set this to 0x7. Mimic them for now. 1527 */ 1528 tCAD = 0x7; 1529 1530 /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */ 1531 sunxi_nand->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD); 1532 1533 /* Convert min_clk_period from picoseconds to nanoseconds */ 1534 min_clk_period = DIV_ROUND_UP(min_clk_period, 1000); 1535 1536 /* 1537 * Unlike what is stated in Allwinner datasheet, the clk_rate should 1538 * be set to (1 / min_clk_period), and not (2 / min_clk_period). 1539 * This new formula was verified with a scope and validated by 1540 * Allwinner engineers. 1541 */ 1542 sunxi_nand->clk_rate = NSEC_PER_SEC / min_clk_period; 1543 real_clk_rate = clk_round_rate(nfc->mod_clk, sunxi_nand->clk_rate); 1544 if (real_clk_rate <= 0) { 1545 dev_err(nfc->dev, "Unable to round clk %lu\n", 1546 sunxi_nand->clk_rate); 1547 return -EINVAL; 1548 } 1549 1550 sunxi_nand->timing_ctl = 0; 1551 1552 /* 1553 * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data 1554 * output cycle timings shall be used if the host drives tRC less than 1555 * 30 ns. We should also use EDO mode if tREA is bigger than tRP. 1556 */ 1557 min_clk_period = NSEC_PER_SEC / real_clk_rate; 1558 if (min_clk_period * 2 < 30 || min_clk_period * 1000 < timings->tREA_max) 1559 sunxi_nand->timing_ctl = NFC_TIMING_CTL_EDO; 1560 1561 return 0; 1562 } 1563 1564 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section, 1565 struct mtd_oob_region *oobregion) 1566 { 1567 struct nand_chip *nand = mtd_to_nand(mtd); 1568 struct nand_ecc_ctrl *ecc = &nand->ecc; 1569 1570 if (section >= ecc->steps) 1571 return -ERANGE; 1572 1573 oobregion->offset = section * (ecc->bytes + 4) + 4; 1574 oobregion->length = ecc->bytes; 1575 1576 return 0; 1577 } 1578 1579 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section, 1580 struct mtd_oob_region *oobregion) 1581 { 1582 struct nand_chip *nand = mtd_to_nand(mtd); 1583 struct nand_ecc_ctrl *ecc = &nand->ecc; 1584 1585 if (section > ecc->steps) 1586 return -ERANGE; 1587 1588 /* 1589 * The first 2 bytes are used for BB markers, hence we 1590 * only have 2 bytes available in the first user data 1591 * section. 1592 */ 1593 if (!section && ecc->mode == NAND_ECC_HW) { 1594 oobregion->offset = 2; 1595 oobregion->length = 2; 1596 1597 return 0; 1598 } 1599 1600 oobregion->offset = section * (ecc->bytes + 4); 1601 1602 if (section < ecc->steps) 1603 oobregion->length = 4; 1604 else 1605 oobregion->offset = mtd->oobsize - oobregion->offset; 1606 1607 return 0; 1608 } 1609 1610 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = { 1611 .ecc = sunxi_nand_ooblayout_ecc, 1612 .free = sunxi_nand_ooblayout_free, 1613 }; 1614 1615 static void sunxi_nand_hw_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc) 1616 { 1617 kfree(ecc->priv); 1618 } 1619 1620 static int sunxi_nand_hw_ecc_ctrl_init(struct nand_chip *nand, 1621 struct nand_ecc_ctrl *ecc, 1622 struct device_node *np) 1623 { 1624 static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 }; 1625 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1626 struct mtd_info *mtd = nand_to_mtd(nand); 1627 struct sunxi_nand_hw_ecc *data; 1628 int nsectors; 1629 int ret; 1630 int i; 1631 1632 if (ecc->options & NAND_ECC_MAXIMIZE) { 1633 int bytes; 1634 1635 ecc->size = 1024; 1636 nsectors = mtd->writesize / ecc->size; 1637 1638 /* Reserve 2 bytes for the BBM */ 1639 bytes = (mtd->oobsize - 2) / nsectors; 1640 1641 /* 4 non-ECC bytes are added before each ECC bytes section */ 1642 bytes -= 4; 1643 1644 /* and bytes has to be even. */ 1645 if (bytes % 2) 1646 bytes--; 1647 1648 ecc->strength = bytes * 8 / fls(8 * ecc->size); 1649 1650 for (i = 0; i < ARRAY_SIZE(strengths); i++) { 1651 if (strengths[i] > ecc->strength) 1652 break; 1653 } 1654 1655 if (!i) 1656 ecc->strength = 0; 1657 else 1658 ecc->strength = strengths[i - 1]; 1659 } 1660 1661 if (ecc->size != 512 && ecc->size != 1024) 1662 return -EINVAL; 1663 1664 data = kzalloc(sizeof(*data), GFP_KERNEL); 1665 if (!data) 1666 return -ENOMEM; 1667 1668 /* Prefer 1k ECC chunk over 512 ones */ 1669 if (ecc->size == 512 && mtd->writesize > 512) { 1670 ecc->size = 1024; 1671 ecc->strength *= 2; 1672 } 1673 1674 /* Add ECC info retrieval from DT */ 1675 for (i = 0; i < ARRAY_SIZE(strengths); i++) { 1676 if (ecc->strength <= strengths[i]) { 1677 /* 1678 * Update ecc->strength value with the actual strength 1679 * that will be used by the ECC engine. 1680 */ 1681 ecc->strength = strengths[i]; 1682 break; 1683 } 1684 } 1685 1686 if (i >= ARRAY_SIZE(strengths)) { 1687 dev_err(nfc->dev, "unsupported strength\n"); 1688 ret = -ENOTSUPP; 1689 goto err; 1690 } 1691 1692 data->mode = i; 1693 1694 /* HW ECC always request ECC bytes for 1024 bytes blocks */ 1695 ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8); 1696 1697 /* HW ECC always work with even numbers of ECC bytes */ 1698 ecc->bytes = ALIGN(ecc->bytes, 2); 1699 1700 nsectors = mtd->writesize / ecc->size; 1701 1702 if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) { 1703 ret = -EINVAL; 1704 goto err; 1705 } 1706 1707 ecc->read_oob = sunxi_nfc_hw_ecc_read_oob; 1708 ecc->write_oob = sunxi_nfc_hw_ecc_write_oob; 1709 mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops); 1710 ecc->priv = data; 1711 1712 if (nfc->dmac) { 1713 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma; 1714 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma; 1715 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma; 1716 nand->options |= NAND_USE_BOUNCE_BUFFER; 1717 } else { 1718 ecc->read_page = sunxi_nfc_hw_ecc_read_page; 1719 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage; 1720 ecc->write_page = sunxi_nfc_hw_ecc_write_page; 1721 } 1722 1723 /* TODO: support DMA for raw accesses and subpage write */ 1724 ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage; 1725 ecc->read_oob_raw = nand_read_oob_std; 1726 ecc->write_oob_raw = nand_write_oob_std; 1727 1728 return 0; 1729 1730 err: 1731 kfree(data); 1732 1733 return ret; 1734 } 1735 1736 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc) 1737 { 1738 switch (ecc->mode) { 1739 case NAND_ECC_HW: 1740 sunxi_nand_hw_ecc_ctrl_cleanup(ecc); 1741 break; 1742 case NAND_ECC_NONE: 1743 default: 1744 break; 1745 } 1746 } 1747 1748 static int sunxi_nand_attach_chip(struct nand_chip *nand) 1749 { 1750 struct nand_ecc_ctrl *ecc = &nand->ecc; 1751 struct device_node *np = nand_get_flash_node(nand); 1752 int ret; 1753 1754 if (nand->bbt_options & NAND_BBT_USE_FLASH) 1755 nand->bbt_options |= NAND_BBT_NO_OOB; 1756 1757 if (nand->options & NAND_NEED_SCRAMBLING) 1758 nand->options |= NAND_NO_SUBPAGE_WRITE; 1759 1760 nand->options |= NAND_SUBPAGE_READ; 1761 1762 if (!ecc->size) { 1763 ecc->size = nand->base.eccreq.step_size; 1764 ecc->strength = nand->base.eccreq.strength; 1765 } 1766 1767 if (!ecc->size || !ecc->strength) 1768 return -EINVAL; 1769 1770 switch (ecc->mode) { 1771 case NAND_ECC_HW: 1772 ret = sunxi_nand_hw_ecc_ctrl_init(nand, ecc, np); 1773 if (ret) 1774 return ret; 1775 break; 1776 case NAND_ECC_NONE: 1777 case NAND_ECC_SOFT: 1778 break; 1779 default: 1780 return -EINVAL; 1781 } 1782 1783 return 0; 1784 } 1785 1786 static int sunxi_nfc_exec_subop(struct nand_chip *nand, 1787 const struct nand_subop *subop) 1788 { 1789 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1790 u32 cmd = 0, extcmd = 0, cnt = 0, addrs[2] = { }; 1791 unsigned int i, j, remaining, start; 1792 void *inbuf = NULL; 1793 int ret; 1794 1795 for (i = 0; i < subop->ninstrs; i++) { 1796 const struct nand_op_instr *instr = &subop->instrs[i]; 1797 1798 switch (instr->type) { 1799 case NAND_OP_CMD_INSTR: 1800 if (cmd & NFC_SEND_CMD1) { 1801 if (WARN_ON(cmd & NFC_SEND_CMD2)) 1802 return -EINVAL; 1803 1804 cmd |= NFC_SEND_CMD2; 1805 extcmd |= instr->ctx.cmd.opcode; 1806 } else { 1807 cmd |= NFC_SEND_CMD1 | 1808 NFC_CMD(instr->ctx.cmd.opcode); 1809 } 1810 break; 1811 1812 case NAND_OP_ADDR_INSTR: 1813 remaining = nand_subop_get_num_addr_cyc(subop, i); 1814 start = nand_subop_get_addr_start_off(subop, i); 1815 for (j = 0; j < 8 && j + start < remaining; j++) { 1816 u32 addr = instr->ctx.addr.addrs[j + start]; 1817 1818 addrs[j / 4] |= addr << (j % 4) * 8; 1819 } 1820 1821 if (j) 1822 cmd |= NFC_SEND_ADR | NFC_ADR_NUM(j); 1823 1824 break; 1825 1826 case NAND_OP_DATA_IN_INSTR: 1827 case NAND_OP_DATA_OUT_INSTR: 1828 start = nand_subop_get_data_start_off(subop, i); 1829 remaining = nand_subop_get_data_len(subop, i); 1830 cnt = min_t(u32, remaining, NFC_SRAM_SIZE); 1831 cmd |= NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD; 1832 1833 if (instr->type == NAND_OP_DATA_OUT_INSTR) { 1834 cmd |= NFC_ACCESS_DIR; 1835 memcpy_toio(nfc->regs + NFC_RAM0_BASE, 1836 instr->ctx.data.buf.out + start, 1837 cnt); 1838 } else { 1839 inbuf = instr->ctx.data.buf.in + start; 1840 } 1841 1842 break; 1843 1844 case NAND_OP_WAITRDY_INSTR: 1845 cmd |= NFC_WAIT_FLAG; 1846 break; 1847 } 1848 } 1849 1850 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1851 if (ret) 1852 return ret; 1853 1854 if (cmd & NFC_SEND_ADR) { 1855 writel(addrs[0], nfc->regs + NFC_REG_ADDR_LOW); 1856 writel(addrs[1], nfc->regs + NFC_REG_ADDR_HIGH); 1857 } 1858 1859 if (cmd & NFC_SEND_CMD2) 1860 writel(extcmd, 1861 nfc->regs + 1862 (cmd & NFC_ACCESS_DIR ? 1863 NFC_REG_WCMD_SET : NFC_REG_RCMD_SET)); 1864 1865 if (cmd & NFC_DATA_TRANS) 1866 writel(cnt, nfc->regs + NFC_REG_CNT); 1867 1868 writel(cmd, nfc->regs + NFC_REG_CMD); 1869 1870 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, 1871 !(cmd & NFC_WAIT_FLAG) && cnt < 64, 1872 0); 1873 if (ret) 1874 return ret; 1875 1876 if (inbuf) 1877 memcpy_fromio(inbuf, nfc->regs + NFC_RAM0_BASE, cnt); 1878 1879 return 0; 1880 } 1881 1882 static int sunxi_nfc_soft_waitrdy(struct nand_chip *nand, 1883 const struct nand_subop *subop) 1884 { 1885 return nand_soft_waitrdy(nand, 1886 subop->instrs[0].ctx.waitrdy.timeout_ms); 1887 } 1888 1889 static const struct nand_op_parser sunxi_nfc_op_parser = NAND_OP_PARSER( 1890 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop, 1891 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1892 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8), 1893 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1894 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true), 1895 NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)), 1896 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop, 1897 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1898 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8), 1899 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024), 1900 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1901 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)), 1902 ); 1903 1904 static const struct nand_op_parser sunxi_nfc_norb_op_parser = NAND_OP_PARSER( 1905 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop, 1906 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1907 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8), 1908 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1909 NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)), 1910 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop, 1911 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1912 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8), 1913 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024), 1914 NAND_OP_PARSER_PAT_CMD_ELEM(true)), 1915 NAND_OP_PARSER_PATTERN(sunxi_nfc_soft_waitrdy, 1916 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)), 1917 ); 1918 1919 static int sunxi_nfc_exec_op(struct nand_chip *nand, 1920 const struct nand_operation *op, bool check_only) 1921 { 1922 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 1923 const struct nand_op_parser *parser; 1924 1925 sunxi_nfc_select_chip(nand, op->cs); 1926 1927 if (sunxi_nand->sels[op->cs].rb >= 0) 1928 parser = &sunxi_nfc_op_parser; 1929 else 1930 parser = &sunxi_nfc_norb_op_parser; 1931 1932 return nand_op_parser_exec_op(nand, parser, op, check_only); 1933 } 1934 1935 static const struct nand_controller_ops sunxi_nand_controller_ops = { 1936 .attach_chip = sunxi_nand_attach_chip, 1937 .setup_data_interface = sunxi_nfc_setup_data_interface, 1938 .exec_op = sunxi_nfc_exec_op, 1939 }; 1940 1941 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc, 1942 struct device_node *np) 1943 { 1944 struct sunxi_nand_chip *sunxi_nand; 1945 struct mtd_info *mtd; 1946 struct nand_chip *nand; 1947 int nsels; 1948 int ret; 1949 int i; 1950 u32 tmp; 1951 1952 if (!of_get_property(np, "reg", &nsels)) 1953 return -EINVAL; 1954 1955 nsels /= sizeof(u32); 1956 if (!nsels) { 1957 dev_err(dev, "invalid reg property size\n"); 1958 return -EINVAL; 1959 } 1960 1961 sunxi_nand = devm_kzalloc(dev, struct_size(sunxi_nand, sels, nsels), 1962 GFP_KERNEL); 1963 if (!sunxi_nand) { 1964 dev_err(dev, "could not allocate chip\n"); 1965 return -ENOMEM; 1966 } 1967 1968 sunxi_nand->nsels = nsels; 1969 1970 for (i = 0; i < nsels; i++) { 1971 ret = of_property_read_u32_index(np, "reg", i, &tmp); 1972 if (ret) { 1973 dev_err(dev, "could not retrieve reg property: %d\n", 1974 ret); 1975 return ret; 1976 } 1977 1978 if (tmp > NFC_MAX_CS) { 1979 dev_err(dev, 1980 "invalid reg value: %u (max CS = 7)\n", 1981 tmp); 1982 return -EINVAL; 1983 } 1984 1985 if (test_and_set_bit(tmp, &nfc->assigned_cs)) { 1986 dev_err(dev, "CS %d already assigned\n", tmp); 1987 return -EINVAL; 1988 } 1989 1990 sunxi_nand->sels[i].cs = tmp; 1991 1992 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) && 1993 tmp < 2) 1994 sunxi_nand->sels[i].rb = tmp; 1995 else 1996 sunxi_nand->sels[i].rb = -1; 1997 } 1998 1999 nand = &sunxi_nand->nand; 2000 /* Default tR value specified in the ONFI spec (chapter 4.15.1) */ 2001 nand->controller = &nfc->controller; 2002 nand->controller->ops = &sunxi_nand_controller_ops; 2003 2004 /* 2005 * Set the ECC mode to the default value in case nothing is specified 2006 * in the DT. 2007 */ 2008 nand->ecc.mode = NAND_ECC_HW; 2009 nand_set_flash_node(nand, np); 2010 2011 mtd = nand_to_mtd(nand); 2012 mtd->dev.parent = dev; 2013 2014 ret = nand_scan(nand, nsels); 2015 if (ret) 2016 return ret; 2017 2018 ret = mtd_device_register(mtd, NULL, 0); 2019 if (ret) { 2020 dev_err(dev, "failed to register mtd device: %d\n", ret); 2021 nand_release(nand); 2022 return ret; 2023 } 2024 2025 list_add_tail(&sunxi_nand->node, &nfc->chips); 2026 2027 return 0; 2028 } 2029 2030 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc) 2031 { 2032 struct device_node *np = dev->of_node; 2033 struct device_node *nand_np; 2034 int nchips = of_get_child_count(np); 2035 int ret; 2036 2037 if (nchips > 8) { 2038 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips); 2039 return -EINVAL; 2040 } 2041 2042 for_each_child_of_node(np, nand_np) { 2043 ret = sunxi_nand_chip_init(dev, nfc, nand_np); 2044 if (ret) { 2045 of_node_put(nand_np); 2046 return ret; 2047 } 2048 } 2049 2050 return 0; 2051 } 2052 2053 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc) 2054 { 2055 struct sunxi_nand_chip *sunxi_nand; 2056 2057 while (!list_empty(&nfc->chips)) { 2058 sunxi_nand = list_first_entry(&nfc->chips, 2059 struct sunxi_nand_chip, 2060 node); 2061 nand_release(&sunxi_nand->nand); 2062 sunxi_nand_ecc_cleanup(&sunxi_nand->nand.ecc); 2063 list_del(&sunxi_nand->node); 2064 } 2065 } 2066 2067 static int sunxi_nfc_probe(struct platform_device *pdev) 2068 { 2069 struct device *dev = &pdev->dev; 2070 struct resource *r; 2071 struct sunxi_nfc *nfc; 2072 int irq; 2073 int ret; 2074 2075 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL); 2076 if (!nfc) 2077 return -ENOMEM; 2078 2079 nfc->dev = dev; 2080 nand_controller_init(&nfc->controller); 2081 INIT_LIST_HEAD(&nfc->chips); 2082 2083 r = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2084 nfc->regs = devm_ioremap_resource(dev, r); 2085 if (IS_ERR(nfc->regs)) 2086 return PTR_ERR(nfc->regs); 2087 2088 irq = platform_get_irq(pdev, 0); 2089 if (irq < 0) { 2090 dev_err(dev, "failed to retrieve irq\n"); 2091 return irq; 2092 } 2093 2094 nfc->ahb_clk = devm_clk_get(dev, "ahb"); 2095 if (IS_ERR(nfc->ahb_clk)) { 2096 dev_err(dev, "failed to retrieve ahb clk\n"); 2097 return PTR_ERR(nfc->ahb_clk); 2098 } 2099 2100 ret = clk_prepare_enable(nfc->ahb_clk); 2101 if (ret) 2102 return ret; 2103 2104 nfc->mod_clk = devm_clk_get(dev, "mod"); 2105 if (IS_ERR(nfc->mod_clk)) { 2106 dev_err(dev, "failed to retrieve mod clk\n"); 2107 ret = PTR_ERR(nfc->mod_clk); 2108 goto out_ahb_clk_unprepare; 2109 } 2110 2111 ret = clk_prepare_enable(nfc->mod_clk); 2112 if (ret) 2113 goto out_ahb_clk_unprepare; 2114 2115 nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb"); 2116 if (IS_ERR(nfc->reset)) { 2117 ret = PTR_ERR(nfc->reset); 2118 goto out_mod_clk_unprepare; 2119 } 2120 2121 ret = reset_control_deassert(nfc->reset); 2122 if (ret) { 2123 dev_err(dev, "reset err %d\n", ret); 2124 goto out_mod_clk_unprepare; 2125 } 2126 2127 nfc->caps = of_device_get_match_data(&pdev->dev); 2128 if (!nfc->caps) { 2129 ret = -EINVAL; 2130 goto out_ahb_reset_reassert; 2131 } 2132 2133 ret = sunxi_nfc_rst(nfc); 2134 if (ret) 2135 goto out_ahb_reset_reassert; 2136 2137 writel(0, nfc->regs + NFC_REG_INT); 2138 ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt, 2139 0, "sunxi-nand", nfc); 2140 if (ret) 2141 goto out_ahb_reset_reassert; 2142 2143 nfc->dmac = dma_request_slave_channel(dev, "rxtx"); 2144 if (nfc->dmac) { 2145 struct dma_slave_config dmac_cfg = { }; 2146 2147 dmac_cfg.src_addr = r->start + nfc->caps->reg_io_data; 2148 dmac_cfg.dst_addr = dmac_cfg.src_addr; 2149 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 2150 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width; 2151 dmac_cfg.src_maxburst = nfc->caps->dma_maxburst; 2152 dmac_cfg.dst_maxburst = nfc->caps->dma_maxburst; 2153 dmaengine_slave_config(nfc->dmac, &dmac_cfg); 2154 } else { 2155 dev_warn(dev, "failed to request rxtx DMA channel\n"); 2156 } 2157 2158 platform_set_drvdata(pdev, nfc); 2159 2160 ret = sunxi_nand_chips_init(dev, nfc); 2161 if (ret) { 2162 dev_err(dev, "failed to init nand chips\n"); 2163 goto out_release_dmac; 2164 } 2165 2166 return 0; 2167 2168 out_release_dmac: 2169 if (nfc->dmac) 2170 dma_release_channel(nfc->dmac); 2171 out_ahb_reset_reassert: 2172 reset_control_assert(nfc->reset); 2173 out_mod_clk_unprepare: 2174 clk_disable_unprepare(nfc->mod_clk); 2175 out_ahb_clk_unprepare: 2176 clk_disable_unprepare(nfc->ahb_clk); 2177 2178 return ret; 2179 } 2180 2181 static int sunxi_nfc_remove(struct platform_device *pdev) 2182 { 2183 struct sunxi_nfc *nfc = platform_get_drvdata(pdev); 2184 2185 sunxi_nand_chips_cleanup(nfc); 2186 2187 reset_control_assert(nfc->reset); 2188 2189 if (nfc->dmac) 2190 dma_release_channel(nfc->dmac); 2191 clk_disable_unprepare(nfc->mod_clk); 2192 clk_disable_unprepare(nfc->ahb_clk); 2193 2194 return 0; 2195 } 2196 2197 static const struct sunxi_nfc_caps sunxi_nfc_a10_caps = { 2198 .reg_io_data = NFC_REG_A10_IO_DATA, 2199 .dma_maxburst = 4, 2200 }; 2201 2202 static const struct sunxi_nfc_caps sunxi_nfc_a23_caps = { 2203 .sram_through_ahb = true, 2204 .reg_io_data = NFC_REG_A23_IO_DATA, 2205 .dma_maxburst = 8, 2206 }; 2207 2208 static const struct of_device_id sunxi_nfc_ids[] = { 2209 { 2210 .compatible = "allwinner,sun4i-a10-nand", 2211 .data = &sunxi_nfc_a10_caps, 2212 }, 2213 { 2214 .compatible = "allwinner,sun8i-a23-nand-controller", 2215 .data = &sunxi_nfc_a23_caps, 2216 }, 2217 { /* sentinel */ } 2218 }; 2219 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids); 2220 2221 static struct platform_driver sunxi_nfc_driver = { 2222 .driver = { 2223 .name = "sunxi_nand", 2224 .of_match_table = sunxi_nfc_ids, 2225 }, 2226 .probe = sunxi_nfc_probe, 2227 .remove = sunxi_nfc_remove, 2228 }; 2229 module_platform_driver(sunxi_nfc_driver); 2230 2231 MODULE_LICENSE("GPL"); 2232 MODULE_AUTHOR("Boris BREZILLON"); 2233 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver"); 2234 MODULE_ALIAS("platform:sunxi_nand"); 2235