1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com> 4 * 5 * Derived from: 6 * https://github.com/yuq/sunxi-nfc-mtd 7 * Copyright (C) 2013 Qiang Yu <yuq825@gmail.com> 8 * 9 * https://github.com/hno/Allwinner-Info 10 * Copyright (C) 2013 Henrik Nordström <Henrik Nordström> 11 * 12 * Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com> 13 * Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org> 14 */ 15 16 #include <linux/dma-mapping.h> 17 #include <linux/slab.h> 18 #include <linux/module.h> 19 #include <linux/moduleparam.h> 20 #include <linux/platform_device.h> 21 #include <linux/of.h> 22 #include <linux/of_device.h> 23 #include <linux/mtd/mtd.h> 24 #include <linux/mtd/rawnand.h> 25 #include <linux/mtd/partitions.h> 26 #include <linux/clk.h> 27 #include <linux/delay.h> 28 #include <linux/dmaengine.h> 29 #include <linux/interrupt.h> 30 #include <linux/iopoll.h> 31 #include <linux/reset.h> 32 33 #define NFC_REG_CTL 0x0000 34 #define NFC_REG_ST 0x0004 35 #define NFC_REG_INT 0x0008 36 #define NFC_REG_TIMING_CTL 0x000C 37 #define NFC_REG_TIMING_CFG 0x0010 38 #define NFC_REG_ADDR_LOW 0x0014 39 #define NFC_REG_ADDR_HIGH 0x0018 40 #define NFC_REG_SECTOR_NUM 0x001C 41 #define NFC_REG_CNT 0x0020 42 #define NFC_REG_CMD 0x0024 43 #define NFC_REG_RCMD_SET 0x0028 44 #define NFC_REG_WCMD_SET 0x002C 45 #define NFC_REG_A10_IO_DATA 0x0030 46 #define NFC_REG_A23_IO_DATA 0x0300 47 #define NFC_REG_ECC_CTL 0x0034 48 #define NFC_REG_ECC_ST 0x0038 49 #define NFC_REG_DEBUG 0x003C 50 #define NFC_REG_ECC_ERR_CNT(x) ((0x0040 + (x)) & ~0x3) 51 #define NFC_REG_USER_DATA(x) (0x0050 + ((x) * 4)) 52 #define NFC_REG_SPARE_AREA 0x00A0 53 #define NFC_REG_PAT_ID 0x00A4 54 #define NFC_REG_MDMA_ADDR 0x00C0 55 #define NFC_REG_MDMA_CNT 0x00C4 56 #define NFC_RAM0_BASE 0x0400 57 #define NFC_RAM1_BASE 0x0800 58 59 /* define bit use in NFC_CTL */ 60 #define NFC_EN BIT(0) 61 #define NFC_RESET BIT(1) 62 #define NFC_BUS_WIDTH_MSK BIT(2) 63 #define NFC_BUS_WIDTH_8 (0 << 2) 64 #define NFC_BUS_WIDTH_16 (1 << 2) 65 #define NFC_RB_SEL_MSK BIT(3) 66 #define NFC_RB_SEL(x) ((x) << 3) 67 #define NFC_CE_SEL_MSK GENMASK(26, 24) 68 #define NFC_CE_SEL(x) ((x) << 24) 69 #define NFC_CE_CTL BIT(6) 70 #define NFC_PAGE_SHIFT_MSK GENMASK(11, 8) 71 #define NFC_PAGE_SHIFT(x) (((x) < 10 ? 0 : (x) - 10) << 8) 72 #define NFC_SAM BIT(12) 73 #define NFC_RAM_METHOD BIT(14) 74 #define NFC_DMA_TYPE_NORMAL BIT(15) 75 #define NFC_DEBUG_CTL BIT(31) 76 77 /* define bit use in NFC_ST */ 78 #define NFC_RB_B2R BIT(0) 79 #define NFC_CMD_INT_FLAG BIT(1) 80 #define NFC_DMA_INT_FLAG BIT(2) 81 #define NFC_CMD_FIFO_STATUS BIT(3) 82 #define NFC_STA BIT(4) 83 #define NFC_NATCH_INT_FLAG BIT(5) 84 #define NFC_RB_STATE(x) BIT(x + 8) 85 86 /* define bit use in NFC_INT */ 87 #define NFC_B2R_INT_ENABLE BIT(0) 88 #define NFC_CMD_INT_ENABLE BIT(1) 89 #define NFC_DMA_INT_ENABLE BIT(2) 90 #define NFC_INT_MASK (NFC_B2R_INT_ENABLE | \ 91 NFC_CMD_INT_ENABLE | \ 92 NFC_DMA_INT_ENABLE) 93 94 /* define bit use in NFC_TIMING_CTL */ 95 #define NFC_TIMING_CTL_EDO BIT(8) 96 97 /* define NFC_TIMING_CFG register layout */ 98 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD) \ 99 (((tWB) & 0x3) | (((tADL) & 0x3) << 2) | \ 100 (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) | \ 101 (((tCAD) & 0x7) << 8)) 102 103 /* define bit use in NFC_CMD */ 104 #define NFC_CMD_LOW_BYTE_MSK GENMASK(7, 0) 105 #define NFC_CMD_HIGH_BYTE_MSK GENMASK(15, 8) 106 #define NFC_CMD(x) (x) 107 #define NFC_ADR_NUM_MSK GENMASK(18, 16) 108 #define NFC_ADR_NUM(x) (((x) - 1) << 16) 109 #define NFC_SEND_ADR BIT(19) 110 #define NFC_ACCESS_DIR BIT(20) 111 #define NFC_DATA_TRANS BIT(21) 112 #define NFC_SEND_CMD1 BIT(22) 113 #define NFC_WAIT_FLAG BIT(23) 114 #define NFC_SEND_CMD2 BIT(24) 115 #define NFC_SEQ BIT(25) 116 #define NFC_DATA_SWAP_METHOD BIT(26) 117 #define NFC_ROW_AUTO_INC BIT(27) 118 #define NFC_SEND_CMD3 BIT(28) 119 #define NFC_SEND_CMD4 BIT(29) 120 #define NFC_CMD_TYPE_MSK GENMASK(31, 30) 121 #define NFC_NORMAL_OP (0 << 30) 122 #define NFC_ECC_OP (1 << 30) 123 #define NFC_PAGE_OP (2U << 30) 124 125 /* define bit use in NFC_RCMD_SET */ 126 #define NFC_READ_CMD_MSK GENMASK(7, 0) 127 #define NFC_RND_READ_CMD0_MSK GENMASK(15, 8) 128 #define NFC_RND_READ_CMD1_MSK GENMASK(23, 16) 129 130 /* define bit use in NFC_WCMD_SET */ 131 #define NFC_PROGRAM_CMD_MSK GENMASK(7, 0) 132 #define NFC_RND_WRITE_CMD_MSK GENMASK(15, 8) 133 #define NFC_READ_CMD0_MSK GENMASK(23, 16) 134 #define NFC_READ_CMD1_MSK GENMASK(31, 24) 135 136 /* define bit use in NFC_ECC_CTL */ 137 #define NFC_ECC_EN BIT(0) 138 #define NFC_ECC_PIPELINE BIT(3) 139 #define NFC_ECC_EXCEPTION BIT(4) 140 #define NFC_ECC_BLOCK_SIZE_MSK BIT(5) 141 #define NFC_ECC_BLOCK_512 BIT(5) 142 #define NFC_RANDOM_EN BIT(9) 143 #define NFC_RANDOM_DIRECTION BIT(10) 144 #define NFC_ECC_MODE_MSK GENMASK(15, 12) 145 #define NFC_ECC_MODE(x) ((x) << 12) 146 #define NFC_RANDOM_SEED_MSK GENMASK(30, 16) 147 #define NFC_RANDOM_SEED(x) ((x) << 16) 148 149 /* define bit use in NFC_ECC_ST */ 150 #define NFC_ECC_ERR(x) BIT(x) 151 #define NFC_ECC_ERR_MSK GENMASK(15, 0) 152 #define NFC_ECC_PAT_FOUND(x) BIT(x + 16) 153 #define NFC_ECC_ERR_CNT(b, x) (((x) >> (((b) % 4) * 8)) & 0xff) 154 155 #define NFC_DEFAULT_TIMEOUT_MS 1000 156 157 #define NFC_SRAM_SIZE 1024 158 159 #define NFC_MAX_CS 7 160 161 /** 162 * struct sunxi_nand_chip_sel - stores information related to NAND Chip Select 163 * 164 * @cs: the NAND CS id used to communicate with a NAND Chip 165 * @rb: the Ready/Busy pin ID. -1 means no R/B pin connected to the NFC 166 */ 167 struct sunxi_nand_chip_sel { 168 u8 cs; 169 s8 rb; 170 }; 171 172 /** 173 * struct sunxi_nand_hw_ecc - stores information related to HW ECC support 174 * 175 * @ecc_ctl: ECC_CTL register value for this NAND chip 176 */ 177 struct sunxi_nand_hw_ecc { 178 u32 ecc_ctl; 179 }; 180 181 /** 182 * struct sunxi_nand_chip - stores NAND chip device related information 183 * 184 * @node: used to store NAND chips into a list 185 * @nand: base NAND chip structure 186 * @ecc: ECC controller structure 187 * @clk_rate: clk_rate required for this NAND chip 188 * @timing_cfg: TIMING_CFG register value for this NAND chip 189 * @timing_ctl: TIMING_CTL register value for this NAND chip 190 * @nsels: number of CS lines required by the NAND chip 191 * @sels: array of CS lines descriptions 192 */ 193 struct sunxi_nand_chip { 194 struct list_head node; 195 struct nand_chip nand; 196 struct sunxi_nand_hw_ecc ecc; 197 unsigned long clk_rate; 198 u32 timing_cfg; 199 u32 timing_ctl; 200 int nsels; 201 struct sunxi_nand_chip_sel sels[]; 202 }; 203 204 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand) 205 { 206 return container_of(nand, struct sunxi_nand_chip, nand); 207 } 208 209 /* 210 * NAND Controller capabilities structure: stores NAND controller capabilities 211 * for distinction between compatible strings. 212 * 213 * @has_mdma: Use mbus dma mode, otherwise general dma 214 * through MBUS on A23/A33 needs extra configuration. 215 * @reg_io_data: I/O data register 216 * @dma_maxburst: DMA maxburst 217 */ 218 struct sunxi_nfc_caps { 219 bool has_mdma; 220 unsigned int reg_io_data; 221 unsigned int dma_maxburst; 222 }; 223 224 /** 225 * struct sunxi_nfc - stores sunxi NAND controller information 226 * 227 * @controller: base controller structure 228 * @dev: parent device (used to print error messages) 229 * @regs: NAND controller registers 230 * @ahb_clk: NAND controller AHB clock 231 * @mod_clk: NAND controller mod clock 232 * @reset: NAND controller reset line 233 * @assigned_cs: bitmask describing already assigned CS lines 234 * @clk_rate: NAND controller current clock rate 235 * @chips: a list containing all the NAND chips attached to this NAND 236 * controller 237 * @complete: a completion object used to wait for NAND controller events 238 * @dmac: the DMA channel attached to the NAND controller 239 * @caps: NAND Controller capabilities 240 */ 241 struct sunxi_nfc { 242 struct nand_controller controller; 243 struct device *dev; 244 void __iomem *regs; 245 struct clk *ahb_clk; 246 struct clk *mod_clk; 247 struct reset_control *reset; 248 unsigned long assigned_cs; 249 unsigned long clk_rate; 250 struct list_head chips; 251 struct completion complete; 252 struct dma_chan *dmac; 253 const struct sunxi_nfc_caps *caps; 254 }; 255 256 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_controller *ctrl) 257 { 258 return container_of(ctrl, struct sunxi_nfc, controller); 259 } 260 261 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id) 262 { 263 struct sunxi_nfc *nfc = dev_id; 264 u32 st = readl(nfc->regs + NFC_REG_ST); 265 u32 ien = readl(nfc->regs + NFC_REG_INT); 266 267 if (!(ien & st)) 268 return IRQ_NONE; 269 270 if ((ien & st) == ien) 271 complete(&nfc->complete); 272 273 writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST); 274 writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT); 275 276 return IRQ_HANDLED; 277 } 278 279 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events, 280 bool use_polling, unsigned int timeout_ms) 281 { 282 int ret; 283 284 if (events & ~NFC_INT_MASK) 285 return -EINVAL; 286 287 if (!timeout_ms) 288 timeout_ms = NFC_DEFAULT_TIMEOUT_MS; 289 290 if (!use_polling) { 291 init_completion(&nfc->complete); 292 293 writel(events, nfc->regs + NFC_REG_INT); 294 295 ret = wait_for_completion_timeout(&nfc->complete, 296 msecs_to_jiffies(timeout_ms)); 297 if (!ret) 298 ret = -ETIMEDOUT; 299 else 300 ret = 0; 301 302 writel(0, nfc->regs + NFC_REG_INT); 303 } else { 304 u32 status; 305 306 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status, 307 (status & events) == events, 1, 308 timeout_ms * 1000); 309 } 310 311 writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST); 312 313 if (ret) 314 dev_err(nfc->dev, "wait interrupt timedout\n"); 315 316 return ret; 317 } 318 319 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc) 320 { 321 u32 status; 322 int ret; 323 324 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status, 325 !(status & NFC_CMD_FIFO_STATUS), 1, 326 NFC_DEFAULT_TIMEOUT_MS * 1000); 327 if (ret) 328 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n"); 329 330 return ret; 331 } 332 333 static int sunxi_nfc_rst(struct sunxi_nfc *nfc) 334 { 335 u32 ctl; 336 int ret; 337 338 writel(0, nfc->regs + NFC_REG_ECC_CTL); 339 writel(NFC_RESET, nfc->regs + NFC_REG_CTL); 340 341 ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl, 342 !(ctl & NFC_RESET), 1, 343 NFC_DEFAULT_TIMEOUT_MS * 1000); 344 if (ret) 345 dev_err(nfc->dev, "wait for NAND controller reset timedout\n"); 346 347 return ret; 348 } 349 350 static int sunxi_nfc_dma_op_prepare(struct sunxi_nfc *nfc, const void *buf, 351 int chunksize, int nchunks, 352 enum dma_data_direction ddir, 353 struct scatterlist *sg) 354 { 355 struct dma_async_tx_descriptor *dmad; 356 enum dma_transfer_direction tdir; 357 dma_cookie_t dmat; 358 int ret; 359 360 if (ddir == DMA_FROM_DEVICE) 361 tdir = DMA_DEV_TO_MEM; 362 else 363 tdir = DMA_MEM_TO_DEV; 364 365 sg_init_one(sg, buf, nchunks * chunksize); 366 ret = dma_map_sg(nfc->dev, sg, 1, ddir); 367 if (!ret) 368 return -ENOMEM; 369 370 if (!nfc->caps->has_mdma) { 371 dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK); 372 if (!dmad) { 373 ret = -EINVAL; 374 goto err_unmap_buf; 375 } 376 } 377 378 writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD, 379 nfc->regs + NFC_REG_CTL); 380 writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM); 381 writel(chunksize, nfc->regs + NFC_REG_CNT); 382 383 if (nfc->caps->has_mdma) { 384 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_DMA_TYPE_NORMAL, 385 nfc->regs + NFC_REG_CTL); 386 writel(chunksize * nchunks, nfc->regs + NFC_REG_MDMA_CNT); 387 writel(sg_dma_address(sg), nfc->regs + NFC_REG_MDMA_ADDR); 388 } else { 389 dmat = dmaengine_submit(dmad); 390 391 ret = dma_submit_error(dmat); 392 if (ret) 393 goto err_clr_dma_flag; 394 } 395 396 return 0; 397 398 err_clr_dma_flag: 399 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD, 400 nfc->regs + NFC_REG_CTL); 401 402 err_unmap_buf: 403 dma_unmap_sg(nfc->dev, sg, 1, ddir); 404 return ret; 405 } 406 407 static void sunxi_nfc_dma_op_cleanup(struct sunxi_nfc *nfc, 408 enum dma_data_direction ddir, 409 struct scatterlist *sg) 410 { 411 dma_unmap_sg(nfc->dev, sg, 1, ddir); 412 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD, 413 nfc->regs + NFC_REG_CTL); 414 } 415 416 static void sunxi_nfc_select_chip(struct nand_chip *nand, unsigned int cs) 417 { 418 struct mtd_info *mtd = nand_to_mtd(nand); 419 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 420 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 421 struct sunxi_nand_chip_sel *sel; 422 u32 ctl; 423 424 if (cs >= sunxi_nand->nsels) 425 return; 426 427 ctl = readl(nfc->regs + NFC_REG_CTL) & 428 ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN); 429 430 sel = &sunxi_nand->sels[cs]; 431 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN | NFC_PAGE_SHIFT(nand->page_shift); 432 if (sel->rb >= 0) 433 ctl |= NFC_RB_SEL(sel->rb); 434 435 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA); 436 437 if (nfc->clk_rate != sunxi_nand->clk_rate) { 438 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate); 439 nfc->clk_rate = sunxi_nand->clk_rate; 440 } 441 442 writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL); 443 writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG); 444 writel(ctl, nfc->regs + NFC_REG_CTL); 445 } 446 447 static void sunxi_nfc_read_buf(struct nand_chip *nand, uint8_t *buf, int len) 448 { 449 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 450 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 451 int ret; 452 int cnt; 453 int offs = 0; 454 u32 tmp; 455 456 while (len > offs) { 457 bool poll = false; 458 459 cnt = min(len - offs, NFC_SRAM_SIZE); 460 461 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 462 if (ret) 463 break; 464 465 writel(cnt, nfc->regs + NFC_REG_CNT); 466 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD; 467 writel(tmp, nfc->regs + NFC_REG_CMD); 468 469 /* Arbitrary limit for polling mode */ 470 if (cnt < 64) 471 poll = true; 472 473 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0); 474 if (ret) 475 break; 476 477 if (buf) 478 memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE, 479 cnt); 480 offs += cnt; 481 } 482 } 483 484 static void sunxi_nfc_write_buf(struct nand_chip *nand, const uint8_t *buf, 485 int len) 486 { 487 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 488 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 489 int ret; 490 int cnt; 491 int offs = 0; 492 u32 tmp; 493 494 while (len > offs) { 495 bool poll = false; 496 497 cnt = min(len - offs, NFC_SRAM_SIZE); 498 499 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 500 if (ret) 501 break; 502 503 writel(cnt, nfc->regs + NFC_REG_CNT); 504 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt); 505 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | 506 NFC_ACCESS_DIR; 507 writel(tmp, nfc->regs + NFC_REG_CMD); 508 509 /* Arbitrary limit for polling mode */ 510 if (cnt < 64) 511 poll = true; 512 513 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0); 514 if (ret) 515 break; 516 517 offs += cnt; 518 } 519 } 520 521 /* These seed values have been extracted from Allwinner's BSP */ 522 static const u16 sunxi_nfc_randomizer_page_seeds[] = { 523 0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72, 524 0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436, 525 0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d, 526 0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130, 527 0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56, 528 0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55, 529 0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb, 530 0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17, 531 0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62, 532 0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064, 533 0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126, 534 0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e, 535 0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3, 536 0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b, 537 0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d, 538 0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db, 539 }; 540 541 /* 542 * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds 543 * have been generated using 544 * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what 545 * the randomizer engine does internally before de/scrambling OOB data. 546 * 547 * Those tables are statically defined to avoid calculating randomizer state 548 * at runtime. 549 */ 550 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = { 551 0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64, 552 0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409, 553 0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617, 554 0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d, 555 0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91, 556 0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d, 557 0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab, 558 0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8, 559 0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8, 560 0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b, 561 0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5, 562 0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a, 563 0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891, 564 0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36, 565 0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd, 566 0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0, 567 }; 568 569 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = { 570 0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6, 571 0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982, 572 0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9, 573 0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07, 574 0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e, 575 0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2, 576 0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c, 577 0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f, 578 0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc, 579 0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e, 580 0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8, 581 0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68, 582 0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d, 583 0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179, 584 0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601, 585 0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd, 586 }; 587 588 static u16 sunxi_nfc_randomizer_step(u16 state, int count) 589 { 590 state &= 0x7fff; 591 592 /* 593 * This loop is just a simple implementation of a Fibonacci LFSR using 594 * the x16 + x15 + 1 polynomial. 595 */ 596 while (count--) 597 state = ((state >> 1) | 598 (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff; 599 600 return state; 601 } 602 603 static u16 sunxi_nfc_randomizer_state(struct nand_chip *nand, int page, 604 bool ecc) 605 { 606 struct mtd_info *mtd = nand_to_mtd(nand); 607 const u16 *seeds = sunxi_nfc_randomizer_page_seeds; 608 int mod = mtd_div_by_ws(mtd->erasesize, mtd); 609 610 if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds)) 611 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds); 612 613 if (ecc) { 614 if (mtd->ecc_step_size == 512) 615 seeds = sunxi_nfc_randomizer_ecc512_seeds; 616 else 617 seeds = sunxi_nfc_randomizer_ecc1024_seeds; 618 } 619 620 return seeds[page % mod]; 621 } 622 623 static void sunxi_nfc_randomizer_config(struct nand_chip *nand, int page, 624 bool ecc) 625 { 626 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 627 u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); 628 u16 state; 629 630 if (!(nand->options & NAND_NEED_SCRAMBLING)) 631 return; 632 633 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); 634 state = sunxi_nfc_randomizer_state(nand, page, ecc); 635 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK; 636 writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL); 637 } 638 639 static void sunxi_nfc_randomizer_enable(struct nand_chip *nand) 640 { 641 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 642 643 if (!(nand->options & NAND_NEED_SCRAMBLING)) 644 return; 645 646 writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN, 647 nfc->regs + NFC_REG_ECC_CTL); 648 } 649 650 static void sunxi_nfc_randomizer_disable(struct nand_chip *nand) 651 { 652 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 653 654 if (!(nand->options & NAND_NEED_SCRAMBLING)) 655 return; 656 657 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN, 658 nfc->regs + NFC_REG_ECC_CTL); 659 } 660 661 static void sunxi_nfc_randomize_bbm(struct nand_chip *nand, int page, u8 *bbm) 662 { 663 u16 state = sunxi_nfc_randomizer_state(nand, page, true); 664 665 bbm[0] ^= state; 666 bbm[1] ^= sunxi_nfc_randomizer_step(state, 8); 667 } 668 669 static void sunxi_nfc_randomizer_write_buf(struct nand_chip *nand, 670 const uint8_t *buf, int len, 671 bool ecc, int page) 672 { 673 sunxi_nfc_randomizer_config(nand, page, ecc); 674 sunxi_nfc_randomizer_enable(nand); 675 sunxi_nfc_write_buf(nand, buf, len); 676 sunxi_nfc_randomizer_disable(nand); 677 } 678 679 static void sunxi_nfc_randomizer_read_buf(struct nand_chip *nand, uint8_t *buf, 680 int len, bool ecc, int page) 681 { 682 sunxi_nfc_randomizer_config(nand, page, ecc); 683 sunxi_nfc_randomizer_enable(nand); 684 sunxi_nfc_read_buf(nand, buf, len); 685 sunxi_nfc_randomizer_disable(nand); 686 } 687 688 static void sunxi_nfc_hw_ecc_enable(struct nand_chip *nand) 689 { 690 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 691 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 692 693 writel(sunxi_nand->ecc.ecc_ctl, nfc->regs + NFC_REG_ECC_CTL); 694 } 695 696 static void sunxi_nfc_hw_ecc_disable(struct nand_chip *nand) 697 { 698 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 699 700 writel(0, nfc->regs + NFC_REG_ECC_CTL); 701 } 702 703 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf) 704 { 705 buf[0] = user_data; 706 buf[1] = user_data >> 8; 707 buf[2] = user_data >> 16; 708 buf[3] = user_data >> 24; 709 } 710 711 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf) 712 { 713 return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24); 714 } 715 716 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct nand_chip *nand, u8 *oob, 717 int step, bool bbm, int page) 718 { 719 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 720 721 sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)), 722 oob); 723 724 /* De-randomize the Bad Block Marker. */ 725 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) 726 sunxi_nfc_randomize_bbm(nand, page, oob); 727 } 728 729 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand, 730 const u8 *oob, int step, 731 bool bbm, int page) 732 { 733 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 734 u8 user_data[4]; 735 736 /* Randomize the Bad Block Marker. */ 737 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) { 738 memcpy(user_data, oob, sizeof(user_data)); 739 sunxi_nfc_randomize_bbm(nand, page, user_data); 740 oob = user_data; 741 } 742 743 writel(sunxi_nfc_buf_to_user_data(oob), 744 nfc->regs + NFC_REG_USER_DATA(step)); 745 } 746 747 static void sunxi_nfc_hw_ecc_update_stats(struct nand_chip *nand, 748 unsigned int *max_bitflips, int ret) 749 { 750 struct mtd_info *mtd = nand_to_mtd(nand); 751 752 if (ret < 0) { 753 mtd->ecc_stats.failed++; 754 } else { 755 mtd->ecc_stats.corrected += ret; 756 *max_bitflips = max_t(unsigned int, *max_bitflips, ret); 757 } 758 } 759 760 static int sunxi_nfc_hw_ecc_correct(struct nand_chip *nand, u8 *data, u8 *oob, 761 int step, u32 status, bool *erased) 762 { 763 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 764 struct nand_ecc_ctrl *ecc = &nand->ecc; 765 u32 tmp; 766 767 *erased = false; 768 769 if (status & NFC_ECC_ERR(step)) 770 return -EBADMSG; 771 772 if (status & NFC_ECC_PAT_FOUND(step)) { 773 u8 pattern; 774 775 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) { 776 pattern = 0x0; 777 } else { 778 pattern = 0xff; 779 *erased = true; 780 } 781 782 if (data) 783 memset(data, pattern, ecc->size); 784 785 if (oob) 786 memset(oob, pattern, ecc->bytes + 4); 787 788 return 0; 789 } 790 791 tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step)); 792 793 return NFC_ECC_ERR_CNT(step, tmp); 794 } 795 796 static int sunxi_nfc_hw_ecc_read_chunk(struct nand_chip *nand, 797 u8 *data, int data_off, 798 u8 *oob, int oob_off, 799 int *cur_off, 800 unsigned int *max_bitflips, 801 bool bbm, bool oob_required, int page) 802 { 803 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 804 struct nand_ecc_ctrl *ecc = &nand->ecc; 805 int raw_mode = 0; 806 bool erased; 807 int ret; 808 809 if (*cur_off != data_off) 810 nand_change_read_column_op(nand, data_off, NULL, 0, false); 811 812 sunxi_nfc_randomizer_read_buf(nand, NULL, ecc->size, false, page); 813 814 if (data_off + ecc->size != oob_off) 815 nand_change_read_column_op(nand, oob_off, NULL, 0, false); 816 817 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 818 if (ret) 819 return ret; 820 821 sunxi_nfc_randomizer_enable(nand); 822 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP, 823 nfc->regs + NFC_REG_CMD); 824 825 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 826 sunxi_nfc_randomizer_disable(nand); 827 if (ret) 828 return ret; 829 830 *cur_off = oob_off + ecc->bytes + 4; 831 832 ret = sunxi_nfc_hw_ecc_correct(nand, data, oob_required ? oob : NULL, 0, 833 readl(nfc->regs + NFC_REG_ECC_ST), 834 &erased); 835 if (erased) 836 return 1; 837 838 if (ret < 0) { 839 /* 840 * Re-read the data with the randomizer disabled to identify 841 * bitflips in erased pages. 842 */ 843 if (nand->options & NAND_NEED_SCRAMBLING) 844 nand_change_read_column_op(nand, data_off, data, 845 ecc->size, false); 846 else 847 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, 848 ecc->size); 849 850 nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4, 851 false); 852 853 ret = nand_check_erased_ecc_chunk(data, ecc->size, 854 oob, ecc->bytes + 4, 855 NULL, 0, ecc->strength); 856 if (ret >= 0) 857 raw_mode = 1; 858 } else { 859 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size); 860 861 if (oob_required) { 862 nand_change_read_column_op(nand, oob_off, NULL, 0, 863 false); 864 sunxi_nfc_randomizer_read_buf(nand, oob, ecc->bytes + 4, 865 true, page); 866 867 sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, 0, 868 bbm, page); 869 } 870 } 871 872 sunxi_nfc_hw_ecc_update_stats(nand, max_bitflips, ret); 873 874 return raw_mode; 875 } 876 877 static void sunxi_nfc_hw_ecc_read_extra_oob(struct nand_chip *nand, 878 u8 *oob, int *cur_off, 879 bool randomize, int page) 880 { 881 struct mtd_info *mtd = nand_to_mtd(nand); 882 struct nand_ecc_ctrl *ecc = &nand->ecc; 883 int offset = ((ecc->bytes + 4) * ecc->steps); 884 int len = mtd->oobsize - offset; 885 886 if (len <= 0) 887 return; 888 889 if (!cur_off || *cur_off != offset) 890 nand_change_read_column_op(nand, mtd->writesize, NULL, 0, 891 false); 892 893 if (!randomize) 894 sunxi_nfc_read_buf(nand, oob + offset, len); 895 else 896 sunxi_nfc_randomizer_read_buf(nand, oob + offset, len, 897 false, page); 898 899 if (cur_off) 900 *cur_off = mtd->oobsize + mtd->writesize; 901 } 902 903 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf, 904 int oob_required, int page, 905 int nchunks) 906 { 907 bool randomized = nand->options & NAND_NEED_SCRAMBLING; 908 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 909 struct mtd_info *mtd = nand_to_mtd(nand); 910 struct nand_ecc_ctrl *ecc = &nand->ecc; 911 unsigned int max_bitflips = 0; 912 int ret, i, raw_mode = 0; 913 struct scatterlist sg; 914 u32 status, wait; 915 916 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 917 if (ret) 918 return ret; 919 920 ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, nchunks, 921 DMA_FROM_DEVICE, &sg); 922 if (ret) 923 return ret; 924 925 sunxi_nfc_hw_ecc_enable(nand); 926 sunxi_nfc_randomizer_config(nand, page, false); 927 sunxi_nfc_randomizer_enable(nand); 928 929 writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) | 930 NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET); 931 932 wait = NFC_CMD_INT_FLAG; 933 934 if (nfc->caps->has_mdma) 935 wait |= NFC_DMA_INT_FLAG; 936 else 937 dma_async_issue_pending(nfc->dmac); 938 939 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS, 940 nfc->regs + NFC_REG_CMD); 941 942 ret = sunxi_nfc_wait_events(nfc, wait, false, 0); 943 if (ret && !nfc->caps->has_mdma) 944 dmaengine_terminate_all(nfc->dmac); 945 946 sunxi_nfc_randomizer_disable(nand); 947 sunxi_nfc_hw_ecc_disable(nand); 948 949 sunxi_nfc_dma_op_cleanup(nfc, DMA_FROM_DEVICE, &sg); 950 951 if (ret) 952 return ret; 953 954 status = readl(nfc->regs + NFC_REG_ECC_ST); 955 956 for (i = 0; i < nchunks; i++) { 957 int data_off = i * ecc->size; 958 int oob_off = i * (ecc->bytes + 4); 959 u8 *data = buf + data_off; 960 u8 *oob = nand->oob_poi + oob_off; 961 bool erased; 962 963 ret = sunxi_nfc_hw_ecc_correct(nand, randomized ? data : NULL, 964 oob_required ? oob : NULL, 965 i, status, &erased); 966 967 /* ECC errors are handled in the second loop. */ 968 if (ret < 0) 969 continue; 970 971 if (oob_required && !erased) { 972 /* TODO: use DMA to retrieve OOB */ 973 nand_change_read_column_op(nand, 974 mtd->writesize + oob_off, 975 oob, ecc->bytes + 4, false); 976 977 sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, i, 978 !i, page); 979 } 980 981 if (erased) 982 raw_mode = 1; 983 984 sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret); 985 } 986 987 if (status & NFC_ECC_ERR_MSK) { 988 for (i = 0; i < nchunks; i++) { 989 int data_off = i * ecc->size; 990 int oob_off = i * (ecc->bytes + 4); 991 u8 *data = buf + data_off; 992 u8 *oob = nand->oob_poi + oob_off; 993 994 if (!(status & NFC_ECC_ERR(i))) 995 continue; 996 997 /* 998 * Re-read the data with the randomizer disabled to 999 * identify bitflips in erased pages. 1000 * TODO: use DMA to read page in raw mode 1001 */ 1002 if (randomized) 1003 nand_change_read_column_op(nand, data_off, 1004 data, ecc->size, 1005 false); 1006 1007 /* TODO: use DMA to retrieve OOB */ 1008 nand_change_read_column_op(nand, 1009 mtd->writesize + oob_off, 1010 oob, ecc->bytes + 4, false); 1011 1012 ret = nand_check_erased_ecc_chunk(data, ecc->size, 1013 oob, ecc->bytes + 4, 1014 NULL, 0, 1015 ecc->strength); 1016 if (ret >= 0) 1017 raw_mode = 1; 1018 1019 sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret); 1020 } 1021 } 1022 1023 if (oob_required) 1024 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi, 1025 NULL, !raw_mode, 1026 page); 1027 1028 return max_bitflips; 1029 } 1030 1031 static int sunxi_nfc_hw_ecc_write_chunk(struct nand_chip *nand, 1032 const u8 *data, int data_off, 1033 const u8 *oob, int oob_off, 1034 int *cur_off, bool bbm, 1035 int page) 1036 { 1037 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1038 struct nand_ecc_ctrl *ecc = &nand->ecc; 1039 int ret; 1040 1041 if (data_off != *cur_off) 1042 nand_change_write_column_op(nand, data_off, NULL, 0, false); 1043 1044 sunxi_nfc_randomizer_write_buf(nand, data, ecc->size, false, page); 1045 1046 if (data_off + ecc->size != oob_off) 1047 nand_change_write_column_op(nand, oob_off, NULL, 0, false); 1048 1049 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1050 if (ret) 1051 return ret; 1052 1053 sunxi_nfc_randomizer_enable(nand); 1054 sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, 0, bbm, page); 1055 1056 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | 1057 NFC_ACCESS_DIR | NFC_ECC_OP, 1058 nfc->regs + NFC_REG_CMD); 1059 1060 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 1061 sunxi_nfc_randomizer_disable(nand); 1062 if (ret) 1063 return ret; 1064 1065 *cur_off = oob_off + ecc->bytes + 4; 1066 1067 return 0; 1068 } 1069 1070 static void sunxi_nfc_hw_ecc_write_extra_oob(struct nand_chip *nand, 1071 u8 *oob, int *cur_off, 1072 int page) 1073 { 1074 struct mtd_info *mtd = nand_to_mtd(nand); 1075 struct nand_ecc_ctrl *ecc = &nand->ecc; 1076 int offset = ((ecc->bytes + 4) * ecc->steps); 1077 int len = mtd->oobsize - offset; 1078 1079 if (len <= 0) 1080 return; 1081 1082 if (!cur_off || *cur_off != offset) 1083 nand_change_write_column_op(nand, offset + mtd->writesize, 1084 NULL, 0, false); 1085 1086 sunxi_nfc_randomizer_write_buf(nand, oob + offset, len, false, page); 1087 1088 if (cur_off) 1089 *cur_off = mtd->oobsize + mtd->writesize; 1090 } 1091 1092 static int sunxi_nfc_hw_ecc_read_page(struct nand_chip *nand, uint8_t *buf, 1093 int oob_required, int page) 1094 { 1095 struct mtd_info *mtd = nand_to_mtd(nand); 1096 struct nand_ecc_ctrl *ecc = &nand->ecc; 1097 unsigned int max_bitflips = 0; 1098 int ret, i, cur_off = 0; 1099 bool raw_mode = false; 1100 1101 sunxi_nfc_select_chip(nand, nand->cur_cs); 1102 1103 nand_read_page_op(nand, page, 0, NULL, 0); 1104 1105 sunxi_nfc_hw_ecc_enable(nand); 1106 1107 for (i = 0; i < ecc->steps; i++) { 1108 int data_off = i * ecc->size; 1109 int oob_off = i * (ecc->bytes + 4); 1110 u8 *data = buf + data_off; 1111 u8 *oob = nand->oob_poi + oob_off; 1112 1113 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off, oob, 1114 oob_off + mtd->writesize, 1115 &cur_off, &max_bitflips, 1116 !i, oob_required, page); 1117 if (ret < 0) 1118 return ret; 1119 else if (ret) 1120 raw_mode = true; 1121 } 1122 1123 if (oob_required) 1124 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi, &cur_off, 1125 !raw_mode, page); 1126 1127 sunxi_nfc_hw_ecc_disable(nand); 1128 1129 return max_bitflips; 1130 } 1131 1132 static int sunxi_nfc_hw_ecc_read_page_dma(struct nand_chip *nand, u8 *buf, 1133 int oob_required, int page) 1134 { 1135 int ret; 1136 1137 sunxi_nfc_select_chip(nand, nand->cur_cs); 1138 1139 nand_read_page_op(nand, page, 0, NULL, 0); 1140 1141 ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, oob_required, page, 1142 nand->ecc.steps); 1143 if (ret >= 0) 1144 return ret; 1145 1146 /* Fallback to PIO mode */ 1147 return sunxi_nfc_hw_ecc_read_page(nand, buf, oob_required, page); 1148 } 1149 1150 static int sunxi_nfc_hw_ecc_read_subpage(struct nand_chip *nand, 1151 u32 data_offs, u32 readlen, 1152 u8 *bufpoi, int page) 1153 { 1154 struct mtd_info *mtd = nand_to_mtd(nand); 1155 struct nand_ecc_ctrl *ecc = &nand->ecc; 1156 int ret, i, cur_off = 0; 1157 unsigned int max_bitflips = 0; 1158 1159 sunxi_nfc_select_chip(nand, nand->cur_cs); 1160 1161 nand_read_page_op(nand, page, 0, NULL, 0); 1162 1163 sunxi_nfc_hw_ecc_enable(nand); 1164 1165 for (i = data_offs / ecc->size; 1166 i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) { 1167 int data_off = i * ecc->size; 1168 int oob_off = i * (ecc->bytes + 4); 1169 u8 *data = bufpoi + data_off; 1170 u8 *oob = nand->oob_poi + oob_off; 1171 1172 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off, 1173 oob, 1174 oob_off + mtd->writesize, 1175 &cur_off, &max_bitflips, !i, 1176 false, page); 1177 if (ret < 0) 1178 return ret; 1179 } 1180 1181 sunxi_nfc_hw_ecc_disable(nand); 1182 1183 return max_bitflips; 1184 } 1185 1186 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct nand_chip *nand, 1187 u32 data_offs, u32 readlen, 1188 u8 *buf, int page) 1189 { 1190 int nchunks = DIV_ROUND_UP(data_offs + readlen, nand->ecc.size); 1191 int ret; 1192 1193 sunxi_nfc_select_chip(nand, nand->cur_cs); 1194 1195 nand_read_page_op(nand, page, 0, NULL, 0); 1196 1197 ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, false, page, nchunks); 1198 if (ret >= 0) 1199 return ret; 1200 1201 /* Fallback to PIO mode */ 1202 return sunxi_nfc_hw_ecc_read_subpage(nand, data_offs, readlen, 1203 buf, page); 1204 } 1205 1206 static int sunxi_nfc_hw_ecc_write_page(struct nand_chip *nand, 1207 const uint8_t *buf, int oob_required, 1208 int page) 1209 { 1210 struct mtd_info *mtd = nand_to_mtd(nand); 1211 struct nand_ecc_ctrl *ecc = &nand->ecc; 1212 int ret, i, cur_off = 0; 1213 1214 sunxi_nfc_select_chip(nand, nand->cur_cs); 1215 1216 nand_prog_page_begin_op(nand, page, 0, NULL, 0); 1217 1218 sunxi_nfc_hw_ecc_enable(nand); 1219 1220 for (i = 0; i < ecc->steps; i++) { 1221 int data_off = i * ecc->size; 1222 int oob_off = i * (ecc->bytes + 4); 1223 const u8 *data = buf + data_off; 1224 const u8 *oob = nand->oob_poi + oob_off; 1225 1226 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob, 1227 oob_off + mtd->writesize, 1228 &cur_off, !i, page); 1229 if (ret) 1230 return ret; 1231 } 1232 1233 if (oob_required || (nand->options & NAND_NEED_SCRAMBLING)) 1234 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi, 1235 &cur_off, page); 1236 1237 sunxi_nfc_hw_ecc_disable(nand); 1238 1239 return nand_prog_page_end_op(nand); 1240 } 1241 1242 static int sunxi_nfc_hw_ecc_write_subpage(struct nand_chip *nand, 1243 u32 data_offs, u32 data_len, 1244 const u8 *buf, int oob_required, 1245 int page) 1246 { 1247 struct mtd_info *mtd = nand_to_mtd(nand); 1248 struct nand_ecc_ctrl *ecc = &nand->ecc; 1249 int ret, i, cur_off = 0; 1250 1251 sunxi_nfc_select_chip(nand, nand->cur_cs); 1252 1253 nand_prog_page_begin_op(nand, page, 0, NULL, 0); 1254 1255 sunxi_nfc_hw_ecc_enable(nand); 1256 1257 for (i = data_offs / ecc->size; 1258 i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) { 1259 int data_off = i * ecc->size; 1260 int oob_off = i * (ecc->bytes + 4); 1261 const u8 *data = buf + data_off; 1262 const u8 *oob = nand->oob_poi + oob_off; 1263 1264 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob, 1265 oob_off + mtd->writesize, 1266 &cur_off, !i, page); 1267 if (ret) 1268 return ret; 1269 } 1270 1271 sunxi_nfc_hw_ecc_disable(nand); 1272 1273 return nand_prog_page_end_op(nand); 1274 } 1275 1276 static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *nand, 1277 const u8 *buf, 1278 int oob_required, 1279 int page) 1280 { 1281 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1282 struct nand_ecc_ctrl *ecc = &nand->ecc; 1283 struct scatterlist sg; 1284 u32 wait; 1285 int ret, i; 1286 1287 sunxi_nfc_select_chip(nand, nand->cur_cs); 1288 1289 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1290 if (ret) 1291 return ret; 1292 1293 ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, ecc->steps, 1294 DMA_TO_DEVICE, &sg); 1295 if (ret) 1296 goto pio_fallback; 1297 1298 for (i = 0; i < ecc->steps; i++) { 1299 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4)); 1300 1301 sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, i, !i, page); 1302 } 1303 1304 nand_prog_page_begin_op(nand, page, 0, NULL, 0); 1305 1306 sunxi_nfc_hw_ecc_enable(nand); 1307 sunxi_nfc_randomizer_config(nand, page, false); 1308 sunxi_nfc_randomizer_enable(nand); 1309 1310 writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG, 1311 nfc->regs + NFC_REG_WCMD_SET); 1312 1313 wait = NFC_CMD_INT_FLAG; 1314 1315 if (nfc->caps->has_mdma) 1316 wait |= NFC_DMA_INT_FLAG; 1317 else 1318 dma_async_issue_pending(nfc->dmac); 1319 1320 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | 1321 NFC_DATA_TRANS | NFC_ACCESS_DIR, 1322 nfc->regs + NFC_REG_CMD); 1323 1324 ret = sunxi_nfc_wait_events(nfc, wait, false, 0); 1325 if (ret && !nfc->caps->has_mdma) 1326 dmaengine_terminate_all(nfc->dmac); 1327 1328 sunxi_nfc_randomizer_disable(nand); 1329 sunxi_nfc_hw_ecc_disable(nand); 1330 1331 sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg); 1332 1333 if (ret) 1334 return ret; 1335 1336 if (oob_required || (nand->options & NAND_NEED_SCRAMBLING)) 1337 /* TODO: use DMA to transfer extra OOB bytes ? */ 1338 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi, 1339 NULL, page); 1340 1341 return nand_prog_page_end_op(nand); 1342 1343 pio_fallback: 1344 return sunxi_nfc_hw_ecc_write_page(nand, buf, oob_required, page); 1345 } 1346 1347 static int sunxi_nfc_hw_ecc_read_oob(struct nand_chip *nand, int page) 1348 { 1349 u8 *buf = nand_get_data_buf(nand); 1350 1351 return nand->ecc.read_page(nand, buf, 1, page); 1352 } 1353 1354 static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *nand, int page) 1355 { 1356 struct mtd_info *mtd = nand_to_mtd(nand); 1357 u8 *buf = nand_get_data_buf(nand); 1358 int ret; 1359 1360 memset(buf, 0xff, mtd->writesize); 1361 ret = nand->ecc.write_page(nand, buf, 1, page); 1362 if (ret) 1363 return ret; 1364 1365 /* Send command to program the OOB data */ 1366 return nand_prog_page_end_op(nand); 1367 } 1368 1369 static const s32 tWB_lut[] = {6, 12, 16, 20}; 1370 static const s32 tRHW_lut[] = {4, 8, 12, 20}; 1371 1372 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration, 1373 u32 clk_period) 1374 { 1375 u32 clk_cycles = DIV_ROUND_UP(duration, clk_period); 1376 int i; 1377 1378 for (i = 0; i < lut_size; i++) { 1379 if (clk_cycles <= lut[i]) 1380 return i; 1381 } 1382 1383 /* Doesn't fit */ 1384 return -EINVAL; 1385 } 1386 1387 #define sunxi_nand_lookup_timing(l, p, c) \ 1388 _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c) 1389 1390 static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline, 1391 const struct nand_interface_config *conf) 1392 { 1393 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 1394 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 1395 const struct nand_sdr_timings *timings; 1396 u32 min_clk_period = 0; 1397 s32 tWB, tADL, tWHR, tRHW, tCAD; 1398 long real_clk_rate; 1399 1400 timings = nand_get_sdr_timings(conf); 1401 if (IS_ERR(timings)) 1402 return -ENOTSUPP; 1403 1404 /* T1 <=> tCLS */ 1405 if (timings->tCLS_min > min_clk_period) 1406 min_clk_period = timings->tCLS_min; 1407 1408 /* T2 <=> tCLH */ 1409 if (timings->tCLH_min > min_clk_period) 1410 min_clk_period = timings->tCLH_min; 1411 1412 /* T3 <=> tCS */ 1413 if (timings->tCS_min > min_clk_period) 1414 min_clk_period = timings->tCS_min; 1415 1416 /* T4 <=> tCH */ 1417 if (timings->tCH_min > min_clk_period) 1418 min_clk_period = timings->tCH_min; 1419 1420 /* T5 <=> tWP */ 1421 if (timings->tWP_min > min_clk_period) 1422 min_clk_period = timings->tWP_min; 1423 1424 /* T6 <=> tWH */ 1425 if (timings->tWH_min > min_clk_period) 1426 min_clk_period = timings->tWH_min; 1427 1428 /* T7 <=> tALS */ 1429 if (timings->tALS_min > min_clk_period) 1430 min_clk_period = timings->tALS_min; 1431 1432 /* T8 <=> tDS */ 1433 if (timings->tDS_min > min_clk_period) 1434 min_clk_period = timings->tDS_min; 1435 1436 /* T9 <=> tDH */ 1437 if (timings->tDH_min > min_clk_period) 1438 min_clk_period = timings->tDH_min; 1439 1440 /* T10 <=> tRR */ 1441 if (timings->tRR_min > (min_clk_period * 3)) 1442 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3); 1443 1444 /* T11 <=> tALH */ 1445 if (timings->tALH_min > min_clk_period) 1446 min_clk_period = timings->tALH_min; 1447 1448 /* T12 <=> tRP */ 1449 if (timings->tRP_min > min_clk_period) 1450 min_clk_period = timings->tRP_min; 1451 1452 /* T13 <=> tREH */ 1453 if (timings->tREH_min > min_clk_period) 1454 min_clk_period = timings->tREH_min; 1455 1456 /* T14 <=> tRC */ 1457 if (timings->tRC_min > (min_clk_period * 2)) 1458 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2); 1459 1460 /* T15 <=> tWC */ 1461 if (timings->tWC_min > (min_clk_period * 2)) 1462 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2); 1463 1464 /* T16 - T19 + tCAD */ 1465 if (timings->tWB_max > (min_clk_period * 20)) 1466 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20); 1467 1468 if (timings->tADL_min > (min_clk_period * 32)) 1469 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32); 1470 1471 if (timings->tWHR_min > (min_clk_period * 32)) 1472 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32); 1473 1474 if (timings->tRHW_min > (min_clk_period * 20)) 1475 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20); 1476 1477 /* 1478 * In non-EDO, tREA should be less than tRP to guarantee that the 1479 * controller does not sample the IO lines too early. Unfortunately, 1480 * the sunxi NAND controller does not allow us to have different 1481 * values for tRP and tREH (tRP = tREH = tRW / 2). 1482 * 1483 * We have 2 options to overcome this limitation: 1484 * 1485 * 1/ Extend tRC to fulfil the tREA <= tRC / 2 constraint 1486 * 2/ Use EDO mode (only works if timings->tRLOH > 0) 1487 */ 1488 if (timings->tREA_max > min_clk_period && !timings->tRLOH_min) 1489 min_clk_period = timings->tREA_max; 1490 1491 tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max, 1492 min_clk_period); 1493 if (tWB < 0) { 1494 dev_err(nfc->dev, "unsupported tWB\n"); 1495 return tWB; 1496 } 1497 1498 tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3; 1499 if (tADL > 3) { 1500 dev_err(nfc->dev, "unsupported tADL\n"); 1501 return -EINVAL; 1502 } 1503 1504 tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3; 1505 if (tWHR > 3) { 1506 dev_err(nfc->dev, "unsupported tWHR\n"); 1507 return -EINVAL; 1508 } 1509 1510 tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min, 1511 min_clk_period); 1512 if (tRHW < 0) { 1513 dev_err(nfc->dev, "unsupported tRHW\n"); 1514 return tRHW; 1515 } 1516 1517 if (csline == NAND_DATA_IFACE_CHECK_ONLY) 1518 return 0; 1519 1520 /* 1521 * TODO: according to ONFI specs this value only applies for DDR NAND, 1522 * but Allwinner seems to set this to 0x7. Mimic them for now. 1523 */ 1524 tCAD = 0x7; 1525 1526 /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */ 1527 sunxi_nand->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD); 1528 1529 /* Convert min_clk_period from picoseconds to nanoseconds */ 1530 min_clk_period = DIV_ROUND_UP(min_clk_period, 1000); 1531 1532 /* 1533 * Unlike what is stated in Allwinner datasheet, the clk_rate should 1534 * be set to (1 / min_clk_period), and not (2 / min_clk_period). 1535 * This new formula was verified with a scope and validated by 1536 * Allwinner engineers. 1537 */ 1538 sunxi_nand->clk_rate = NSEC_PER_SEC / min_clk_period; 1539 real_clk_rate = clk_round_rate(nfc->mod_clk, sunxi_nand->clk_rate); 1540 if (real_clk_rate <= 0) { 1541 dev_err(nfc->dev, "Unable to round clk %lu\n", 1542 sunxi_nand->clk_rate); 1543 return -EINVAL; 1544 } 1545 1546 sunxi_nand->timing_ctl = 0; 1547 1548 /* 1549 * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data 1550 * output cycle timings shall be used if the host drives tRC less than 1551 * 30 ns. We should also use EDO mode if tREA is bigger than tRP. 1552 */ 1553 min_clk_period = NSEC_PER_SEC / real_clk_rate; 1554 if (min_clk_period * 2 < 30 || min_clk_period * 1000 < timings->tREA_max) 1555 sunxi_nand->timing_ctl = NFC_TIMING_CTL_EDO; 1556 1557 return 0; 1558 } 1559 1560 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section, 1561 struct mtd_oob_region *oobregion) 1562 { 1563 struct nand_chip *nand = mtd_to_nand(mtd); 1564 struct nand_ecc_ctrl *ecc = &nand->ecc; 1565 1566 if (section >= ecc->steps) 1567 return -ERANGE; 1568 1569 oobregion->offset = section * (ecc->bytes + 4) + 4; 1570 oobregion->length = ecc->bytes; 1571 1572 return 0; 1573 } 1574 1575 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section, 1576 struct mtd_oob_region *oobregion) 1577 { 1578 struct nand_chip *nand = mtd_to_nand(mtd); 1579 struct nand_ecc_ctrl *ecc = &nand->ecc; 1580 1581 if (section > ecc->steps) 1582 return -ERANGE; 1583 1584 /* 1585 * The first 2 bytes are used for BB markers, hence we 1586 * only have 2 bytes available in the first user data 1587 * section. 1588 */ 1589 if (!section && ecc->engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) { 1590 oobregion->offset = 2; 1591 oobregion->length = 2; 1592 1593 return 0; 1594 } 1595 1596 /* 1597 * The controller does not provide access to OOB bytes 1598 * past the end of the ECC data. 1599 */ 1600 if (section == ecc->steps && ecc->engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) 1601 return -ERANGE; 1602 1603 oobregion->offset = section * (ecc->bytes + 4); 1604 1605 if (section < ecc->steps) 1606 oobregion->length = 4; 1607 else 1608 oobregion->length = mtd->oobsize - oobregion->offset; 1609 1610 return 0; 1611 } 1612 1613 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = { 1614 .ecc = sunxi_nand_ooblayout_ecc, 1615 .free = sunxi_nand_ooblayout_free, 1616 }; 1617 1618 static int sunxi_nand_hw_ecc_ctrl_init(struct nand_chip *nand, 1619 struct nand_ecc_ctrl *ecc, 1620 struct device_node *np) 1621 { 1622 static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 }; 1623 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 1624 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1625 struct mtd_info *mtd = nand_to_mtd(nand); 1626 struct nand_device *nanddev = mtd_to_nanddev(mtd); 1627 int nsectors; 1628 int i; 1629 1630 if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) { 1631 int bytes; 1632 1633 ecc->size = 1024; 1634 nsectors = mtd->writesize / ecc->size; 1635 1636 /* Reserve 2 bytes for the BBM */ 1637 bytes = (mtd->oobsize - 2) / nsectors; 1638 1639 /* 4 non-ECC bytes are added before each ECC bytes section */ 1640 bytes -= 4; 1641 1642 /* and bytes has to be even. */ 1643 if (bytes % 2) 1644 bytes--; 1645 1646 ecc->strength = bytes * 8 / fls(8 * ecc->size); 1647 1648 for (i = 0; i < ARRAY_SIZE(strengths); i++) { 1649 if (strengths[i] > ecc->strength) 1650 break; 1651 } 1652 1653 if (!i) 1654 ecc->strength = 0; 1655 else 1656 ecc->strength = strengths[i - 1]; 1657 } 1658 1659 if (ecc->size != 512 && ecc->size != 1024) 1660 return -EINVAL; 1661 1662 /* Prefer 1k ECC chunk over 512 ones */ 1663 if (ecc->size == 512 && mtd->writesize > 512) { 1664 ecc->size = 1024; 1665 ecc->strength *= 2; 1666 } 1667 1668 /* Add ECC info retrieval from DT */ 1669 for (i = 0; i < ARRAY_SIZE(strengths); i++) { 1670 if (ecc->strength <= strengths[i]) { 1671 /* 1672 * Update ecc->strength value with the actual strength 1673 * that will be used by the ECC engine. 1674 */ 1675 ecc->strength = strengths[i]; 1676 break; 1677 } 1678 } 1679 1680 if (i >= ARRAY_SIZE(strengths)) { 1681 dev_err(nfc->dev, "unsupported strength\n"); 1682 return -ENOTSUPP; 1683 } 1684 1685 /* HW ECC always request ECC bytes for 1024 bytes blocks */ 1686 ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8); 1687 1688 /* HW ECC always work with even numbers of ECC bytes */ 1689 ecc->bytes = ALIGN(ecc->bytes, 2); 1690 1691 nsectors = mtd->writesize / ecc->size; 1692 1693 if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) 1694 return -EINVAL; 1695 1696 ecc->read_oob = sunxi_nfc_hw_ecc_read_oob; 1697 ecc->write_oob = sunxi_nfc_hw_ecc_write_oob; 1698 mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops); 1699 1700 if (nfc->dmac || nfc->caps->has_mdma) { 1701 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma; 1702 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma; 1703 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma; 1704 nand->options |= NAND_USES_DMA; 1705 } else { 1706 ecc->read_page = sunxi_nfc_hw_ecc_read_page; 1707 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage; 1708 ecc->write_page = sunxi_nfc_hw_ecc_write_page; 1709 } 1710 1711 /* TODO: support DMA for raw accesses and subpage write */ 1712 ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage; 1713 ecc->read_oob_raw = nand_read_oob_std; 1714 ecc->write_oob_raw = nand_write_oob_std; 1715 1716 sunxi_nand->ecc.ecc_ctl = NFC_ECC_MODE(i) | NFC_ECC_EXCEPTION | 1717 NFC_ECC_PIPELINE | NFC_ECC_EN; 1718 1719 if (ecc->size == 512) 1720 sunxi_nand->ecc.ecc_ctl |= NFC_ECC_BLOCK_512; 1721 1722 return 0; 1723 } 1724 1725 static int sunxi_nand_attach_chip(struct nand_chip *nand) 1726 { 1727 const struct nand_ecc_props *requirements = 1728 nanddev_get_ecc_requirements(&nand->base); 1729 struct nand_ecc_ctrl *ecc = &nand->ecc; 1730 struct device_node *np = nand_get_flash_node(nand); 1731 int ret; 1732 1733 if (nand->bbt_options & NAND_BBT_USE_FLASH) 1734 nand->bbt_options |= NAND_BBT_NO_OOB; 1735 1736 if (nand->options & NAND_NEED_SCRAMBLING) 1737 nand->options |= NAND_NO_SUBPAGE_WRITE; 1738 1739 nand->options |= NAND_SUBPAGE_READ; 1740 1741 if (!ecc->size) { 1742 ecc->size = requirements->step_size; 1743 ecc->strength = requirements->strength; 1744 } 1745 1746 if (!ecc->size || !ecc->strength) 1747 return -EINVAL; 1748 1749 switch (ecc->engine_type) { 1750 case NAND_ECC_ENGINE_TYPE_ON_HOST: 1751 ret = sunxi_nand_hw_ecc_ctrl_init(nand, ecc, np); 1752 if (ret) 1753 return ret; 1754 break; 1755 case NAND_ECC_ENGINE_TYPE_NONE: 1756 case NAND_ECC_ENGINE_TYPE_SOFT: 1757 break; 1758 default: 1759 return -EINVAL; 1760 } 1761 1762 return 0; 1763 } 1764 1765 static int sunxi_nfc_exec_subop(struct nand_chip *nand, 1766 const struct nand_subop *subop) 1767 { 1768 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1769 u32 cmd = 0, extcmd = 0, cnt = 0, addrs[2] = { }; 1770 unsigned int i, j, remaining, start; 1771 void *inbuf = NULL; 1772 int ret; 1773 1774 for (i = 0; i < subop->ninstrs; i++) { 1775 const struct nand_op_instr *instr = &subop->instrs[i]; 1776 1777 switch (instr->type) { 1778 case NAND_OP_CMD_INSTR: 1779 if (cmd & NFC_SEND_CMD1) { 1780 if (WARN_ON(cmd & NFC_SEND_CMD2)) 1781 return -EINVAL; 1782 1783 cmd |= NFC_SEND_CMD2; 1784 extcmd |= instr->ctx.cmd.opcode; 1785 } else { 1786 cmd |= NFC_SEND_CMD1 | 1787 NFC_CMD(instr->ctx.cmd.opcode); 1788 } 1789 break; 1790 1791 case NAND_OP_ADDR_INSTR: 1792 remaining = nand_subop_get_num_addr_cyc(subop, i); 1793 start = nand_subop_get_addr_start_off(subop, i); 1794 for (j = 0; j < 8 && j + start < remaining; j++) { 1795 u32 addr = instr->ctx.addr.addrs[j + start]; 1796 1797 addrs[j / 4] |= addr << (j % 4) * 8; 1798 } 1799 1800 if (j) 1801 cmd |= NFC_SEND_ADR | NFC_ADR_NUM(j); 1802 1803 break; 1804 1805 case NAND_OP_DATA_IN_INSTR: 1806 case NAND_OP_DATA_OUT_INSTR: 1807 start = nand_subop_get_data_start_off(subop, i); 1808 remaining = nand_subop_get_data_len(subop, i); 1809 cnt = min_t(u32, remaining, NFC_SRAM_SIZE); 1810 cmd |= NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD; 1811 1812 if (instr->type == NAND_OP_DATA_OUT_INSTR) { 1813 cmd |= NFC_ACCESS_DIR; 1814 memcpy_toio(nfc->regs + NFC_RAM0_BASE, 1815 instr->ctx.data.buf.out + start, 1816 cnt); 1817 } else { 1818 inbuf = instr->ctx.data.buf.in + start; 1819 } 1820 1821 break; 1822 1823 case NAND_OP_WAITRDY_INSTR: 1824 cmd |= NFC_WAIT_FLAG; 1825 break; 1826 } 1827 } 1828 1829 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1830 if (ret) 1831 return ret; 1832 1833 if (cmd & NFC_SEND_ADR) { 1834 writel(addrs[0], nfc->regs + NFC_REG_ADDR_LOW); 1835 writel(addrs[1], nfc->regs + NFC_REG_ADDR_HIGH); 1836 } 1837 1838 if (cmd & NFC_SEND_CMD2) 1839 writel(extcmd, 1840 nfc->regs + 1841 (cmd & NFC_ACCESS_DIR ? 1842 NFC_REG_WCMD_SET : NFC_REG_RCMD_SET)); 1843 1844 if (cmd & NFC_DATA_TRANS) 1845 writel(cnt, nfc->regs + NFC_REG_CNT); 1846 1847 writel(cmd, nfc->regs + NFC_REG_CMD); 1848 1849 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, 1850 !(cmd & NFC_WAIT_FLAG) && cnt < 64, 1851 0); 1852 if (ret) 1853 return ret; 1854 1855 if (inbuf) 1856 memcpy_fromio(inbuf, nfc->regs + NFC_RAM0_BASE, cnt); 1857 1858 return 0; 1859 } 1860 1861 static int sunxi_nfc_soft_waitrdy(struct nand_chip *nand, 1862 const struct nand_subop *subop) 1863 { 1864 return nand_soft_waitrdy(nand, 1865 subop->instrs[0].ctx.waitrdy.timeout_ms); 1866 } 1867 1868 static const struct nand_op_parser sunxi_nfc_op_parser = NAND_OP_PARSER( 1869 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop, 1870 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1871 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8), 1872 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1873 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true), 1874 NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)), 1875 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop, 1876 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1877 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8), 1878 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024), 1879 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1880 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)), 1881 ); 1882 1883 static const struct nand_op_parser sunxi_nfc_norb_op_parser = NAND_OP_PARSER( 1884 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop, 1885 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1886 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8), 1887 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1888 NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)), 1889 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop, 1890 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1891 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8), 1892 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024), 1893 NAND_OP_PARSER_PAT_CMD_ELEM(true)), 1894 NAND_OP_PARSER_PATTERN(sunxi_nfc_soft_waitrdy, 1895 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)), 1896 ); 1897 1898 static int sunxi_nfc_exec_op(struct nand_chip *nand, 1899 const struct nand_operation *op, bool check_only) 1900 { 1901 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 1902 const struct nand_op_parser *parser; 1903 1904 if (!check_only) 1905 sunxi_nfc_select_chip(nand, op->cs); 1906 1907 if (sunxi_nand->sels[op->cs].rb >= 0) 1908 parser = &sunxi_nfc_op_parser; 1909 else 1910 parser = &sunxi_nfc_norb_op_parser; 1911 1912 return nand_op_parser_exec_op(nand, parser, op, check_only); 1913 } 1914 1915 static const struct nand_controller_ops sunxi_nand_controller_ops = { 1916 .attach_chip = sunxi_nand_attach_chip, 1917 .setup_interface = sunxi_nfc_setup_interface, 1918 .exec_op = sunxi_nfc_exec_op, 1919 }; 1920 1921 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc) 1922 { 1923 struct sunxi_nand_chip *sunxi_nand; 1924 struct nand_chip *chip; 1925 int ret; 1926 1927 while (!list_empty(&nfc->chips)) { 1928 sunxi_nand = list_first_entry(&nfc->chips, 1929 struct sunxi_nand_chip, 1930 node); 1931 chip = &sunxi_nand->nand; 1932 ret = mtd_device_unregister(nand_to_mtd(chip)); 1933 WARN_ON(ret); 1934 nand_cleanup(chip); 1935 list_del(&sunxi_nand->node); 1936 } 1937 } 1938 1939 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc, 1940 struct device_node *np) 1941 { 1942 struct sunxi_nand_chip *sunxi_nand; 1943 struct mtd_info *mtd; 1944 struct nand_chip *nand; 1945 int nsels; 1946 int ret; 1947 int i; 1948 u32 tmp; 1949 1950 if (!of_get_property(np, "reg", &nsels)) 1951 return -EINVAL; 1952 1953 nsels /= sizeof(u32); 1954 if (!nsels) { 1955 dev_err(dev, "invalid reg property size\n"); 1956 return -EINVAL; 1957 } 1958 1959 sunxi_nand = devm_kzalloc(dev, struct_size(sunxi_nand, sels, nsels), 1960 GFP_KERNEL); 1961 if (!sunxi_nand) 1962 return -ENOMEM; 1963 1964 sunxi_nand->nsels = nsels; 1965 1966 for (i = 0; i < nsels; i++) { 1967 ret = of_property_read_u32_index(np, "reg", i, &tmp); 1968 if (ret) { 1969 dev_err(dev, "could not retrieve reg property: %d\n", 1970 ret); 1971 return ret; 1972 } 1973 1974 if (tmp > NFC_MAX_CS) { 1975 dev_err(dev, 1976 "invalid reg value: %u (max CS = 7)\n", 1977 tmp); 1978 return -EINVAL; 1979 } 1980 1981 if (test_and_set_bit(tmp, &nfc->assigned_cs)) { 1982 dev_err(dev, "CS %d already assigned\n", tmp); 1983 return -EINVAL; 1984 } 1985 1986 sunxi_nand->sels[i].cs = tmp; 1987 1988 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) && 1989 tmp < 2) 1990 sunxi_nand->sels[i].rb = tmp; 1991 else 1992 sunxi_nand->sels[i].rb = -1; 1993 } 1994 1995 nand = &sunxi_nand->nand; 1996 /* Default tR value specified in the ONFI spec (chapter 4.15.1) */ 1997 nand->controller = &nfc->controller; 1998 nand->controller->ops = &sunxi_nand_controller_ops; 1999 2000 /* 2001 * Set the ECC mode to the default value in case nothing is specified 2002 * in the DT. 2003 */ 2004 nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; 2005 nand_set_flash_node(nand, np); 2006 2007 mtd = nand_to_mtd(nand); 2008 mtd->dev.parent = dev; 2009 2010 ret = nand_scan(nand, nsels); 2011 if (ret) 2012 return ret; 2013 2014 ret = mtd_device_register(mtd, NULL, 0); 2015 if (ret) { 2016 dev_err(dev, "failed to register mtd device: %d\n", ret); 2017 nand_cleanup(nand); 2018 return ret; 2019 } 2020 2021 list_add_tail(&sunxi_nand->node, &nfc->chips); 2022 2023 return 0; 2024 } 2025 2026 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc) 2027 { 2028 struct device_node *np = dev->of_node; 2029 struct device_node *nand_np; 2030 int ret; 2031 2032 for_each_child_of_node(np, nand_np) { 2033 ret = sunxi_nand_chip_init(dev, nfc, nand_np); 2034 if (ret) { 2035 of_node_put(nand_np); 2036 sunxi_nand_chips_cleanup(nfc); 2037 return ret; 2038 } 2039 } 2040 2041 return 0; 2042 } 2043 2044 static int sunxi_nfc_dma_init(struct sunxi_nfc *nfc, struct resource *r) 2045 { 2046 int ret; 2047 2048 if (nfc->caps->has_mdma) 2049 return 0; 2050 2051 nfc->dmac = dma_request_chan(nfc->dev, "rxtx"); 2052 if (IS_ERR(nfc->dmac)) { 2053 ret = PTR_ERR(nfc->dmac); 2054 if (ret == -EPROBE_DEFER) 2055 return ret; 2056 2057 /* Ignore errors to fall back to PIO mode */ 2058 dev_warn(nfc->dev, "failed to request rxtx DMA channel: %d\n", ret); 2059 nfc->dmac = NULL; 2060 } else { 2061 struct dma_slave_config dmac_cfg = { }; 2062 2063 dmac_cfg.src_addr = r->start + nfc->caps->reg_io_data; 2064 dmac_cfg.dst_addr = dmac_cfg.src_addr; 2065 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 2066 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width; 2067 dmac_cfg.src_maxburst = nfc->caps->dma_maxburst; 2068 dmac_cfg.dst_maxburst = nfc->caps->dma_maxburst; 2069 dmaengine_slave_config(nfc->dmac, &dmac_cfg); 2070 } 2071 return 0; 2072 } 2073 2074 static int sunxi_nfc_probe(struct platform_device *pdev) 2075 { 2076 struct device *dev = &pdev->dev; 2077 struct resource *r; 2078 struct sunxi_nfc *nfc; 2079 int irq; 2080 int ret; 2081 2082 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL); 2083 if (!nfc) 2084 return -ENOMEM; 2085 2086 nfc->dev = dev; 2087 nand_controller_init(&nfc->controller); 2088 INIT_LIST_HEAD(&nfc->chips); 2089 2090 r = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2091 nfc->regs = devm_ioremap_resource(dev, r); 2092 if (IS_ERR(nfc->regs)) 2093 return PTR_ERR(nfc->regs); 2094 2095 irq = platform_get_irq(pdev, 0); 2096 if (irq < 0) 2097 return irq; 2098 2099 nfc->ahb_clk = devm_clk_get(dev, "ahb"); 2100 if (IS_ERR(nfc->ahb_clk)) { 2101 dev_err(dev, "failed to retrieve ahb clk\n"); 2102 return PTR_ERR(nfc->ahb_clk); 2103 } 2104 2105 ret = clk_prepare_enable(nfc->ahb_clk); 2106 if (ret) 2107 return ret; 2108 2109 nfc->mod_clk = devm_clk_get(dev, "mod"); 2110 if (IS_ERR(nfc->mod_clk)) { 2111 dev_err(dev, "failed to retrieve mod clk\n"); 2112 ret = PTR_ERR(nfc->mod_clk); 2113 goto out_ahb_clk_unprepare; 2114 } 2115 2116 ret = clk_prepare_enable(nfc->mod_clk); 2117 if (ret) 2118 goto out_ahb_clk_unprepare; 2119 2120 nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb"); 2121 if (IS_ERR(nfc->reset)) { 2122 ret = PTR_ERR(nfc->reset); 2123 goto out_mod_clk_unprepare; 2124 } 2125 2126 ret = reset_control_deassert(nfc->reset); 2127 if (ret) { 2128 dev_err(dev, "reset err %d\n", ret); 2129 goto out_mod_clk_unprepare; 2130 } 2131 2132 nfc->caps = of_device_get_match_data(&pdev->dev); 2133 if (!nfc->caps) { 2134 ret = -EINVAL; 2135 goto out_ahb_reset_reassert; 2136 } 2137 2138 ret = sunxi_nfc_rst(nfc); 2139 if (ret) 2140 goto out_ahb_reset_reassert; 2141 2142 writel(0, nfc->regs + NFC_REG_INT); 2143 ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt, 2144 0, "sunxi-nand", nfc); 2145 if (ret) 2146 goto out_ahb_reset_reassert; 2147 2148 ret = sunxi_nfc_dma_init(nfc, r); 2149 2150 if (ret) 2151 goto out_ahb_reset_reassert; 2152 2153 platform_set_drvdata(pdev, nfc); 2154 2155 ret = sunxi_nand_chips_init(dev, nfc); 2156 if (ret) { 2157 dev_err(dev, "failed to init nand chips\n"); 2158 goto out_release_dmac; 2159 } 2160 2161 return 0; 2162 2163 out_release_dmac: 2164 if (nfc->dmac) 2165 dma_release_channel(nfc->dmac); 2166 out_ahb_reset_reassert: 2167 reset_control_assert(nfc->reset); 2168 out_mod_clk_unprepare: 2169 clk_disable_unprepare(nfc->mod_clk); 2170 out_ahb_clk_unprepare: 2171 clk_disable_unprepare(nfc->ahb_clk); 2172 2173 return ret; 2174 } 2175 2176 static int sunxi_nfc_remove(struct platform_device *pdev) 2177 { 2178 struct sunxi_nfc *nfc = platform_get_drvdata(pdev); 2179 2180 sunxi_nand_chips_cleanup(nfc); 2181 2182 reset_control_assert(nfc->reset); 2183 2184 if (nfc->dmac) 2185 dma_release_channel(nfc->dmac); 2186 clk_disable_unprepare(nfc->mod_clk); 2187 clk_disable_unprepare(nfc->ahb_clk); 2188 2189 return 0; 2190 } 2191 2192 static const struct sunxi_nfc_caps sunxi_nfc_a10_caps = { 2193 .reg_io_data = NFC_REG_A10_IO_DATA, 2194 .dma_maxburst = 4, 2195 }; 2196 2197 static const struct sunxi_nfc_caps sunxi_nfc_a23_caps = { 2198 .has_mdma = true, 2199 .reg_io_data = NFC_REG_A23_IO_DATA, 2200 .dma_maxburst = 8, 2201 }; 2202 2203 static const struct of_device_id sunxi_nfc_ids[] = { 2204 { 2205 .compatible = "allwinner,sun4i-a10-nand", 2206 .data = &sunxi_nfc_a10_caps, 2207 }, 2208 { 2209 .compatible = "allwinner,sun8i-a23-nand-controller", 2210 .data = &sunxi_nfc_a23_caps, 2211 }, 2212 { /* sentinel */ } 2213 }; 2214 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids); 2215 2216 static struct platform_driver sunxi_nfc_driver = { 2217 .driver = { 2218 .name = "sunxi_nand", 2219 .of_match_table = sunxi_nfc_ids, 2220 }, 2221 .probe = sunxi_nfc_probe, 2222 .remove = sunxi_nfc_remove, 2223 }; 2224 module_platform_driver(sunxi_nfc_driver); 2225 2226 MODULE_LICENSE("GPL"); 2227 MODULE_AUTHOR("Boris BREZILLON"); 2228 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver"); 2229 MODULE_ALIAS("platform:sunxi_nand"); 2230