1 /* 2 * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com> 3 * 4 * Derived from: 5 * https://github.com/yuq/sunxi-nfc-mtd 6 * Copyright (C) 2013 Qiang Yu <yuq825@gmail.com> 7 * 8 * https://github.com/hno/Allwinner-Info 9 * Copyright (C) 2013 Henrik Nordström <Henrik Nordström> 10 * 11 * Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com> 12 * Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org> 13 * 14 * This program is free software; you can redistribute it and/or modify 15 * it under the terms of the GNU General Public License as published by 16 * the Free Software Foundation; either version 2 of the License, or 17 * (at your option) any later version. 18 * 19 * This program is distributed in the hope that it will be useful, 20 * but WITHOUT ANY WARRANTY; without even the implied warranty of 21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 * GNU General Public License for more details. 23 */ 24 25 #include <linux/dma-mapping.h> 26 #include <linux/slab.h> 27 #include <linux/module.h> 28 #include <linux/moduleparam.h> 29 #include <linux/platform_device.h> 30 #include <linux/of.h> 31 #include <linux/of_device.h> 32 #include <linux/mtd/mtd.h> 33 #include <linux/mtd/rawnand.h> 34 #include <linux/mtd/partitions.h> 35 #include <linux/clk.h> 36 #include <linux/delay.h> 37 #include <linux/dmaengine.h> 38 #include <linux/interrupt.h> 39 #include <linux/iopoll.h> 40 #include <linux/reset.h> 41 42 #define NFC_REG_CTL 0x0000 43 #define NFC_REG_ST 0x0004 44 #define NFC_REG_INT 0x0008 45 #define NFC_REG_TIMING_CTL 0x000C 46 #define NFC_REG_TIMING_CFG 0x0010 47 #define NFC_REG_ADDR_LOW 0x0014 48 #define NFC_REG_ADDR_HIGH 0x0018 49 #define NFC_REG_SECTOR_NUM 0x001C 50 #define NFC_REG_CNT 0x0020 51 #define NFC_REG_CMD 0x0024 52 #define NFC_REG_RCMD_SET 0x0028 53 #define NFC_REG_WCMD_SET 0x002C 54 #define NFC_REG_IO_DATA 0x0030 55 #define NFC_REG_ECC_CTL 0x0034 56 #define NFC_REG_ECC_ST 0x0038 57 #define NFC_REG_DEBUG 0x003C 58 #define NFC_REG_ECC_ERR_CNT(x) ((0x0040 + (x)) & ~0x3) 59 #define NFC_REG_USER_DATA(x) (0x0050 + ((x) * 4)) 60 #define NFC_REG_SPARE_AREA 0x00A0 61 #define NFC_REG_PAT_ID 0x00A4 62 #define NFC_RAM0_BASE 0x0400 63 #define NFC_RAM1_BASE 0x0800 64 65 /* define bit use in NFC_CTL */ 66 #define NFC_EN BIT(0) 67 #define NFC_RESET BIT(1) 68 #define NFC_BUS_WIDTH_MSK BIT(2) 69 #define NFC_BUS_WIDTH_8 (0 << 2) 70 #define NFC_BUS_WIDTH_16 (1 << 2) 71 #define NFC_RB_SEL_MSK BIT(3) 72 #define NFC_RB_SEL(x) ((x) << 3) 73 #define NFC_CE_SEL_MSK GENMASK(26, 24) 74 #define NFC_CE_SEL(x) ((x) << 24) 75 #define NFC_CE_CTL BIT(6) 76 #define NFC_PAGE_SHIFT_MSK GENMASK(11, 8) 77 #define NFC_PAGE_SHIFT(x) (((x) < 10 ? 0 : (x) - 10) << 8) 78 #define NFC_SAM BIT(12) 79 #define NFC_RAM_METHOD BIT(14) 80 #define NFC_DEBUG_CTL BIT(31) 81 82 /* define bit use in NFC_ST */ 83 #define NFC_RB_B2R BIT(0) 84 #define NFC_CMD_INT_FLAG BIT(1) 85 #define NFC_DMA_INT_FLAG BIT(2) 86 #define NFC_CMD_FIFO_STATUS BIT(3) 87 #define NFC_STA BIT(4) 88 #define NFC_NATCH_INT_FLAG BIT(5) 89 #define NFC_RB_STATE(x) BIT(x + 8) 90 91 /* define bit use in NFC_INT */ 92 #define NFC_B2R_INT_ENABLE BIT(0) 93 #define NFC_CMD_INT_ENABLE BIT(1) 94 #define NFC_DMA_INT_ENABLE BIT(2) 95 #define NFC_INT_MASK (NFC_B2R_INT_ENABLE | \ 96 NFC_CMD_INT_ENABLE | \ 97 NFC_DMA_INT_ENABLE) 98 99 /* define bit use in NFC_TIMING_CTL */ 100 #define NFC_TIMING_CTL_EDO BIT(8) 101 102 /* define NFC_TIMING_CFG register layout */ 103 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD) \ 104 (((tWB) & 0x3) | (((tADL) & 0x3) << 2) | \ 105 (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) | \ 106 (((tCAD) & 0x7) << 8)) 107 108 /* define bit use in NFC_CMD */ 109 #define NFC_CMD_LOW_BYTE_MSK GENMASK(7, 0) 110 #define NFC_CMD_HIGH_BYTE_MSK GENMASK(15, 8) 111 #define NFC_CMD(x) (x) 112 #define NFC_ADR_NUM_MSK GENMASK(18, 16) 113 #define NFC_ADR_NUM(x) (((x) - 1) << 16) 114 #define NFC_SEND_ADR BIT(19) 115 #define NFC_ACCESS_DIR BIT(20) 116 #define NFC_DATA_TRANS BIT(21) 117 #define NFC_SEND_CMD1 BIT(22) 118 #define NFC_WAIT_FLAG BIT(23) 119 #define NFC_SEND_CMD2 BIT(24) 120 #define NFC_SEQ BIT(25) 121 #define NFC_DATA_SWAP_METHOD BIT(26) 122 #define NFC_ROW_AUTO_INC BIT(27) 123 #define NFC_SEND_CMD3 BIT(28) 124 #define NFC_SEND_CMD4 BIT(29) 125 #define NFC_CMD_TYPE_MSK GENMASK(31, 30) 126 #define NFC_NORMAL_OP (0 << 30) 127 #define NFC_ECC_OP (1 << 30) 128 #define NFC_PAGE_OP (2U << 30) 129 130 /* define bit use in NFC_RCMD_SET */ 131 #define NFC_READ_CMD_MSK GENMASK(7, 0) 132 #define NFC_RND_READ_CMD0_MSK GENMASK(15, 8) 133 #define NFC_RND_READ_CMD1_MSK GENMASK(23, 16) 134 135 /* define bit use in NFC_WCMD_SET */ 136 #define NFC_PROGRAM_CMD_MSK GENMASK(7, 0) 137 #define NFC_RND_WRITE_CMD_MSK GENMASK(15, 8) 138 #define NFC_READ_CMD0_MSK GENMASK(23, 16) 139 #define NFC_READ_CMD1_MSK GENMASK(31, 24) 140 141 /* define bit use in NFC_ECC_CTL */ 142 #define NFC_ECC_EN BIT(0) 143 #define NFC_ECC_PIPELINE BIT(3) 144 #define NFC_ECC_EXCEPTION BIT(4) 145 #define NFC_ECC_BLOCK_SIZE_MSK BIT(5) 146 #define NFC_ECC_BLOCK_512 BIT(5) 147 #define NFC_RANDOM_EN BIT(9) 148 #define NFC_RANDOM_DIRECTION BIT(10) 149 #define NFC_ECC_MODE_MSK GENMASK(15, 12) 150 #define NFC_ECC_MODE(x) ((x) << 12) 151 #define NFC_RANDOM_SEED_MSK GENMASK(30, 16) 152 #define NFC_RANDOM_SEED(x) ((x) << 16) 153 154 /* define bit use in NFC_ECC_ST */ 155 #define NFC_ECC_ERR(x) BIT(x) 156 #define NFC_ECC_ERR_MSK GENMASK(15, 0) 157 #define NFC_ECC_PAT_FOUND(x) BIT(x + 16) 158 #define NFC_ECC_ERR_CNT(b, x) (((x) >> (((b) % 4) * 8)) & 0xff) 159 160 #define NFC_DEFAULT_TIMEOUT_MS 1000 161 162 #define NFC_SRAM_SIZE 1024 163 164 #define NFC_MAX_CS 7 165 166 /* 167 * Chip Select structure: stores information related to NAND Chip Select 168 * 169 * @cs: the NAND CS id used to communicate with a NAND Chip 170 * @rb: the Ready/Busy pin ID. -1 means no R/B pin connected to the 171 * NFC 172 */ 173 struct sunxi_nand_chip_sel { 174 u8 cs; 175 s8 rb; 176 }; 177 178 /* 179 * sunxi HW ECC infos: stores information related to HW ECC support 180 * 181 * @mode: the sunxi ECC mode field deduced from ECC requirements 182 */ 183 struct sunxi_nand_hw_ecc { 184 int mode; 185 }; 186 187 /* 188 * NAND chip structure: stores NAND chip device related information 189 * 190 * @node: used to store NAND chips into a list 191 * @nand: base NAND chip structure 192 * @mtd: base MTD structure 193 * @clk_rate: clk_rate required for this NAND chip 194 * @timing_cfg TIMING_CFG register value for this NAND chip 195 * @selected: current active CS 196 * @nsels: number of CS lines required by the NAND chip 197 * @sels: array of CS lines descriptions 198 */ 199 struct sunxi_nand_chip { 200 struct list_head node; 201 struct nand_chip nand; 202 unsigned long clk_rate; 203 u32 timing_cfg; 204 u32 timing_ctl; 205 int selected; 206 int addr_cycles; 207 u32 addr[2]; 208 int cmd_cycles; 209 u8 cmd[2]; 210 int nsels; 211 struct sunxi_nand_chip_sel sels[0]; 212 }; 213 214 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand) 215 { 216 return container_of(nand, struct sunxi_nand_chip, nand); 217 } 218 219 /* 220 * NAND Controller structure: stores sunxi NAND controller information 221 * 222 * @controller: base controller structure 223 * @dev: parent device (used to print error messages) 224 * @regs: NAND controller registers 225 * @ahb_clk: NAND Controller AHB clock 226 * @mod_clk: NAND Controller mod clock 227 * @assigned_cs: bitmask describing already assigned CS lines 228 * @clk_rate: NAND controller current clock rate 229 * @chips: a list containing all the NAND chips attached to 230 * this NAND controller 231 * @complete: a completion object used to wait for NAND 232 * controller events 233 */ 234 struct sunxi_nfc { 235 struct nand_controller controller; 236 struct device *dev; 237 void __iomem *regs; 238 struct clk *ahb_clk; 239 struct clk *mod_clk; 240 struct reset_control *reset; 241 unsigned long assigned_cs; 242 unsigned long clk_rate; 243 struct list_head chips; 244 struct completion complete; 245 struct dma_chan *dmac; 246 }; 247 248 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_controller *ctrl) 249 { 250 return container_of(ctrl, struct sunxi_nfc, controller); 251 } 252 253 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id) 254 { 255 struct sunxi_nfc *nfc = dev_id; 256 u32 st = readl(nfc->regs + NFC_REG_ST); 257 u32 ien = readl(nfc->regs + NFC_REG_INT); 258 259 if (!(ien & st)) 260 return IRQ_NONE; 261 262 if ((ien & st) == ien) 263 complete(&nfc->complete); 264 265 writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST); 266 writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT); 267 268 return IRQ_HANDLED; 269 } 270 271 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events, 272 bool use_polling, unsigned int timeout_ms) 273 { 274 int ret; 275 276 if (events & ~NFC_INT_MASK) 277 return -EINVAL; 278 279 if (!timeout_ms) 280 timeout_ms = NFC_DEFAULT_TIMEOUT_MS; 281 282 if (!use_polling) { 283 init_completion(&nfc->complete); 284 285 writel(events, nfc->regs + NFC_REG_INT); 286 287 ret = wait_for_completion_timeout(&nfc->complete, 288 msecs_to_jiffies(timeout_ms)); 289 if (!ret) 290 ret = -ETIMEDOUT; 291 else 292 ret = 0; 293 294 writel(0, nfc->regs + NFC_REG_INT); 295 } else { 296 u32 status; 297 298 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status, 299 (status & events) == events, 1, 300 timeout_ms * 1000); 301 } 302 303 writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST); 304 305 if (ret) 306 dev_err(nfc->dev, "wait interrupt timedout\n"); 307 308 return ret; 309 } 310 311 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc) 312 { 313 u32 status; 314 int ret; 315 316 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status, 317 !(status & NFC_CMD_FIFO_STATUS), 1, 318 NFC_DEFAULT_TIMEOUT_MS * 1000); 319 if (ret) 320 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n"); 321 322 return ret; 323 } 324 325 static int sunxi_nfc_rst(struct sunxi_nfc *nfc) 326 { 327 u32 ctl; 328 int ret; 329 330 writel(0, nfc->regs + NFC_REG_ECC_CTL); 331 writel(NFC_RESET, nfc->regs + NFC_REG_CTL); 332 333 ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl, 334 !(ctl & NFC_RESET), 1, 335 NFC_DEFAULT_TIMEOUT_MS * 1000); 336 if (ret) 337 dev_err(nfc->dev, "wait for NAND controller reset timedout\n"); 338 339 return ret; 340 } 341 342 static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf, 343 int chunksize, int nchunks, 344 enum dma_data_direction ddir, 345 struct scatterlist *sg) 346 { 347 struct nand_chip *nand = mtd_to_nand(mtd); 348 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 349 struct dma_async_tx_descriptor *dmad; 350 enum dma_transfer_direction tdir; 351 dma_cookie_t dmat; 352 int ret; 353 354 if (ddir == DMA_FROM_DEVICE) 355 tdir = DMA_DEV_TO_MEM; 356 else 357 tdir = DMA_MEM_TO_DEV; 358 359 sg_init_one(sg, buf, nchunks * chunksize); 360 ret = dma_map_sg(nfc->dev, sg, 1, ddir); 361 if (!ret) 362 return -ENOMEM; 363 364 dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK); 365 if (!dmad) { 366 ret = -EINVAL; 367 goto err_unmap_buf; 368 } 369 370 writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD, 371 nfc->regs + NFC_REG_CTL); 372 writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM); 373 writel(chunksize, nfc->regs + NFC_REG_CNT); 374 dmat = dmaengine_submit(dmad); 375 376 ret = dma_submit_error(dmat); 377 if (ret) 378 goto err_clr_dma_flag; 379 380 return 0; 381 382 err_clr_dma_flag: 383 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD, 384 nfc->regs + NFC_REG_CTL); 385 386 err_unmap_buf: 387 dma_unmap_sg(nfc->dev, sg, 1, ddir); 388 return ret; 389 } 390 391 static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd, 392 enum dma_data_direction ddir, 393 struct scatterlist *sg) 394 { 395 struct nand_chip *nand = mtd_to_nand(mtd); 396 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 397 398 dma_unmap_sg(nfc->dev, sg, 1, ddir); 399 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD, 400 nfc->regs + NFC_REG_CTL); 401 } 402 403 static int sunxi_nfc_dev_ready(struct nand_chip *nand) 404 { 405 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 406 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 407 u32 mask; 408 409 if (sunxi_nand->selected < 0) 410 return 0; 411 412 if (sunxi_nand->sels[sunxi_nand->selected].rb < 0) { 413 dev_err(nfc->dev, "cannot check R/B NAND status!\n"); 414 return 0; 415 } 416 417 mask = NFC_RB_STATE(sunxi_nand->sels[sunxi_nand->selected].rb); 418 419 return !!(readl(nfc->regs + NFC_REG_ST) & mask); 420 } 421 422 static void sunxi_nfc_select_chip(struct nand_chip *nand, int chip) 423 { 424 struct mtd_info *mtd = nand_to_mtd(nand); 425 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 426 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 427 struct sunxi_nand_chip_sel *sel; 428 u32 ctl; 429 430 if (chip > 0 && chip >= sunxi_nand->nsels) 431 return; 432 433 if (chip == sunxi_nand->selected) 434 return; 435 436 ctl = readl(nfc->regs + NFC_REG_CTL) & 437 ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN); 438 439 if (chip >= 0) { 440 sel = &sunxi_nand->sels[chip]; 441 442 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN | 443 NFC_PAGE_SHIFT(nand->page_shift); 444 if (sel->rb < 0) { 445 nand->legacy.dev_ready = NULL; 446 } else { 447 nand->legacy.dev_ready = sunxi_nfc_dev_ready; 448 ctl |= NFC_RB_SEL(sel->rb); 449 } 450 451 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA); 452 453 if (nfc->clk_rate != sunxi_nand->clk_rate) { 454 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate); 455 nfc->clk_rate = sunxi_nand->clk_rate; 456 } 457 } 458 459 writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL); 460 writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG); 461 writel(ctl, nfc->regs + NFC_REG_CTL); 462 463 sunxi_nand->selected = chip; 464 } 465 466 static void sunxi_nfc_read_buf(struct nand_chip *nand, uint8_t *buf, int len) 467 { 468 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 469 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 470 int ret; 471 int cnt; 472 int offs = 0; 473 u32 tmp; 474 475 while (len > offs) { 476 bool poll = false; 477 478 cnt = min(len - offs, NFC_SRAM_SIZE); 479 480 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 481 if (ret) 482 break; 483 484 writel(cnt, nfc->regs + NFC_REG_CNT); 485 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD; 486 writel(tmp, nfc->regs + NFC_REG_CMD); 487 488 /* Arbitrary limit for polling mode */ 489 if (cnt < 64) 490 poll = true; 491 492 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0); 493 if (ret) 494 break; 495 496 if (buf) 497 memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE, 498 cnt); 499 offs += cnt; 500 } 501 } 502 503 static void sunxi_nfc_write_buf(struct nand_chip *nand, const uint8_t *buf, 504 int len) 505 { 506 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 507 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 508 int ret; 509 int cnt; 510 int offs = 0; 511 u32 tmp; 512 513 while (len > offs) { 514 bool poll = false; 515 516 cnt = min(len - offs, NFC_SRAM_SIZE); 517 518 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 519 if (ret) 520 break; 521 522 writel(cnt, nfc->regs + NFC_REG_CNT); 523 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt); 524 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | 525 NFC_ACCESS_DIR; 526 writel(tmp, nfc->regs + NFC_REG_CMD); 527 528 /* Arbitrary limit for polling mode */ 529 if (cnt < 64) 530 poll = true; 531 532 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0); 533 if (ret) 534 break; 535 536 offs += cnt; 537 } 538 } 539 540 static uint8_t sunxi_nfc_read_byte(struct nand_chip *nand) 541 { 542 uint8_t ret = 0; 543 544 sunxi_nfc_read_buf(nand, &ret, 1); 545 546 return ret; 547 } 548 549 static void sunxi_nfc_cmd_ctrl(struct nand_chip *nand, int dat, 550 unsigned int ctrl) 551 { 552 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 553 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 554 int ret; 555 556 if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) && 557 !(ctrl & (NAND_CLE | NAND_ALE))) { 558 u32 cmd = 0; 559 560 if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles) 561 return; 562 563 if (sunxi_nand->cmd_cycles--) 564 cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0]; 565 566 if (sunxi_nand->cmd_cycles--) { 567 cmd |= NFC_SEND_CMD2; 568 writel(sunxi_nand->cmd[1], 569 nfc->regs + NFC_REG_RCMD_SET); 570 } 571 572 sunxi_nand->cmd_cycles = 0; 573 574 if (sunxi_nand->addr_cycles) { 575 cmd |= NFC_SEND_ADR | 576 NFC_ADR_NUM(sunxi_nand->addr_cycles); 577 writel(sunxi_nand->addr[0], 578 nfc->regs + NFC_REG_ADDR_LOW); 579 } 580 581 if (sunxi_nand->addr_cycles > 4) 582 writel(sunxi_nand->addr[1], 583 nfc->regs + NFC_REG_ADDR_HIGH); 584 585 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 586 if (ret) 587 return; 588 589 writel(cmd, nfc->regs + NFC_REG_CMD); 590 sunxi_nand->addr[0] = 0; 591 sunxi_nand->addr[1] = 0; 592 sunxi_nand->addr_cycles = 0; 593 sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0); 594 } 595 596 if (ctrl & NAND_CLE) { 597 sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat; 598 } else if (ctrl & NAND_ALE) { 599 sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |= 600 dat << ((sunxi_nand->addr_cycles % 4) * 8); 601 sunxi_nand->addr_cycles++; 602 } 603 } 604 605 /* These seed values have been extracted from Allwinner's BSP */ 606 static const u16 sunxi_nfc_randomizer_page_seeds[] = { 607 0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72, 608 0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436, 609 0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d, 610 0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130, 611 0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56, 612 0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55, 613 0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb, 614 0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17, 615 0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62, 616 0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064, 617 0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126, 618 0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e, 619 0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3, 620 0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b, 621 0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d, 622 0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db, 623 }; 624 625 /* 626 * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds 627 * have been generated using 628 * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what 629 * the randomizer engine does internally before de/scrambling OOB data. 630 * 631 * Those tables are statically defined to avoid calculating randomizer state 632 * at runtime. 633 */ 634 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = { 635 0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64, 636 0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409, 637 0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617, 638 0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d, 639 0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91, 640 0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d, 641 0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab, 642 0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8, 643 0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8, 644 0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b, 645 0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5, 646 0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a, 647 0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891, 648 0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36, 649 0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd, 650 0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0, 651 }; 652 653 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = { 654 0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6, 655 0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982, 656 0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9, 657 0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07, 658 0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e, 659 0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2, 660 0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c, 661 0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f, 662 0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc, 663 0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e, 664 0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8, 665 0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68, 666 0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d, 667 0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179, 668 0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601, 669 0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd, 670 }; 671 672 static u16 sunxi_nfc_randomizer_step(u16 state, int count) 673 { 674 state &= 0x7fff; 675 676 /* 677 * This loop is just a simple implementation of a Fibonacci LFSR using 678 * the x16 + x15 + 1 polynomial. 679 */ 680 while (count--) 681 state = ((state >> 1) | 682 (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff; 683 684 return state; 685 } 686 687 static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc) 688 { 689 const u16 *seeds = sunxi_nfc_randomizer_page_seeds; 690 int mod = mtd_div_by_ws(mtd->erasesize, mtd); 691 692 if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds)) 693 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds); 694 695 if (ecc) { 696 if (mtd->ecc_step_size == 512) 697 seeds = sunxi_nfc_randomizer_ecc512_seeds; 698 else 699 seeds = sunxi_nfc_randomizer_ecc1024_seeds; 700 } 701 702 return seeds[page % mod]; 703 } 704 705 static void sunxi_nfc_randomizer_config(struct mtd_info *mtd, 706 int page, bool ecc) 707 { 708 struct nand_chip *nand = mtd_to_nand(mtd); 709 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 710 u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); 711 u16 state; 712 713 if (!(nand->options & NAND_NEED_SCRAMBLING)) 714 return; 715 716 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); 717 state = sunxi_nfc_randomizer_state(mtd, page, ecc); 718 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK; 719 writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL); 720 } 721 722 static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd) 723 { 724 struct nand_chip *nand = mtd_to_nand(mtd); 725 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 726 727 if (!(nand->options & NAND_NEED_SCRAMBLING)) 728 return; 729 730 writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN, 731 nfc->regs + NFC_REG_ECC_CTL); 732 } 733 734 static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd) 735 { 736 struct nand_chip *nand = mtd_to_nand(mtd); 737 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 738 739 if (!(nand->options & NAND_NEED_SCRAMBLING)) 740 return; 741 742 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN, 743 nfc->regs + NFC_REG_ECC_CTL); 744 } 745 746 static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm) 747 { 748 u16 state = sunxi_nfc_randomizer_state(mtd, page, true); 749 750 bbm[0] ^= state; 751 bbm[1] ^= sunxi_nfc_randomizer_step(state, 8); 752 } 753 754 static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd, 755 const uint8_t *buf, int len, 756 bool ecc, int page) 757 { 758 sunxi_nfc_randomizer_config(mtd, page, ecc); 759 sunxi_nfc_randomizer_enable(mtd); 760 sunxi_nfc_write_buf(mtd_to_nand(mtd), buf, len); 761 sunxi_nfc_randomizer_disable(mtd); 762 } 763 764 static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf, 765 int len, bool ecc, int page) 766 { 767 sunxi_nfc_randomizer_config(mtd, page, ecc); 768 sunxi_nfc_randomizer_enable(mtd); 769 sunxi_nfc_read_buf(mtd_to_nand(mtd), buf, len); 770 sunxi_nfc_randomizer_disable(mtd); 771 } 772 773 static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd) 774 { 775 struct nand_chip *nand = mtd_to_nand(mtd); 776 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 777 struct sunxi_nand_hw_ecc *data = nand->ecc.priv; 778 u32 ecc_ctl; 779 780 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); 781 ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE | 782 NFC_ECC_BLOCK_SIZE_MSK); 783 ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION | 784 NFC_ECC_PIPELINE; 785 786 if (nand->ecc.size == 512) 787 ecc_ctl |= NFC_ECC_BLOCK_512; 788 789 writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL); 790 } 791 792 static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd) 793 { 794 struct nand_chip *nand = mtd_to_nand(mtd); 795 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 796 797 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN, 798 nfc->regs + NFC_REG_ECC_CTL); 799 } 800 801 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf) 802 { 803 buf[0] = user_data; 804 buf[1] = user_data >> 8; 805 buf[2] = user_data >> 16; 806 buf[3] = user_data >> 24; 807 } 808 809 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf) 810 { 811 return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24); 812 } 813 814 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob, 815 int step, bool bbm, int page) 816 { 817 struct nand_chip *nand = mtd_to_nand(mtd); 818 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 819 820 sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)), 821 oob); 822 823 /* De-randomize the Bad Block Marker. */ 824 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) 825 sunxi_nfc_randomize_bbm(mtd, page, oob); 826 } 827 828 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd, 829 const u8 *oob, int step, 830 bool bbm, int page) 831 { 832 struct nand_chip *nand = mtd_to_nand(mtd); 833 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 834 u8 user_data[4]; 835 836 /* Randomize the Bad Block Marker. */ 837 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) { 838 memcpy(user_data, oob, sizeof(user_data)); 839 sunxi_nfc_randomize_bbm(mtd, page, user_data); 840 oob = user_data; 841 } 842 843 writel(sunxi_nfc_buf_to_user_data(oob), 844 nfc->regs + NFC_REG_USER_DATA(step)); 845 } 846 847 static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd, 848 unsigned int *max_bitflips, int ret) 849 { 850 if (ret < 0) { 851 mtd->ecc_stats.failed++; 852 } else { 853 mtd->ecc_stats.corrected += ret; 854 *max_bitflips = max_t(unsigned int, *max_bitflips, ret); 855 } 856 } 857 858 static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob, 859 int step, u32 status, bool *erased) 860 { 861 struct nand_chip *nand = mtd_to_nand(mtd); 862 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 863 struct nand_ecc_ctrl *ecc = &nand->ecc; 864 u32 tmp; 865 866 *erased = false; 867 868 if (status & NFC_ECC_ERR(step)) 869 return -EBADMSG; 870 871 if (status & NFC_ECC_PAT_FOUND(step)) { 872 u8 pattern; 873 874 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) { 875 pattern = 0x0; 876 } else { 877 pattern = 0xff; 878 *erased = true; 879 } 880 881 if (data) 882 memset(data, pattern, ecc->size); 883 884 if (oob) 885 memset(oob, pattern, ecc->bytes + 4); 886 887 return 0; 888 } 889 890 tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step)); 891 892 return NFC_ECC_ERR_CNT(step, tmp); 893 } 894 895 static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd, 896 u8 *data, int data_off, 897 u8 *oob, int oob_off, 898 int *cur_off, 899 unsigned int *max_bitflips, 900 bool bbm, bool oob_required, int page) 901 { 902 struct nand_chip *nand = mtd_to_nand(mtd); 903 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 904 struct nand_ecc_ctrl *ecc = &nand->ecc; 905 int raw_mode = 0; 906 bool erased; 907 int ret; 908 909 if (*cur_off != data_off) 910 nand_change_read_column_op(nand, data_off, NULL, 0, false); 911 912 sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page); 913 914 if (data_off + ecc->size != oob_off) 915 nand_change_read_column_op(nand, oob_off, NULL, 0, false); 916 917 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 918 if (ret) 919 return ret; 920 921 sunxi_nfc_randomizer_enable(mtd); 922 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP, 923 nfc->regs + NFC_REG_CMD); 924 925 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 926 sunxi_nfc_randomizer_disable(mtd); 927 if (ret) 928 return ret; 929 930 *cur_off = oob_off + ecc->bytes + 4; 931 932 ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0, 933 readl(nfc->regs + NFC_REG_ECC_ST), 934 &erased); 935 if (erased) 936 return 1; 937 938 if (ret < 0) { 939 /* 940 * Re-read the data with the randomizer disabled to identify 941 * bitflips in erased pages. 942 */ 943 if (nand->options & NAND_NEED_SCRAMBLING) 944 nand_change_read_column_op(nand, data_off, data, 945 ecc->size, false); 946 else 947 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, 948 ecc->size); 949 950 nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4, 951 false); 952 953 ret = nand_check_erased_ecc_chunk(data, ecc->size, 954 oob, ecc->bytes + 4, 955 NULL, 0, ecc->strength); 956 if (ret >= 0) 957 raw_mode = 1; 958 } else { 959 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size); 960 961 if (oob_required) { 962 nand_change_read_column_op(nand, oob_off, NULL, 0, 963 false); 964 sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4, 965 true, page); 966 967 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0, 968 bbm, page); 969 } 970 } 971 972 sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret); 973 974 return raw_mode; 975 } 976 977 static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd, 978 u8 *oob, int *cur_off, 979 bool randomize, int page) 980 { 981 struct nand_chip *nand = mtd_to_nand(mtd); 982 struct nand_ecc_ctrl *ecc = &nand->ecc; 983 int offset = ((ecc->bytes + 4) * ecc->steps); 984 int len = mtd->oobsize - offset; 985 986 if (len <= 0) 987 return; 988 989 if (!cur_off || *cur_off != offset) 990 nand_change_read_column_op(nand, mtd->writesize, NULL, 0, 991 false); 992 993 if (!randomize) 994 sunxi_nfc_read_buf(nand, oob + offset, len); 995 else 996 sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len, 997 false, page); 998 999 if (cur_off) 1000 *cur_off = mtd->oobsize + mtd->writesize; 1001 } 1002 1003 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf, 1004 int oob_required, int page, 1005 int nchunks) 1006 { 1007 struct nand_chip *nand = mtd_to_nand(mtd); 1008 bool randomized = nand->options & NAND_NEED_SCRAMBLING; 1009 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1010 struct nand_ecc_ctrl *ecc = &nand->ecc; 1011 unsigned int max_bitflips = 0; 1012 int ret, i, raw_mode = 0; 1013 struct scatterlist sg; 1014 u32 status; 1015 1016 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1017 if (ret) 1018 return ret; 1019 1020 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks, 1021 DMA_FROM_DEVICE, &sg); 1022 if (ret) 1023 return ret; 1024 1025 sunxi_nfc_hw_ecc_enable(mtd); 1026 sunxi_nfc_randomizer_config(mtd, page, false); 1027 sunxi_nfc_randomizer_enable(mtd); 1028 1029 writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) | 1030 NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET); 1031 1032 dma_async_issue_pending(nfc->dmac); 1033 1034 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS, 1035 nfc->regs + NFC_REG_CMD); 1036 1037 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 1038 if (ret) 1039 dmaengine_terminate_all(nfc->dmac); 1040 1041 sunxi_nfc_randomizer_disable(mtd); 1042 sunxi_nfc_hw_ecc_disable(mtd); 1043 1044 sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg); 1045 1046 if (ret) 1047 return ret; 1048 1049 status = readl(nfc->regs + NFC_REG_ECC_ST); 1050 1051 for (i = 0; i < nchunks; i++) { 1052 int data_off = i * ecc->size; 1053 int oob_off = i * (ecc->bytes + 4); 1054 u8 *data = buf + data_off; 1055 u8 *oob = nand->oob_poi + oob_off; 1056 bool erased; 1057 1058 ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL, 1059 oob_required ? oob : NULL, 1060 i, status, &erased); 1061 1062 /* ECC errors are handled in the second loop. */ 1063 if (ret < 0) 1064 continue; 1065 1066 if (oob_required && !erased) { 1067 /* TODO: use DMA to retrieve OOB */ 1068 nand_change_read_column_op(nand, 1069 mtd->writesize + oob_off, 1070 oob, ecc->bytes + 4, false); 1071 1072 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i, 1073 !i, page); 1074 } 1075 1076 if (erased) 1077 raw_mode = 1; 1078 1079 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret); 1080 } 1081 1082 if (status & NFC_ECC_ERR_MSK) { 1083 for (i = 0; i < nchunks; i++) { 1084 int data_off = i * ecc->size; 1085 int oob_off = i * (ecc->bytes + 4); 1086 u8 *data = buf + data_off; 1087 u8 *oob = nand->oob_poi + oob_off; 1088 1089 if (!(status & NFC_ECC_ERR(i))) 1090 continue; 1091 1092 /* 1093 * Re-read the data with the randomizer disabled to 1094 * identify bitflips in erased pages. 1095 * TODO: use DMA to read page in raw mode 1096 */ 1097 if (randomized) 1098 nand_change_read_column_op(nand, data_off, 1099 data, ecc->size, 1100 false); 1101 1102 /* TODO: use DMA to retrieve OOB */ 1103 nand_change_read_column_op(nand, 1104 mtd->writesize + oob_off, 1105 oob, ecc->bytes + 4, false); 1106 1107 ret = nand_check_erased_ecc_chunk(data, ecc->size, 1108 oob, ecc->bytes + 4, 1109 NULL, 0, 1110 ecc->strength); 1111 if (ret >= 0) 1112 raw_mode = 1; 1113 1114 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret); 1115 } 1116 } 1117 1118 if (oob_required) 1119 sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi, 1120 NULL, !raw_mode, 1121 page); 1122 1123 return max_bitflips; 1124 } 1125 1126 static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd, 1127 const u8 *data, int data_off, 1128 const u8 *oob, int oob_off, 1129 int *cur_off, bool bbm, 1130 int page) 1131 { 1132 struct nand_chip *nand = mtd_to_nand(mtd); 1133 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1134 struct nand_ecc_ctrl *ecc = &nand->ecc; 1135 int ret; 1136 1137 if (data_off != *cur_off) 1138 nand_change_write_column_op(nand, data_off, NULL, 0, false); 1139 1140 sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page); 1141 1142 if (data_off + ecc->size != oob_off) 1143 nand_change_write_column_op(nand, oob_off, NULL, 0, false); 1144 1145 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1146 if (ret) 1147 return ret; 1148 1149 sunxi_nfc_randomizer_enable(mtd); 1150 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page); 1151 1152 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | 1153 NFC_ACCESS_DIR | NFC_ECC_OP, 1154 nfc->regs + NFC_REG_CMD); 1155 1156 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 1157 sunxi_nfc_randomizer_disable(mtd); 1158 if (ret) 1159 return ret; 1160 1161 *cur_off = oob_off + ecc->bytes + 4; 1162 1163 return 0; 1164 } 1165 1166 static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd, 1167 u8 *oob, int *cur_off, 1168 int page) 1169 { 1170 struct nand_chip *nand = mtd_to_nand(mtd); 1171 struct nand_ecc_ctrl *ecc = &nand->ecc; 1172 int offset = ((ecc->bytes + 4) * ecc->steps); 1173 int len = mtd->oobsize - offset; 1174 1175 if (len <= 0) 1176 return; 1177 1178 if (!cur_off || *cur_off != offset) 1179 nand_change_write_column_op(nand, offset + mtd->writesize, 1180 NULL, 0, false); 1181 1182 sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page); 1183 1184 if (cur_off) 1185 *cur_off = mtd->oobsize + mtd->writesize; 1186 } 1187 1188 static int sunxi_nfc_hw_ecc_read_page(struct nand_chip *chip, uint8_t *buf, 1189 int oob_required, int page) 1190 { 1191 struct mtd_info *mtd = nand_to_mtd(chip); 1192 struct nand_ecc_ctrl *ecc = &chip->ecc; 1193 unsigned int max_bitflips = 0; 1194 int ret, i, cur_off = 0; 1195 bool raw_mode = false; 1196 1197 nand_read_page_op(chip, page, 0, NULL, 0); 1198 1199 sunxi_nfc_hw_ecc_enable(mtd); 1200 1201 for (i = 0; i < ecc->steps; i++) { 1202 int data_off = i * ecc->size; 1203 int oob_off = i * (ecc->bytes + 4); 1204 u8 *data = buf + data_off; 1205 u8 *oob = chip->oob_poi + oob_off; 1206 1207 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob, 1208 oob_off + mtd->writesize, 1209 &cur_off, &max_bitflips, 1210 !i, oob_required, page); 1211 if (ret < 0) 1212 return ret; 1213 else if (ret) 1214 raw_mode = true; 1215 } 1216 1217 if (oob_required) 1218 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off, 1219 !raw_mode, page); 1220 1221 sunxi_nfc_hw_ecc_disable(mtd); 1222 1223 return max_bitflips; 1224 } 1225 1226 static int sunxi_nfc_hw_ecc_read_page_dma(struct nand_chip *chip, u8 *buf, 1227 int oob_required, int page) 1228 { 1229 struct mtd_info *mtd = nand_to_mtd(chip); 1230 int ret; 1231 1232 nand_read_page_op(chip, page, 0, NULL, 0); 1233 1234 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page, 1235 chip->ecc.steps); 1236 if (ret >= 0) 1237 return ret; 1238 1239 /* Fallback to PIO mode */ 1240 return sunxi_nfc_hw_ecc_read_page(chip, buf, oob_required, page); 1241 } 1242 1243 static int sunxi_nfc_hw_ecc_read_subpage(struct nand_chip *chip, 1244 u32 data_offs, u32 readlen, 1245 u8 *bufpoi, int page) 1246 { 1247 struct mtd_info *mtd = nand_to_mtd(chip); 1248 struct nand_ecc_ctrl *ecc = &chip->ecc; 1249 int ret, i, cur_off = 0; 1250 unsigned int max_bitflips = 0; 1251 1252 nand_read_page_op(chip, page, 0, NULL, 0); 1253 1254 sunxi_nfc_hw_ecc_enable(mtd); 1255 1256 for (i = data_offs / ecc->size; 1257 i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) { 1258 int data_off = i * ecc->size; 1259 int oob_off = i * (ecc->bytes + 4); 1260 u8 *data = bufpoi + data_off; 1261 u8 *oob = chip->oob_poi + oob_off; 1262 1263 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, 1264 oob, 1265 oob_off + mtd->writesize, 1266 &cur_off, &max_bitflips, !i, 1267 false, page); 1268 if (ret < 0) 1269 return ret; 1270 } 1271 1272 sunxi_nfc_hw_ecc_disable(mtd); 1273 1274 return max_bitflips; 1275 } 1276 1277 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct nand_chip *chip, 1278 u32 data_offs, u32 readlen, 1279 u8 *buf, int page) 1280 { 1281 struct mtd_info *mtd = nand_to_mtd(chip); 1282 int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size); 1283 int ret; 1284 1285 nand_read_page_op(chip, page, 0, NULL, 0); 1286 1287 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks); 1288 if (ret >= 0) 1289 return ret; 1290 1291 /* Fallback to PIO mode */ 1292 return sunxi_nfc_hw_ecc_read_subpage(chip, data_offs, readlen, 1293 buf, page); 1294 } 1295 1296 static int sunxi_nfc_hw_ecc_write_page(struct nand_chip *chip, 1297 const uint8_t *buf, int oob_required, 1298 int page) 1299 { 1300 struct mtd_info *mtd = nand_to_mtd(chip); 1301 struct nand_ecc_ctrl *ecc = &chip->ecc; 1302 int ret, i, cur_off = 0; 1303 1304 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1305 1306 sunxi_nfc_hw_ecc_enable(mtd); 1307 1308 for (i = 0; i < ecc->steps; i++) { 1309 int data_off = i * ecc->size; 1310 int oob_off = i * (ecc->bytes + 4); 1311 const u8 *data = buf + data_off; 1312 const u8 *oob = chip->oob_poi + oob_off; 1313 1314 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob, 1315 oob_off + mtd->writesize, 1316 &cur_off, !i, page); 1317 if (ret) 1318 return ret; 1319 } 1320 1321 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING)) 1322 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi, 1323 &cur_off, page); 1324 1325 sunxi_nfc_hw_ecc_disable(mtd); 1326 1327 return nand_prog_page_end_op(chip); 1328 } 1329 1330 static int sunxi_nfc_hw_ecc_write_subpage(struct nand_chip *chip, 1331 u32 data_offs, u32 data_len, 1332 const u8 *buf, int oob_required, 1333 int page) 1334 { 1335 struct mtd_info *mtd = nand_to_mtd(chip); 1336 struct nand_ecc_ctrl *ecc = &chip->ecc; 1337 int ret, i, cur_off = 0; 1338 1339 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1340 1341 sunxi_nfc_hw_ecc_enable(mtd); 1342 1343 for (i = data_offs / ecc->size; 1344 i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) { 1345 int data_off = i * ecc->size; 1346 int oob_off = i * (ecc->bytes + 4); 1347 const u8 *data = buf + data_off; 1348 const u8 *oob = chip->oob_poi + oob_off; 1349 1350 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob, 1351 oob_off + mtd->writesize, 1352 &cur_off, !i, page); 1353 if (ret) 1354 return ret; 1355 } 1356 1357 sunxi_nfc_hw_ecc_disable(mtd); 1358 1359 return nand_prog_page_end_op(chip); 1360 } 1361 1362 static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *chip, 1363 const u8 *buf, 1364 int oob_required, 1365 int page) 1366 { 1367 struct mtd_info *mtd = nand_to_mtd(chip); 1368 struct nand_chip *nand = mtd_to_nand(mtd); 1369 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1370 struct nand_ecc_ctrl *ecc = &nand->ecc; 1371 struct scatterlist sg; 1372 int ret, i; 1373 1374 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1375 if (ret) 1376 return ret; 1377 1378 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps, 1379 DMA_TO_DEVICE, &sg); 1380 if (ret) 1381 goto pio_fallback; 1382 1383 for (i = 0; i < ecc->steps; i++) { 1384 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4)); 1385 1386 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page); 1387 } 1388 1389 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1390 1391 sunxi_nfc_hw_ecc_enable(mtd); 1392 sunxi_nfc_randomizer_config(mtd, page, false); 1393 sunxi_nfc_randomizer_enable(mtd); 1394 1395 writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG, 1396 nfc->regs + NFC_REG_WCMD_SET); 1397 1398 dma_async_issue_pending(nfc->dmac); 1399 1400 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | 1401 NFC_DATA_TRANS | NFC_ACCESS_DIR, 1402 nfc->regs + NFC_REG_CMD); 1403 1404 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 1405 if (ret) 1406 dmaengine_terminate_all(nfc->dmac); 1407 1408 sunxi_nfc_randomizer_disable(mtd); 1409 sunxi_nfc_hw_ecc_disable(mtd); 1410 1411 sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg); 1412 1413 if (ret) 1414 return ret; 1415 1416 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING)) 1417 /* TODO: use DMA to transfer extra OOB bytes ? */ 1418 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi, 1419 NULL, page); 1420 1421 return nand_prog_page_end_op(chip); 1422 1423 pio_fallback: 1424 return sunxi_nfc_hw_ecc_write_page(chip, buf, oob_required, page); 1425 } 1426 1427 static int sunxi_nfc_hw_ecc_read_oob(struct nand_chip *chip, int page) 1428 { 1429 chip->pagebuf = -1; 1430 1431 return chip->ecc.read_page(chip, chip->data_buf, 1, page); 1432 } 1433 1434 static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *chip, int page) 1435 { 1436 struct mtd_info *mtd = nand_to_mtd(chip); 1437 int ret; 1438 1439 chip->pagebuf = -1; 1440 1441 memset(chip->data_buf, 0xff, mtd->writesize); 1442 ret = chip->ecc.write_page(chip, chip->data_buf, 1, page); 1443 if (ret) 1444 return ret; 1445 1446 /* Send command to program the OOB data */ 1447 return nand_prog_page_end_op(chip); 1448 } 1449 1450 static const s32 tWB_lut[] = {6, 12, 16, 20}; 1451 static const s32 tRHW_lut[] = {4, 8, 12, 20}; 1452 1453 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration, 1454 u32 clk_period) 1455 { 1456 u32 clk_cycles = DIV_ROUND_UP(duration, clk_period); 1457 int i; 1458 1459 for (i = 0; i < lut_size; i++) { 1460 if (clk_cycles <= lut[i]) 1461 return i; 1462 } 1463 1464 /* Doesn't fit */ 1465 return -EINVAL; 1466 } 1467 1468 #define sunxi_nand_lookup_timing(l, p, c) \ 1469 _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c) 1470 1471 static int sunxi_nfc_setup_data_interface(struct nand_chip *nand, int csline, 1472 const struct nand_data_interface *conf) 1473 { 1474 struct sunxi_nand_chip *chip = to_sunxi_nand(nand); 1475 struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller); 1476 const struct nand_sdr_timings *timings; 1477 u32 min_clk_period = 0; 1478 s32 tWB, tADL, tWHR, tRHW, tCAD; 1479 long real_clk_rate; 1480 1481 timings = nand_get_sdr_timings(conf); 1482 if (IS_ERR(timings)) 1483 return -ENOTSUPP; 1484 1485 /* T1 <=> tCLS */ 1486 if (timings->tCLS_min > min_clk_period) 1487 min_clk_period = timings->tCLS_min; 1488 1489 /* T2 <=> tCLH */ 1490 if (timings->tCLH_min > min_clk_period) 1491 min_clk_period = timings->tCLH_min; 1492 1493 /* T3 <=> tCS */ 1494 if (timings->tCS_min > min_clk_period) 1495 min_clk_period = timings->tCS_min; 1496 1497 /* T4 <=> tCH */ 1498 if (timings->tCH_min > min_clk_period) 1499 min_clk_period = timings->tCH_min; 1500 1501 /* T5 <=> tWP */ 1502 if (timings->tWP_min > min_clk_period) 1503 min_clk_period = timings->tWP_min; 1504 1505 /* T6 <=> tWH */ 1506 if (timings->tWH_min > min_clk_period) 1507 min_clk_period = timings->tWH_min; 1508 1509 /* T7 <=> tALS */ 1510 if (timings->tALS_min > min_clk_period) 1511 min_clk_period = timings->tALS_min; 1512 1513 /* T8 <=> tDS */ 1514 if (timings->tDS_min > min_clk_period) 1515 min_clk_period = timings->tDS_min; 1516 1517 /* T9 <=> tDH */ 1518 if (timings->tDH_min > min_clk_period) 1519 min_clk_period = timings->tDH_min; 1520 1521 /* T10 <=> tRR */ 1522 if (timings->tRR_min > (min_clk_period * 3)) 1523 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3); 1524 1525 /* T11 <=> tALH */ 1526 if (timings->tALH_min > min_clk_period) 1527 min_clk_period = timings->tALH_min; 1528 1529 /* T12 <=> tRP */ 1530 if (timings->tRP_min > min_clk_period) 1531 min_clk_period = timings->tRP_min; 1532 1533 /* T13 <=> tREH */ 1534 if (timings->tREH_min > min_clk_period) 1535 min_clk_period = timings->tREH_min; 1536 1537 /* T14 <=> tRC */ 1538 if (timings->tRC_min > (min_clk_period * 2)) 1539 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2); 1540 1541 /* T15 <=> tWC */ 1542 if (timings->tWC_min > (min_clk_period * 2)) 1543 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2); 1544 1545 /* T16 - T19 + tCAD */ 1546 if (timings->tWB_max > (min_clk_period * 20)) 1547 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20); 1548 1549 if (timings->tADL_min > (min_clk_period * 32)) 1550 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32); 1551 1552 if (timings->tWHR_min > (min_clk_period * 32)) 1553 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32); 1554 1555 if (timings->tRHW_min > (min_clk_period * 20)) 1556 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20); 1557 1558 tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max, 1559 min_clk_period); 1560 if (tWB < 0) { 1561 dev_err(nfc->dev, "unsupported tWB\n"); 1562 return tWB; 1563 } 1564 1565 tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3; 1566 if (tADL > 3) { 1567 dev_err(nfc->dev, "unsupported tADL\n"); 1568 return -EINVAL; 1569 } 1570 1571 tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3; 1572 if (tWHR > 3) { 1573 dev_err(nfc->dev, "unsupported tWHR\n"); 1574 return -EINVAL; 1575 } 1576 1577 tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min, 1578 min_clk_period); 1579 if (tRHW < 0) { 1580 dev_err(nfc->dev, "unsupported tRHW\n"); 1581 return tRHW; 1582 } 1583 1584 if (csline == NAND_DATA_IFACE_CHECK_ONLY) 1585 return 0; 1586 1587 /* 1588 * TODO: according to ONFI specs this value only applies for DDR NAND, 1589 * but Allwinner seems to set this to 0x7. Mimic them for now. 1590 */ 1591 tCAD = 0x7; 1592 1593 /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */ 1594 chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD); 1595 1596 /* Convert min_clk_period from picoseconds to nanoseconds */ 1597 min_clk_period = DIV_ROUND_UP(min_clk_period, 1000); 1598 1599 /* 1600 * Unlike what is stated in Allwinner datasheet, the clk_rate should 1601 * be set to (1 / min_clk_period), and not (2 / min_clk_period). 1602 * This new formula was verified with a scope and validated by 1603 * Allwinner engineers. 1604 */ 1605 chip->clk_rate = NSEC_PER_SEC / min_clk_period; 1606 real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate); 1607 if (real_clk_rate <= 0) { 1608 dev_err(nfc->dev, "Unable to round clk %lu\n", chip->clk_rate); 1609 return -EINVAL; 1610 } 1611 1612 /* 1613 * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data 1614 * output cycle timings shall be used if the host drives tRC less than 1615 * 30 ns. 1616 */ 1617 min_clk_period = NSEC_PER_SEC / real_clk_rate; 1618 chip->timing_ctl = ((min_clk_period * 2) < 30) ? 1619 NFC_TIMING_CTL_EDO : 0; 1620 1621 return 0; 1622 } 1623 1624 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section, 1625 struct mtd_oob_region *oobregion) 1626 { 1627 struct nand_chip *nand = mtd_to_nand(mtd); 1628 struct nand_ecc_ctrl *ecc = &nand->ecc; 1629 1630 if (section >= ecc->steps) 1631 return -ERANGE; 1632 1633 oobregion->offset = section * (ecc->bytes + 4) + 4; 1634 oobregion->length = ecc->bytes; 1635 1636 return 0; 1637 } 1638 1639 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section, 1640 struct mtd_oob_region *oobregion) 1641 { 1642 struct nand_chip *nand = mtd_to_nand(mtd); 1643 struct nand_ecc_ctrl *ecc = &nand->ecc; 1644 1645 if (section > ecc->steps) 1646 return -ERANGE; 1647 1648 /* 1649 * The first 2 bytes are used for BB markers, hence we 1650 * only have 2 bytes available in the first user data 1651 * section. 1652 */ 1653 if (!section && ecc->mode == NAND_ECC_HW) { 1654 oobregion->offset = 2; 1655 oobregion->length = 2; 1656 1657 return 0; 1658 } 1659 1660 oobregion->offset = section * (ecc->bytes + 4); 1661 1662 if (section < ecc->steps) 1663 oobregion->length = 4; 1664 else 1665 oobregion->offset = mtd->oobsize - oobregion->offset; 1666 1667 return 0; 1668 } 1669 1670 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = { 1671 .ecc = sunxi_nand_ooblayout_ecc, 1672 .free = sunxi_nand_ooblayout_free, 1673 }; 1674 1675 static void sunxi_nand_hw_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc) 1676 { 1677 kfree(ecc->priv); 1678 } 1679 1680 static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd, 1681 struct nand_ecc_ctrl *ecc, 1682 struct device_node *np) 1683 { 1684 static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 }; 1685 struct nand_chip *nand = mtd_to_nand(mtd); 1686 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 1687 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 1688 struct sunxi_nand_hw_ecc *data; 1689 int nsectors; 1690 int ret; 1691 int i; 1692 1693 if (ecc->options & NAND_ECC_MAXIMIZE) { 1694 int bytes; 1695 1696 ecc->size = 1024; 1697 nsectors = mtd->writesize / ecc->size; 1698 1699 /* Reserve 2 bytes for the BBM */ 1700 bytes = (mtd->oobsize - 2) / nsectors; 1701 1702 /* 4 non-ECC bytes are added before each ECC bytes section */ 1703 bytes -= 4; 1704 1705 /* and bytes has to be even. */ 1706 if (bytes % 2) 1707 bytes--; 1708 1709 ecc->strength = bytes * 8 / fls(8 * ecc->size); 1710 1711 for (i = 0; i < ARRAY_SIZE(strengths); i++) { 1712 if (strengths[i] > ecc->strength) 1713 break; 1714 } 1715 1716 if (!i) 1717 ecc->strength = 0; 1718 else 1719 ecc->strength = strengths[i - 1]; 1720 } 1721 1722 if (ecc->size != 512 && ecc->size != 1024) 1723 return -EINVAL; 1724 1725 data = kzalloc(sizeof(*data), GFP_KERNEL); 1726 if (!data) 1727 return -ENOMEM; 1728 1729 /* Prefer 1k ECC chunk over 512 ones */ 1730 if (ecc->size == 512 && mtd->writesize > 512) { 1731 ecc->size = 1024; 1732 ecc->strength *= 2; 1733 } 1734 1735 /* Add ECC info retrieval from DT */ 1736 for (i = 0; i < ARRAY_SIZE(strengths); i++) { 1737 if (ecc->strength <= strengths[i]) { 1738 /* 1739 * Update ecc->strength value with the actual strength 1740 * that will be used by the ECC engine. 1741 */ 1742 ecc->strength = strengths[i]; 1743 break; 1744 } 1745 } 1746 1747 if (i >= ARRAY_SIZE(strengths)) { 1748 dev_err(nfc->dev, "unsupported strength\n"); 1749 ret = -ENOTSUPP; 1750 goto err; 1751 } 1752 1753 data->mode = i; 1754 1755 /* HW ECC always request ECC bytes for 1024 bytes blocks */ 1756 ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8); 1757 1758 /* HW ECC always work with even numbers of ECC bytes */ 1759 ecc->bytes = ALIGN(ecc->bytes, 2); 1760 1761 nsectors = mtd->writesize / ecc->size; 1762 1763 if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) { 1764 ret = -EINVAL; 1765 goto err; 1766 } 1767 1768 ecc->read_oob = sunxi_nfc_hw_ecc_read_oob; 1769 ecc->write_oob = sunxi_nfc_hw_ecc_write_oob; 1770 mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops); 1771 ecc->priv = data; 1772 1773 if (nfc->dmac) { 1774 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma; 1775 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma; 1776 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma; 1777 nand->options |= NAND_USE_BOUNCE_BUFFER; 1778 } else { 1779 ecc->read_page = sunxi_nfc_hw_ecc_read_page; 1780 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage; 1781 ecc->write_page = sunxi_nfc_hw_ecc_write_page; 1782 } 1783 1784 /* TODO: support DMA for raw accesses and subpage write */ 1785 ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage; 1786 ecc->read_oob_raw = nand_read_oob_std; 1787 ecc->write_oob_raw = nand_write_oob_std; 1788 1789 return 0; 1790 1791 err: 1792 kfree(data); 1793 1794 return ret; 1795 } 1796 1797 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc) 1798 { 1799 switch (ecc->mode) { 1800 case NAND_ECC_HW: 1801 sunxi_nand_hw_ecc_ctrl_cleanup(ecc); 1802 break; 1803 case NAND_ECC_NONE: 1804 default: 1805 break; 1806 } 1807 } 1808 1809 static int sunxi_nand_attach_chip(struct nand_chip *nand) 1810 { 1811 struct mtd_info *mtd = nand_to_mtd(nand); 1812 struct nand_ecc_ctrl *ecc = &nand->ecc; 1813 struct device_node *np = nand_get_flash_node(nand); 1814 int ret; 1815 1816 if (nand->bbt_options & NAND_BBT_USE_FLASH) 1817 nand->bbt_options |= NAND_BBT_NO_OOB; 1818 1819 if (nand->options & NAND_NEED_SCRAMBLING) 1820 nand->options |= NAND_NO_SUBPAGE_WRITE; 1821 1822 nand->options |= NAND_SUBPAGE_READ; 1823 1824 if (!ecc->size) { 1825 ecc->size = nand->ecc_step_ds; 1826 ecc->strength = nand->ecc_strength_ds; 1827 } 1828 1829 if (!ecc->size || !ecc->strength) 1830 return -EINVAL; 1831 1832 switch (ecc->mode) { 1833 case NAND_ECC_HW: 1834 ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np); 1835 if (ret) 1836 return ret; 1837 break; 1838 case NAND_ECC_NONE: 1839 case NAND_ECC_SOFT: 1840 break; 1841 default: 1842 return -EINVAL; 1843 } 1844 1845 return 0; 1846 } 1847 1848 static const struct nand_controller_ops sunxi_nand_controller_ops = { 1849 .attach_chip = sunxi_nand_attach_chip, 1850 .setup_data_interface = sunxi_nfc_setup_data_interface, 1851 }; 1852 1853 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc, 1854 struct device_node *np) 1855 { 1856 struct sunxi_nand_chip *chip; 1857 struct mtd_info *mtd; 1858 struct nand_chip *nand; 1859 int nsels; 1860 int ret; 1861 int i; 1862 u32 tmp; 1863 1864 if (!of_get_property(np, "reg", &nsels)) 1865 return -EINVAL; 1866 1867 nsels /= sizeof(u32); 1868 if (!nsels) { 1869 dev_err(dev, "invalid reg property size\n"); 1870 return -EINVAL; 1871 } 1872 1873 chip = devm_kzalloc(dev, 1874 sizeof(*chip) + 1875 (nsels * sizeof(struct sunxi_nand_chip_sel)), 1876 GFP_KERNEL); 1877 if (!chip) { 1878 dev_err(dev, "could not allocate chip\n"); 1879 return -ENOMEM; 1880 } 1881 1882 chip->nsels = nsels; 1883 chip->selected = -1; 1884 1885 for (i = 0; i < nsels; i++) { 1886 ret = of_property_read_u32_index(np, "reg", i, &tmp); 1887 if (ret) { 1888 dev_err(dev, "could not retrieve reg property: %d\n", 1889 ret); 1890 return ret; 1891 } 1892 1893 if (tmp > NFC_MAX_CS) { 1894 dev_err(dev, 1895 "invalid reg value: %u (max CS = 7)\n", 1896 tmp); 1897 return -EINVAL; 1898 } 1899 1900 if (test_and_set_bit(tmp, &nfc->assigned_cs)) { 1901 dev_err(dev, "CS %d already assigned\n", tmp); 1902 return -EINVAL; 1903 } 1904 1905 chip->sels[i].cs = tmp; 1906 1907 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) && 1908 tmp < 2) 1909 chip->sels[i].rb = tmp; 1910 else 1911 chip->sels[i].rb = -1; 1912 } 1913 1914 nand = &chip->nand; 1915 /* Default tR value specified in the ONFI spec (chapter 4.15.1) */ 1916 nand->legacy.chip_delay = 200; 1917 nand->controller = &nfc->controller; 1918 nand->controller->ops = &sunxi_nand_controller_ops; 1919 1920 /* 1921 * Set the ECC mode to the default value in case nothing is specified 1922 * in the DT. 1923 */ 1924 nand->ecc.mode = NAND_ECC_HW; 1925 nand_set_flash_node(nand, np); 1926 nand->legacy.select_chip = sunxi_nfc_select_chip; 1927 nand->legacy.cmd_ctrl = sunxi_nfc_cmd_ctrl; 1928 nand->legacy.read_buf = sunxi_nfc_read_buf; 1929 nand->legacy.write_buf = sunxi_nfc_write_buf; 1930 nand->legacy.read_byte = sunxi_nfc_read_byte; 1931 1932 mtd = nand_to_mtd(nand); 1933 mtd->dev.parent = dev; 1934 1935 ret = nand_scan(nand, nsels); 1936 if (ret) 1937 return ret; 1938 1939 ret = mtd_device_register(mtd, NULL, 0); 1940 if (ret) { 1941 dev_err(dev, "failed to register mtd device: %d\n", ret); 1942 nand_release(nand); 1943 return ret; 1944 } 1945 1946 list_add_tail(&chip->node, &nfc->chips); 1947 1948 return 0; 1949 } 1950 1951 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc) 1952 { 1953 struct device_node *np = dev->of_node; 1954 struct device_node *nand_np; 1955 int nchips = of_get_child_count(np); 1956 int ret; 1957 1958 if (nchips > 8) { 1959 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips); 1960 return -EINVAL; 1961 } 1962 1963 for_each_child_of_node(np, nand_np) { 1964 ret = sunxi_nand_chip_init(dev, nfc, nand_np); 1965 if (ret) { 1966 of_node_put(nand_np); 1967 return ret; 1968 } 1969 } 1970 1971 return 0; 1972 } 1973 1974 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc) 1975 { 1976 struct sunxi_nand_chip *chip; 1977 1978 while (!list_empty(&nfc->chips)) { 1979 chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip, 1980 node); 1981 nand_release(&chip->nand); 1982 sunxi_nand_ecc_cleanup(&chip->nand.ecc); 1983 list_del(&chip->node); 1984 } 1985 } 1986 1987 static int sunxi_nfc_probe(struct platform_device *pdev) 1988 { 1989 struct device *dev = &pdev->dev; 1990 struct resource *r; 1991 struct sunxi_nfc *nfc; 1992 int irq; 1993 int ret; 1994 1995 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL); 1996 if (!nfc) 1997 return -ENOMEM; 1998 1999 nfc->dev = dev; 2000 nand_controller_init(&nfc->controller); 2001 INIT_LIST_HEAD(&nfc->chips); 2002 2003 r = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2004 nfc->regs = devm_ioremap_resource(dev, r); 2005 if (IS_ERR(nfc->regs)) 2006 return PTR_ERR(nfc->regs); 2007 2008 irq = platform_get_irq(pdev, 0); 2009 if (irq < 0) { 2010 dev_err(dev, "failed to retrieve irq\n"); 2011 return irq; 2012 } 2013 2014 nfc->ahb_clk = devm_clk_get(dev, "ahb"); 2015 if (IS_ERR(nfc->ahb_clk)) { 2016 dev_err(dev, "failed to retrieve ahb clk\n"); 2017 return PTR_ERR(nfc->ahb_clk); 2018 } 2019 2020 ret = clk_prepare_enable(nfc->ahb_clk); 2021 if (ret) 2022 return ret; 2023 2024 nfc->mod_clk = devm_clk_get(dev, "mod"); 2025 if (IS_ERR(nfc->mod_clk)) { 2026 dev_err(dev, "failed to retrieve mod clk\n"); 2027 ret = PTR_ERR(nfc->mod_clk); 2028 goto out_ahb_clk_unprepare; 2029 } 2030 2031 ret = clk_prepare_enable(nfc->mod_clk); 2032 if (ret) 2033 goto out_ahb_clk_unprepare; 2034 2035 nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb"); 2036 if (IS_ERR(nfc->reset)) { 2037 ret = PTR_ERR(nfc->reset); 2038 goto out_mod_clk_unprepare; 2039 } 2040 2041 ret = reset_control_deassert(nfc->reset); 2042 if (ret) { 2043 dev_err(dev, "reset err %d\n", ret); 2044 goto out_mod_clk_unprepare; 2045 } 2046 2047 ret = sunxi_nfc_rst(nfc); 2048 if (ret) 2049 goto out_ahb_reset_reassert; 2050 2051 writel(0, nfc->regs + NFC_REG_INT); 2052 ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt, 2053 0, "sunxi-nand", nfc); 2054 if (ret) 2055 goto out_ahb_reset_reassert; 2056 2057 nfc->dmac = dma_request_slave_channel(dev, "rxtx"); 2058 if (nfc->dmac) { 2059 struct dma_slave_config dmac_cfg = { }; 2060 2061 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA; 2062 dmac_cfg.dst_addr = dmac_cfg.src_addr; 2063 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 2064 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width; 2065 dmac_cfg.src_maxburst = 4; 2066 dmac_cfg.dst_maxburst = 4; 2067 dmaengine_slave_config(nfc->dmac, &dmac_cfg); 2068 } else { 2069 dev_warn(dev, "failed to request rxtx DMA channel\n"); 2070 } 2071 2072 platform_set_drvdata(pdev, nfc); 2073 2074 ret = sunxi_nand_chips_init(dev, nfc); 2075 if (ret) { 2076 dev_err(dev, "failed to init nand chips\n"); 2077 goto out_release_dmac; 2078 } 2079 2080 return 0; 2081 2082 out_release_dmac: 2083 if (nfc->dmac) 2084 dma_release_channel(nfc->dmac); 2085 out_ahb_reset_reassert: 2086 reset_control_assert(nfc->reset); 2087 out_mod_clk_unprepare: 2088 clk_disable_unprepare(nfc->mod_clk); 2089 out_ahb_clk_unprepare: 2090 clk_disable_unprepare(nfc->ahb_clk); 2091 2092 return ret; 2093 } 2094 2095 static int sunxi_nfc_remove(struct platform_device *pdev) 2096 { 2097 struct sunxi_nfc *nfc = platform_get_drvdata(pdev); 2098 2099 sunxi_nand_chips_cleanup(nfc); 2100 2101 reset_control_assert(nfc->reset); 2102 2103 if (nfc->dmac) 2104 dma_release_channel(nfc->dmac); 2105 clk_disable_unprepare(nfc->mod_clk); 2106 clk_disable_unprepare(nfc->ahb_clk); 2107 2108 return 0; 2109 } 2110 2111 static const struct of_device_id sunxi_nfc_ids[] = { 2112 { .compatible = "allwinner,sun4i-a10-nand" }, 2113 { /* sentinel */ } 2114 }; 2115 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids); 2116 2117 static struct platform_driver sunxi_nfc_driver = { 2118 .driver = { 2119 .name = "sunxi_nand", 2120 .of_match_table = sunxi_nfc_ids, 2121 }, 2122 .probe = sunxi_nfc_probe, 2123 .remove = sunxi_nfc_remove, 2124 }; 2125 module_platform_driver(sunxi_nfc_driver); 2126 2127 MODULE_LICENSE("GPL v2"); 2128 MODULE_AUTHOR("Boris BREZILLON"); 2129 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver"); 2130 MODULE_ALIAS("platform:sunxi_nand"); 2131