1 /* 2 * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com> 3 * 4 * Derived from: 5 * https://github.com/yuq/sunxi-nfc-mtd 6 * Copyright (C) 2013 Qiang Yu <yuq825@gmail.com> 7 * 8 * https://github.com/hno/Allwinner-Info 9 * Copyright (C) 2013 Henrik Nordström <Henrik Nordström> 10 * 11 * Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com> 12 * Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org> 13 * 14 * This program is free software; you can redistribute it and/or modify 15 * it under the terms of the GNU General Public License as published by 16 * the Free Software Foundation; either version 2 of the License, or 17 * (at your option) any later version. 18 * 19 * This program is distributed in the hope that it will be useful, 20 * but WITHOUT ANY WARRANTY; without even the implied warranty of 21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 * GNU General Public License for more details. 23 */ 24 25 #include <linux/dma-mapping.h> 26 #include <linux/slab.h> 27 #include <linux/module.h> 28 #include <linux/moduleparam.h> 29 #include <linux/platform_device.h> 30 #include <linux/of.h> 31 #include <linux/of_device.h> 32 #include <linux/of_gpio.h> 33 #include <linux/mtd/mtd.h> 34 #include <linux/mtd/rawnand.h> 35 #include <linux/mtd/partitions.h> 36 #include <linux/clk.h> 37 #include <linux/delay.h> 38 #include <linux/dmaengine.h> 39 #include <linux/gpio.h> 40 #include <linux/interrupt.h> 41 #include <linux/iopoll.h> 42 #include <linux/reset.h> 43 44 #define NFC_REG_CTL 0x0000 45 #define NFC_REG_ST 0x0004 46 #define NFC_REG_INT 0x0008 47 #define NFC_REG_TIMING_CTL 0x000C 48 #define NFC_REG_TIMING_CFG 0x0010 49 #define NFC_REG_ADDR_LOW 0x0014 50 #define NFC_REG_ADDR_HIGH 0x0018 51 #define NFC_REG_SECTOR_NUM 0x001C 52 #define NFC_REG_CNT 0x0020 53 #define NFC_REG_CMD 0x0024 54 #define NFC_REG_RCMD_SET 0x0028 55 #define NFC_REG_WCMD_SET 0x002C 56 #define NFC_REG_IO_DATA 0x0030 57 #define NFC_REG_ECC_CTL 0x0034 58 #define NFC_REG_ECC_ST 0x0038 59 #define NFC_REG_DEBUG 0x003C 60 #define NFC_REG_ECC_ERR_CNT(x) ((0x0040 + (x)) & ~0x3) 61 #define NFC_REG_USER_DATA(x) (0x0050 + ((x) * 4)) 62 #define NFC_REG_SPARE_AREA 0x00A0 63 #define NFC_REG_PAT_ID 0x00A4 64 #define NFC_RAM0_BASE 0x0400 65 #define NFC_RAM1_BASE 0x0800 66 67 /* define bit use in NFC_CTL */ 68 #define NFC_EN BIT(0) 69 #define NFC_RESET BIT(1) 70 #define NFC_BUS_WIDTH_MSK BIT(2) 71 #define NFC_BUS_WIDTH_8 (0 << 2) 72 #define NFC_BUS_WIDTH_16 (1 << 2) 73 #define NFC_RB_SEL_MSK BIT(3) 74 #define NFC_RB_SEL(x) ((x) << 3) 75 #define NFC_CE_SEL_MSK GENMASK(26, 24) 76 #define NFC_CE_SEL(x) ((x) << 24) 77 #define NFC_CE_CTL BIT(6) 78 #define NFC_PAGE_SHIFT_MSK GENMASK(11, 8) 79 #define NFC_PAGE_SHIFT(x) (((x) < 10 ? 0 : (x) - 10) << 8) 80 #define NFC_SAM BIT(12) 81 #define NFC_RAM_METHOD BIT(14) 82 #define NFC_DEBUG_CTL BIT(31) 83 84 /* define bit use in NFC_ST */ 85 #define NFC_RB_B2R BIT(0) 86 #define NFC_CMD_INT_FLAG BIT(1) 87 #define NFC_DMA_INT_FLAG BIT(2) 88 #define NFC_CMD_FIFO_STATUS BIT(3) 89 #define NFC_STA BIT(4) 90 #define NFC_NATCH_INT_FLAG BIT(5) 91 #define NFC_RB_STATE(x) BIT(x + 8) 92 93 /* define bit use in NFC_INT */ 94 #define NFC_B2R_INT_ENABLE BIT(0) 95 #define NFC_CMD_INT_ENABLE BIT(1) 96 #define NFC_DMA_INT_ENABLE BIT(2) 97 #define NFC_INT_MASK (NFC_B2R_INT_ENABLE | \ 98 NFC_CMD_INT_ENABLE | \ 99 NFC_DMA_INT_ENABLE) 100 101 /* define bit use in NFC_TIMING_CTL */ 102 #define NFC_TIMING_CTL_EDO BIT(8) 103 104 /* define NFC_TIMING_CFG register layout */ 105 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD) \ 106 (((tWB) & 0x3) | (((tADL) & 0x3) << 2) | \ 107 (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) | \ 108 (((tCAD) & 0x7) << 8)) 109 110 /* define bit use in NFC_CMD */ 111 #define NFC_CMD_LOW_BYTE_MSK GENMASK(7, 0) 112 #define NFC_CMD_HIGH_BYTE_MSK GENMASK(15, 8) 113 #define NFC_CMD(x) (x) 114 #define NFC_ADR_NUM_MSK GENMASK(18, 16) 115 #define NFC_ADR_NUM(x) (((x) - 1) << 16) 116 #define NFC_SEND_ADR BIT(19) 117 #define NFC_ACCESS_DIR BIT(20) 118 #define NFC_DATA_TRANS BIT(21) 119 #define NFC_SEND_CMD1 BIT(22) 120 #define NFC_WAIT_FLAG BIT(23) 121 #define NFC_SEND_CMD2 BIT(24) 122 #define NFC_SEQ BIT(25) 123 #define NFC_DATA_SWAP_METHOD BIT(26) 124 #define NFC_ROW_AUTO_INC BIT(27) 125 #define NFC_SEND_CMD3 BIT(28) 126 #define NFC_SEND_CMD4 BIT(29) 127 #define NFC_CMD_TYPE_MSK GENMASK(31, 30) 128 #define NFC_NORMAL_OP (0 << 30) 129 #define NFC_ECC_OP (1 << 30) 130 #define NFC_PAGE_OP (2 << 30) 131 132 /* define bit use in NFC_RCMD_SET */ 133 #define NFC_READ_CMD_MSK GENMASK(7, 0) 134 #define NFC_RND_READ_CMD0_MSK GENMASK(15, 8) 135 #define NFC_RND_READ_CMD1_MSK GENMASK(23, 16) 136 137 /* define bit use in NFC_WCMD_SET */ 138 #define NFC_PROGRAM_CMD_MSK GENMASK(7, 0) 139 #define NFC_RND_WRITE_CMD_MSK GENMASK(15, 8) 140 #define NFC_READ_CMD0_MSK GENMASK(23, 16) 141 #define NFC_READ_CMD1_MSK GENMASK(31, 24) 142 143 /* define bit use in NFC_ECC_CTL */ 144 #define NFC_ECC_EN BIT(0) 145 #define NFC_ECC_PIPELINE BIT(3) 146 #define NFC_ECC_EXCEPTION BIT(4) 147 #define NFC_ECC_BLOCK_SIZE_MSK BIT(5) 148 #define NFC_ECC_BLOCK_512 BIT(5) 149 #define NFC_RANDOM_EN BIT(9) 150 #define NFC_RANDOM_DIRECTION BIT(10) 151 #define NFC_ECC_MODE_MSK GENMASK(15, 12) 152 #define NFC_ECC_MODE(x) ((x) << 12) 153 #define NFC_RANDOM_SEED_MSK GENMASK(30, 16) 154 #define NFC_RANDOM_SEED(x) ((x) << 16) 155 156 /* define bit use in NFC_ECC_ST */ 157 #define NFC_ECC_ERR(x) BIT(x) 158 #define NFC_ECC_ERR_MSK GENMASK(15, 0) 159 #define NFC_ECC_PAT_FOUND(x) BIT(x + 16) 160 #define NFC_ECC_ERR_CNT(b, x) (((x) >> (((b) % 4) * 8)) & 0xff) 161 162 #define NFC_DEFAULT_TIMEOUT_MS 1000 163 164 #define NFC_SRAM_SIZE 1024 165 166 #define NFC_MAX_CS 7 167 168 /* 169 * Chip Select structure: stores information related to NAND Chip Select 170 * 171 * @cs: the NAND CS id used to communicate with a NAND Chip 172 * @rb: the Ready/Busy pin ID. -1 means no R/B pin connected to the 173 * NFC 174 */ 175 struct sunxi_nand_chip_sel { 176 u8 cs; 177 s8 rb; 178 }; 179 180 /* 181 * sunxi HW ECC infos: stores information related to HW ECC support 182 * 183 * @mode: the sunxi ECC mode field deduced from ECC requirements 184 */ 185 struct sunxi_nand_hw_ecc { 186 int mode; 187 }; 188 189 /* 190 * NAND chip structure: stores NAND chip device related information 191 * 192 * @node: used to store NAND chips into a list 193 * @nand: base NAND chip structure 194 * @mtd: base MTD structure 195 * @clk_rate: clk_rate required for this NAND chip 196 * @timing_cfg TIMING_CFG register value for this NAND chip 197 * @selected: current active CS 198 * @nsels: number of CS lines required by the NAND chip 199 * @sels: array of CS lines descriptions 200 */ 201 struct sunxi_nand_chip { 202 struct list_head node; 203 struct nand_chip nand; 204 unsigned long clk_rate; 205 u32 timing_cfg; 206 u32 timing_ctl; 207 int selected; 208 int addr_cycles; 209 u32 addr[2]; 210 int cmd_cycles; 211 u8 cmd[2]; 212 int nsels; 213 struct sunxi_nand_chip_sel sels[0]; 214 }; 215 216 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand) 217 { 218 return container_of(nand, struct sunxi_nand_chip, nand); 219 } 220 221 /* 222 * NAND Controller structure: stores sunxi NAND controller information 223 * 224 * @controller: base controller structure 225 * @dev: parent device (used to print error messages) 226 * @regs: NAND controller registers 227 * @ahb_clk: NAND Controller AHB clock 228 * @mod_clk: NAND Controller mod clock 229 * @assigned_cs: bitmask describing already assigned CS lines 230 * @clk_rate: NAND controller current clock rate 231 * @chips: a list containing all the NAND chips attached to 232 * this NAND controller 233 * @complete: a completion object used to wait for NAND 234 * controller events 235 */ 236 struct sunxi_nfc { 237 struct nand_hw_control controller; 238 struct device *dev; 239 void __iomem *regs; 240 struct clk *ahb_clk; 241 struct clk *mod_clk; 242 struct reset_control *reset; 243 unsigned long assigned_cs; 244 unsigned long clk_rate; 245 struct list_head chips; 246 struct completion complete; 247 struct dma_chan *dmac; 248 }; 249 250 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_hw_control *ctrl) 251 { 252 return container_of(ctrl, struct sunxi_nfc, controller); 253 } 254 255 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id) 256 { 257 struct sunxi_nfc *nfc = dev_id; 258 u32 st = readl(nfc->regs + NFC_REG_ST); 259 u32 ien = readl(nfc->regs + NFC_REG_INT); 260 261 if (!(ien & st)) 262 return IRQ_NONE; 263 264 if ((ien & st) == ien) 265 complete(&nfc->complete); 266 267 writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST); 268 writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT); 269 270 return IRQ_HANDLED; 271 } 272 273 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events, 274 bool use_polling, unsigned int timeout_ms) 275 { 276 int ret; 277 278 if (events & ~NFC_INT_MASK) 279 return -EINVAL; 280 281 if (!timeout_ms) 282 timeout_ms = NFC_DEFAULT_TIMEOUT_MS; 283 284 if (!use_polling) { 285 init_completion(&nfc->complete); 286 287 writel(events, nfc->regs + NFC_REG_INT); 288 289 ret = wait_for_completion_timeout(&nfc->complete, 290 msecs_to_jiffies(timeout_ms)); 291 if (!ret) 292 ret = -ETIMEDOUT; 293 else 294 ret = 0; 295 296 writel(0, nfc->regs + NFC_REG_INT); 297 } else { 298 u32 status; 299 300 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status, 301 (status & events) == events, 1, 302 timeout_ms * 1000); 303 } 304 305 writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST); 306 307 if (ret) 308 dev_err(nfc->dev, "wait interrupt timedout\n"); 309 310 return ret; 311 } 312 313 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc) 314 { 315 u32 status; 316 int ret; 317 318 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status, 319 !(status & NFC_CMD_FIFO_STATUS), 1, 320 NFC_DEFAULT_TIMEOUT_MS * 1000); 321 if (ret) 322 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n"); 323 324 return ret; 325 } 326 327 static int sunxi_nfc_rst(struct sunxi_nfc *nfc) 328 { 329 u32 ctl; 330 int ret; 331 332 writel(0, nfc->regs + NFC_REG_ECC_CTL); 333 writel(NFC_RESET, nfc->regs + NFC_REG_CTL); 334 335 ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl, 336 !(ctl & NFC_RESET), 1, 337 NFC_DEFAULT_TIMEOUT_MS * 1000); 338 if (ret) 339 dev_err(nfc->dev, "wait for NAND controller reset timedout\n"); 340 341 return ret; 342 } 343 344 static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf, 345 int chunksize, int nchunks, 346 enum dma_data_direction ddir, 347 struct scatterlist *sg) 348 { 349 struct nand_chip *nand = mtd_to_nand(mtd); 350 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 351 struct dma_async_tx_descriptor *dmad; 352 enum dma_transfer_direction tdir; 353 dma_cookie_t dmat; 354 int ret; 355 356 if (ddir == DMA_FROM_DEVICE) 357 tdir = DMA_DEV_TO_MEM; 358 else 359 tdir = DMA_MEM_TO_DEV; 360 361 sg_init_one(sg, buf, nchunks * chunksize); 362 ret = dma_map_sg(nfc->dev, sg, 1, ddir); 363 if (!ret) 364 return -ENOMEM; 365 366 dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK); 367 if (!dmad) { 368 ret = -EINVAL; 369 goto err_unmap_buf; 370 } 371 372 writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD, 373 nfc->regs + NFC_REG_CTL); 374 writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM); 375 writel(chunksize, nfc->regs + NFC_REG_CNT); 376 dmat = dmaengine_submit(dmad); 377 378 ret = dma_submit_error(dmat); 379 if (ret) 380 goto err_clr_dma_flag; 381 382 return 0; 383 384 err_clr_dma_flag: 385 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD, 386 nfc->regs + NFC_REG_CTL); 387 388 err_unmap_buf: 389 dma_unmap_sg(nfc->dev, sg, 1, ddir); 390 return ret; 391 } 392 393 static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd, 394 enum dma_data_direction ddir, 395 struct scatterlist *sg) 396 { 397 struct nand_chip *nand = mtd_to_nand(mtd); 398 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 399 400 dma_unmap_sg(nfc->dev, sg, 1, ddir); 401 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD, 402 nfc->regs + NFC_REG_CTL); 403 } 404 405 static int sunxi_nfc_dev_ready(struct mtd_info *mtd) 406 { 407 struct nand_chip *nand = mtd_to_nand(mtd); 408 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 409 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 410 u32 mask; 411 412 if (sunxi_nand->selected < 0) 413 return 0; 414 415 if (sunxi_nand->sels[sunxi_nand->selected].rb < 0) { 416 dev_err(nfc->dev, "cannot check R/B NAND status!\n"); 417 return 0; 418 } 419 420 mask = NFC_RB_STATE(sunxi_nand->sels[sunxi_nand->selected].rb); 421 422 return !!(readl(nfc->regs + NFC_REG_ST) & mask); 423 } 424 425 static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip) 426 { 427 struct nand_chip *nand = mtd_to_nand(mtd); 428 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 429 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 430 struct sunxi_nand_chip_sel *sel; 431 u32 ctl; 432 433 if (chip > 0 && chip >= sunxi_nand->nsels) 434 return; 435 436 if (chip == sunxi_nand->selected) 437 return; 438 439 ctl = readl(nfc->regs + NFC_REG_CTL) & 440 ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN); 441 442 if (chip >= 0) { 443 sel = &sunxi_nand->sels[chip]; 444 445 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN | 446 NFC_PAGE_SHIFT(nand->page_shift); 447 if (sel->rb < 0) { 448 nand->dev_ready = NULL; 449 } else { 450 nand->dev_ready = sunxi_nfc_dev_ready; 451 ctl |= NFC_RB_SEL(sel->rb); 452 } 453 454 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA); 455 456 if (nfc->clk_rate != sunxi_nand->clk_rate) { 457 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate); 458 nfc->clk_rate = sunxi_nand->clk_rate; 459 } 460 } 461 462 writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL); 463 writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG); 464 writel(ctl, nfc->regs + NFC_REG_CTL); 465 466 sunxi_nand->selected = chip; 467 } 468 469 static void sunxi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) 470 { 471 struct nand_chip *nand = mtd_to_nand(mtd); 472 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 473 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 474 int ret; 475 int cnt; 476 int offs = 0; 477 u32 tmp; 478 479 while (len > offs) { 480 bool poll = false; 481 482 cnt = min(len - offs, NFC_SRAM_SIZE); 483 484 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 485 if (ret) 486 break; 487 488 writel(cnt, nfc->regs + NFC_REG_CNT); 489 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD; 490 writel(tmp, nfc->regs + NFC_REG_CMD); 491 492 /* Arbitrary limit for polling mode */ 493 if (cnt < 64) 494 poll = true; 495 496 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0); 497 if (ret) 498 break; 499 500 if (buf) 501 memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE, 502 cnt); 503 offs += cnt; 504 } 505 } 506 507 static void sunxi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf, 508 int len) 509 { 510 struct nand_chip *nand = mtd_to_nand(mtd); 511 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 512 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 513 int ret; 514 int cnt; 515 int offs = 0; 516 u32 tmp; 517 518 while (len > offs) { 519 bool poll = false; 520 521 cnt = min(len - offs, NFC_SRAM_SIZE); 522 523 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 524 if (ret) 525 break; 526 527 writel(cnt, nfc->regs + NFC_REG_CNT); 528 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt); 529 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | 530 NFC_ACCESS_DIR; 531 writel(tmp, nfc->regs + NFC_REG_CMD); 532 533 /* Arbitrary limit for polling mode */ 534 if (cnt < 64) 535 poll = true; 536 537 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0); 538 if (ret) 539 break; 540 541 offs += cnt; 542 } 543 } 544 545 static uint8_t sunxi_nfc_read_byte(struct mtd_info *mtd) 546 { 547 uint8_t ret; 548 549 sunxi_nfc_read_buf(mtd, &ret, 1); 550 551 return ret; 552 } 553 554 static void sunxi_nfc_cmd_ctrl(struct mtd_info *mtd, int dat, 555 unsigned int ctrl) 556 { 557 struct nand_chip *nand = mtd_to_nand(mtd); 558 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 559 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 560 int ret; 561 562 if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) && 563 !(ctrl & (NAND_CLE | NAND_ALE))) { 564 u32 cmd = 0; 565 566 if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles) 567 return; 568 569 if (sunxi_nand->cmd_cycles--) 570 cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0]; 571 572 if (sunxi_nand->cmd_cycles--) { 573 cmd |= NFC_SEND_CMD2; 574 writel(sunxi_nand->cmd[1], 575 nfc->regs + NFC_REG_RCMD_SET); 576 } 577 578 sunxi_nand->cmd_cycles = 0; 579 580 if (sunxi_nand->addr_cycles) { 581 cmd |= NFC_SEND_ADR | 582 NFC_ADR_NUM(sunxi_nand->addr_cycles); 583 writel(sunxi_nand->addr[0], 584 nfc->regs + NFC_REG_ADDR_LOW); 585 } 586 587 if (sunxi_nand->addr_cycles > 4) 588 writel(sunxi_nand->addr[1], 589 nfc->regs + NFC_REG_ADDR_HIGH); 590 591 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 592 if (ret) 593 return; 594 595 writel(cmd, nfc->regs + NFC_REG_CMD); 596 sunxi_nand->addr[0] = 0; 597 sunxi_nand->addr[1] = 0; 598 sunxi_nand->addr_cycles = 0; 599 sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0); 600 } 601 602 if (ctrl & NAND_CLE) { 603 sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat; 604 } else if (ctrl & NAND_ALE) { 605 sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |= 606 dat << ((sunxi_nand->addr_cycles % 4) * 8); 607 sunxi_nand->addr_cycles++; 608 } 609 } 610 611 /* These seed values have been extracted from Allwinner's BSP */ 612 static const u16 sunxi_nfc_randomizer_page_seeds[] = { 613 0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72, 614 0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436, 615 0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d, 616 0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130, 617 0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56, 618 0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55, 619 0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb, 620 0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17, 621 0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62, 622 0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064, 623 0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126, 624 0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e, 625 0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3, 626 0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b, 627 0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d, 628 0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db, 629 }; 630 631 /* 632 * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds 633 * have been generated using 634 * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what 635 * the randomizer engine does internally before de/scrambling OOB data. 636 * 637 * Those tables are statically defined to avoid calculating randomizer state 638 * at runtime. 639 */ 640 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = { 641 0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64, 642 0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409, 643 0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617, 644 0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d, 645 0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91, 646 0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d, 647 0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab, 648 0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8, 649 0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8, 650 0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b, 651 0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5, 652 0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a, 653 0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891, 654 0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36, 655 0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd, 656 0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0, 657 }; 658 659 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = { 660 0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6, 661 0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982, 662 0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9, 663 0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07, 664 0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e, 665 0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2, 666 0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c, 667 0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f, 668 0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc, 669 0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e, 670 0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8, 671 0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68, 672 0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d, 673 0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179, 674 0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601, 675 0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd, 676 }; 677 678 static u16 sunxi_nfc_randomizer_step(u16 state, int count) 679 { 680 state &= 0x7fff; 681 682 /* 683 * This loop is just a simple implementation of a Fibonacci LFSR using 684 * the x16 + x15 + 1 polynomial. 685 */ 686 while (count--) 687 state = ((state >> 1) | 688 (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff; 689 690 return state; 691 } 692 693 static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc) 694 { 695 const u16 *seeds = sunxi_nfc_randomizer_page_seeds; 696 int mod = mtd_div_by_ws(mtd->erasesize, mtd); 697 698 if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds)) 699 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds); 700 701 if (ecc) { 702 if (mtd->ecc_step_size == 512) 703 seeds = sunxi_nfc_randomizer_ecc512_seeds; 704 else 705 seeds = sunxi_nfc_randomizer_ecc1024_seeds; 706 } 707 708 return seeds[page % mod]; 709 } 710 711 static void sunxi_nfc_randomizer_config(struct mtd_info *mtd, 712 int page, bool ecc) 713 { 714 struct nand_chip *nand = mtd_to_nand(mtd); 715 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 716 u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); 717 u16 state; 718 719 if (!(nand->options & NAND_NEED_SCRAMBLING)) 720 return; 721 722 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); 723 state = sunxi_nfc_randomizer_state(mtd, page, ecc); 724 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK; 725 writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL); 726 } 727 728 static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd) 729 { 730 struct nand_chip *nand = mtd_to_nand(mtd); 731 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 732 733 if (!(nand->options & NAND_NEED_SCRAMBLING)) 734 return; 735 736 writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN, 737 nfc->regs + NFC_REG_ECC_CTL); 738 } 739 740 static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd) 741 { 742 struct nand_chip *nand = mtd_to_nand(mtd); 743 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 744 745 if (!(nand->options & NAND_NEED_SCRAMBLING)) 746 return; 747 748 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN, 749 nfc->regs + NFC_REG_ECC_CTL); 750 } 751 752 static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm) 753 { 754 u16 state = sunxi_nfc_randomizer_state(mtd, page, true); 755 756 bbm[0] ^= state; 757 bbm[1] ^= sunxi_nfc_randomizer_step(state, 8); 758 } 759 760 static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd, 761 const uint8_t *buf, int len, 762 bool ecc, int page) 763 { 764 sunxi_nfc_randomizer_config(mtd, page, ecc); 765 sunxi_nfc_randomizer_enable(mtd); 766 sunxi_nfc_write_buf(mtd, buf, len); 767 sunxi_nfc_randomizer_disable(mtd); 768 } 769 770 static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf, 771 int len, bool ecc, int page) 772 { 773 sunxi_nfc_randomizer_config(mtd, page, ecc); 774 sunxi_nfc_randomizer_enable(mtd); 775 sunxi_nfc_read_buf(mtd, buf, len); 776 sunxi_nfc_randomizer_disable(mtd); 777 } 778 779 static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd) 780 { 781 struct nand_chip *nand = mtd_to_nand(mtd); 782 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 783 struct sunxi_nand_hw_ecc *data = nand->ecc.priv; 784 u32 ecc_ctl; 785 786 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); 787 ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE | 788 NFC_ECC_BLOCK_SIZE_MSK); 789 ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION | 790 NFC_ECC_PIPELINE; 791 792 if (nand->ecc.size == 512) 793 ecc_ctl |= NFC_ECC_BLOCK_512; 794 795 writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL); 796 } 797 798 static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd) 799 { 800 struct nand_chip *nand = mtd_to_nand(mtd); 801 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 802 803 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN, 804 nfc->regs + NFC_REG_ECC_CTL); 805 } 806 807 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf) 808 { 809 buf[0] = user_data; 810 buf[1] = user_data >> 8; 811 buf[2] = user_data >> 16; 812 buf[3] = user_data >> 24; 813 } 814 815 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf) 816 { 817 return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24); 818 } 819 820 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob, 821 int step, bool bbm, int page) 822 { 823 struct nand_chip *nand = mtd_to_nand(mtd); 824 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 825 826 sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)), 827 oob); 828 829 /* De-randomize the Bad Block Marker. */ 830 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) 831 sunxi_nfc_randomize_bbm(mtd, page, oob); 832 } 833 834 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd, 835 const u8 *oob, int step, 836 bool bbm, int page) 837 { 838 struct nand_chip *nand = mtd_to_nand(mtd); 839 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 840 u8 user_data[4]; 841 842 /* Randomize the Bad Block Marker. */ 843 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) { 844 memcpy(user_data, oob, sizeof(user_data)); 845 sunxi_nfc_randomize_bbm(mtd, page, user_data); 846 oob = user_data; 847 } 848 849 writel(sunxi_nfc_buf_to_user_data(oob), 850 nfc->regs + NFC_REG_USER_DATA(step)); 851 } 852 853 static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd, 854 unsigned int *max_bitflips, int ret) 855 { 856 if (ret < 0) { 857 mtd->ecc_stats.failed++; 858 } else { 859 mtd->ecc_stats.corrected += ret; 860 *max_bitflips = max_t(unsigned int, *max_bitflips, ret); 861 } 862 } 863 864 static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob, 865 int step, u32 status, bool *erased) 866 { 867 struct nand_chip *nand = mtd_to_nand(mtd); 868 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 869 struct nand_ecc_ctrl *ecc = &nand->ecc; 870 u32 tmp; 871 872 *erased = false; 873 874 if (status & NFC_ECC_ERR(step)) 875 return -EBADMSG; 876 877 if (status & NFC_ECC_PAT_FOUND(step)) { 878 u8 pattern; 879 880 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) { 881 pattern = 0x0; 882 } else { 883 pattern = 0xff; 884 *erased = true; 885 } 886 887 if (data) 888 memset(data, pattern, ecc->size); 889 890 if (oob) 891 memset(oob, pattern, ecc->bytes + 4); 892 893 return 0; 894 } 895 896 tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step)); 897 898 return NFC_ECC_ERR_CNT(step, tmp); 899 } 900 901 static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd, 902 u8 *data, int data_off, 903 u8 *oob, int oob_off, 904 int *cur_off, 905 unsigned int *max_bitflips, 906 bool bbm, bool oob_required, int page) 907 { 908 struct nand_chip *nand = mtd_to_nand(mtd); 909 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 910 struct nand_ecc_ctrl *ecc = &nand->ecc; 911 int raw_mode = 0; 912 bool erased; 913 int ret; 914 915 if (*cur_off != data_off) 916 nand_change_read_column_op(nand, data_off, NULL, 0, false); 917 918 sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page); 919 920 if (data_off + ecc->size != oob_off) 921 nand_change_read_column_op(nand, oob_off, NULL, 0, false); 922 923 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 924 if (ret) 925 return ret; 926 927 sunxi_nfc_randomizer_enable(mtd); 928 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP, 929 nfc->regs + NFC_REG_CMD); 930 931 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 932 sunxi_nfc_randomizer_disable(mtd); 933 if (ret) 934 return ret; 935 936 *cur_off = oob_off + ecc->bytes + 4; 937 938 ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0, 939 readl(nfc->regs + NFC_REG_ECC_ST), 940 &erased); 941 if (erased) 942 return 1; 943 944 if (ret < 0) { 945 /* 946 * Re-read the data with the randomizer disabled to identify 947 * bitflips in erased pages. 948 */ 949 if (nand->options & NAND_NEED_SCRAMBLING) 950 nand_change_read_column_op(nand, data_off, data, 951 ecc->size, false); 952 else 953 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, 954 ecc->size); 955 956 nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4, 957 false); 958 959 ret = nand_check_erased_ecc_chunk(data, ecc->size, 960 oob, ecc->bytes + 4, 961 NULL, 0, ecc->strength); 962 if (ret >= 0) 963 raw_mode = 1; 964 } else { 965 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size); 966 967 if (oob_required) { 968 nand_change_read_column_op(nand, oob_off, NULL, 0, 969 false); 970 sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4, 971 true, page); 972 973 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0, 974 bbm, page); 975 } 976 } 977 978 sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret); 979 980 return raw_mode; 981 } 982 983 static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd, 984 u8 *oob, int *cur_off, 985 bool randomize, int page) 986 { 987 struct nand_chip *nand = mtd_to_nand(mtd); 988 struct nand_ecc_ctrl *ecc = &nand->ecc; 989 int offset = ((ecc->bytes + 4) * ecc->steps); 990 int len = mtd->oobsize - offset; 991 992 if (len <= 0) 993 return; 994 995 if (!cur_off || *cur_off != offset) 996 nand_change_read_column_op(nand, mtd->writesize, NULL, 0, 997 false); 998 999 if (!randomize) 1000 sunxi_nfc_read_buf(mtd, oob + offset, len); 1001 else 1002 sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len, 1003 false, page); 1004 1005 if (cur_off) 1006 *cur_off = mtd->oobsize + mtd->writesize; 1007 } 1008 1009 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf, 1010 int oob_required, int page, 1011 int nchunks) 1012 { 1013 struct nand_chip *nand = mtd_to_nand(mtd); 1014 bool randomized = nand->options & NAND_NEED_SCRAMBLING; 1015 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1016 struct nand_ecc_ctrl *ecc = &nand->ecc; 1017 unsigned int max_bitflips = 0; 1018 int ret, i, raw_mode = 0; 1019 struct scatterlist sg; 1020 u32 status; 1021 1022 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1023 if (ret) 1024 return ret; 1025 1026 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks, 1027 DMA_FROM_DEVICE, &sg); 1028 if (ret) 1029 return ret; 1030 1031 sunxi_nfc_hw_ecc_enable(mtd); 1032 sunxi_nfc_randomizer_config(mtd, page, false); 1033 sunxi_nfc_randomizer_enable(mtd); 1034 1035 writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) | 1036 NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET); 1037 1038 dma_async_issue_pending(nfc->dmac); 1039 1040 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS, 1041 nfc->regs + NFC_REG_CMD); 1042 1043 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 1044 if (ret) 1045 dmaengine_terminate_all(nfc->dmac); 1046 1047 sunxi_nfc_randomizer_disable(mtd); 1048 sunxi_nfc_hw_ecc_disable(mtd); 1049 1050 sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg); 1051 1052 if (ret) 1053 return ret; 1054 1055 status = readl(nfc->regs + NFC_REG_ECC_ST); 1056 1057 for (i = 0; i < nchunks; i++) { 1058 int data_off = i * ecc->size; 1059 int oob_off = i * (ecc->bytes + 4); 1060 u8 *data = buf + data_off; 1061 u8 *oob = nand->oob_poi + oob_off; 1062 bool erased; 1063 1064 ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL, 1065 oob_required ? oob : NULL, 1066 i, status, &erased); 1067 1068 /* ECC errors are handled in the second loop. */ 1069 if (ret < 0) 1070 continue; 1071 1072 if (oob_required && !erased) { 1073 /* TODO: use DMA to retrieve OOB */ 1074 nand_change_read_column_op(nand, 1075 mtd->writesize + oob_off, 1076 oob, ecc->bytes + 4, false); 1077 1078 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i, 1079 !i, page); 1080 } 1081 1082 if (erased) 1083 raw_mode = 1; 1084 1085 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret); 1086 } 1087 1088 if (status & NFC_ECC_ERR_MSK) { 1089 for (i = 0; i < nchunks; i++) { 1090 int data_off = i * ecc->size; 1091 int oob_off = i * (ecc->bytes + 4); 1092 u8 *data = buf + data_off; 1093 u8 *oob = nand->oob_poi + oob_off; 1094 1095 if (!(status & NFC_ECC_ERR(i))) 1096 continue; 1097 1098 /* 1099 * Re-read the data with the randomizer disabled to 1100 * identify bitflips in erased pages. 1101 * TODO: use DMA to read page in raw mode 1102 */ 1103 if (randomized) 1104 nand_change_read_column_op(nand, data_off, 1105 data, ecc->size, 1106 false); 1107 1108 /* TODO: use DMA to retrieve OOB */ 1109 nand_change_read_column_op(nand, 1110 mtd->writesize + oob_off, 1111 oob, ecc->bytes + 4, false); 1112 1113 ret = nand_check_erased_ecc_chunk(data, ecc->size, 1114 oob, ecc->bytes + 4, 1115 NULL, 0, 1116 ecc->strength); 1117 if (ret >= 0) 1118 raw_mode = 1; 1119 1120 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret); 1121 } 1122 } 1123 1124 if (oob_required) 1125 sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi, 1126 NULL, !raw_mode, 1127 page); 1128 1129 return max_bitflips; 1130 } 1131 1132 static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd, 1133 const u8 *data, int data_off, 1134 const u8 *oob, int oob_off, 1135 int *cur_off, bool bbm, 1136 int page) 1137 { 1138 struct nand_chip *nand = mtd_to_nand(mtd); 1139 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1140 struct nand_ecc_ctrl *ecc = &nand->ecc; 1141 int ret; 1142 1143 if (data_off != *cur_off) 1144 nand_change_write_column_op(nand, data_off, NULL, 0, false); 1145 1146 sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page); 1147 1148 if (data_off + ecc->size != oob_off) 1149 nand_change_write_column_op(nand, oob_off, NULL, 0, false); 1150 1151 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1152 if (ret) 1153 return ret; 1154 1155 sunxi_nfc_randomizer_enable(mtd); 1156 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page); 1157 1158 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | 1159 NFC_ACCESS_DIR | NFC_ECC_OP, 1160 nfc->regs + NFC_REG_CMD); 1161 1162 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 1163 sunxi_nfc_randomizer_disable(mtd); 1164 if (ret) 1165 return ret; 1166 1167 *cur_off = oob_off + ecc->bytes + 4; 1168 1169 return 0; 1170 } 1171 1172 static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd, 1173 u8 *oob, int *cur_off, 1174 int page) 1175 { 1176 struct nand_chip *nand = mtd_to_nand(mtd); 1177 struct nand_ecc_ctrl *ecc = &nand->ecc; 1178 int offset = ((ecc->bytes + 4) * ecc->steps); 1179 int len = mtd->oobsize - offset; 1180 1181 if (len <= 0) 1182 return; 1183 1184 if (!cur_off || *cur_off != offset) 1185 nand_change_write_column_op(nand, offset + mtd->writesize, 1186 NULL, 0, false); 1187 1188 sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page); 1189 1190 if (cur_off) 1191 *cur_off = mtd->oobsize + mtd->writesize; 1192 } 1193 1194 static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd, 1195 struct nand_chip *chip, uint8_t *buf, 1196 int oob_required, int page) 1197 { 1198 struct nand_ecc_ctrl *ecc = &chip->ecc; 1199 unsigned int max_bitflips = 0; 1200 int ret, i, cur_off = 0; 1201 bool raw_mode = false; 1202 1203 nand_read_page_op(chip, page, 0, NULL, 0); 1204 1205 sunxi_nfc_hw_ecc_enable(mtd); 1206 1207 for (i = 0; i < ecc->steps; i++) { 1208 int data_off = i * ecc->size; 1209 int oob_off = i * (ecc->bytes + 4); 1210 u8 *data = buf + data_off; 1211 u8 *oob = chip->oob_poi + oob_off; 1212 1213 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob, 1214 oob_off + mtd->writesize, 1215 &cur_off, &max_bitflips, 1216 !i, oob_required, page); 1217 if (ret < 0) 1218 return ret; 1219 else if (ret) 1220 raw_mode = true; 1221 } 1222 1223 if (oob_required) 1224 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off, 1225 !raw_mode, page); 1226 1227 sunxi_nfc_hw_ecc_disable(mtd); 1228 1229 return max_bitflips; 1230 } 1231 1232 static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd, 1233 struct nand_chip *chip, u8 *buf, 1234 int oob_required, int page) 1235 { 1236 int ret; 1237 1238 nand_read_page_op(chip, page, 0, NULL, 0); 1239 1240 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page, 1241 chip->ecc.steps); 1242 if (ret >= 0) 1243 return ret; 1244 1245 /* Fallback to PIO mode */ 1246 return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page); 1247 } 1248 1249 static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd, 1250 struct nand_chip *chip, 1251 u32 data_offs, u32 readlen, 1252 u8 *bufpoi, int page) 1253 { 1254 struct nand_ecc_ctrl *ecc = &chip->ecc; 1255 int ret, i, cur_off = 0; 1256 unsigned int max_bitflips = 0; 1257 1258 nand_read_page_op(chip, page, 0, NULL, 0); 1259 1260 sunxi_nfc_hw_ecc_enable(mtd); 1261 1262 for (i = data_offs / ecc->size; 1263 i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) { 1264 int data_off = i * ecc->size; 1265 int oob_off = i * (ecc->bytes + 4); 1266 u8 *data = bufpoi + data_off; 1267 u8 *oob = chip->oob_poi + oob_off; 1268 1269 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, 1270 oob, 1271 oob_off + mtd->writesize, 1272 &cur_off, &max_bitflips, !i, 1273 false, page); 1274 if (ret < 0) 1275 return ret; 1276 } 1277 1278 sunxi_nfc_hw_ecc_disable(mtd); 1279 1280 return max_bitflips; 1281 } 1282 1283 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd, 1284 struct nand_chip *chip, 1285 u32 data_offs, u32 readlen, 1286 u8 *buf, int page) 1287 { 1288 int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size); 1289 int ret; 1290 1291 nand_read_page_op(chip, page, 0, NULL, 0); 1292 1293 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks); 1294 if (ret >= 0) 1295 return ret; 1296 1297 /* Fallback to PIO mode */ 1298 return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen, 1299 buf, page); 1300 } 1301 1302 static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd, 1303 struct nand_chip *chip, 1304 const uint8_t *buf, int oob_required, 1305 int page) 1306 { 1307 struct nand_ecc_ctrl *ecc = &chip->ecc; 1308 int ret, i, cur_off = 0; 1309 1310 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1311 1312 sunxi_nfc_hw_ecc_enable(mtd); 1313 1314 for (i = 0; i < ecc->steps; i++) { 1315 int data_off = i * ecc->size; 1316 int oob_off = i * (ecc->bytes + 4); 1317 const u8 *data = buf + data_off; 1318 const u8 *oob = chip->oob_poi + oob_off; 1319 1320 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob, 1321 oob_off + mtd->writesize, 1322 &cur_off, !i, page); 1323 if (ret) 1324 return ret; 1325 } 1326 1327 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING)) 1328 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi, 1329 &cur_off, page); 1330 1331 sunxi_nfc_hw_ecc_disable(mtd); 1332 1333 return nand_prog_page_end_op(chip); 1334 } 1335 1336 static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd, 1337 struct nand_chip *chip, 1338 u32 data_offs, u32 data_len, 1339 const u8 *buf, int oob_required, 1340 int page) 1341 { 1342 struct nand_ecc_ctrl *ecc = &chip->ecc; 1343 int ret, i, cur_off = 0; 1344 1345 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1346 1347 sunxi_nfc_hw_ecc_enable(mtd); 1348 1349 for (i = data_offs / ecc->size; 1350 i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) { 1351 int data_off = i * ecc->size; 1352 int oob_off = i * (ecc->bytes + 4); 1353 const u8 *data = buf + data_off; 1354 const u8 *oob = chip->oob_poi + oob_off; 1355 1356 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob, 1357 oob_off + mtd->writesize, 1358 &cur_off, !i, page); 1359 if (ret) 1360 return ret; 1361 } 1362 1363 sunxi_nfc_hw_ecc_disable(mtd); 1364 1365 return nand_prog_page_end_op(chip); 1366 } 1367 1368 static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd, 1369 struct nand_chip *chip, 1370 const u8 *buf, 1371 int oob_required, 1372 int page) 1373 { 1374 struct nand_chip *nand = mtd_to_nand(mtd); 1375 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1376 struct nand_ecc_ctrl *ecc = &nand->ecc; 1377 struct scatterlist sg; 1378 int ret, i; 1379 1380 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1381 if (ret) 1382 return ret; 1383 1384 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps, 1385 DMA_TO_DEVICE, &sg); 1386 if (ret) 1387 goto pio_fallback; 1388 1389 for (i = 0; i < ecc->steps; i++) { 1390 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4)); 1391 1392 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page); 1393 } 1394 1395 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1396 1397 sunxi_nfc_hw_ecc_enable(mtd); 1398 sunxi_nfc_randomizer_config(mtd, page, false); 1399 sunxi_nfc_randomizer_enable(mtd); 1400 1401 writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG, 1402 nfc->regs + NFC_REG_RCMD_SET); 1403 1404 dma_async_issue_pending(nfc->dmac); 1405 1406 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | 1407 NFC_DATA_TRANS | NFC_ACCESS_DIR, 1408 nfc->regs + NFC_REG_CMD); 1409 1410 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 1411 if (ret) 1412 dmaengine_terminate_all(nfc->dmac); 1413 1414 sunxi_nfc_randomizer_disable(mtd); 1415 sunxi_nfc_hw_ecc_disable(mtd); 1416 1417 sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg); 1418 1419 if (ret) 1420 return ret; 1421 1422 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING)) 1423 /* TODO: use DMA to transfer extra OOB bytes ? */ 1424 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi, 1425 NULL, page); 1426 1427 return nand_prog_page_end_op(chip); 1428 1429 pio_fallback: 1430 return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page); 1431 } 1432 1433 static int sunxi_nfc_hw_ecc_read_oob(struct mtd_info *mtd, 1434 struct nand_chip *chip, 1435 int page) 1436 { 1437 chip->pagebuf = -1; 1438 1439 return chip->ecc.read_page(mtd, chip, chip->data_buf, 1, page); 1440 } 1441 1442 static int sunxi_nfc_hw_ecc_write_oob(struct mtd_info *mtd, 1443 struct nand_chip *chip, 1444 int page) 1445 { 1446 int ret; 1447 1448 chip->pagebuf = -1; 1449 1450 memset(chip->data_buf, 0xff, mtd->writesize); 1451 ret = chip->ecc.write_page(mtd, chip, chip->data_buf, 1, page); 1452 if (ret) 1453 return ret; 1454 1455 /* Send command to program the OOB data */ 1456 return nand_prog_page_end_op(chip); 1457 } 1458 1459 static const s32 tWB_lut[] = {6, 12, 16, 20}; 1460 static const s32 tRHW_lut[] = {4, 8, 12, 20}; 1461 1462 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration, 1463 u32 clk_period) 1464 { 1465 u32 clk_cycles = DIV_ROUND_UP(duration, clk_period); 1466 int i; 1467 1468 for (i = 0; i < lut_size; i++) { 1469 if (clk_cycles <= lut[i]) 1470 return i; 1471 } 1472 1473 /* Doesn't fit */ 1474 return -EINVAL; 1475 } 1476 1477 #define sunxi_nand_lookup_timing(l, p, c) \ 1478 _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c) 1479 1480 static int sunxi_nfc_setup_data_interface(struct mtd_info *mtd, int csline, 1481 const struct nand_data_interface *conf) 1482 { 1483 struct nand_chip *nand = mtd_to_nand(mtd); 1484 struct sunxi_nand_chip *chip = to_sunxi_nand(nand); 1485 struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller); 1486 const struct nand_sdr_timings *timings; 1487 u32 min_clk_period = 0; 1488 s32 tWB, tADL, tWHR, tRHW, tCAD; 1489 long real_clk_rate; 1490 1491 timings = nand_get_sdr_timings(conf); 1492 if (IS_ERR(timings)) 1493 return -ENOTSUPP; 1494 1495 /* T1 <=> tCLS */ 1496 if (timings->tCLS_min > min_clk_period) 1497 min_clk_period = timings->tCLS_min; 1498 1499 /* T2 <=> tCLH */ 1500 if (timings->tCLH_min > min_clk_period) 1501 min_clk_period = timings->tCLH_min; 1502 1503 /* T3 <=> tCS */ 1504 if (timings->tCS_min > min_clk_period) 1505 min_clk_period = timings->tCS_min; 1506 1507 /* T4 <=> tCH */ 1508 if (timings->tCH_min > min_clk_period) 1509 min_clk_period = timings->tCH_min; 1510 1511 /* T5 <=> tWP */ 1512 if (timings->tWP_min > min_clk_period) 1513 min_clk_period = timings->tWP_min; 1514 1515 /* T6 <=> tWH */ 1516 if (timings->tWH_min > min_clk_period) 1517 min_clk_period = timings->tWH_min; 1518 1519 /* T7 <=> tALS */ 1520 if (timings->tALS_min > min_clk_period) 1521 min_clk_period = timings->tALS_min; 1522 1523 /* T8 <=> tDS */ 1524 if (timings->tDS_min > min_clk_period) 1525 min_clk_period = timings->tDS_min; 1526 1527 /* T9 <=> tDH */ 1528 if (timings->tDH_min > min_clk_period) 1529 min_clk_period = timings->tDH_min; 1530 1531 /* T10 <=> tRR */ 1532 if (timings->tRR_min > (min_clk_period * 3)) 1533 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3); 1534 1535 /* T11 <=> tALH */ 1536 if (timings->tALH_min > min_clk_period) 1537 min_clk_period = timings->tALH_min; 1538 1539 /* T12 <=> tRP */ 1540 if (timings->tRP_min > min_clk_period) 1541 min_clk_period = timings->tRP_min; 1542 1543 /* T13 <=> tREH */ 1544 if (timings->tREH_min > min_clk_period) 1545 min_clk_period = timings->tREH_min; 1546 1547 /* T14 <=> tRC */ 1548 if (timings->tRC_min > (min_clk_period * 2)) 1549 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2); 1550 1551 /* T15 <=> tWC */ 1552 if (timings->tWC_min > (min_clk_period * 2)) 1553 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2); 1554 1555 /* T16 - T19 + tCAD */ 1556 if (timings->tWB_max > (min_clk_period * 20)) 1557 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20); 1558 1559 if (timings->tADL_min > (min_clk_period * 32)) 1560 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32); 1561 1562 if (timings->tWHR_min > (min_clk_period * 32)) 1563 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32); 1564 1565 if (timings->tRHW_min > (min_clk_period * 20)) 1566 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20); 1567 1568 tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max, 1569 min_clk_period); 1570 if (tWB < 0) { 1571 dev_err(nfc->dev, "unsupported tWB\n"); 1572 return tWB; 1573 } 1574 1575 tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3; 1576 if (tADL > 3) { 1577 dev_err(nfc->dev, "unsupported tADL\n"); 1578 return -EINVAL; 1579 } 1580 1581 tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3; 1582 if (tWHR > 3) { 1583 dev_err(nfc->dev, "unsupported tWHR\n"); 1584 return -EINVAL; 1585 } 1586 1587 tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min, 1588 min_clk_period); 1589 if (tRHW < 0) { 1590 dev_err(nfc->dev, "unsupported tRHW\n"); 1591 return tRHW; 1592 } 1593 1594 if (csline == NAND_DATA_IFACE_CHECK_ONLY) 1595 return 0; 1596 1597 /* 1598 * TODO: according to ONFI specs this value only applies for DDR NAND, 1599 * but Allwinner seems to set this to 0x7. Mimic them for now. 1600 */ 1601 tCAD = 0x7; 1602 1603 /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */ 1604 chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD); 1605 1606 /* Convert min_clk_period from picoseconds to nanoseconds */ 1607 min_clk_period = DIV_ROUND_UP(min_clk_period, 1000); 1608 1609 /* 1610 * Unlike what is stated in Allwinner datasheet, the clk_rate should 1611 * be set to (1 / min_clk_period), and not (2 / min_clk_period). 1612 * This new formula was verified with a scope and validated by 1613 * Allwinner engineers. 1614 */ 1615 chip->clk_rate = NSEC_PER_SEC / min_clk_period; 1616 real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate); 1617 if (real_clk_rate <= 0) { 1618 dev_err(nfc->dev, "Unable to round clk %lu\n", chip->clk_rate); 1619 return -EINVAL; 1620 } 1621 1622 /* 1623 * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data 1624 * output cycle timings shall be used if the host drives tRC less than 1625 * 30 ns. 1626 */ 1627 min_clk_period = NSEC_PER_SEC / real_clk_rate; 1628 chip->timing_ctl = ((min_clk_period * 2) < 30) ? 1629 NFC_TIMING_CTL_EDO : 0; 1630 1631 return 0; 1632 } 1633 1634 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section, 1635 struct mtd_oob_region *oobregion) 1636 { 1637 struct nand_chip *nand = mtd_to_nand(mtd); 1638 struct nand_ecc_ctrl *ecc = &nand->ecc; 1639 1640 if (section >= ecc->steps) 1641 return -ERANGE; 1642 1643 oobregion->offset = section * (ecc->bytes + 4) + 4; 1644 oobregion->length = ecc->bytes; 1645 1646 return 0; 1647 } 1648 1649 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section, 1650 struct mtd_oob_region *oobregion) 1651 { 1652 struct nand_chip *nand = mtd_to_nand(mtd); 1653 struct nand_ecc_ctrl *ecc = &nand->ecc; 1654 1655 if (section > ecc->steps) 1656 return -ERANGE; 1657 1658 /* 1659 * The first 2 bytes are used for BB markers, hence we 1660 * only have 2 bytes available in the first user data 1661 * section. 1662 */ 1663 if (!section && ecc->mode == NAND_ECC_HW) { 1664 oobregion->offset = 2; 1665 oobregion->length = 2; 1666 1667 return 0; 1668 } 1669 1670 oobregion->offset = section * (ecc->bytes + 4); 1671 1672 if (section < ecc->steps) 1673 oobregion->length = 4; 1674 else 1675 oobregion->offset = mtd->oobsize - oobregion->offset; 1676 1677 return 0; 1678 } 1679 1680 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = { 1681 .ecc = sunxi_nand_ooblayout_ecc, 1682 .free = sunxi_nand_ooblayout_free, 1683 }; 1684 1685 static void sunxi_nand_hw_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc) 1686 { 1687 kfree(ecc->priv); 1688 } 1689 1690 static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd, 1691 struct nand_ecc_ctrl *ecc, 1692 struct device_node *np) 1693 { 1694 static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 }; 1695 struct nand_chip *nand = mtd_to_nand(mtd); 1696 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 1697 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 1698 struct sunxi_nand_hw_ecc *data; 1699 int nsectors; 1700 int ret; 1701 int i; 1702 1703 if (ecc->options & NAND_ECC_MAXIMIZE) { 1704 int bytes; 1705 1706 ecc->size = 1024; 1707 nsectors = mtd->writesize / ecc->size; 1708 1709 /* Reserve 2 bytes for the BBM */ 1710 bytes = (mtd->oobsize - 2) / nsectors; 1711 1712 /* 4 non-ECC bytes are added before each ECC bytes section */ 1713 bytes -= 4; 1714 1715 /* and bytes has to be even. */ 1716 if (bytes % 2) 1717 bytes--; 1718 1719 ecc->strength = bytes * 8 / fls(8 * ecc->size); 1720 1721 for (i = 0; i < ARRAY_SIZE(strengths); i++) { 1722 if (strengths[i] > ecc->strength) 1723 break; 1724 } 1725 1726 if (!i) 1727 ecc->strength = 0; 1728 else 1729 ecc->strength = strengths[i - 1]; 1730 } 1731 1732 if (ecc->size != 512 && ecc->size != 1024) 1733 return -EINVAL; 1734 1735 data = kzalloc(sizeof(*data), GFP_KERNEL); 1736 if (!data) 1737 return -ENOMEM; 1738 1739 /* Prefer 1k ECC chunk over 512 ones */ 1740 if (ecc->size == 512 && mtd->writesize > 512) { 1741 ecc->size = 1024; 1742 ecc->strength *= 2; 1743 } 1744 1745 /* Add ECC info retrieval from DT */ 1746 for (i = 0; i < ARRAY_SIZE(strengths); i++) { 1747 if (ecc->strength <= strengths[i]) { 1748 /* 1749 * Update ecc->strength value with the actual strength 1750 * that will be used by the ECC engine. 1751 */ 1752 ecc->strength = strengths[i]; 1753 break; 1754 } 1755 } 1756 1757 if (i >= ARRAY_SIZE(strengths)) { 1758 dev_err(nfc->dev, "unsupported strength\n"); 1759 ret = -ENOTSUPP; 1760 goto err; 1761 } 1762 1763 data->mode = i; 1764 1765 /* HW ECC always request ECC bytes for 1024 bytes blocks */ 1766 ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8); 1767 1768 /* HW ECC always work with even numbers of ECC bytes */ 1769 ecc->bytes = ALIGN(ecc->bytes, 2); 1770 1771 nsectors = mtd->writesize / ecc->size; 1772 1773 if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) { 1774 ret = -EINVAL; 1775 goto err; 1776 } 1777 1778 ecc->read_oob = sunxi_nfc_hw_ecc_read_oob; 1779 ecc->write_oob = sunxi_nfc_hw_ecc_write_oob; 1780 mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops); 1781 ecc->priv = data; 1782 1783 if (nfc->dmac) { 1784 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma; 1785 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma; 1786 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma; 1787 nand->options |= NAND_USE_BOUNCE_BUFFER; 1788 } else { 1789 ecc->read_page = sunxi_nfc_hw_ecc_read_page; 1790 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage; 1791 ecc->write_page = sunxi_nfc_hw_ecc_write_page; 1792 } 1793 1794 /* TODO: support DMA for raw accesses and subpage write */ 1795 ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage; 1796 ecc->read_oob_raw = nand_read_oob_std; 1797 ecc->write_oob_raw = nand_write_oob_std; 1798 1799 return 0; 1800 1801 err: 1802 kfree(data); 1803 1804 return ret; 1805 } 1806 1807 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc) 1808 { 1809 switch (ecc->mode) { 1810 case NAND_ECC_HW: 1811 sunxi_nand_hw_ecc_ctrl_cleanup(ecc); 1812 break; 1813 case NAND_ECC_NONE: 1814 default: 1815 break; 1816 } 1817 } 1818 1819 static int sunxi_nand_ecc_init(struct mtd_info *mtd, struct nand_ecc_ctrl *ecc, 1820 struct device_node *np) 1821 { 1822 struct nand_chip *nand = mtd_to_nand(mtd); 1823 int ret; 1824 1825 if (!ecc->size) { 1826 ecc->size = nand->ecc_step_ds; 1827 ecc->strength = nand->ecc_strength_ds; 1828 } 1829 1830 if (!ecc->size || !ecc->strength) 1831 return -EINVAL; 1832 1833 switch (ecc->mode) { 1834 case NAND_ECC_HW: 1835 ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np); 1836 if (ret) 1837 return ret; 1838 break; 1839 case NAND_ECC_NONE: 1840 case NAND_ECC_SOFT: 1841 break; 1842 default: 1843 return -EINVAL; 1844 } 1845 1846 return 0; 1847 } 1848 1849 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc, 1850 struct device_node *np) 1851 { 1852 struct sunxi_nand_chip *chip; 1853 struct mtd_info *mtd; 1854 struct nand_chip *nand; 1855 int nsels; 1856 int ret; 1857 int i; 1858 u32 tmp; 1859 1860 if (!of_get_property(np, "reg", &nsels)) 1861 return -EINVAL; 1862 1863 nsels /= sizeof(u32); 1864 if (!nsels) { 1865 dev_err(dev, "invalid reg property size\n"); 1866 return -EINVAL; 1867 } 1868 1869 chip = devm_kzalloc(dev, 1870 sizeof(*chip) + 1871 (nsels * sizeof(struct sunxi_nand_chip_sel)), 1872 GFP_KERNEL); 1873 if (!chip) { 1874 dev_err(dev, "could not allocate chip\n"); 1875 return -ENOMEM; 1876 } 1877 1878 chip->nsels = nsels; 1879 chip->selected = -1; 1880 1881 for (i = 0; i < nsels; i++) { 1882 ret = of_property_read_u32_index(np, "reg", i, &tmp); 1883 if (ret) { 1884 dev_err(dev, "could not retrieve reg property: %d\n", 1885 ret); 1886 return ret; 1887 } 1888 1889 if (tmp > NFC_MAX_CS) { 1890 dev_err(dev, 1891 "invalid reg value: %u (max CS = 7)\n", 1892 tmp); 1893 return -EINVAL; 1894 } 1895 1896 if (test_and_set_bit(tmp, &nfc->assigned_cs)) { 1897 dev_err(dev, "CS %d already assigned\n", tmp); 1898 return -EINVAL; 1899 } 1900 1901 chip->sels[i].cs = tmp; 1902 1903 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) && 1904 tmp < 2) 1905 chip->sels[i].rb = tmp; 1906 else 1907 chip->sels[i].rb = -1; 1908 } 1909 1910 nand = &chip->nand; 1911 /* Default tR value specified in the ONFI spec (chapter 4.15.1) */ 1912 nand->chip_delay = 200; 1913 nand->controller = &nfc->controller; 1914 /* 1915 * Set the ECC mode to the default value in case nothing is specified 1916 * in the DT. 1917 */ 1918 nand->ecc.mode = NAND_ECC_HW; 1919 nand_set_flash_node(nand, np); 1920 nand->select_chip = sunxi_nfc_select_chip; 1921 nand->cmd_ctrl = sunxi_nfc_cmd_ctrl; 1922 nand->read_buf = sunxi_nfc_read_buf; 1923 nand->write_buf = sunxi_nfc_write_buf; 1924 nand->read_byte = sunxi_nfc_read_byte; 1925 nand->setup_data_interface = sunxi_nfc_setup_data_interface; 1926 1927 mtd = nand_to_mtd(nand); 1928 mtd->dev.parent = dev; 1929 1930 ret = nand_scan_ident(mtd, nsels, NULL); 1931 if (ret) 1932 return ret; 1933 1934 if (nand->bbt_options & NAND_BBT_USE_FLASH) 1935 nand->bbt_options |= NAND_BBT_NO_OOB; 1936 1937 if (nand->options & NAND_NEED_SCRAMBLING) 1938 nand->options |= NAND_NO_SUBPAGE_WRITE; 1939 1940 nand->options |= NAND_SUBPAGE_READ; 1941 1942 ret = sunxi_nand_ecc_init(mtd, &nand->ecc, np); 1943 if (ret) { 1944 dev_err(dev, "ECC init failed: %d\n", ret); 1945 return ret; 1946 } 1947 1948 ret = nand_scan_tail(mtd); 1949 if (ret) { 1950 dev_err(dev, "nand_scan_tail failed: %d\n", ret); 1951 return ret; 1952 } 1953 1954 ret = mtd_device_register(mtd, NULL, 0); 1955 if (ret) { 1956 dev_err(dev, "failed to register mtd device: %d\n", ret); 1957 nand_release(mtd); 1958 return ret; 1959 } 1960 1961 list_add_tail(&chip->node, &nfc->chips); 1962 1963 return 0; 1964 } 1965 1966 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc) 1967 { 1968 struct device_node *np = dev->of_node; 1969 struct device_node *nand_np; 1970 int nchips = of_get_child_count(np); 1971 int ret; 1972 1973 if (nchips > 8) { 1974 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips); 1975 return -EINVAL; 1976 } 1977 1978 for_each_child_of_node(np, nand_np) { 1979 ret = sunxi_nand_chip_init(dev, nfc, nand_np); 1980 if (ret) { 1981 of_node_put(nand_np); 1982 return ret; 1983 } 1984 } 1985 1986 return 0; 1987 } 1988 1989 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc) 1990 { 1991 struct sunxi_nand_chip *chip; 1992 1993 while (!list_empty(&nfc->chips)) { 1994 chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip, 1995 node); 1996 nand_release(nand_to_mtd(&chip->nand)); 1997 sunxi_nand_ecc_cleanup(&chip->nand.ecc); 1998 list_del(&chip->node); 1999 } 2000 } 2001 2002 static int sunxi_nfc_probe(struct platform_device *pdev) 2003 { 2004 struct device *dev = &pdev->dev; 2005 struct resource *r; 2006 struct sunxi_nfc *nfc; 2007 int irq; 2008 int ret; 2009 2010 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL); 2011 if (!nfc) 2012 return -ENOMEM; 2013 2014 nfc->dev = dev; 2015 nand_hw_control_init(&nfc->controller); 2016 INIT_LIST_HEAD(&nfc->chips); 2017 2018 r = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2019 nfc->regs = devm_ioremap_resource(dev, r); 2020 if (IS_ERR(nfc->regs)) 2021 return PTR_ERR(nfc->regs); 2022 2023 irq = platform_get_irq(pdev, 0); 2024 if (irq < 0) { 2025 dev_err(dev, "failed to retrieve irq\n"); 2026 return irq; 2027 } 2028 2029 nfc->ahb_clk = devm_clk_get(dev, "ahb"); 2030 if (IS_ERR(nfc->ahb_clk)) { 2031 dev_err(dev, "failed to retrieve ahb clk\n"); 2032 return PTR_ERR(nfc->ahb_clk); 2033 } 2034 2035 ret = clk_prepare_enable(nfc->ahb_clk); 2036 if (ret) 2037 return ret; 2038 2039 nfc->mod_clk = devm_clk_get(dev, "mod"); 2040 if (IS_ERR(nfc->mod_clk)) { 2041 dev_err(dev, "failed to retrieve mod clk\n"); 2042 ret = PTR_ERR(nfc->mod_clk); 2043 goto out_ahb_clk_unprepare; 2044 } 2045 2046 ret = clk_prepare_enable(nfc->mod_clk); 2047 if (ret) 2048 goto out_ahb_clk_unprepare; 2049 2050 nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb"); 2051 if (IS_ERR(nfc->reset)) { 2052 ret = PTR_ERR(nfc->reset); 2053 goto out_mod_clk_unprepare; 2054 } 2055 2056 ret = reset_control_deassert(nfc->reset); 2057 if (ret) { 2058 dev_err(dev, "reset err %d\n", ret); 2059 goto out_mod_clk_unprepare; 2060 } 2061 2062 ret = sunxi_nfc_rst(nfc); 2063 if (ret) 2064 goto out_ahb_reset_reassert; 2065 2066 writel(0, nfc->regs + NFC_REG_INT); 2067 ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt, 2068 0, "sunxi-nand", nfc); 2069 if (ret) 2070 goto out_ahb_reset_reassert; 2071 2072 nfc->dmac = dma_request_slave_channel(dev, "rxtx"); 2073 if (nfc->dmac) { 2074 struct dma_slave_config dmac_cfg = { }; 2075 2076 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA; 2077 dmac_cfg.dst_addr = dmac_cfg.src_addr; 2078 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 2079 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width; 2080 dmac_cfg.src_maxburst = 4; 2081 dmac_cfg.dst_maxburst = 4; 2082 dmaengine_slave_config(nfc->dmac, &dmac_cfg); 2083 } else { 2084 dev_warn(dev, "failed to request rxtx DMA channel\n"); 2085 } 2086 2087 platform_set_drvdata(pdev, nfc); 2088 2089 ret = sunxi_nand_chips_init(dev, nfc); 2090 if (ret) { 2091 dev_err(dev, "failed to init nand chips\n"); 2092 goto out_release_dmac; 2093 } 2094 2095 return 0; 2096 2097 out_release_dmac: 2098 if (nfc->dmac) 2099 dma_release_channel(nfc->dmac); 2100 out_ahb_reset_reassert: 2101 reset_control_assert(nfc->reset); 2102 out_mod_clk_unprepare: 2103 clk_disable_unprepare(nfc->mod_clk); 2104 out_ahb_clk_unprepare: 2105 clk_disable_unprepare(nfc->ahb_clk); 2106 2107 return ret; 2108 } 2109 2110 static int sunxi_nfc_remove(struct platform_device *pdev) 2111 { 2112 struct sunxi_nfc *nfc = platform_get_drvdata(pdev); 2113 2114 sunxi_nand_chips_cleanup(nfc); 2115 2116 reset_control_assert(nfc->reset); 2117 2118 if (nfc->dmac) 2119 dma_release_channel(nfc->dmac); 2120 clk_disable_unprepare(nfc->mod_clk); 2121 clk_disable_unprepare(nfc->ahb_clk); 2122 2123 return 0; 2124 } 2125 2126 static const struct of_device_id sunxi_nfc_ids[] = { 2127 { .compatible = "allwinner,sun4i-a10-nand" }, 2128 { /* sentinel */ } 2129 }; 2130 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids); 2131 2132 static struct platform_driver sunxi_nfc_driver = { 2133 .driver = { 2134 .name = "sunxi_nand", 2135 .of_match_table = sunxi_nfc_ids, 2136 }, 2137 .probe = sunxi_nfc_probe, 2138 .remove = sunxi_nfc_remove, 2139 }; 2140 module_platform_driver(sunxi_nfc_driver); 2141 2142 MODULE_LICENSE("GPL v2"); 2143 MODULE_AUTHOR("Boris BREZILLON"); 2144 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver"); 2145 MODULE_ALIAS("platform:sunxi_nand"); 2146