1 /* 2 * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com> 3 * 4 * Derived from: 5 * https://github.com/yuq/sunxi-nfc-mtd 6 * Copyright (C) 2013 Qiang Yu <yuq825@gmail.com> 7 * 8 * https://github.com/hno/Allwinner-Info 9 * Copyright (C) 2013 Henrik Nordström <Henrik Nordström> 10 * 11 * Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com> 12 * Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org> 13 * 14 * This program is free software; you can redistribute it and/or modify 15 * it under the terms of the GNU General Public License as published by 16 * the Free Software Foundation; either version 2 of the License, or 17 * (at your option) any later version. 18 * 19 * This program is distributed in the hope that it will be useful, 20 * but WITHOUT ANY WARRANTY; without even the implied warranty of 21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 * GNU General Public License for more details. 23 */ 24 25 #include <linux/dma-mapping.h> 26 #include <linux/slab.h> 27 #include <linux/module.h> 28 #include <linux/moduleparam.h> 29 #include <linux/platform_device.h> 30 #include <linux/of.h> 31 #include <linux/of_device.h> 32 #include <linux/mtd/mtd.h> 33 #include <linux/mtd/rawnand.h> 34 #include <linux/mtd/partitions.h> 35 #include <linux/clk.h> 36 #include <linux/delay.h> 37 #include <linux/dmaengine.h> 38 #include <linux/interrupt.h> 39 #include <linux/iopoll.h> 40 #include <linux/reset.h> 41 42 #define NFC_REG_CTL 0x0000 43 #define NFC_REG_ST 0x0004 44 #define NFC_REG_INT 0x0008 45 #define NFC_REG_TIMING_CTL 0x000C 46 #define NFC_REG_TIMING_CFG 0x0010 47 #define NFC_REG_ADDR_LOW 0x0014 48 #define NFC_REG_ADDR_HIGH 0x0018 49 #define NFC_REG_SECTOR_NUM 0x001C 50 #define NFC_REG_CNT 0x0020 51 #define NFC_REG_CMD 0x0024 52 #define NFC_REG_RCMD_SET 0x0028 53 #define NFC_REG_WCMD_SET 0x002C 54 #define NFC_REG_IO_DATA 0x0030 55 #define NFC_REG_ECC_CTL 0x0034 56 #define NFC_REG_ECC_ST 0x0038 57 #define NFC_REG_DEBUG 0x003C 58 #define NFC_REG_ECC_ERR_CNT(x) ((0x0040 + (x)) & ~0x3) 59 #define NFC_REG_USER_DATA(x) (0x0050 + ((x) * 4)) 60 #define NFC_REG_SPARE_AREA 0x00A0 61 #define NFC_REG_PAT_ID 0x00A4 62 #define NFC_RAM0_BASE 0x0400 63 #define NFC_RAM1_BASE 0x0800 64 65 /* define bit use in NFC_CTL */ 66 #define NFC_EN BIT(0) 67 #define NFC_RESET BIT(1) 68 #define NFC_BUS_WIDTH_MSK BIT(2) 69 #define NFC_BUS_WIDTH_8 (0 << 2) 70 #define NFC_BUS_WIDTH_16 (1 << 2) 71 #define NFC_RB_SEL_MSK BIT(3) 72 #define NFC_RB_SEL(x) ((x) << 3) 73 #define NFC_CE_SEL_MSK GENMASK(26, 24) 74 #define NFC_CE_SEL(x) ((x) << 24) 75 #define NFC_CE_CTL BIT(6) 76 #define NFC_PAGE_SHIFT_MSK GENMASK(11, 8) 77 #define NFC_PAGE_SHIFT(x) (((x) < 10 ? 0 : (x) - 10) << 8) 78 #define NFC_SAM BIT(12) 79 #define NFC_RAM_METHOD BIT(14) 80 #define NFC_DEBUG_CTL BIT(31) 81 82 /* define bit use in NFC_ST */ 83 #define NFC_RB_B2R BIT(0) 84 #define NFC_CMD_INT_FLAG BIT(1) 85 #define NFC_DMA_INT_FLAG BIT(2) 86 #define NFC_CMD_FIFO_STATUS BIT(3) 87 #define NFC_STA BIT(4) 88 #define NFC_NATCH_INT_FLAG BIT(5) 89 #define NFC_RB_STATE(x) BIT(x + 8) 90 91 /* define bit use in NFC_INT */ 92 #define NFC_B2R_INT_ENABLE BIT(0) 93 #define NFC_CMD_INT_ENABLE BIT(1) 94 #define NFC_DMA_INT_ENABLE BIT(2) 95 #define NFC_INT_MASK (NFC_B2R_INT_ENABLE | \ 96 NFC_CMD_INT_ENABLE | \ 97 NFC_DMA_INT_ENABLE) 98 99 /* define bit use in NFC_TIMING_CTL */ 100 #define NFC_TIMING_CTL_EDO BIT(8) 101 102 /* define NFC_TIMING_CFG register layout */ 103 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD) \ 104 (((tWB) & 0x3) | (((tADL) & 0x3) << 2) | \ 105 (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) | \ 106 (((tCAD) & 0x7) << 8)) 107 108 /* define bit use in NFC_CMD */ 109 #define NFC_CMD_LOW_BYTE_MSK GENMASK(7, 0) 110 #define NFC_CMD_HIGH_BYTE_MSK GENMASK(15, 8) 111 #define NFC_CMD(x) (x) 112 #define NFC_ADR_NUM_MSK GENMASK(18, 16) 113 #define NFC_ADR_NUM(x) (((x) - 1) << 16) 114 #define NFC_SEND_ADR BIT(19) 115 #define NFC_ACCESS_DIR BIT(20) 116 #define NFC_DATA_TRANS BIT(21) 117 #define NFC_SEND_CMD1 BIT(22) 118 #define NFC_WAIT_FLAG BIT(23) 119 #define NFC_SEND_CMD2 BIT(24) 120 #define NFC_SEQ BIT(25) 121 #define NFC_DATA_SWAP_METHOD BIT(26) 122 #define NFC_ROW_AUTO_INC BIT(27) 123 #define NFC_SEND_CMD3 BIT(28) 124 #define NFC_SEND_CMD4 BIT(29) 125 #define NFC_CMD_TYPE_MSK GENMASK(31, 30) 126 #define NFC_NORMAL_OP (0 << 30) 127 #define NFC_ECC_OP (1 << 30) 128 #define NFC_PAGE_OP (2U << 30) 129 130 /* define bit use in NFC_RCMD_SET */ 131 #define NFC_READ_CMD_MSK GENMASK(7, 0) 132 #define NFC_RND_READ_CMD0_MSK GENMASK(15, 8) 133 #define NFC_RND_READ_CMD1_MSK GENMASK(23, 16) 134 135 /* define bit use in NFC_WCMD_SET */ 136 #define NFC_PROGRAM_CMD_MSK GENMASK(7, 0) 137 #define NFC_RND_WRITE_CMD_MSK GENMASK(15, 8) 138 #define NFC_READ_CMD0_MSK GENMASK(23, 16) 139 #define NFC_READ_CMD1_MSK GENMASK(31, 24) 140 141 /* define bit use in NFC_ECC_CTL */ 142 #define NFC_ECC_EN BIT(0) 143 #define NFC_ECC_PIPELINE BIT(3) 144 #define NFC_ECC_EXCEPTION BIT(4) 145 #define NFC_ECC_BLOCK_SIZE_MSK BIT(5) 146 #define NFC_ECC_BLOCK_512 BIT(5) 147 #define NFC_RANDOM_EN BIT(9) 148 #define NFC_RANDOM_DIRECTION BIT(10) 149 #define NFC_ECC_MODE_MSK GENMASK(15, 12) 150 #define NFC_ECC_MODE(x) ((x) << 12) 151 #define NFC_RANDOM_SEED_MSK GENMASK(30, 16) 152 #define NFC_RANDOM_SEED(x) ((x) << 16) 153 154 /* define bit use in NFC_ECC_ST */ 155 #define NFC_ECC_ERR(x) BIT(x) 156 #define NFC_ECC_ERR_MSK GENMASK(15, 0) 157 #define NFC_ECC_PAT_FOUND(x) BIT(x + 16) 158 #define NFC_ECC_ERR_CNT(b, x) (((x) >> (((b) % 4) * 8)) & 0xff) 159 160 #define NFC_DEFAULT_TIMEOUT_MS 1000 161 162 #define NFC_SRAM_SIZE 1024 163 164 #define NFC_MAX_CS 7 165 166 /* 167 * Chip Select structure: stores information related to NAND Chip Select 168 * 169 * @cs: the NAND CS id used to communicate with a NAND Chip 170 * @rb: the Ready/Busy pin ID. -1 means no R/B pin connected to the 171 * NFC 172 */ 173 struct sunxi_nand_chip_sel { 174 u8 cs; 175 s8 rb; 176 }; 177 178 /* 179 * sunxi HW ECC infos: stores information related to HW ECC support 180 * 181 * @mode: the sunxi ECC mode field deduced from ECC requirements 182 */ 183 struct sunxi_nand_hw_ecc { 184 int mode; 185 }; 186 187 /* 188 * NAND chip structure: stores NAND chip device related information 189 * 190 * @node: used to store NAND chips into a list 191 * @nand: base NAND chip structure 192 * @mtd: base MTD structure 193 * @clk_rate: clk_rate required for this NAND chip 194 * @timing_cfg TIMING_CFG register value for this NAND chip 195 * @selected: current active CS 196 * @nsels: number of CS lines required by the NAND chip 197 * @sels: array of CS lines descriptions 198 */ 199 struct sunxi_nand_chip { 200 struct list_head node; 201 struct nand_chip nand; 202 unsigned long clk_rate; 203 u32 timing_cfg; 204 u32 timing_ctl; 205 int selected; 206 int addr_cycles; 207 u32 addr[2]; 208 int cmd_cycles; 209 u8 cmd[2]; 210 int nsels; 211 struct sunxi_nand_chip_sel sels[0]; 212 }; 213 214 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand) 215 { 216 return container_of(nand, struct sunxi_nand_chip, nand); 217 } 218 219 /* 220 * NAND Controller structure: stores sunxi NAND controller information 221 * 222 * @controller: base controller structure 223 * @dev: parent device (used to print error messages) 224 * @regs: NAND controller registers 225 * @ahb_clk: NAND Controller AHB clock 226 * @mod_clk: NAND Controller mod clock 227 * @assigned_cs: bitmask describing already assigned CS lines 228 * @clk_rate: NAND controller current clock rate 229 * @chips: a list containing all the NAND chips attached to 230 * this NAND controller 231 * @complete: a completion object used to wait for NAND 232 * controller events 233 */ 234 struct sunxi_nfc { 235 struct nand_controller controller; 236 struct device *dev; 237 void __iomem *regs; 238 struct clk *ahb_clk; 239 struct clk *mod_clk; 240 struct reset_control *reset; 241 unsigned long assigned_cs; 242 unsigned long clk_rate; 243 struct list_head chips; 244 struct completion complete; 245 struct dma_chan *dmac; 246 }; 247 248 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_controller *ctrl) 249 { 250 return container_of(ctrl, struct sunxi_nfc, controller); 251 } 252 253 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id) 254 { 255 struct sunxi_nfc *nfc = dev_id; 256 u32 st = readl(nfc->regs + NFC_REG_ST); 257 u32 ien = readl(nfc->regs + NFC_REG_INT); 258 259 if (!(ien & st)) 260 return IRQ_NONE; 261 262 if ((ien & st) == ien) 263 complete(&nfc->complete); 264 265 writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST); 266 writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT); 267 268 return IRQ_HANDLED; 269 } 270 271 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events, 272 bool use_polling, unsigned int timeout_ms) 273 { 274 int ret; 275 276 if (events & ~NFC_INT_MASK) 277 return -EINVAL; 278 279 if (!timeout_ms) 280 timeout_ms = NFC_DEFAULT_TIMEOUT_MS; 281 282 if (!use_polling) { 283 init_completion(&nfc->complete); 284 285 writel(events, nfc->regs + NFC_REG_INT); 286 287 ret = wait_for_completion_timeout(&nfc->complete, 288 msecs_to_jiffies(timeout_ms)); 289 if (!ret) 290 ret = -ETIMEDOUT; 291 else 292 ret = 0; 293 294 writel(0, nfc->regs + NFC_REG_INT); 295 } else { 296 u32 status; 297 298 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status, 299 (status & events) == events, 1, 300 timeout_ms * 1000); 301 } 302 303 writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST); 304 305 if (ret) 306 dev_err(nfc->dev, "wait interrupt timedout\n"); 307 308 return ret; 309 } 310 311 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc) 312 { 313 u32 status; 314 int ret; 315 316 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status, 317 !(status & NFC_CMD_FIFO_STATUS), 1, 318 NFC_DEFAULT_TIMEOUT_MS * 1000); 319 if (ret) 320 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n"); 321 322 return ret; 323 } 324 325 static int sunxi_nfc_rst(struct sunxi_nfc *nfc) 326 { 327 u32 ctl; 328 int ret; 329 330 writel(0, nfc->regs + NFC_REG_ECC_CTL); 331 writel(NFC_RESET, nfc->regs + NFC_REG_CTL); 332 333 ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl, 334 !(ctl & NFC_RESET), 1, 335 NFC_DEFAULT_TIMEOUT_MS * 1000); 336 if (ret) 337 dev_err(nfc->dev, "wait for NAND controller reset timedout\n"); 338 339 return ret; 340 } 341 342 static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf, 343 int chunksize, int nchunks, 344 enum dma_data_direction ddir, 345 struct scatterlist *sg) 346 { 347 struct nand_chip *nand = mtd_to_nand(mtd); 348 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 349 struct dma_async_tx_descriptor *dmad; 350 enum dma_transfer_direction tdir; 351 dma_cookie_t dmat; 352 int ret; 353 354 if (ddir == DMA_FROM_DEVICE) 355 tdir = DMA_DEV_TO_MEM; 356 else 357 tdir = DMA_MEM_TO_DEV; 358 359 sg_init_one(sg, buf, nchunks * chunksize); 360 ret = dma_map_sg(nfc->dev, sg, 1, ddir); 361 if (!ret) 362 return -ENOMEM; 363 364 dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK); 365 if (!dmad) { 366 ret = -EINVAL; 367 goto err_unmap_buf; 368 } 369 370 writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD, 371 nfc->regs + NFC_REG_CTL); 372 writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM); 373 writel(chunksize, nfc->regs + NFC_REG_CNT); 374 dmat = dmaengine_submit(dmad); 375 376 ret = dma_submit_error(dmat); 377 if (ret) 378 goto err_clr_dma_flag; 379 380 return 0; 381 382 err_clr_dma_flag: 383 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD, 384 nfc->regs + NFC_REG_CTL); 385 386 err_unmap_buf: 387 dma_unmap_sg(nfc->dev, sg, 1, ddir); 388 return ret; 389 } 390 391 static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd, 392 enum dma_data_direction ddir, 393 struct scatterlist *sg) 394 { 395 struct nand_chip *nand = mtd_to_nand(mtd); 396 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 397 398 dma_unmap_sg(nfc->dev, sg, 1, ddir); 399 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD, 400 nfc->regs + NFC_REG_CTL); 401 } 402 403 static int sunxi_nfc_dev_ready(struct mtd_info *mtd) 404 { 405 struct nand_chip *nand = mtd_to_nand(mtd); 406 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 407 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 408 u32 mask; 409 410 if (sunxi_nand->selected < 0) 411 return 0; 412 413 if (sunxi_nand->sels[sunxi_nand->selected].rb < 0) { 414 dev_err(nfc->dev, "cannot check R/B NAND status!\n"); 415 return 0; 416 } 417 418 mask = NFC_RB_STATE(sunxi_nand->sels[sunxi_nand->selected].rb); 419 420 return !!(readl(nfc->regs + NFC_REG_ST) & mask); 421 } 422 423 static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip) 424 { 425 struct nand_chip *nand = mtd_to_nand(mtd); 426 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 427 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 428 struct sunxi_nand_chip_sel *sel; 429 u32 ctl; 430 431 if (chip > 0 && chip >= sunxi_nand->nsels) 432 return; 433 434 if (chip == sunxi_nand->selected) 435 return; 436 437 ctl = readl(nfc->regs + NFC_REG_CTL) & 438 ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN); 439 440 if (chip >= 0) { 441 sel = &sunxi_nand->sels[chip]; 442 443 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN | 444 NFC_PAGE_SHIFT(nand->page_shift); 445 if (sel->rb < 0) { 446 nand->dev_ready = NULL; 447 } else { 448 nand->dev_ready = sunxi_nfc_dev_ready; 449 ctl |= NFC_RB_SEL(sel->rb); 450 } 451 452 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA); 453 454 if (nfc->clk_rate != sunxi_nand->clk_rate) { 455 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate); 456 nfc->clk_rate = sunxi_nand->clk_rate; 457 } 458 } 459 460 writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL); 461 writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG); 462 writel(ctl, nfc->regs + NFC_REG_CTL); 463 464 sunxi_nand->selected = chip; 465 } 466 467 static void sunxi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) 468 { 469 struct nand_chip *nand = mtd_to_nand(mtd); 470 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 471 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 472 int ret; 473 int cnt; 474 int offs = 0; 475 u32 tmp; 476 477 while (len > offs) { 478 bool poll = false; 479 480 cnt = min(len - offs, NFC_SRAM_SIZE); 481 482 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 483 if (ret) 484 break; 485 486 writel(cnt, nfc->regs + NFC_REG_CNT); 487 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD; 488 writel(tmp, nfc->regs + NFC_REG_CMD); 489 490 /* Arbitrary limit for polling mode */ 491 if (cnt < 64) 492 poll = true; 493 494 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0); 495 if (ret) 496 break; 497 498 if (buf) 499 memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE, 500 cnt); 501 offs += cnt; 502 } 503 } 504 505 static void sunxi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf, 506 int len) 507 { 508 struct nand_chip *nand = mtd_to_nand(mtd); 509 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 510 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 511 int ret; 512 int cnt; 513 int offs = 0; 514 u32 tmp; 515 516 while (len > offs) { 517 bool poll = false; 518 519 cnt = min(len - offs, NFC_SRAM_SIZE); 520 521 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 522 if (ret) 523 break; 524 525 writel(cnt, nfc->regs + NFC_REG_CNT); 526 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt); 527 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | 528 NFC_ACCESS_DIR; 529 writel(tmp, nfc->regs + NFC_REG_CMD); 530 531 /* Arbitrary limit for polling mode */ 532 if (cnt < 64) 533 poll = true; 534 535 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0); 536 if (ret) 537 break; 538 539 offs += cnt; 540 } 541 } 542 543 static uint8_t sunxi_nfc_read_byte(struct mtd_info *mtd) 544 { 545 uint8_t ret = 0; 546 547 sunxi_nfc_read_buf(mtd, &ret, 1); 548 549 return ret; 550 } 551 552 static void sunxi_nfc_cmd_ctrl(struct mtd_info *mtd, int dat, 553 unsigned int ctrl) 554 { 555 struct nand_chip *nand = mtd_to_nand(mtd); 556 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 557 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 558 int ret; 559 560 if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) && 561 !(ctrl & (NAND_CLE | NAND_ALE))) { 562 u32 cmd = 0; 563 564 if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles) 565 return; 566 567 if (sunxi_nand->cmd_cycles--) 568 cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0]; 569 570 if (sunxi_nand->cmd_cycles--) { 571 cmd |= NFC_SEND_CMD2; 572 writel(sunxi_nand->cmd[1], 573 nfc->regs + NFC_REG_RCMD_SET); 574 } 575 576 sunxi_nand->cmd_cycles = 0; 577 578 if (sunxi_nand->addr_cycles) { 579 cmd |= NFC_SEND_ADR | 580 NFC_ADR_NUM(sunxi_nand->addr_cycles); 581 writel(sunxi_nand->addr[0], 582 nfc->regs + NFC_REG_ADDR_LOW); 583 } 584 585 if (sunxi_nand->addr_cycles > 4) 586 writel(sunxi_nand->addr[1], 587 nfc->regs + NFC_REG_ADDR_HIGH); 588 589 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 590 if (ret) 591 return; 592 593 writel(cmd, nfc->regs + NFC_REG_CMD); 594 sunxi_nand->addr[0] = 0; 595 sunxi_nand->addr[1] = 0; 596 sunxi_nand->addr_cycles = 0; 597 sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0); 598 } 599 600 if (ctrl & NAND_CLE) { 601 sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat; 602 } else if (ctrl & NAND_ALE) { 603 sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |= 604 dat << ((sunxi_nand->addr_cycles % 4) * 8); 605 sunxi_nand->addr_cycles++; 606 } 607 } 608 609 /* These seed values have been extracted from Allwinner's BSP */ 610 static const u16 sunxi_nfc_randomizer_page_seeds[] = { 611 0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72, 612 0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436, 613 0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d, 614 0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130, 615 0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56, 616 0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55, 617 0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb, 618 0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17, 619 0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62, 620 0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064, 621 0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126, 622 0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e, 623 0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3, 624 0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b, 625 0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d, 626 0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db, 627 }; 628 629 /* 630 * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds 631 * have been generated using 632 * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what 633 * the randomizer engine does internally before de/scrambling OOB data. 634 * 635 * Those tables are statically defined to avoid calculating randomizer state 636 * at runtime. 637 */ 638 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = { 639 0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64, 640 0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409, 641 0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617, 642 0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d, 643 0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91, 644 0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d, 645 0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab, 646 0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8, 647 0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8, 648 0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b, 649 0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5, 650 0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a, 651 0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891, 652 0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36, 653 0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd, 654 0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0, 655 }; 656 657 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = { 658 0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6, 659 0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982, 660 0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9, 661 0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07, 662 0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e, 663 0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2, 664 0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c, 665 0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f, 666 0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc, 667 0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e, 668 0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8, 669 0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68, 670 0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d, 671 0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179, 672 0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601, 673 0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd, 674 }; 675 676 static u16 sunxi_nfc_randomizer_step(u16 state, int count) 677 { 678 state &= 0x7fff; 679 680 /* 681 * This loop is just a simple implementation of a Fibonacci LFSR using 682 * the x16 + x15 + 1 polynomial. 683 */ 684 while (count--) 685 state = ((state >> 1) | 686 (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff; 687 688 return state; 689 } 690 691 static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc) 692 { 693 const u16 *seeds = sunxi_nfc_randomizer_page_seeds; 694 int mod = mtd_div_by_ws(mtd->erasesize, mtd); 695 696 if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds)) 697 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds); 698 699 if (ecc) { 700 if (mtd->ecc_step_size == 512) 701 seeds = sunxi_nfc_randomizer_ecc512_seeds; 702 else 703 seeds = sunxi_nfc_randomizer_ecc1024_seeds; 704 } 705 706 return seeds[page % mod]; 707 } 708 709 static void sunxi_nfc_randomizer_config(struct mtd_info *mtd, 710 int page, bool ecc) 711 { 712 struct nand_chip *nand = mtd_to_nand(mtd); 713 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 714 u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); 715 u16 state; 716 717 if (!(nand->options & NAND_NEED_SCRAMBLING)) 718 return; 719 720 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); 721 state = sunxi_nfc_randomizer_state(mtd, page, ecc); 722 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK; 723 writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL); 724 } 725 726 static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd) 727 { 728 struct nand_chip *nand = mtd_to_nand(mtd); 729 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 730 731 if (!(nand->options & NAND_NEED_SCRAMBLING)) 732 return; 733 734 writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN, 735 nfc->regs + NFC_REG_ECC_CTL); 736 } 737 738 static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd) 739 { 740 struct nand_chip *nand = mtd_to_nand(mtd); 741 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 742 743 if (!(nand->options & NAND_NEED_SCRAMBLING)) 744 return; 745 746 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN, 747 nfc->regs + NFC_REG_ECC_CTL); 748 } 749 750 static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm) 751 { 752 u16 state = sunxi_nfc_randomizer_state(mtd, page, true); 753 754 bbm[0] ^= state; 755 bbm[1] ^= sunxi_nfc_randomizer_step(state, 8); 756 } 757 758 static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd, 759 const uint8_t *buf, int len, 760 bool ecc, int page) 761 { 762 sunxi_nfc_randomizer_config(mtd, page, ecc); 763 sunxi_nfc_randomizer_enable(mtd); 764 sunxi_nfc_write_buf(mtd, buf, len); 765 sunxi_nfc_randomizer_disable(mtd); 766 } 767 768 static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf, 769 int len, bool ecc, int page) 770 { 771 sunxi_nfc_randomizer_config(mtd, page, ecc); 772 sunxi_nfc_randomizer_enable(mtd); 773 sunxi_nfc_read_buf(mtd, buf, len); 774 sunxi_nfc_randomizer_disable(mtd); 775 } 776 777 static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd) 778 { 779 struct nand_chip *nand = mtd_to_nand(mtd); 780 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 781 struct sunxi_nand_hw_ecc *data = nand->ecc.priv; 782 u32 ecc_ctl; 783 784 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); 785 ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE | 786 NFC_ECC_BLOCK_SIZE_MSK); 787 ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION | 788 NFC_ECC_PIPELINE; 789 790 if (nand->ecc.size == 512) 791 ecc_ctl |= NFC_ECC_BLOCK_512; 792 793 writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL); 794 } 795 796 static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd) 797 { 798 struct nand_chip *nand = mtd_to_nand(mtd); 799 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 800 801 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN, 802 nfc->regs + NFC_REG_ECC_CTL); 803 } 804 805 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf) 806 { 807 buf[0] = user_data; 808 buf[1] = user_data >> 8; 809 buf[2] = user_data >> 16; 810 buf[3] = user_data >> 24; 811 } 812 813 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf) 814 { 815 return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24); 816 } 817 818 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob, 819 int step, bool bbm, int page) 820 { 821 struct nand_chip *nand = mtd_to_nand(mtd); 822 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 823 824 sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)), 825 oob); 826 827 /* De-randomize the Bad Block Marker. */ 828 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) 829 sunxi_nfc_randomize_bbm(mtd, page, oob); 830 } 831 832 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd, 833 const u8 *oob, int step, 834 bool bbm, int page) 835 { 836 struct nand_chip *nand = mtd_to_nand(mtd); 837 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 838 u8 user_data[4]; 839 840 /* Randomize the Bad Block Marker. */ 841 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) { 842 memcpy(user_data, oob, sizeof(user_data)); 843 sunxi_nfc_randomize_bbm(mtd, page, user_data); 844 oob = user_data; 845 } 846 847 writel(sunxi_nfc_buf_to_user_data(oob), 848 nfc->regs + NFC_REG_USER_DATA(step)); 849 } 850 851 static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd, 852 unsigned int *max_bitflips, int ret) 853 { 854 if (ret < 0) { 855 mtd->ecc_stats.failed++; 856 } else { 857 mtd->ecc_stats.corrected += ret; 858 *max_bitflips = max_t(unsigned int, *max_bitflips, ret); 859 } 860 } 861 862 static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob, 863 int step, u32 status, bool *erased) 864 { 865 struct nand_chip *nand = mtd_to_nand(mtd); 866 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 867 struct nand_ecc_ctrl *ecc = &nand->ecc; 868 u32 tmp; 869 870 *erased = false; 871 872 if (status & NFC_ECC_ERR(step)) 873 return -EBADMSG; 874 875 if (status & NFC_ECC_PAT_FOUND(step)) { 876 u8 pattern; 877 878 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) { 879 pattern = 0x0; 880 } else { 881 pattern = 0xff; 882 *erased = true; 883 } 884 885 if (data) 886 memset(data, pattern, ecc->size); 887 888 if (oob) 889 memset(oob, pattern, ecc->bytes + 4); 890 891 return 0; 892 } 893 894 tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step)); 895 896 return NFC_ECC_ERR_CNT(step, tmp); 897 } 898 899 static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd, 900 u8 *data, int data_off, 901 u8 *oob, int oob_off, 902 int *cur_off, 903 unsigned int *max_bitflips, 904 bool bbm, bool oob_required, int page) 905 { 906 struct nand_chip *nand = mtd_to_nand(mtd); 907 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 908 struct nand_ecc_ctrl *ecc = &nand->ecc; 909 int raw_mode = 0; 910 bool erased; 911 int ret; 912 913 if (*cur_off != data_off) 914 nand_change_read_column_op(nand, data_off, NULL, 0, false); 915 916 sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page); 917 918 if (data_off + ecc->size != oob_off) 919 nand_change_read_column_op(nand, oob_off, NULL, 0, false); 920 921 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 922 if (ret) 923 return ret; 924 925 sunxi_nfc_randomizer_enable(mtd); 926 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP, 927 nfc->regs + NFC_REG_CMD); 928 929 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 930 sunxi_nfc_randomizer_disable(mtd); 931 if (ret) 932 return ret; 933 934 *cur_off = oob_off + ecc->bytes + 4; 935 936 ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0, 937 readl(nfc->regs + NFC_REG_ECC_ST), 938 &erased); 939 if (erased) 940 return 1; 941 942 if (ret < 0) { 943 /* 944 * Re-read the data with the randomizer disabled to identify 945 * bitflips in erased pages. 946 */ 947 if (nand->options & NAND_NEED_SCRAMBLING) 948 nand_change_read_column_op(nand, data_off, data, 949 ecc->size, false); 950 else 951 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, 952 ecc->size); 953 954 nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4, 955 false); 956 957 ret = nand_check_erased_ecc_chunk(data, ecc->size, 958 oob, ecc->bytes + 4, 959 NULL, 0, ecc->strength); 960 if (ret >= 0) 961 raw_mode = 1; 962 } else { 963 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size); 964 965 if (oob_required) { 966 nand_change_read_column_op(nand, oob_off, NULL, 0, 967 false); 968 sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4, 969 true, page); 970 971 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0, 972 bbm, page); 973 } 974 } 975 976 sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret); 977 978 return raw_mode; 979 } 980 981 static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd, 982 u8 *oob, int *cur_off, 983 bool randomize, int page) 984 { 985 struct nand_chip *nand = mtd_to_nand(mtd); 986 struct nand_ecc_ctrl *ecc = &nand->ecc; 987 int offset = ((ecc->bytes + 4) * ecc->steps); 988 int len = mtd->oobsize - offset; 989 990 if (len <= 0) 991 return; 992 993 if (!cur_off || *cur_off != offset) 994 nand_change_read_column_op(nand, mtd->writesize, NULL, 0, 995 false); 996 997 if (!randomize) 998 sunxi_nfc_read_buf(mtd, oob + offset, len); 999 else 1000 sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len, 1001 false, page); 1002 1003 if (cur_off) 1004 *cur_off = mtd->oobsize + mtd->writesize; 1005 } 1006 1007 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf, 1008 int oob_required, int page, 1009 int nchunks) 1010 { 1011 struct nand_chip *nand = mtd_to_nand(mtd); 1012 bool randomized = nand->options & NAND_NEED_SCRAMBLING; 1013 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1014 struct nand_ecc_ctrl *ecc = &nand->ecc; 1015 unsigned int max_bitflips = 0; 1016 int ret, i, raw_mode = 0; 1017 struct scatterlist sg; 1018 u32 status; 1019 1020 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1021 if (ret) 1022 return ret; 1023 1024 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks, 1025 DMA_FROM_DEVICE, &sg); 1026 if (ret) 1027 return ret; 1028 1029 sunxi_nfc_hw_ecc_enable(mtd); 1030 sunxi_nfc_randomizer_config(mtd, page, false); 1031 sunxi_nfc_randomizer_enable(mtd); 1032 1033 writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) | 1034 NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET); 1035 1036 dma_async_issue_pending(nfc->dmac); 1037 1038 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS, 1039 nfc->regs + NFC_REG_CMD); 1040 1041 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 1042 if (ret) 1043 dmaengine_terminate_all(nfc->dmac); 1044 1045 sunxi_nfc_randomizer_disable(mtd); 1046 sunxi_nfc_hw_ecc_disable(mtd); 1047 1048 sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg); 1049 1050 if (ret) 1051 return ret; 1052 1053 status = readl(nfc->regs + NFC_REG_ECC_ST); 1054 1055 for (i = 0; i < nchunks; i++) { 1056 int data_off = i * ecc->size; 1057 int oob_off = i * (ecc->bytes + 4); 1058 u8 *data = buf + data_off; 1059 u8 *oob = nand->oob_poi + oob_off; 1060 bool erased; 1061 1062 ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL, 1063 oob_required ? oob : NULL, 1064 i, status, &erased); 1065 1066 /* ECC errors are handled in the second loop. */ 1067 if (ret < 0) 1068 continue; 1069 1070 if (oob_required && !erased) { 1071 /* TODO: use DMA to retrieve OOB */ 1072 nand_change_read_column_op(nand, 1073 mtd->writesize + oob_off, 1074 oob, ecc->bytes + 4, false); 1075 1076 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i, 1077 !i, page); 1078 } 1079 1080 if (erased) 1081 raw_mode = 1; 1082 1083 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret); 1084 } 1085 1086 if (status & NFC_ECC_ERR_MSK) { 1087 for (i = 0; i < nchunks; i++) { 1088 int data_off = i * ecc->size; 1089 int oob_off = i * (ecc->bytes + 4); 1090 u8 *data = buf + data_off; 1091 u8 *oob = nand->oob_poi + oob_off; 1092 1093 if (!(status & NFC_ECC_ERR(i))) 1094 continue; 1095 1096 /* 1097 * Re-read the data with the randomizer disabled to 1098 * identify bitflips in erased pages. 1099 * TODO: use DMA to read page in raw mode 1100 */ 1101 if (randomized) 1102 nand_change_read_column_op(nand, data_off, 1103 data, ecc->size, 1104 false); 1105 1106 /* TODO: use DMA to retrieve OOB */ 1107 nand_change_read_column_op(nand, 1108 mtd->writesize + oob_off, 1109 oob, ecc->bytes + 4, false); 1110 1111 ret = nand_check_erased_ecc_chunk(data, ecc->size, 1112 oob, ecc->bytes + 4, 1113 NULL, 0, 1114 ecc->strength); 1115 if (ret >= 0) 1116 raw_mode = 1; 1117 1118 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret); 1119 } 1120 } 1121 1122 if (oob_required) 1123 sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi, 1124 NULL, !raw_mode, 1125 page); 1126 1127 return max_bitflips; 1128 } 1129 1130 static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd, 1131 const u8 *data, int data_off, 1132 const u8 *oob, int oob_off, 1133 int *cur_off, bool bbm, 1134 int page) 1135 { 1136 struct nand_chip *nand = mtd_to_nand(mtd); 1137 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1138 struct nand_ecc_ctrl *ecc = &nand->ecc; 1139 int ret; 1140 1141 if (data_off != *cur_off) 1142 nand_change_write_column_op(nand, data_off, NULL, 0, false); 1143 1144 sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page); 1145 1146 if (data_off + ecc->size != oob_off) 1147 nand_change_write_column_op(nand, oob_off, NULL, 0, false); 1148 1149 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1150 if (ret) 1151 return ret; 1152 1153 sunxi_nfc_randomizer_enable(mtd); 1154 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page); 1155 1156 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | 1157 NFC_ACCESS_DIR | NFC_ECC_OP, 1158 nfc->regs + NFC_REG_CMD); 1159 1160 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 1161 sunxi_nfc_randomizer_disable(mtd); 1162 if (ret) 1163 return ret; 1164 1165 *cur_off = oob_off + ecc->bytes + 4; 1166 1167 return 0; 1168 } 1169 1170 static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd, 1171 u8 *oob, int *cur_off, 1172 int page) 1173 { 1174 struct nand_chip *nand = mtd_to_nand(mtd); 1175 struct nand_ecc_ctrl *ecc = &nand->ecc; 1176 int offset = ((ecc->bytes + 4) * ecc->steps); 1177 int len = mtd->oobsize - offset; 1178 1179 if (len <= 0) 1180 return; 1181 1182 if (!cur_off || *cur_off != offset) 1183 nand_change_write_column_op(nand, offset + mtd->writesize, 1184 NULL, 0, false); 1185 1186 sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page); 1187 1188 if (cur_off) 1189 *cur_off = mtd->oobsize + mtd->writesize; 1190 } 1191 1192 static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd, 1193 struct nand_chip *chip, uint8_t *buf, 1194 int oob_required, int page) 1195 { 1196 struct nand_ecc_ctrl *ecc = &chip->ecc; 1197 unsigned int max_bitflips = 0; 1198 int ret, i, cur_off = 0; 1199 bool raw_mode = false; 1200 1201 nand_read_page_op(chip, page, 0, NULL, 0); 1202 1203 sunxi_nfc_hw_ecc_enable(mtd); 1204 1205 for (i = 0; i < ecc->steps; i++) { 1206 int data_off = i * ecc->size; 1207 int oob_off = i * (ecc->bytes + 4); 1208 u8 *data = buf + data_off; 1209 u8 *oob = chip->oob_poi + oob_off; 1210 1211 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob, 1212 oob_off + mtd->writesize, 1213 &cur_off, &max_bitflips, 1214 !i, oob_required, page); 1215 if (ret < 0) 1216 return ret; 1217 else if (ret) 1218 raw_mode = true; 1219 } 1220 1221 if (oob_required) 1222 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off, 1223 !raw_mode, page); 1224 1225 sunxi_nfc_hw_ecc_disable(mtd); 1226 1227 return max_bitflips; 1228 } 1229 1230 static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd, 1231 struct nand_chip *chip, u8 *buf, 1232 int oob_required, int page) 1233 { 1234 int ret; 1235 1236 nand_read_page_op(chip, page, 0, NULL, 0); 1237 1238 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page, 1239 chip->ecc.steps); 1240 if (ret >= 0) 1241 return ret; 1242 1243 /* Fallback to PIO mode */ 1244 return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page); 1245 } 1246 1247 static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd, 1248 struct nand_chip *chip, 1249 u32 data_offs, u32 readlen, 1250 u8 *bufpoi, int page) 1251 { 1252 struct nand_ecc_ctrl *ecc = &chip->ecc; 1253 int ret, i, cur_off = 0; 1254 unsigned int max_bitflips = 0; 1255 1256 nand_read_page_op(chip, page, 0, NULL, 0); 1257 1258 sunxi_nfc_hw_ecc_enable(mtd); 1259 1260 for (i = data_offs / ecc->size; 1261 i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) { 1262 int data_off = i * ecc->size; 1263 int oob_off = i * (ecc->bytes + 4); 1264 u8 *data = bufpoi + data_off; 1265 u8 *oob = chip->oob_poi + oob_off; 1266 1267 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, 1268 oob, 1269 oob_off + mtd->writesize, 1270 &cur_off, &max_bitflips, !i, 1271 false, page); 1272 if (ret < 0) 1273 return ret; 1274 } 1275 1276 sunxi_nfc_hw_ecc_disable(mtd); 1277 1278 return max_bitflips; 1279 } 1280 1281 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd, 1282 struct nand_chip *chip, 1283 u32 data_offs, u32 readlen, 1284 u8 *buf, int page) 1285 { 1286 int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size); 1287 int ret; 1288 1289 nand_read_page_op(chip, page, 0, NULL, 0); 1290 1291 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks); 1292 if (ret >= 0) 1293 return ret; 1294 1295 /* Fallback to PIO mode */ 1296 return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen, 1297 buf, page); 1298 } 1299 1300 static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd, 1301 struct nand_chip *chip, 1302 const uint8_t *buf, int oob_required, 1303 int page) 1304 { 1305 struct nand_ecc_ctrl *ecc = &chip->ecc; 1306 int ret, i, cur_off = 0; 1307 1308 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1309 1310 sunxi_nfc_hw_ecc_enable(mtd); 1311 1312 for (i = 0; i < ecc->steps; i++) { 1313 int data_off = i * ecc->size; 1314 int oob_off = i * (ecc->bytes + 4); 1315 const u8 *data = buf + data_off; 1316 const u8 *oob = chip->oob_poi + oob_off; 1317 1318 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob, 1319 oob_off + mtd->writesize, 1320 &cur_off, !i, page); 1321 if (ret) 1322 return ret; 1323 } 1324 1325 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING)) 1326 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi, 1327 &cur_off, page); 1328 1329 sunxi_nfc_hw_ecc_disable(mtd); 1330 1331 return nand_prog_page_end_op(chip); 1332 } 1333 1334 static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd, 1335 struct nand_chip *chip, 1336 u32 data_offs, u32 data_len, 1337 const u8 *buf, int oob_required, 1338 int page) 1339 { 1340 struct nand_ecc_ctrl *ecc = &chip->ecc; 1341 int ret, i, cur_off = 0; 1342 1343 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1344 1345 sunxi_nfc_hw_ecc_enable(mtd); 1346 1347 for (i = data_offs / ecc->size; 1348 i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) { 1349 int data_off = i * ecc->size; 1350 int oob_off = i * (ecc->bytes + 4); 1351 const u8 *data = buf + data_off; 1352 const u8 *oob = chip->oob_poi + oob_off; 1353 1354 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob, 1355 oob_off + mtd->writesize, 1356 &cur_off, !i, page); 1357 if (ret) 1358 return ret; 1359 } 1360 1361 sunxi_nfc_hw_ecc_disable(mtd); 1362 1363 return nand_prog_page_end_op(chip); 1364 } 1365 1366 static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd, 1367 struct nand_chip *chip, 1368 const u8 *buf, 1369 int oob_required, 1370 int page) 1371 { 1372 struct nand_chip *nand = mtd_to_nand(mtd); 1373 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1374 struct nand_ecc_ctrl *ecc = &nand->ecc; 1375 struct scatterlist sg; 1376 int ret, i; 1377 1378 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1379 if (ret) 1380 return ret; 1381 1382 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps, 1383 DMA_TO_DEVICE, &sg); 1384 if (ret) 1385 goto pio_fallback; 1386 1387 for (i = 0; i < ecc->steps; i++) { 1388 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4)); 1389 1390 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page); 1391 } 1392 1393 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1394 1395 sunxi_nfc_hw_ecc_enable(mtd); 1396 sunxi_nfc_randomizer_config(mtd, page, false); 1397 sunxi_nfc_randomizer_enable(mtd); 1398 1399 writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG, 1400 nfc->regs + NFC_REG_RCMD_SET); 1401 1402 dma_async_issue_pending(nfc->dmac); 1403 1404 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | 1405 NFC_DATA_TRANS | NFC_ACCESS_DIR, 1406 nfc->regs + NFC_REG_CMD); 1407 1408 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 1409 if (ret) 1410 dmaengine_terminate_all(nfc->dmac); 1411 1412 sunxi_nfc_randomizer_disable(mtd); 1413 sunxi_nfc_hw_ecc_disable(mtd); 1414 1415 sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg); 1416 1417 if (ret) 1418 return ret; 1419 1420 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING)) 1421 /* TODO: use DMA to transfer extra OOB bytes ? */ 1422 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi, 1423 NULL, page); 1424 1425 return nand_prog_page_end_op(chip); 1426 1427 pio_fallback: 1428 return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page); 1429 } 1430 1431 static int sunxi_nfc_hw_ecc_read_oob(struct mtd_info *mtd, 1432 struct nand_chip *chip, 1433 int page) 1434 { 1435 chip->pagebuf = -1; 1436 1437 return chip->ecc.read_page(mtd, chip, chip->data_buf, 1, page); 1438 } 1439 1440 static int sunxi_nfc_hw_ecc_write_oob(struct mtd_info *mtd, 1441 struct nand_chip *chip, 1442 int page) 1443 { 1444 int ret; 1445 1446 chip->pagebuf = -1; 1447 1448 memset(chip->data_buf, 0xff, mtd->writesize); 1449 ret = chip->ecc.write_page(mtd, chip, chip->data_buf, 1, page); 1450 if (ret) 1451 return ret; 1452 1453 /* Send command to program the OOB data */ 1454 return nand_prog_page_end_op(chip); 1455 } 1456 1457 static const s32 tWB_lut[] = {6, 12, 16, 20}; 1458 static const s32 tRHW_lut[] = {4, 8, 12, 20}; 1459 1460 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration, 1461 u32 clk_period) 1462 { 1463 u32 clk_cycles = DIV_ROUND_UP(duration, clk_period); 1464 int i; 1465 1466 for (i = 0; i < lut_size; i++) { 1467 if (clk_cycles <= lut[i]) 1468 return i; 1469 } 1470 1471 /* Doesn't fit */ 1472 return -EINVAL; 1473 } 1474 1475 #define sunxi_nand_lookup_timing(l, p, c) \ 1476 _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c) 1477 1478 static int sunxi_nfc_setup_data_interface(struct mtd_info *mtd, int csline, 1479 const struct nand_data_interface *conf) 1480 { 1481 struct nand_chip *nand = mtd_to_nand(mtd); 1482 struct sunxi_nand_chip *chip = to_sunxi_nand(nand); 1483 struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller); 1484 const struct nand_sdr_timings *timings; 1485 u32 min_clk_period = 0; 1486 s32 tWB, tADL, tWHR, tRHW, tCAD; 1487 long real_clk_rate; 1488 1489 timings = nand_get_sdr_timings(conf); 1490 if (IS_ERR(timings)) 1491 return -ENOTSUPP; 1492 1493 /* T1 <=> tCLS */ 1494 if (timings->tCLS_min > min_clk_period) 1495 min_clk_period = timings->tCLS_min; 1496 1497 /* T2 <=> tCLH */ 1498 if (timings->tCLH_min > min_clk_period) 1499 min_clk_period = timings->tCLH_min; 1500 1501 /* T3 <=> tCS */ 1502 if (timings->tCS_min > min_clk_period) 1503 min_clk_period = timings->tCS_min; 1504 1505 /* T4 <=> tCH */ 1506 if (timings->tCH_min > min_clk_period) 1507 min_clk_period = timings->tCH_min; 1508 1509 /* T5 <=> tWP */ 1510 if (timings->tWP_min > min_clk_period) 1511 min_clk_period = timings->tWP_min; 1512 1513 /* T6 <=> tWH */ 1514 if (timings->tWH_min > min_clk_period) 1515 min_clk_period = timings->tWH_min; 1516 1517 /* T7 <=> tALS */ 1518 if (timings->tALS_min > min_clk_period) 1519 min_clk_period = timings->tALS_min; 1520 1521 /* T8 <=> tDS */ 1522 if (timings->tDS_min > min_clk_period) 1523 min_clk_period = timings->tDS_min; 1524 1525 /* T9 <=> tDH */ 1526 if (timings->tDH_min > min_clk_period) 1527 min_clk_period = timings->tDH_min; 1528 1529 /* T10 <=> tRR */ 1530 if (timings->tRR_min > (min_clk_period * 3)) 1531 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3); 1532 1533 /* T11 <=> tALH */ 1534 if (timings->tALH_min > min_clk_period) 1535 min_clk_period = timings->tALH_min; 1536 1537 /* T12 <=> tRP */ 1538 if (timings->tRP_min > min_clk_period) 1539 min_clk_period = timings->tRP_min; 1540 1541 /* T13 <=> tREH */ 1542 if (timings->tREH_min > min_clk_period) 1543 min_clk_period = timings->tREH_min; 1544 1545 /* T14 <=> tRC */ 1546 if (timings->tRC_min > (min_clk_period * 2)) 1547 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2); 1548 1549 /* T15 <=> tWC */ 1550 if (timings->tWC_min > (min_clk_period * 2)) 1551 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2); 1552 1553 /* T16 - T19 + tCAD */ 1554 if (timings->tWB_max > (min_clk_period * 20)) 1555 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20); 1556 1557 if (timings->tADL_min > (min_clk_period * 32)) 1558 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32); 1559 1560 if (timings->tWHR_min > (min_clk_period * 32)) 1561 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32); 1562 1563 if (timings->tRHW_min > (min_clk_period * 20)) 1564 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20); 1565 1566 tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max, 1567 min_clk_period); 1568 if (tWB < 0) { 1569 dev_err(nfc->dev, "unsupported tWB\n"); 1570 return tWB; 1571 } 1572 1573 tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3; 1574 if (tADL > 3) { 1575 dev_err(nfc->dev, "unsupported tADL\n"); 1576 return -EINVAL; 1577 } 1578 1579 tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3; 1580 if (tWHR > 3) { 1581 dev_err(nfc->dev, "unsupported tWHR\n"); 1582 return -EINVAL; 1583 } 1584 1585 tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min, 1586 min_clk_period); 1587 if (tRHW < 0) { 1588 dev_err(nfc->dev, "unsupported tRHW\n"); 1589 return tRHW; 1590 } 1591 1592 if (csline == NAND_DATA_IFACE_CHECK_ONLY) 1593 return 0; 1594 1595 /* 1596 * TODO: according to ONFI specs this value only applies for DDR NAND, 1597 * but Allwinner seems to set this to 0x7. Mimic them for now. 1598 */ 1599 tCAD = 0x7; 1600 1601 /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */ 1602 chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD); 1603 1604 /* Convert min_clk_period from picoseconds to nanoseconds */ 1605 min_clk_period = DIV_ROUND_UP(min_clk_period, 1000); 1606 1607 /* 1608 * Unlike what is stated in Allwinner datasheet, the clk_rate should 1609 * be set to (1 / min_clk_period), and not (2 / min_clk_period). 1610 * This new formula was verified with a scope and validated by 1611 * Allwinner engineers. 1612 */ 1613 chip->clk_rate = NSEC_PER_SEC / min_clk_period; 1614 real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate); 1615 if (real_clk_rate <= 0) { 1616 dev_err(nfc->dev, "Unable to round clk %lu\n", chip->clk_rate); 1617 return -EINVAL; 1618 } 1619 1620 /* 1621 * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data 1622 * output cycle timings shall be used if the host drives tRC less than 1623 * 30 ns. 1624 */ 1625 min_clk_period = NSEC_PER_SEC / real_clk_rate; 1626 chip->timing_ctl = ((min_clk_period * 2) < 30) ? 1627 NFC_TIMING_CTL_EDO : 0; 1628 1629 return 0; 1630 } 1631 1632 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section, 1633 struct mtd_oob_region *oobregion) 1634 { 1635 struct nand_chip *nand = mtd_to_nand(mtd); 1636 struct nand_ecc_ctrl *ecc = &nand->ecc; 1637 1638 if (section >= ecc->steps) 1639 return -ERANGE; 1640 1641 oobregion->offset = section * (ecc->bytes + 4) + 4; 1642 oobregion->length = ecc->bytes; 1643 1644 return 0; 1645 } 1646 1647 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section, 1648 struct mtd_oob_region *oobregion) 1649 { 1650 struct nand_chip *nand = mtd_to_nand(mtd); 1651 struct nand_ecc_ctrl *ecc = &nand->ecc; 1652 1653 if (section > ecc->steps) 1654 return -ERANGE; 1655 1656 /* 1657 * The first 2 bytes are used for BB markers, hence we 1658 * only have 2 bytes available in the first user data 1659 * section. 1660 */ 1661 if (!section && ecc->mode == NAND_ECC_HW) { 1662 oobregion->offset = 2; 1663 oobregion->length = 2; 1664 1665 return 0; 1666 } 1667 1668 oobregion->offset = section * (ecc->bytes + 4); 1669 1670 if (section < ecc->steps) 1671 oobregion->length = 4; 1672 else 1673 oobregion->offset = mtd->oobsize - oobregion->offset; 1674 1675 return 0; 1676 } 1677 1678 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = { 1679 .ecc = sunxi_nand_ooblayout_ecc, 1680 .free = sunxi_nand_ooblayout_free, 1681 }; 1682 1683 static void sunxi_nand_hw_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc) 1684 { 1685 kfree(ecc->priv); 1686 } 1687 1688 static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd, 1689 struct nand_ecc_ctrl *ecc, 1690 struct device_node *np) 1691 { 1692 static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 }; 1693 struct nand_chip *nand = mtd_to_nand(mtd); 1694 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 1695 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 1696 struct sunxi_nand_hw_ecc *data; 1697 int nsectors; 1698 int ret; 1699 int i; 1700 1701 if (ecc->options & NAND_ECC_MAXIMIZE) { 1702 int bytes; 1703 1704 ecc->size = 1024; 1705 nsectors = mtd->writesize / ecc->size; 1706 1707 /* Reserve 2 bytes for the BBM */ 1708 bytes = (mtd->oobsize - 2) / nsectors; 1709 1710 /* 4 non-ECC bytes are added before each ECC bytes section */ 1711 bytes -= 4; 1712 1713 /* and bytes has to be even. */ 1714 if (bytes % 2) 1715 bytes--; 1716 1717 ecc->strength = bytes * 8 / fls(8 * ecc->size); 1718 1719 for (i = 0; i < ARRAY_SIZE(strengths); i++) { 1720 if (strengths[i] > ecc->strength) 1721 break; 1722 } 1723 1724 if (!i) 1725 ecc->strength = 0; 1726 else 1727 ecc->strength = strengths[i - 1]; 1728 } 1729 1730 if (ecc->size != 512 && ecc->size != 1024) 1731 return -EINVAL; 1732 1733 data = kzalloc(sizeof(*data), GFP_KERNEL); 1734 if (!data) 1735 return -ENOMEM; 1736 1737 /* Prefer 1k ECC chunk over 512 ones */ 1738 if (ecc->size == 512 && mtd->writesize > 512) { 1739 ecc->size = 1024; 1740 ecc->strength *= 2; 1741 } 1742 1743 /* Add ECC info retrieval from DT */ 1744 for (i = 0; i < ARRAY_SIZE(strengths); i++) { 1745 if (ecc->strength <= strengths[i]) { 1746 /* 1747 * Update ecc->strength value with the actual strength 1748 * that will be used by the ECC engine. 1749 */ 1750 ecc->strength = strengths[i]; 1751 break; 1752 } 1753 } 1754 1755 if (i >= ARRAY_SIZE(strengths)) { 1756 dev_err(nfc->dev, "unsupported strength\n"); 1757 ret = -ENOTSUPP; 1758 goto err; 1759 } 1760 1761 data->mode = i; 1762 1763 /* HW ECC always request ECC bytes for 1024 bytes blocks */ 1764 ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8); 1765 1766 /* HW ECC always work with even numbers of ECC bytes */ 1767 ecc->bytes = ALIGN(ecc->bytes, 2); 1768 1769 nsectors = mtd->writesize / ecc->size; 1770 1771 if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) { 1772 ret = -EINVAL; 1773 goto err; 1774 } 1775 1776 ecc->read_oob = sunxi_nfc_hw_ecc_read_oob; 1777 ecc->write_oob = sunxi_nfc_hw_ecc_write_oob; 1778 mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops); 1779 ecc->priv = data; 1780 1781 if (nfc->dmac) { 1782 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma; 1783 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma; 1784 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma; 1785 nand->options |= NAND_USE_BOUNCE_BUFFER; 1786 } else { 1787 ecc->read_page = sunxi_nfc_hw_ecc_read_page; 1788 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage; 1789 ecc->write_page = sunxi_nfc_hw_ecc_write_page; 1790 } 1791 1792 /* TODO: support DMA for raw accesses and subpage write */ 1793 ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage; 1794 ecc->read_oob_raw = nand_read_oob_std; 1795 ecc->write_oob_raw = nand_write_oob_std; 1796 1797 return 0; 1798 1799 err: 1800 kfree(data); 1801 1802 return ret; 1803 } 1804 1805 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc) 1806 { 1807 switch (ecc->mode) { 1808 case NAND_ECC_HW: 1809 sunxi_nand_hw_ecc_ctrl_cleanup(ecc); 1810 break; 1811 case NAND_ECC_NONE: 1812 default: 1813 break; 1814 } 1815 } 1816 1817 static int sunxi_nand_attach_chip(struct nand_chip *nand) 1818 { 1819 struct mtd_info *mtd = nand_to_mtd(nand); 1820 struct nand_ecc_ctrl *ecc = &nand->ecc; 1821 struct device_node *np = nand_get_flash_node(nand); 1822 int ret; 1823 1824 if (nand->bbt_options & NAND_BBT_USE_FLASH) 1825 nand->bbt_options |= NAND_BBT_NO_OOB; 1826 1827 if (nand->options & NAND_NEED_SCRAMBLING) 1828 nand->options |= NAND_NO_SUBPAGE_WRITE; 1829 1830 nand->options |= NAND_SUBPAGE_READ; 1831 1832 if (!ecc->size) { 1833 ecc->size = nand->ecc_step_ds; 1834 ecc->strength = nand->ecc_strength_ds; 1835 } 1836 1837 if (!ecc->size || !ecc->strength) 1838 return -EINVAL; 1839 1840 switch (ecc->mode) { 1841 case NAND_ECC_HW: 1842 ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np); 1843 if (ret) 1844 return ret; 1845 break; 1846 case NAND_ECC_NONE: 1847 case NAND_ECC_SOFT: 1848 break; 1849 default: 1850 return -EINVAL; 1851 } 1852 1853 return 0; 1854 } 1855 1856 static const struct nand_controller_ops sunxi_nand_controller_ops = { 1857 .attach_chip = sunxi_nand_attach_chip, 1858 }; 1859 1860 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc, 1861 struct device_node *np) 1862 { 1863 struct sunxi_nand_chip *chip; 1864 struct mtd_info *mtd; 1865 struct nand_chip *nand; 1866 int nsels; 1867 int ret; 1868 int i; 1869 u32 tmp; 1870 1871 if (!of_get_property(np, "reg", &nsels)) 1872 return -EINVAL; 1873 1874 nsels /= sizeof(u32); 1875 if (!nsels) { 1876 dev_err(dev, "invalid reg property size\n"); 1877 return -EINVAL; 1878 } 1879 1880 chip = devm_kzalloc(dev, 1881 sizeof(*chip) + 1882 (nsels * sizeof(struct sunxi_nand_chip_sel)), 1883 GFP_KERNEL); 1884 if (!chip) { 1885 dev_err(dev, "could not allocate chip\n"); 1886 return -ENOMEM; 1887 } 1888 1889 chip->nsels = nsels; 1890 chip->selected = -1; 1891 1892 for (i = 0; i < nsels; i++) { 1893 ret = of_property_read_u32_index(np, "reg", i, &tmp); 1894 if (ret) { 1895 dev_err(dev, "could not retrieve reg property: %d\n", 1896 ret); 1897 return ret; 1898 } 1899 1900 if (tmp > NFC_MAX_CS) { 1901 dev_err(dev, 1902 "invalid reg value: %u (max CS = 7)\n", 1903 tmp); 1904 return -EINVAL; 1905 } 1906 1907 if (test_and_set_bit(tmp, &nfc->assigned_cs)) { 1908 dev_err(dev, "CS %d already assigned\n", tmp); 1909 return -EINVAL; 1910 } 1911 1912 chip->sels[i].cs = tmp; 1913 1914 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) && 1915 tmp < 2) 1916 chip->sels[i].rb = tmp; 1917 else 1918 chip->sels[i].rb = -1; 1919 } 1920 1921 nand = &chip->nand; 1922 /* Default tR value specified in the ONFI spec (chapter 4.15.1) */ 1923 nand->chip_delay = 200; 1924 nand->controller = &nfc->controller; 1925 nand->controller->ops = &sunxi_nand_controller_ops; 1926 1927 /* 1928 * Set the ECC mode to the default value in case nothing is specified 1929 * in the DT. 1930 */ 1931 nand->ecc.mode = NAND_ECC_HW; 1932 nand_set_flash_node(nand, np); 1933 nand->select_chip = sunxi_nfc_select_chip; 1934 nand->cmd_ctrl = sunxi_nfc_cmd_ctrl; 1935 nand->read_buf = sunxi_nfc_read_buf; 1936 nand->write_buf = sunxi_nfc_write_buf; 1937 nand->read_byte = sunxi_nfc_read_byte; 1938 nand->setup_data_interface = sunxi_nfc_setup_data_interface; 1939 1940 mtd = nand_to_mtd(nand); 1941 mtd->dev.parent = dev; 1942 1943 ret = nand_scan(mtd, nsels); 1944 if (ret) 1945 return ret; 1946 1947 ret = mtd_device_register(mtd, NULL, 0); 1948 if (ret) { 1949 dev_err(dev, "failed to register mtd device: %d\n", ret); 1950 nand_release(mtd); 1951 return ret; 1952 } 1953 1954 list_add_tail(&chip->node, &nfc->chips); 1955 1956 return 0; 1957 } 1958 1959 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc) 1960 { 1961 struct device_node *np = dev->of_node; 1962 struct device_node *nand_np; 1963 int nchips = of_get_child_count(np); 1964 int ret; 1965 1966 if (nchips > 8) { 1967 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips); 1968 return -EINVAL; 1969 } 1970 1971 for_each_child_of_node(np, nand_np) { 1972 ret = sunxi_nand_chip_init(dev, nfc, nand_np); 1973 if (ret) { 1974 of_node_put(nand_np); 1975 return ret; 1976 } 1977 } 1978 1979 return 0; 1980 } 1981 1982 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc) 1983 { 1984 struct sunxi_nand_chip *chip; 1985 1986 while (!list_empty(&nfc->chips)) { 1987 chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip, 1988 node); 1989 nand_release(nand_to_mtd(&chip->nand)); 1990 sunxi_nand_ecc_cleanup(&chip->nand.ecc); 1991 list_del(&chip->node); 1992 } 1993 } 1994 1995 static int sunxi_nfc_probe(struct platform_device *pdev) 1996 { 1997 struct device *dev = &pdev->dev; 1998 struct resource *r; 1999 struct sunxi_nfc *nfc; 2000 int irq; 2001 int ret; 2002 2003 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL); 2004 if (!nfc) 2005 return -ENOMEM; 2006 2007 nfc->dev = dev; 2008 nand_controller_init(&nfc->controller); 2009 INIT_LIST_HEAD(&nfc->chips); 2010 2011 r = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2012 nfc->regs = devm_ioremap_resource(dev, r); 2013 if (IS_ERR(nfc->regs)) 2014 return PTR_ERR(nfc->regs); 2015 2016 irq = platform_get_irq(pdev, 0); 2017 if (irq < 0) { 2018 dev_err(dev, "failed to retrieve irq\n"); 2019 return irq; 2020 } 2021 2022 nfc->ahb_clk = devm_clk_get(dev, "ahb"); 2023 if (IS_ERR(nfc->ahb_clk)) { 2024 dev_err(dev, "failed to retrieve ahb clk\n"); 2025 return PTR_ERR(nfc->ahb_clk); 2026 } 2027 2028 ret = clk_prepare_enable(nfc->ahb_clk); 2029 if (ret) 2030 return ret; 2031 2032 nfc->mod_clk = devm_clk_get(dev, "mod"); 2033 if (IS_ERR(nfc->mod_clk)) { 2034 dev_err(dev, "failed to retrieve mod clk\n"); 2035 ret = PTR_ERR(nfc->mod_clk); 2036 goto out_ahb_clk_unprepare; 2037 } 2038 2039 ret = clk_prepare_enable(nfc->mod_clk); 2040 if (ret) 2041 goto out_ahb_clk_unprepare; 2042 2043 nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb"); 2044 if (IS_ERR(nfc->reset)) { 2045 ret = PTR_ERR(nfc->reset); 2046 goto out_mod_clk_unprepare; 2047 } 2048 2049 ret = reset_control_deassert(nfc->reset); 2050 if (ret) { 2051 dev_err(dev, "reset err %d\n", ret); 2052 goto out_mod_clk_unprepare; 2053 } 2054 2055 ret = sunxi_nfc_rst(nfc); 2056 if (ret) 2057 goto out_ahb_reset_reassert; 2058 2059 writel(0, nfc->regs + NFC_REG_INT); 2060 ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt, 2061 0, "sunxi-nand", nfc); 2062 if (ret) 2063 goto out_ahb_reset_reassert; 2064 2065 nfc->dmac = dma_request_slave_channel(dev, "rxtx"); 2066 if (nfc->dmac) { 2067 struct dma_slave_config dmac_cfg = { }; 2068 2069 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA; 2070 dmac_cfg.dst_addr = dmac_cfg.src_addr; 2071 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 2072 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width; 2073 dmac_cfg.src_maxburst = 4; 2074 dmac_cfg.dst_maxburst = 4; 2075 dmaengine_slave_config(nfc->dmac, &dmac_cfg); 2076 } else { 2077 dev_warn(dev, "failed to request rxtx DMA channel\n"); 2078 } 2079 2080 platform_set_drvdata(pdev, nfc); 2081 2082 ret = sunxi_nand_chips_init(dev, nfc); 2083 if (ret) { 2084 dev_err(dev, "failed to init nand chips\n"); 2085 goto out_release_dmac; 2086 } 2087 2088 return 0; 2089 2090 out_release_dmac: 2091 if (nfc->dmac) 2092 dma_release_channel(nfc->dmac); 2093 out_ahb_reset_reassert: 2094 reset_control_assert(nfc->reset); 2095 out_mod_clk_unprepare: 2096 clk_disable_unprepare(nfc->mod_clk); 2097 out_ahb_clk_unprepare: 2098 clk_disable_unprepare(nfc->ahb_clk); 2099 2100 return ret; 2101 } 2102 2103 static int sunxi_nfc_remove(struct platform_device *pdev) 2104 { 2105 struct sunxi_nfc *nfc = platform_get_drvdata(pdev); 2106 2107 sunxi_nand_chips_cleanup(nfc); 2108 2109 reset_control_assert(nfc->reset); 2110 2111 if (nfc->dmac) 2112 dma_release_channel(nfc->dmac); 2113 clk_disable_unprepare(nfc->mod_clk); 2114 clk_disable_unprepare(nfc->ahb_clk); 2115 2116 return 0; 2117 } 2118 2119 static const struct of_device_id sunxi_nfc_ids[] = { 2120 { .compatible = "allwinner,sun4i-a10-nand" }, 2121 { /* sentinel */ } 2122 }; 2123 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids); 2124 2125 static struct platform_driver sunxi_nfc_driver = { 2126 .driver = { 2127 .name = "sunxi_nand", 2128 .of_match_table = sunxi_nfc_ids, 2129 }, 2130 .probe = sunxi_nfc_probe, 2131 .remove = sunxi_nfc_remove, 2132 }; 2133 module_platform_driver(sunxi_nfc_driver); 2134 2135 MODULE_LICENSE("GPL v2"); 2136 MODULE_AUTHOR("Boris BREZILLON"); 2137 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver"); 2138 MODULE_ALIAS("platform:sunxi_nand"); 2139