1 /* 2 * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com> 3 * 4 * Derived from: 5 * https://github.com/yuq/sunxi-nfc-mtd 6 * Copyright (C) 2013 Qiang Yu <yuq825@gmail.com> 7 * 8 * https://github.com/hno/Allwinner-Info 9 * Copyright (C) 2013 Henrik Nordström <Henrik Nordström> 10 * 11 * Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com> 12 * Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org> 13 * 14 * This program is free software; you can redistribute it and/or modify 15 * it under the terms of the GNU General Public License as published by 16 * the Free Software Foundation; either version 2 of the License, or 17 * (at your option) any later version. 18 * 19 * This program is distributed in the hope that it will be useful, 20 * but WITHOUT ANY WARRANTY; without even the implied warranty of 21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 * GNU General Public License for more details. 23 */ 24 25 #include <linux/dma-mapping.h> 26 #include <linux/slab.h> 27 #include <linux/module.h> 28 #include <linux/moduleparam.h> 29 #include <linux/platform_device.h> 30 #include <linux/of.h> 31 #include <linux/of_device.h> 32 #include <linux/of_gpio.h> 33 #include <linux/mtd/mtd.h> 34 #include <linux/mtd/rawnand.h> 35 #include <linux/mtd/partitions.h> 36 #include <linux/clk.h> 37 #include <linux/delay.h> 38 #include <linux/dmaengine.h> 39 #include <linux/gpio.h> 40 #include <linux/interrupt.h> 41 #include <linux/iopoll.h> 42 #include <linux/reset.h> 43 44 #define NFC_REG_CTL 0x0000 45 #define NFC_REG_ST 0x0004 46 #define NFC_REG_INT 0x0008 47 #define NFC_REG_TIMING_CTL 0x000C 48 #define NFC_REG_TIMING_CFG 0x0010 49 #define NFC_REG_ADDR_LOW 0x0014 50 #define NFC_REG_ADDR_HIGH 0x0018 51 #define NFC_REG_SECTOR_NUM 0x001C 52 #define NFC_REG_CNT 0x0020 53 #define NFC_REG_CMD 0x0024 54 #define NFC_REG_RCMD_SET 0x0028 55 #define NFC_REG_WCMD_SET 0x002C 56 #define NFC_REG_IO_DATA 0x0030 57 #define NFC_REG_ECC_CTL 0x0034 58 #define NFC_REG_ECC_ST 0x0038 59 #define NFC_REG_DEBUG 0x003C 60 #define NFC_REG_ECC_ERR_CNT(x) ((0x0040 + (x)) & ~0x3) 61 #define NFC_REG_USER_DATA(x) (0x0050 + ((x) * 4)) 62 #define NFC_REG_SPARE_AREA 0x00A0 63 #define NFC_REG_PAT_ID 0x00A4 64 #define NFC_RAM0_BASE 0x0400 65 #define NFC_RAM1_BASE 0x0800 66 67 /* define bit use in NFC_CTL */ 68 #define NFC_EN BIT(0) 69 #define NFC_RESET BIT(1) 70 #define NFC_BUS_WIDTH_MSK BIT(2) 71 #define NFC_BUS_WIDTH_8 (0 << 2) 72 #define NFC_BUS_WIDTH_16 (1 << 2) 73 #define NFC_RB_SEL_MSK BIT(3) 74 #define NFC_RB_SEL(x) ((x) << 3) 75 #define NFC_CE_SEL_MSK GENMASK(26, 24) 76 #define NFC_CE_SEL(x) ((x) << 24) 77 #define NFC_CE_CTL BIT(6) 78 #define NFC_PAGE_SHIFT_MSK GENMASK(11, 8) 79 #define NFC_PAGE_SHIFT(x) (((x) < 10 ? 0 : (x) - 10) << 8) 80 #define NFC_SAM BIT(12) 81 #define NFC_RAM_METHOD BIT(14) 82 #define NFC_DEBUG_CTL BIT(31) 83 84 /* define bit use in NFC_ST */ 85 #define NFC_RB_B2R BIT(0) 86 #define NFC_CMD_INT_FLAG BIT(1) 87 #define NFC_DMA_INT_FLAG BIT(2) 88 #define NFC_CMD_FIFO_STATUS BIT(3) 89 #define NFC_STA BIT(4) 90 #define NFC_NATCH_INT_FLAG BIT(5) 91 #define NFC_RB_STATE(x) BIT(x + 8) 92 93 /* define bit use in NFC_INT */ 94 #define NFC_B2R_INT_ENABLE BIT(0) 95 #define NFC_CMD_INT_ENABLE BIT(1) 96 #define NFC_DMA_INT_ENABLE BIT(2) 97 #define NFC_INT_MASK (NFC_B2R_INT_ENABLE | \ 98 NFC_CMD_INT_ENABLE | \ 99 NFC_DMA_INT_ENABLE) 100 101 /* define bit use in NFC_TIMING_CTL */ 102 #define NFC_TIMING_CTL_EDO BIT(8) 103 104 /* define NFC_TIMING_CFG register layout */ 105 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD) \ 106 (((tWB) & 0x3) | (((tADL) & 0x3) << 2) | \ 107 (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) | \ 108 (((tCAD) & 0x7) << 8)) 109 110 /* define bit use in NFC_CMD */ 111 #define NFC_CMD_LOW_BYTE_MSK GENMASK(7, 0) 112 #define NFC_CMD_HIGH_BYTE_MSK GENMASK(15, 8) 113 #define NFC_CMD(x) (x) 114 #define NFC_ADR_NUM_MSK GENMASK(18, 16) 115 #define NFC_ADR_NUM(x) (((x) - 1) << 16) 116 #define NFC_SEND_ADR BIT(19) 117 #define NFC_ACCESS_DIR BIT(20) 118 #define NFC_DATA_TRANS BIT(21) 119 #define NFC_SEND_CMD1 BIT(22) 120 #define NFC_WAIT_FLAG BIT(23) 121 #define NFC_SEND_CMD2 BIT(24) 122 #define NFC_SEQ BIT(25) 123 #define NFC_DATA_SWAP_METHOD BIT(26) 124 #define NFC_ROW_AUTO_INC BIT(27) 125 #define NFC_SEND_CMD3 BIT(28) 126 #define NFC_SEND_CMD4 BIT(29) 127 #define NFC_CMD_TYPE_MSK GENMASK(31, 30) 128 #define NFC_NORMAL_OP (0 << 30) 129 #define NFC_ECC_OP (1 << 30) 130 #define NFC_PAGE_OP (2 << 30) 131 132 /* define bit use in NFC_RCMD_SET */ 133 #define NFC_READ_CMD_MSK GENMASK(7, 0) 134 #define NFC_RND_READ_CMD0_MSK GENMASK(15, 8) 135 #define NFC_RND_READ_CMD1_MSK GENMASK(23, 16) 136 137 /* define bit use in NFC_WCMD_SET */ 138 #define NFC_PROGRAM_CMD_MSK GENMASK(7, 0) 139 #define NFC_RND_WRITE_CMD_MSK GENMASK(15, 8) 140 #define NFC_READ_CMD0_MSK GENMASK(23, 16) 141 #define NFC_READ_CMD1_MSK GENMASK(31, 24) 142 143 /* define bit use in NFC_ECC_CTL */ 144 #define NFC_ECC_EN BIT(0) 145 #define NFC_ECC_PIPELINE BIT(3) 146 #define NFC_ECC_EXCEPTION BIT(4) 147 #define NFC_ECC_BLOCK_SIZE_MSK BIT(5) 148 #define NFC_ECC_BLOCK_512 BIT(5) 149 #define NFC_RANDOM_EN BIT(9) 150 #define NFC_RANDOM_DIRECTION BIT(10) 151 #define NFC_ECC_MODE_MSK GENMASK(15, 12) 152 #define NFC_ECC_MODE(x) ((x) << 12) 153 #define NFC_RANDOM_SEED_MSK GENMASK(30, 16) 154 #define NFC_RANDOM_SEED(x) ((x) << 16) 155 156 /* define bit use in NFC_ECC_ST */ 157 #define NFC_ECC_ERR(x) BIT(x) 158 #define NFC_ECC_ERR_MSK GENMASK(15, 0) 159 #define NFC_ECC_PAT_FOUND(x) BIT(x + 16) 160 #define NFC_ECC_ERR_CNT(b, x) (((x) >> (((b) % 4) * 8)) & 0xff) 161 162 #define NFC_DEFAULT_TIMEOUT_MS 1000 163 164 #define NFC_SRAM_SIZE 1024 165 166 #define NFC_MAX_CS 7 167 168 /* 169 * Ready/Busy detection type: describes the Ready/Busy detection modes 170 * 171 * @RB_NONE: no external detection available, rely on STATUS command 172 * and software timeouts 173 * @RB_NATIVE: use sunxi NAND controller Ready/Busy support. The Ready/Busy 174 * pin of the NAND flash chip must be connected to one of the 175 * native NAND R/B pins (those which can be muxed to the NAND 176 * Controller) 177 * @RB_GPIO: use a simple GPIO to handle Ready/Busy status. The Ready/Busy 178 * pin of the NAND flash chip must be connected to a GPIO capable 179 * pin. 180 */ 181 enum sunxi_nand_rb_type { 182 RB_NONE, 183 RB_NATIVE, 184 RB_GPIO, 185 }; 186 187 /* 188 * Ready/Busy structure: stores information related to Ready/Busy detection 189 * 190 * @type: the Ready/Busy detection mode 191 * @info: information related to the R/B detection mode. Either a gpio 192 * id or a native R/B id (those supported by the NAND controller). 193 */ 194 struct sunxi_nand_rb { 195 enum sunxi_nand_rb_type type; 196 union { 197 int gpio; 198 int nativeid; 199 } info; 200 }; 201 202 /* 203 * Chip Select structure: stores information related to NAND Chip Select 204 * 205 * @cs: the NAND CS id used to communicate with a NAND Chip 206 * @rb: the Ready/Busy description 207 */ 208 struct sunxi_nand_chip_sel { 209 u8 cs; 210 struct sunxi_nand_rb rb; 211 }; 212 213 /* 214 * sunxi HW ECC infos: stores information related to HW ECC support 215 * 216 * @mode: the sunxi ECC mode field deduced from ECC requirements 217 */ 218 struct sunxi_nand_hw_ecc { 219 int mode; 220 }; 221 222 /* 223 * NAND chip structure: stores NAND chip device related information 224 * 225 * @node: used to store NAND chips into a list 226 * @nand: base NAND chip structure 227 * @mtd: base MTD structure 228 * @clk_rate: clk_rate required for this NAND chip 229 * @timing_cfg TIMING_CFG register value for this NAND chip 230 * @selected: current active CS 231 * @nsels: number of CS lines required by the NAND chip 232 * @sels: array of CS lines descriptions 233 */ 234 struct sunxi_nand_chip { 235 struct list_head node; 236 struct nand_chip nand; 237 unsigned long clk_rate; 238 u32 timing_cfg; 239 u32 timing_ctl; 240 int selected; 241 int addr_cycles; 242 u32 addr[2]; 243 int cmd_cycles; 244 u8 cmd[2]; 245 int nsels; 246 struct sunxi_nand_chip_sel sels[0]; 247 }; 248 249 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand) 250 { 251 return container_of(nand, struct sunxi_nand_chip, nand); 252 } 253 254 /* 255 * NAND Controller structure: stores sunxi NAND controller information 256 * 257 * @controller: base controller structure 258 * @dev: parent device (used to print error messages) 259 * @regs: NAND controller registers 260 * @ahb_clk: NAND Controller AHB clock 261 * @mod_clk: NAND Controller mod clock 262 * @assigned_cs: bitmask describing already assigned CS lines 263 * @clk_rate: NAND controller current clock rate 264 * @chips: a list containing all the NAND chips attached to 265 * this NAND controller 266 * @complete: a completion object used to wait for NAND 267 * controller events 268 */ 269 struct sunxi_nfc { 270 struct nand_hw_control controller; 271 struct device *dev; 272 void __iomem *regs; 273 struct clk *ahb_clk; 274 struct clk *mod_clk; 275 struct reset_control *reset; 276 unsigned long assigned_cs; 277 unsigned long clk_rate; 278 struct list_head chips; 279 struct completion complete; 280 struct dma_chan *dmac; 281 }; 282 283 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_hw_control *ctrl) 284 { 285 return container_of(ctrl, struct sunxi_nfc, controller); 286 } 287 288 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id) 289 { 290 struct sunxi_nfc *nfc = dev_id; 291 u32 st = readl(nfc->regs + NFC_REG_ST); 292 u32 ien = readl(nfc->regs + NFC_REG_INT); 293 294 if (!(ien & st)) 295 return IRQ_NONE; 296 297 if ((ien & st) == ien) 298 complete(&nfc->complete); 299 300 writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST); 301 writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT); 302 303 return IRQ_HANDLED; 304 } 305 306 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events, 307 bool use_polling, unsigned int timeout_ms) 308 { 309 int ret; 310 311 if (events & ~NFC_INT_MASK) 312 return -EINVAL; 313 314 if (!timeout_ms) 315 timeout_ms = NFC_DEFAULT_TIMEOUT_MS; 316 317 if (!use_polling) { 318 init_completion(&nfc->complete); 319 320 writel(events, nfc->regs + NFC_REG_INT); 321 322 ret = wait_for_completion_timeout(&nfc->complete, 323 msecs_to_jiffies(timeout_ms)); 324 if (!ret) 325 ret = -ETIMEDOUT; 326 else 327 ret = 0; 328 329 writel(0, nfc->regs + NFC_REG_INT); 330 } else { 331 u32 status; 332 333 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status, 334 (status & events) == events, 1, 335 timeout_ms * 1000); 336 } 337 338 writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST); 339 340 if (ret) 341 dev_err(nfc->dev, "wait interrupt timedout\n"); 342 343 return ret; 344 } 345 346 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc) 347 { 348 u32 status; 349 int ret; 350 351 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status, 352 !(status & NFC_CMD_FIFO_STATUS), 1, 353 NFC_DEFAULT_TIMEOUT_MS * 1000); 354 if (ret) 355 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n"); 356 357 return ret; 358 } 359 360 static int sunxi_nfc_rst(struct sunxi_nfc *nfc) 361 { 362 u32 ctl; 363 int ret; 364 365 writel(0, nfc->regs + NFC_REG_ECC_CTL); 366 writel(NFC_RESET, nfc->regs + NFC_REG_CTL); 367 368 ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl, 369 !(ctl & NFC_RESET), 1, 370 NFC_DEFAULT_TIMEOUT_MS * 1000); 371 if (ret) 372 dev_err(nfc->dev, "wait for NAND controller reset timedout\n"); 373 374 return ret; 375 } 376 377 static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf, 378 int chunksize, int nchunks, 379 enum dma_data_direction ddir, 380 struct scatterlist *sg) 381 { 382 struct nand_chip *nand = mtd_to_nand(mtd); 383 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 384 struct dma_async_tx_descriptor *dmad; 385 enum dma_transfer_direction tdir; 386 dma_cookie_t dmat; 387 int ret; 388 389 if (ddir == DMA_FROM_DEVICE) 390 tdir = DMA_DEV_TO_MEM; 391 else 392 tdir = DMA_MEM_TO_DEV; 393 394 sg_init_one(sg, buf, nchunks * chunksize); 395 ret = dma_map_sg(nfc->dev, sg, 1, ddir); 396 if (!ret) 397 return -ENOMEM; 398 399 dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK); 400 if (!dmad) { 401 ret = -EINVAL; 402 goto err_unmap_buf; 403 } 404 405 writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD, 406 nfc->regs + NFC_REG_CTL); 407 writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM); 408 writel(chunksize, nfc->regs + NFC_REG_CNT); 409 dmat = dmaengine_submit(dmad); 410 411 ret = dma_submit_error(dmat); 412 if (ret) 413 goto err_clr_dma_flag; 414 415 return 0; 416 417 err_clr_dma_flag: 418 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD, 419 nfc->regs + NFC_REG_CTL); 420 421 err_unmap_buf: 422 dma_unmap_sg(nfc->dev, sg, 1, ddir); 423 return ret; 424 } 425 426 static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd, 427 enum dma_data_direction ddir, 428 struct scatterlist *sg) 429 { 430 struct nand_chip *nand = mtd_to_nand(mtd); 431 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 432 433 dma_unmap_sg(nfc->dev, sg, 1, ddir); 434 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD, 435 nfc->regs + NFC_REG_CTL); 436 } 437 438 static int sunxi_nfc_dev_ready(struct mtd_info *mtd) 439 { 440 struct nand_chip *nand = mtd_to_nand(mtd); 441 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 442 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 443 struct sunxi_nand_rb *rb; 444 int ret; 445 446 if (sunxi_nand->selected < 0) 447 return 0; 448 449 rb = &sunxi_nand->sels[sunxi_nand->selected].rb; 450 451 switch (rb->type) { 452 case RB_NATIVE: 453 ret = !!(readl(nfc->regs + NFC_REG_ST) & 454 NFC_RB_STATE(rb->info.nativeid)); 455 break; 456 case RB_GPIO: 457 ret = gpio_get_value(rb->info.gpio); 458 break; 459 case RB_NONE: 460 default: 461 ret = 0; 462 dev_err(nfc->dev, "cannot check R/B NAND status!\n"); 463 break; 464 } 465 466 return ret; 467 } 468 469 static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip) 470 { 471 struct nand_chip *nand = mtd_to_nand(mtd); 472 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 473 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 474 struct sunxi_nand_chip_sel *sel; 475 u32 ctl; 476 477 if (chip > 0 && chip >= sunxi_nand->nsels) 478 return; 479 480 if (chip == sunxi_nand->selected) 481 return; 482 483 ctl = readl(nfc->regs + NFC_REG_CTL) & 484 ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN); 485 486 if (chip >= 0) { 487 sel = &sunxi_nand->sels[chip]; 488 489 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN | 490 NFC_PAGE_SHIFT(nand->page_shift); 491 if (sel->rb.type == RB_NONE) { 492 nand->dev_ready = NULL; 493 } else { 494 nand->dev_ready = sunxi_nfc_dev_ready; 495 if (sel->rb.type == RB_NATIVE) 496 ctl |= NFC_RB_SEL(sel->rb.info.nativeid); 497 } 498 499 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA); 500 501 if (nfc->clk_rate != sunxi_nand->clk_rate) { 502 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate); 503 nfc->clk_rate = sunxi_nand->clk_rate; 504 } 505 } 506 507 writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL); 508 writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG); 509 writel(ctl, nfc->regs + NFC_REG_CTL); 510 511 sunxi_nand->selected = chip; 512 } 513 514 static void sunxi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) 515 { 516 struct nand_chip *nand = mtd_to_nand(mtd); 517 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 518 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 519 int ret; 520 int cnt; 521 int offs = 0; 522 u32 tmp; 523 524 while (len > offs) { 525 bool poll = false; 526 527 cnt = min(len - offs, NFC_SRAM_SIZE); 528 529 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 530 if (ret) 531 break; 532 533 writel(cnt, nfc->regs + NFC_REG_CNT); 534 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD; 535 writel(tmp, nfc->regs + NFC_REG_CMD); 536 537 /* Arbitrary limit for polling mode */ 538 if (cnt < 64) 539 poll = true; 540 541 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0); 542 if (ret) 543 break; 544 545 if (buf) 546 memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE, 547 cnt); 548 offs += cnt; 549 } 550 } 551 552 static void sunxi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf, 553 int len) 554 { 555 struct nand_chip *nand = mtd_to_nand(mtd); 556 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 557 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 558 int ret; 559 int cnt; 560 int offs = 0; 561 u32 tmp; 562 563 while (len > offs) { 564 bool poll = false; 565 566 cnt = min(len - offs, NFC_SRAM_SIZE); 567 568 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 569 if (ret) 570 break; 571 572 writel(cnt, nfc->regs + NFC_REG_CNT); 573 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt); 574 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | 575 NFC_ACCESS_DIR; 576 writel(tmp, nfc->regs + NFC_REG_CMD); 577 578 /* Arbitrary limit for polling mode */ 579 if (cnt < 64) 580 poll = true; 581 582 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0); 583 if (ret) 584 break; 585 586 offs += cnt; 587 } 588 } 589 590 static uint8_t sunxi_nfc_read_byte(struct mtd_info *mtd) 591 { 592 uint8_t ret; 593 594 sunxi_nfc_read_buf(mtd, &ret, 1); 595 596 return ret; 597 } 598 599 static void sunxi_nfc_cmd_ctrl(struct mtd_info *mtd, int dat, 600 unsigned int ctrl) 601 { 602 struct nand_chip *nand = mtd_to_nand(mtd); 603 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 604 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 605 int ret; 606 607 if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) && 608 !(ctrl & (NAND_CLE | NAND_ALE))) { 609 u32 cmd = 0; 610 611 if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles) 612 return; 613 614 if (sunxi_nand->cmd_cycles--) 615 cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0]; 616 617 if (sunxi_nand->cmd_cycles--) { 618 cmd |= NFC_SEND_CMD2; 619 writel(sunxi_nand->cmd[1], 620 nfc->regs + NFC_REG_RCMD_SET); 621 } 622 623 sunxi_nand->cmd_cycles = 0; 624 625 if (sunxi_nand->addr_cycles) { 626 cmd |= NFC_SEND_ADR | 627 NFC_ADR_NUM(sunxi_nand->addr_cycles); 628 writel(sunxi_nand->addr[0], 629 nfc->regs + NFC_REG_ADDR_LOW); 630 } 631 632 if (sunxi_nand->addr_cycles > 4) 633 writel(sunxi_nand->addr[1], 634 nfc->regs + NFC_REG_ADDR_HIGH); 635 636 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 637 if (ret) 638 return; 639 640 writel(cmd, nfc->regs + NFC_REG_CMD); 641 sunxi_nand->addr[0] = 0; 642 sunxi_nand->addr[1] = 0; 643 sunxi_nand->addr_cycles = 0; 644 sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0); 645 } 646 647 if (ctrl & NAND_CLE) { 648 sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat; 649 } else if (ctrl & NAND_ALE) { 650 sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |= 651 dat << ((sunxi_nand->addr_cycles % 4) * 8); 652 sunxi_nand->addr_cycles++; 653 } 654 } 655 656 /* These seed values have been extracted from Allwinner's BSP */ 657 static const u16 sunxi_nfc_randomizer_page_seeds[] = { 658 0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72, 659 0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436, 660 0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d, 661 0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130, 662 0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56, 663 0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55, 664 0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb, 665 0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17, 666 0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62, 667 0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064, 668 0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126, 669 0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e, 670 0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3, 671 0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b, 672 0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d, 673 0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db, 674 }; 675 676 /* 677 * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds 678 * have been generated using 679 * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what 680 * the randomizer engine does internally before de/scrambling OOB data. 681 * 682 * Those tables are statically defined to avoid calculating randomizer state 683 * at runtime. 684 */ 685 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = { 686 0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64, 687 0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409, 688 0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617, 689 0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d, 690 0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91, 691 0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d, 692 0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab, 693 0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8, 694 0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8, 695 0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b, 696 0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5, 697 0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a, 698 0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891, 699 0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36, 700 0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd, 701 0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0, 702 }; 703 704 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = { 705 0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6, 706 0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982, 707 0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9, 708 0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07, 709 0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e, 710 0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2, 711 0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c, 712 0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f, 713 0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc, 714 0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e, 715 0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8, 716 0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68, 717 0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d, 718 0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179, 719 0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601, 720 0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd, 721 }; 722 723 static u16 sunxi_nfc_randomizer_step(u16 state, int count) 724 { 725 state &= 0x7fff; 726 727 /* 728 * This loop is just a simple implementation of a Fibonacci LFSR using 729 * the x16 + x15 + 1 polynomial. 730 */ 731 while (count--) 732 state = ((state >> 1) | 733 (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff; 734 735 return state; 736 } 737 738 static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc) 739 { 740 const u16 *seeds = sunxi_nfc_randomizer_page_seeds; 741 int mod = mtd_div_by_ws(mtd->erasesize, mtd); 742 743 if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds)) 744 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds); 745 746 if (ecc) { 747 if (mtd->ecc_step_size == 512) 748 seeds = sunxi_nfc_randomizer_ecc512_seeds; 749 else 750 seeds = sunxi_nfc_randomizer_ecc1024_seeds; 751 } 752 753 return seeds[page % mod]; 754 } 755 756 static void sunxi_nfc_randomizer_config(struct mtd_info *mtd, 757 int page, bool ecc) 758 { 759 struct nand_chip *nand = mtd_to_nand(mtd); 760 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 761 u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); 762 u16 state; 763 764 if (!(nand->options & NAND_NEED_SCRAMBLING)) 765 return; 766 767 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); 768 state = sunxi_nfc_randomizer_state(mtd, page, ecc); 769 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK; 770 writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL); 771 } 772 773 static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd) 774 { 775 struct nand_chip *nand = mtd_to_nand(mtd); 776 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 777 778 if (!(nand->options & NAND_NEED_SCRAMBLING)) 779 return; 780 781 writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN, 782 nfc->regs + NFC_REG_ECC_CTL); 783 } 784 785 static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd) 786 { 787 struct nand_chip *nand = mtd_to_nand(mtd); 788 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 789 790 if (!(nand->options & NAND_NEED_SCRAMBLING)) 791 return; 792 793 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN, 794 nfc->regs + NFC_REG_ECC_CTL); 795 } 796 797 static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm) 798 { 799 u16 state = sunxi_nfc_randomizer_state(mtd, page, true); 800 801 bbm[0] ^= state; 802 bbm[1] ^= sunxi_nfc_randomizer_step(state, 8); 803 } 804 805 static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd, 806 const uint8_t *buf, int len, 807 bool ecc, int page) 808 { 809 sunxi_nfc_randomizer_config(mtd, page, ecc); 810 sunxi_nfc_randomizer_enable(mtd); 811 sunxi_nfc_write_buf(mtd, buf, len); 812 sunxi_nfc_randomizer_disable(mtd); 813 } 814 815 static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf, 816 int len, bool ecc, int page) 817 { 818 sunxi_nfc_randomizer_config(mtd, page, ecc); 819 sunxi_nfc_randomizer_enable(mtd); 820 sunxi_nfc_read_buf(mtd, buf, len); 821 sunxi_nfc_randomizer_disable(mtd); 822 } 823 824 static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd) 825 { 826 struct nand_chip *nand = mtd_to_nand(mtd); 827 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 828 struct sunxi_nand_hw_ecc *data = nand->ecc.priv; 829 u32 ecc_ctl; 830 831 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); 832 ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE | 833 NFC_ECC_BLOCK_SIZE_MSK); 834 ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION | 835 NFC_ECC_PIPELINE; 836 837 if (nand->ecc.size == 512) 838 ecc_ctl |= NFC_ECC_BLOCK_512; 839 840 writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL); 841 } 842 843 static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd) 844 { 845 struct nand_chip *nand = mtd_to_nand(mtd); 846 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 847 848 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN, 849 nfc->regs + NFC_REG_ECC_CTL); 850 } 851 852 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf) 853 { 854 buf[0] = user_data; 855 buf[1] = user_data >> 8; 856 buf[2] = user_data >> 16; 857 buf[3] = user_data >> 24; 858 } 859 860 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf) 861 { 862 return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24); 863 } 864 865 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob, 866 int step, bool bbm, int page) 867 { 868 struct nand_chip *nand = mtd_to_nand(mtd); 869 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 870 871 sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)), 872 oob); 873 874 /* De-randomize the Bad Block Marker. */ 875 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) 876 sunxi_nfc_randomize_bbm(mtd, page, oob); 877 } 878 879 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd, 880 const u8 *oob, int step, 881 bool bbm, int page) 882 { 883 struct nand_chip *nand = mtd_to_nand(mtd); 884 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 885 u8 user_data[4]; 886 887 /* Randomize the Bad Block Marker. */ 888 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) { 889 memcpy(user_data, oob, sizeof(user_data)); 890 sunxi_nfc_randomize_bbm(mtd, page, user_data); 891 oob = user_data; 892 } 893 894 writel(sunxi_nfc_buf_to_user_data(oob), 895 nfc->regs + NFC_REG_USER_DATA(step)); 896 } 897 898 static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd, 899 unsigned int *max_bitflips, int ret) 900 { 901 if (ret < 0) { 902 mtd->ecc_stats.failed++; 903 } else { 904 mtd->ecc_stats.corrected += ret; 905 *max_bitflips = max_t(unsigned int, *max_bitflips, ret); 906 } 907 } 908 909 static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob, 910 int step, u32 status, bool *erased) 911 { 912 struct nand_chip *nand = mtd_to_nand(mtd); 913 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 914 struct nand_ecc_ctrl *ecc = &nand->ecc; 915 u32 tmp; 916 917 *erased = false; 918 919 if (status & NFC_ECC_ERR(step)) 920 return -EBADMSG; 921 922 if (status & NFC_ECC_PAT_FOUND(step)) { 923 u8 pattern; 924 925 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) { 926 pattern = 0x0; 927 } else { 928 pattern = 0xff; 929 *erased = true; 930 } 931 932 if (data) 933 memset(data, pattern, ecc->size); 934 935 if (oob) 936 memset(oob, pattern, ecc->bytes + 4); 937 938 return 0; 939 } 940 941 tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step)); 942 943 return NFC_ECC_ERR_CNT(step, tmp); 944 } 945 946 static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd, 947 u8 *data, int data_off, 948 u8 *oob, int oob_off, 949 int *cur_off, 950 unsigned int *max_bitflips, 951 bool bbm, bool oob_required, int page) 952 { 953 struct nand_chip *nand = mtd_to_nand(mtd); 954 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 955 struct nand_ecc_ctrl *ecc = &nand->ecc; 956 int raw_mode = 0; 957 bool erased; 958 int ret; 959 960 if (*cur_off != data_off) 961 nand_change_read_column_op(nand, data_off, NULL, 0, false); 962 963 sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page); 964 965 if (data_off + ecc->size != oob_off) 966 nand_change_read_column_op(nand, oob_off, NULL, 0, false); 967 968 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 969 if (ret) 970 return ret; 971 972 sunxi_nfc_randomizer_enable(mtd); 973 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP, 974 nfc->regs + NFC_REG_CMD); 975 976 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 977 sunxi_nfc_randomizer_disable(mtd); 978 if (ret) 979 return ret; 980 981 *cur_off = oob_off + ecc->bytes + 4; 982 983 ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0, 984 readl(nfc->regs + NFC_REG_ECC_ST), 985 &erased); 986 if (erased) 987 return 1; 988 989 if (ret < 0) { 990 /* 991 * Re-read the data with the randomizer disabled to identify 992 * bitflips in erased pages. 993 */ 994 if (nand->options & NAND_NEED_SCRAMBLING) 995 nand_change_read_column_op(nand, data_off, data, 996 ecc->size, false); 997 else 998 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, 999 ecc->size); 1000 1001 nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4, 1002 false); 1003 1004 ret = nand_check_erased_ecc_chunk(data, ecc->size, 1005 oob, ecc->bytes + 4, 1006 NULL, 0, ecc->strength); 1007 if (ret >= 0) 1008 raw_mode = 1; 1009 } else { 1010 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size); 1011 1012 if (oob_required) { 1013 nand_change_read_column_op(nand, oob_off, NULL, 0, 1014 false); 1015 sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4, 1016 true, page); 1017 1018 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0, 1019 bbm, page); 1020 } 1021 } 1022 1023 sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret); 1024 1025 return raw_mode; 1026 } 1027 1028 static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd, 1029 u8 *oob, int *cur_off, 1030 bool randomize, int page) 1031 { 1032 struct nand_chip *nand = mtd_to_nand(mtd); 1033 struct nand_ecc_ctrl *ecc = &nand->ecc; 1034 int offset = ((ecc->bytes + 4) * ecc->steps); 1035 int len = mtd->oobsize - offset; 1036 1037 if (len <= 0) 1038 return; 1039 1040 if (!cur_off || *cur_off != offset) 1041 nand_change_read_column_op(nand, mtd->writesize, NULL, 0, 1042 false); 1043 1044 if (!randomize) 1045 sunxi_nfc_read_buf(mtd, oob + offset, len); 1046 else 1047 sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len, 1048 false, page); 1049 1050 if (cur_off) 1051 *cur_off = mtd->oobsize + mtd->writesize; 1052 } 1053 1054 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf, 1055 int oob_required, int page, 1056 int nchunks) 1057 { 1058 struct nand_chip *nand = mtd_to_nand(mtd); 1059 bool randomized = nand->options & NAND_NEED_SCRAMBLING; 1060 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1061 struct nand_ecc_ctrl *ecc = &nand->ecc; 1062 unsigned int max_bitflips = 0; 1063 int ret, i, raw_mode = 0; 1064 struct scatterlist sg; 1065 u32 status; 1066 1067 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1068 if (ret) 1069 return ret; 1070 1071 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks, 1072 DMA_FROM_DEVICE, &sg); 1073 if (ret) 1074 return ret; 1075 1076 sunxi_nfc_hw_ecc_enable(mtd); 1077 sunxi_nfc_randomizer_config(mtd, page, false); 1078 sunxi_nfc_randomizer_enable(mtd); 1079 1080 writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) | 1081 NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET); 1082 1083 dma_async_issue_pending(nfc->dmac); 1084 1085 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS, 1086 nfc->regs + NFC_REG_CMD); 1087 1088 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 1089 if (ret) 1090 dmaengine_terminate_all(nfc->dmac); 1091 1092 sunxi_nfc_randomizer_disable(mtd); 1093 sunxi_nfc_hw_ecc_disable(mtd); 1094 1095 sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg); 1096 1097 if (ret) 1098 return ret; 1099 1100 status = readl(nfc->regs + NFC_REG_ECC_ST); 1101 1102 for (i = 0; i < nchunks; i++) { 1103 int data_off = i * ecc->size; 1104 int oob_off = i * (ecc->bytes + 4); 1105 u8 *data = buf + data_off; 1106 u8 *oob = nand->oob_poi + oob_off; 1107 bool erased; 1108 1109 ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL, 1110 oob_required ? oob : NULL, 1111 i, status, &erased); 1112 1113 /* ECC errors are handled in the second loop. */ 1114 if (ret < 0) 1115 continue; 1116 1117 if (oob_required && !erased) { 1118 /* TODO: use DMA to retrieve OOB */ 1119 nand_change_read_column_op(nand, 1120 mtd->writesize + oob_off, 1121 oob, ecc->bytes + 4, false); 1122 1123 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i, 1124 !i, page); 1125 } 1126 1127 if (erased) 1128 raw_mode = 1; 1129 1130 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret); 1131 } 1132 1133 if (status & NFC_ECC_ERR_MSK) { 1134 for (i = 0; i < nchunks; i++) { 1135 int data_off = i * ecc->size; 1136 int oob_off = i * (ecc->bytes + 4); 1137 u8 *data = buf + data_off; 1138 u8 *oob = nand->oob_poi + oob_off; 1139 1140 if (!(status & NFC_ECC_ERR(i))) 1141 continue; 1142 1143 /* 1144 * Re-read the data with the randomizer disabled to 1145 * identify bitflips in erased pages. 1146 * TODO: use DMA to read page in raw mode 1147 */ 1148 if (randomized) 1149 nand_change_read_column_op(nand, data_off, 1150 data, ecc->size, 1151 false); 1152 1153 /* TODO: use DMA to retrieve OOB */ 1154 nand_change_read_column_op(nand, 1155 mtd->writesize + oob_off, 1156 oob, ecc->bytes + 4, false); 1157 1158 ret = nand_check_erased_ecc_chunk(data, ecc->size, 1159 oob, ecc->bytes + 4, 1160 NULL, 0, 1161 ecc->strength); 1162 if (ret >= 0) 1163 raw_mode = 1; 1164 1165 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret); 1166 } 1167 } 1168 1169 if (oob_required) 1170 sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi, 1171 NULL, !raw_mode, 1172 page); 1173 1174 return max_bitflips; 1175 } 1176 1177 static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd, 1178 const u8 *data, int data_off, 1179 const u8 *oob, int oob_off, 1180 int *cur_off, bool bbm, 1181 int page) 1182 { 1183 struct nand_chip *nand = mtd_to_nand(mtd); 1184 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1185 struct nand_ecc_ctrl *ecc = &nand->ecc; 1186 int ret; 1187 1188 if (data_off != *cur_off) 1189 nand_change_write_column_op(nand, data_off, NULL, 0, false); 1190 1191 sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page); 1192 1193 if (data_off + ecc->size != oob_off) 1194 nand_change_write_column_op(nand, oob_off, NULL, 0, false); 1195 1196 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1197 if (ret) 1198 return ret; 1199 1200 sunxi_nfc_randomizer_enable(mtd); 1201 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page); 1202 1203 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | 1204 NFC_ACCESS_DIR | NFC_ECC_OP, 1205 nfc->regs + NFC_REG_CMD); 1206 1207 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 1208 sunxi_nfc_randomizer_disable(mtd); 1209 if (ret) 1210 return ret; 1211 1212 *cur_off = oob_off + ecc->bytes + 4; 1213 1214 return 0; 1215 } 1216 1217 static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd, 1218 u8 *oob, int *cur_off, 1219 int page) 1220 { 1221 struct nand_chip *nand = mtd_to_nand(mtd); 1222 struct nand_ecc_ctrl *ecc = &nand->ecc; 1223 int offset = ((ecc->bytes + 4) * ecc->steps); 1224 int len = mtd->oobsize - offset; 1225 1226 if (len <= 0) 1227 return; 1228 1229 if (!cur_off || *cur_off != offset) 1230 nand_change_write_column_op(nand, offset + mtd->writesize, 1231 NULL, 0, false); 1232 1233 sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page); 1234 1235 if (cur_off) 1236 *cur_off = mtd->oobsize + mtd->writesize; 1237 } 1238 1239 static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd, 1240 struct nand_chip *chip, uint8_t *buf, 1241 int oob_required, int page) 1242 { 1243 struct nand_ecc_ctrl *ecc = &chip->ecc; 1244 unsigned int max_bitflips = 0; 1245 int ret, i, cur_off = 0; 1246 bool raw_mode = false; 1247 1248 nand_read_page_op(chip, page, 0, NULL, 0); 1249 1250 sunxi_nfc_hw_ecc_enable(mtd); 1251 1252 for (i = 0; i < ecc->steps; i++) { 1253 int data_off = i * ecc->size; 1254 int oob_off = i * (ecc->bytes + 4); 1255 u8 *data = buf + data_off; 1256 u8 *oob = chip->oob_poi + oob_off; 1257 1258 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob, 1259 oob_off + mtd->writesize, 1260 &cur_off, &max_bitflips, 1261 !i, oob_required, page); 1262 if (ret < 0) 1263 return ret; 1264 else if (ret) 1265 raw_mode = true; 1266 } 1267 1268 if (oob_required) 1269 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off, 1270 !raw_mode, page); 1271 1272 sunxi_nfc_hw_ecc_disable(mtd); 1273 1274 return max_bitflips; 1275 } 1276 1277 static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd, 1278 struct nand_chip *chip, u8 *buf, 1279 int oob_required, int page) 1280 { 1281 int ret; 1282 1283 nand_read_page_op(chip, page, 0, NULL, 0); 1284 1285 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page, 1286 chip->ecc.steps); 1287 if (ret >= 0) 1288 return ret; 1289 1290 /* Fallback to PIO mode */ 1291 return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page); 1292 } 1293 1294 static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd, 1295 struct nand_chip *chip, 1296 u32 data_offs, u32 readlen, 1297 u8 *bufpoi, int page) 1298 { 1299 struct nand_ecc_ctrl *ecc = &chip->ecc; 1300 int ret, i, cur_off = 0; 1301 unsigned int max_bitflips = 0; 1302 1303 nand_read_page_op(chip, page, 0, NULL, 0); 1304 1305 sunxi_nfc_hw_ecc_enable(mtd); 1306 1307 for (i = data_offs / ecc->size; 1308 i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) { 1309 int data_off = i * ecc->size; 1310 int oob_off = i * (ecc->bytes + 4); 1311 u8 *data = bufpoi + data_off; 1312 u8 *oob = chip->oob_poi + oob_off; 1313 1314 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, 1315 oob, 1316 oob_off + mtd->writesize, 1317 &cur_off, &max_bitflips, !i, 1318 false, page); 1319 if (ret < 0) 1320 return ret; 1321 } 1322 1323 sunxi_nfc_hw_ecc_disable(mtd); 1324 1325 return max_bitflips; 1326 } 1327 1328 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd, 1329 struct nand_chip *chip, 1330 u32 data_offs, u32 readlen, 1331 u8 *buf, int page) 1332 { 1333 int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size); 1334 int ret; 1335 1336 nand_read_page_op(chip, page, 0, NULL, 0); 1337 1338 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks); 1339 if (ret >= 0) 1340 return ret; 1341 1342 /* Fallback to PIO mode */ 1343 return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen, 1344 buf, page); 1345 } 1346 1347 static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd, 1348 struct nand_chip *chip, 1349 const uint8_t *buf, int oob_required, 1350 int page) 1351 { 1352 struct nand_ecc_ctrl *ecc = &chip->ecc; 1353 int ret, i, cur_off = 0; 1354 1355 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1356 1357 sunxi_nfc_hw_ecc_enable(mtd); 1358 1359 for (i = 0; i < ecc->steps; i++) { 1360 int data_off = i * ecc->size; 1361 int oob_off = i * (ecc->bytes + 4); 1362 const u8 *data = buf + data_off; 1363 const u8 *oob = chip->oob_poi + oob_off; 1364 1365 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob, 1366 oob_off + mtd->writesize, 1367 &cur_off, !i, page); 1368 if (ret) 1369 return ret; 1370 } 1371 1372 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING)) 1373 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi, 1374 &cur_off, page); 1375 1376 sunxi_nfc_hw_ecc_disable(mtd); 1377 1378 return nand_prog_page_end_op(chip); 1379 } 1380 1381 static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd, 1382 struct nand_chip *chip, 1383 u32 data_offs, u32 data_len, 1384 const u8 *buf, int oob_required, 1385 int page) 1386 { 1387 struct nand_ecc_ctrl *ecc = &chip->ecc; 1388 int ret, i, cur_off = 0; 1389 1390 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1391 1392 sunxi_nfc_hw_ecc_enable(mtd); 1393 1394 for (i = data_offs / ecc->size; 1395 i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) { 1396 int data_off = i * ecc->size; 1397 int oob_off = i * (ecc->bytes + 4); 1398 const u8 *data = buf + data_off; 1399 const u8 *oob = chip->oob_poi + oob_off; 1400 1401 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob, 1402 oob_off + mtd->writesize, 1403 &cur_off, !i, page); 1404 if (ret) 1405 return ret; 1406 } 1407 1408 sunxi_nfc_hw_ecc_disable(mtd); 1409 1410 return nand_prog_page_end_op(chip); 1411 } 1412 1413 static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd, 1414 struct nand_chip *chip, 1415 const u8 *buf, 1416 int oob_required, 1417 int page) 1418 { 1419 struct nand_chip *nand = mtd_to_nand(mtd); 1420 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1421 struct nand_ecc_ctrl *ecc = &nand->ecc; 1422 struct scatterlist sg; 1423 int ret, i; 1424 1425 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1426 if (ret) 1427 return ret; 1428 1429 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps, 1430 DMA_TO_DEVICE, &sg); 1431 if (ret) 1432 goto pio_fallback; 1433 1434 for (i = 0; i < ecc->steps; i++) { 1435 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4)); 1436 1437 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page); 1438 } 1439 1440 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1441 1442 sunxi_nfc_hw_ecc_enable(mtd); 1443 sunxi_nfc_randomizer_config(mtd, page, false); 1444 sunxi_nfc_randomizer_enable(mtd); 1445 1446 writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG, 1447 nfc->regs + NFC_REG_RCMD_SET); 1448 1449 dma_async_issue_pending(nfc->dmac); 1450 1451 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | 1452 NFC_DATA_TRANS | NFC_ACCESS_DIR, 1453 nfc->regs + NFC_REG_CMD); 1454 1455 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 1456 if (ret) 1457 dmaengine_terminate_all(nfc->dmac); 1458 1459 sunxi_nfc_randomizer_disable(mtd); 1460 sunxi_nfc_hw_ecc_disable(mtd); 1461 1462 sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg); 1463 1464 if (ret) 1465 return ret; 1466 1467 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING)) 1468 /* TODO: use DMA to transfer extra OOB bytes ? */ 1469 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi, 1470 NULL, page); 1471 1472 return nand_prog_page_end_op(chip); 1473 1474 pio_fallback: 1475 return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page); 1476 } 1477 1478 static int sunxi_nfc_hw_ecc_read_oob(struct mtd_info *mtd, 1479 struct nand_chip *chip, 1480 int page) 1481 { 1482 chip->pagebuf = -1; 1483 1484 return chip->ecc.read_page(mtd, chip, chip->data_buf, 1, page); 1485 } 1486 1487 static int sunxi_nfc_hw_ecc_write_oob(struct mtd_info *mtd, 1488 struct nand_chip *chip, 1489 int page) 1490 { 1491 int ret; 1492 1493 chip->pagebuf = -1; 1494 1495 memset(chip->data_buf, 0xff, mtd->writesize); 1496 ret = chip->ecc.write_page(mtd, chip, chip->data_buf, 1, page); 1497 if (ret) 1498 return ret; 1499 1500 /* Send command to program the OOB data */ 1501 return nand_prog_page_end_op(chip); 1502 } 1503 1504 static const s32 tWB_lut[] = {6, 12, 16, 20}; 1505 static const s32 tRHW_lut[] = {4, 8, 12, 20}; 1506 1507 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration, 1508 u32 clk_period) 1509 { 1510 u32 clk_cycles = DIV_ROUND_UP(duration, clk_period); 1511 int i; 1512 1513 for (i = 0; i < lut_size; i++) { 1514 if (clk_cycles <= lut[i]) 1515 return i; 1516 } 1517 1518 /* Doesn't fit */ 1519 return -EINVAL; 1520 } 1521 1522 #define sunxi_nand_lookup_timing(l, p, c) \ 1523 _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c) 1524 1525 static int sunxi_nfc_setup_data_interface(struct mtd_info *mtd, int csline, 1526 const struct nand_data_interface *conf) 1527 { 1528 struct nand_chip *nand = mtd_to_nand(mtd); 1529 struct sunxi_nand_chip *chip = to_sunxi_nand(nand); 1530 struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller); 1531 const struct nand_sdr_timings *timings; 1532 u32 min_clk_period = 0; 1533 s32 tWB, tADL, tWHR, tRHW, tCAD; 1534 long real_clk_rate; 1535 1536 timings = nand_get_sdr_timings(conf); 1537 if (IS_ERR(timings)) 1538 return -ENOTSUPP; 1539 1540 /* T1 <=> tCLS */ 1541 if (timings->tCLS_min > min_clk_period) 1542 min_clk_period = timings->tCLS_min; 1543 1544 /* T2 <=> tCLH */ 1545 if (timings->tCLH_min > min_clk_period) 1546 min_clk_period = timings->tCLH_min; 1547 1548 /* T3 <=> tCS */ 1549 if (timings->tCS_min > min_clk_period) 1550 min_clk_period = timings->tCS_min; 1551 1552 /* T4 <=> tCH */ 1553 if (timings->tCH_min > min_clk_period) 1554 min_clk_period = timings->tCH_min; 1555 1556 /* T5 <=> tWP */ 1557 if (timings->tWP_min > min_clk_period) 1558 min_clk_period = timings->tWP_min; 1559 1560 /* T6 <=> tWH */ 1561 if (timings->tWH_min > min_clk_period) 1562 min_clk_period = timings->tWH_min; 1563 1564 /* T7 <=> tALS */ 1565 if (timings->tALS_min > min_clk_period) 1566 min_clk_period = timings->tALS_min; 1567 1568 /* T8 <=> tDS */ 1569 if (timings->tDS_min > min_clk_period) 1570 min_clk_period = timings->tDS_min; 1571 1572 /* T9 <=> tDH */ 1573 if (timings->tDH_min > min_clk_period) 1574 min_clk_period = timings->tDH_min; 1575 1576 /* T10 <=> tRR */ 1577 if (timings->tRR_min > (min_clk_period * 3)) 1578 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3); 1579 1580 /* T11 <=> tALH */ 1581 if (timings->tALH_min > min_clk_period) 1582 min_clk_period = timings->tALH_min; 1583 1584 /* T12 <=> tRP */ 1585 if (timings->tRP_min > min_clk_period) 1586 min_clk_period = timings->tRP_min; 1587 1588 /* T13 <=> tREH */ 1589 if (timings->tREH_min > min_clk_period) 1590 min_clk_period = timings->tREH_min; 1591 1592 /* T14 <=> tRC */ 1593 if (timings->tRC_min > (min_clk_period * 2)) 1594 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2); 1595 1596 /* T15 <=> tWC */ 1597 if (timings->tWC_min > (min_clk_period * 2)) 1598 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2); 1599 1600 /* T16 - T19 + tCAD */ 1601 if (timings->tWB_max > (min_clk_period * 20)) 1602 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20); 1603 1604 if (timings->tADL_min > (min_clk_period * 32)) 1605 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32); 1606 1607 if (timings->tWHR_min > (min_clk_period * 32)) 1608 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32); 1609 1610 if (timings->tRHW_min > (min_clk_period * 20)) 1611 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20); 1612 1613 tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max, 1614 min_clk_period); 1615 if (tWB < 0) { 1616 dev_err(nfc->dev, "unsupported tWB\n"); 1617 return tWB; 1618 } 1619 1620 tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3; 1621 if (tADL > 3) { 1622 dev_err(nfc->dev, "unsupported tADL\n"); 1623 return -EINVAL; 1624 } 1625 1626 tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3; 1627 if (tWHR > 3) { 1628 dev_err(nfc->dev, "unsupported tWHR\n"); 1629 return -EINVAL; 1630 } 1631 1632 tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min, 1633 min_clk_period); 1634 if (tRHW < 0) { 1635 dev_err(nfc->dev, "unsupported tRHW\n"); 1636 return tRHW; 1637 } 1638 1639 if (csline == NAND_DATA_IFACE_CHECK_ONLY) 1640 return 0; 1641 1642 /* 1643 * TODO: according to ONFI specs this value only applies for DDR NAND, 1644 * but Allwinner seems to set this to 0x7. Mimic them for now. 1645 */ 1646 tCAD = 0x7; 1647 1648 /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */ 1649 chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD); 1650 1651 /* Convert min_clk_period from picoseconds to nanoseconds */ 1652 min_clk_period = DIV_ROUND_UP(min_clk_period, 1000); 1653 1654 /* 1655 * Unlike what is stated in Allwinner datasheet, the clk_rate should 1656 * be set to (1 / min_clk_period), and not (2 / min_clk_period). 1657 * This new formula was verified with a scope and validated by 1658 * Allwinner engineers. 1659 */ 1660 chip->clk_rate = NSEC_PER_SEC / min_clk_period; 1661 real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate); 1662 if (real_clk_rate <= 0) { 1663 dev_err(nfc->dev, "Unable to round clk %lu\n", chip->clk_rate); 1664 return -EINVAL; 1665 } 1666 1667 /* 1668 * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data 1669 * output cycle timings shall be used if the host drives tRC less than 1670 * 30 ns. 1671 */ 1672 min_clk_period = NSEC_PER_SEC / real_clk_rate; 1673 chip->timing_ctl = ((min_clk_period * 2) < 30) ? 1674 NFC_TIMING_CTL_EDO : 0; 1675 1676 return 0; 1677 } 1678 1679 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section, 1680 struct mtd_oob_region *oobregion) 1681 { 1682 struct nand_chip *nand = mtd_to_nand(mtd); 1683 struct nand_ecc_ctrl *ecc = &nand->ecc; 1684 1685 if (section >= ecc->steps) 1686 return -ERANGE; 1687 1688 oobregion->offset = section * (ecc->bytes + 4) + 4; 1689 oobregion->length = ecc->bytes; 1690 1691 return 0; 1692 } 1693 1694 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section, 1695 struct mtd_oob_region *oobregion) 1696 { 1697 struct nand_chip *nand = mtd_to_nand(mtd); 1698 struct nand_ecc_ctrl *ecc = &nand->ecc; 1699 1700 if (section > ecc->steps) 1701 return -ERANGE; 1702 1703 /* 1704 * The first 2 bytes are used for BB markers, hence we 1705 * only have 2 bytes available in the first user data 1706 * section. 1707 */ 1708 if (!section && ecc->mode == NAND_ECC_HW) { 1709 oobregion->offset = 2; 1710 oobregion->length = 2; 1711 1712 return 0; 1713 } 1714 1715 oobregion->offset = section * (ecc->bytes + 4); 1716 1717 if (section < ecc->steps) 1718 oobregion->length = 4; 1719 else 1720 oobregion->offset = mtd->oobsize - oobregion->offset; 1721 1722 return 0; 1723 } 1724 1725 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = { 1726 .ecc = sunxi_nand_ooblayout_ecc, 1727 .free = sunxi_nand_ooblayout_free, 1728 }; 1729 1730 static void sunxi_nand_hw_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc) 1731 { 1732 kfree(ecc->priv); 1733 } 1734 1735 static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd, 1736 struct nand_ecc_ctrl *ecc, 1737 struct device_node *np) 1738 { 1739 static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 }; 1740 struct nand_chip *nand = mtd_to_nand(mtd); 1741 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 1742 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 1743 struct sunxi_nand_hw_ecc *data; 1744 int nsectors; 1745 int ret; 1746 int i; 1747 1748 if (ecc->options & NAND_ECC_MAXIMIZE) { 1749 int bytes; 1750 1751 ecc->size = 1024; 1752 nsectors = mtd->writesize / ecc->size; 1753 1754 /* Reserve 2 bytes for the BBM */ 1755 bytes = (mtd->oobsize - 2) / nsectors; 1756 1757 /* 4 non-ECC bytes are added before each ECC bytes section */ 1758 bytes -= 4; 1759 1760 /* and bytes has to be even. */ 1761 if (bytes % 2) 1762 bytes--; 1763 1764 ecc->strength = bytes * 8 / fls(8 * ecc->size); 1765 1766 for (i = 0; i < ARRAY_SIZE(strengths); i++) { 1767 if (strengths[i] > ecc->strength) 1768 break; 1769 } 1770 1771 if (!i) 1772 ecc->strength = 0; 1773 else 1774 ecc->strength = strengths[i - 1]; 1775 } 1776 1777 if (ecc->size != 512 && ecc->size != 1024) 1778 return -EINVAL; 1779 1780 data = kzalloc(sizeof(*data), GFP_KERNEL); 1781 if (!data) 1782 return -ENOMEM; 1783 1784 /* Prefer 1k ECC chunk over 512 ones */ 1785 if (ecc->size == 512 && mtd->writesize > 512) { 1786 ecc->size = 1024; 1787 ecc->strength *= 2; 1788 } 1789 1790 /* Add ECC info retrieval from DT */ 1791 for (i = 0; i < ARRAY_SIZE(strengths); i++) { 1792 if (ecc->strength <= strengths[i]) { 1793 /* 1794 * Update ecc->strength value with the actual strength 1795 * that will be used by the ECC engine. 1796 */ 1797 ecc->strength = strengths[i]; 1798 break; 1799 } 1800 } 1801 1802 if (i >= ARRAY_SIZE(strengths)) { 1803 dev_err(nfc->dev, "unsupported strength\n"); 1804 ret = -ENOTSUPP; 1805 goto err; 1806 } 1807 1808 data->mode = i; 1809 1810 /* HW ECC always request ECC bytes for 1024 bytes blocks */ 1811 ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8); 1812 1813 /* HW ECC always work with even numbers of ECC bytes */ 1814 ecc->bytes = ALIGN(ecc->bytes, 2); 1815 1816 nsectors = mtd->writesize / ecc->size; 1817 1818 if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) { 1819 ret = -EINVAL; 1820 goto err; 1821 } 1822 1823 ecc->read_oob = sunxi_nfc_hw_ecc_read_oob; 1824 ecc->write_oob = sunxi_nfc_hw_ecc_write_oob; 1825 mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops); 1826 ecc->priv = data; 1827 1828 if (nfc->dmac) { 1829 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma; 1830 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma; 1831 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma; 1832 nand->options |= NAND_USE_BOUNCE_BUFFER; 1833 } else { 1834 ecc->read_page = sunxi_nfc_hw_ecc_read_page; 1835 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage; 1836 ecc->write_page = sunxi_nfc_hw_ecc_write_page; 1837 } 1838 1839 /* TODO: support DMA for raw accesses and subpage write */ 1840 ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage; 1841 ecc->read_oob_raw = nand_read_oob_std; 1842 ecc->write_oob_raw = nand_write_oob_std; 1843 1844 return 0; 1845 1846 err: 1847 kfree(data); 1848 1849 return ret; 1850 } 1851 1852 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc) 1853 { 1854 switch (ecc->mode) { 1855 case NAND_ECC_HW: 1856 sunxi_nand_hw_ecc_ctrl_cleanup(ecc); 1857 break; 1858 case NAND_ECC_NONE: 1859 default: 1860 break; 1861 } 1862 } 1863 1864 static int sunxi_nand_ecc_init(struct mtd_info *mtd, struct nand_ecc_ctrl *ecc, 1865 struct device_node *np) 1866 { 1867 struct nand_chip *nand = mtd_to_nand(mtd); 1868 int ret; 1869 1870 if (!ecc->size) { 1871 ecc->size = nand->ecc_step_ds; 1872 ecc->strength = nand->ecc_strength_ds; 1873 } 1874 1875 if (!ecc->size || !ecc->strength) 1876 return -EINVAL; 1877 1878 switch (ecc->mode) { 1879 case NAND_ECC_HW: 1880 ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np); 1881 if (ret) 1882 return ret; 1883 break; 1884 case NAND_ECC_NONE: 1885 case NAND_ECC_SOFT: 1886 break; 1887 default: 1888 return -EINVAL; 1889 } 1890 1891 return 0; 1892 } 1893 1894 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc, 1895 struct device_node *np) 1896 { 1897 struct sunxi_nand_chip *chip; 1898 struct mtd_info *mtd; 1899 struct nand_chip *nand; 1900 int nsels; 1901 int ret; 1902 int i; 1903 u32 tmp; 1904 1905 if (!of_get_property(np, "reg", &nsels)) 1906 return -EINVAL; 1907 1908 nsels /= sizeof(u32); 1909 if (!nsels) { 1910 dev_err(dev, "invalid reg property size\n"); 1911 return -EINVAL; 1912 } 1913 1914 chip = devm_kzalloc(dev, 1915 sizeof(*chip) + 1916 (nsels * sizeof(struct sunxi_nand_chip_sel)), 1917 GFP_KERNEL); 1918 if (!chip) { 1919 dev_err(dev, "could not allocate chip\n"); 1920 return -ENOMEM; 1921 } 1922 1923 chip->nsels = nsels; 1924 chip->selected = -1; 1925 1926 for (i = 0; i < nsels; i++) { 1927 ret = of_property_read_u32_index(np, "reg", i, &tmp); 1928 if (ret) { 1929 dev_err(dev, "could not retrieve reg property: %d\n", 1930 ret); 1931 return ret; 1932 } 1933 1934 if (tmp > NFC_MAX_CS) { 1935 dev_err(dev, 1936 "invalid reg value: %u (max CS = 7)\n", 1937 tmp); 1938 return -EINVAL; 1939 } 1940 1941 if (test_and_set_bit(tmp, &nfc->assigned_cs)) { 1942 dev_err(dev, "CS %d already assigned\n", tmp); 1943 return -EINVAL; 1944 } 1945 1946 chip->sels[i].cs = tmp; 1947 1948 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) && 1949 tmp < 2) { 1950 chip->sels[i].rb.type = RB_NATIVE; 1951 chip->sels[i].rb.info.nativeid = tmp; 1952 } else { 1953 ret = of_get_named_gpio(np, "rb-gpios", i); 1954 if (ret >= 0) { 1955 tmp = ret; 1956 chip->sels[i].rb.type = RB_GPIO; 1957 chip->sels[i].rb.info.gpio = tmp; 1958 ret = devm_gpio_request(dev, tmp, "nand-rb"); 1959 if (ret) 1960 return ret; 1961 1962 ret = gpio_direction_input(tmp); 1963 if (ret) 1964 return ret; 1965 } else { 1966 chip->sels[i].rb.type = RB_NONE; 1967 } 1968 } 1969 } 1970 1971 nand = &chip->nand; 1972 /* Default tR value specified in the ONFI spec (chapter 4.15.1) */ 1973 nand->chip_delay = 200; 1974 nand->controller = &nfc->controller; 1975 /* 1976 * Set the ECC mode to the default value in case nothing is specified 1977 * in the DT. 1978 */ 1979 nand->ecc.mode = NAND_ECC_HW; 1980 nand_set_flash_node(nand, np); 1981 nand->select_chip = sunxi_nfc_select_chip; 1982 nand->cmd_ctrl = sunxi_nfc_cmd_ctrl; 1983 nand->read_buf = sunxi_nfc_read_buf; 1984 nand->write_buf = sunxi_nfc_write_buf; 1985 nand->read_byte = sunxi_nfc_read_byte; 1986 nand->setup_data_interface = sunxi_nfc_setup_data_interface; 1987 1988 mtd = nand_to_mtd(nand); 1989 mtd->dev.parent = dev; 1990 1991 ret = nand_scan_ident(mtd, nsels, NULL); 1992 if (ret) 1993 return ret; 1994 1995 if (nand->bbt_options & NAND_BBT_USE_FLASH) 1996 nand->bbt_options |= NAND_BBT_NO_OOB; 1997 1998 if (nand->options & NAND_NEED_SCRAMBLING) 1999 nand->options |= NAND_NO_SUBPAGE_WRITE; 2000 2001 nand->options |= NAND_SUBPAGE_READ; 2002 2003 ret = sunxi_nand_ecc_init(mtd, &nand->ecc, np); 2004 if (ret) { 2005 dev_err(dev, "ECC init failed: %d\n", ret); 2006 return ret; 2007 } 2008 2009 ret = nand_scan_tail(mtd); 2010 if (ret) { 2011 dev_err(dev, "nand_scan_tail failed: %d\n", ret); 2012 return ret; 2013 } 2014 2015 ret = mtd_device_register(mtd, NULL, 0); 2016 if (ret) { 2017 dev_err(dev, "failed to register mtd device: %d\n", ret); 2018 nand_release(mtd); 2019 return ret; 2020 } 2021 2022 list_add_tail(&chip->node, &nfc->chips); 2023 2024 return 0; 2025 } 2026 2027 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc) 2028 { 2029 struct device_node *np = dev->of_node; 2030 struct device_node *nand_np; 2031 int nchips = of_get_child_count(np); 2032 int ret; 2033 2034 if (nchips > 8) { 2035 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips); 2036 return -EINVAL; 2037 } 2038 2039 for_each_child_of_node(np, nand_np) { 2040 ret = sunxi_nand_chip_init(dev, nfc, nand_np); 2041 if (ret) { 2042 of_node_put(nand_np); 2043 return ret; 2044 } 2045 } 2046 2047 return 0; 2048 } 2049 2050 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc) 2051 { 2052 struct sunxi_nand_chip *chip; 2053 2054 while (!list_empty(&nfc->chips)) { 2055 chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip, 2056 node); 2057 nand_release(nand_to_mtd(&chip->nand)); 2058 sunxi_nand_ecc_cleanup(&chip->nand.ecc); 2059 list_del(&chip->node); 2060 } 2061 } 2062 2063 static int sunxi_nfc_probe(struct platform_device *pdev) 2064 { 2065 struct device *dev = &pdev->dev; 2066 struct resource *r; 2067 struct sunxi_nfc *nfc; 2068 int irq; 2069 int ret; 2070 2071 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL); 2072 if (!nfc) 2073 return -ENOMEM; 2074 2075 nfc->dev = dev; 2076 nand_hw_control_init(&nfc->controller); 2077 INIT_LIST_HEAD(&nfc->chips); 2078 2079 r = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2080 nfc->regs = devm_ioremap_resource(dev, r); 2081 if (IS_ERR(nfc->regs)) 2082 return PTR_ERR(nfc->regs); 2083 2084 irq = platform_get_irq(pdev, 0); 2085 if (irq < 0) { 2086 dev_err(dev, "failed to retrieve irq\n"); 2087 return irq; 2088 } 2089 2090 nfc->ahb_clk = devm_clk_get(dev, "ahb"); 2091 if (IS_ERR(nfc->ahb_clk)) { 2092 dev_err(dev, "failed to retrieve ahb clk\n"); 2093 return PTR_ERR(nfc->ahb_clk); 2094 } 2095 2096 ret = clk_prepare_enable(nfc->ahb_clk); 2097 if (ret) 2098 return ret; 2099 2100 nfc->mod_clk = devm_clk_get(dev, "mod"); 2101 if (IS_ERR(nfc->mod_clk)) { 2102 dev_err(dev, "failed to retrieve mod clk\n"); 2103 ret = PTR_ERR(nfc->mod_clk); 2104 goto out_ahb_clk_unprepare; 2105 } 2106 2107 ret = clk_prepare_enable(nfc->mod_clk); 2108 if (ret) 2109 goto out_ahb_clk_unprepare; 2110 2111 nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb"); 2112 if (IS_ERR(nfc->reset)) { 2113 ret = PTR_ERR(nfc->reset); 2114 goto out_mod_clk_unprepare; 2115 } 2116 2117 ret = reset_control_deassert(nfc->reset); 2118 if (ret) { 2119 dev_err(dev, "reset err %d\n", ret); 2120 goto out_mod_clk_unprepare; 2121 } 2122 2123 ret = sunxi_nfc_rst(nfc); 2124 if (ret) 2125 goto out_ahb_reset_reassert; 2126 2127 writel(0, nfc->regs + NFC_REG_INT); 2128 ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt, 2129 0, "sunxi-nand", nfc); 2130 if (ret) 2131 goto out_ahb_reset_reassert; 2132 2133 nfc->dmac = dma_request_slave_channel(dev, "rxtx"); 2134 if (nfc->dmac) { 2135 struct dma_slave_config dmac_cfg = { }; 2136 2137 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA; 2138 dmac_cfg.dst_addr = dmac_cfg.src_addr; 2139 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 2140 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width; 2141 dmac_cfg.src_maxburst = 4; 2142 dmac_cfg.dst_maxburst = 4; 2143 dmaengine_slave_config(nfc->dmac, &dmac_cfg); 2144 } else { 2145 dev_warn(dev, "failed to request rxtx DMA channel\n"); 2146 } 2147 2148 platform_set_drvdata(pdev, nfc); 2149 2150 ret = sunxi_nand_chips_init(dev, nfc); 2151 if (ret) { 2152 dev_err(dev, "failed to init nand chips\n"); 2153 goto out_release_dmac; 2154 } 2155 2156 return 0; 2157 2158 out_release_dmac: 2159 if (nfc->dmac) 2160 dma_release_channel(nfc->dmac); 2161 out_ahb_reset_reassert: 2162 reset_control_assert(nfc->reset); 2163 out_mod_clk_unprepare: 2164 clk_disable_unprepare(nfc->mod_clk); 2165 out_ahb_clk_unprepare: 2166 clk_disable_unprepare(nfc->ahb_clk); 2167 2168 return ret; 2169 } 2170 2171 static int sunxi_nfc_remove(struct platform_device *pdev) 2172 { 2173 struct sunxi_nfc *nfc = platform_get_drvdata(pdev); 2174 2175 sunxi_nand_chips_cleanup(nfc); 2176 2177 reset_control_assert(nfc->reset); 2178 2179 if (nfc->dmac) 2180 dma_release_channel(nfc->dmac); 2181 clk_disable_unprepare(nfc->mod_clk); 2182 clk_disable_unprepare(nfc->ahb_clk); 2183 2184 return 0; 2185 } 2186 2187 static const struct of_device_id sunxi_nfc_ids[] = { 2188 { .compatible = "allwinner,sun4i-a10-nand" }, 2189 { /* sentinel */ } 2190 }; 2191 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids); 2192 2193 static struct platform_driver sunxi_nfc_driver = { 2194 .driver = { 2195 .name = "sunxi_nand", 2196 .of_match_table = sunxi_nfc_ids, 2197 }, 2198 .probe = sunxi_nfc_probe, 2199 .remove = sunxi_nfc_remove, 2200 }; 2201 module_platform_driver(sunxi_nfc_driver); 2202 2203 MODULE_LICENSE("GPL v2"); 2204 MODULE_AUTHOR("Boris BREZILLON"); 2205 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver"); 2206 MODULE_ALIAS("platform:sunxi_nand"); 2207