1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com> 4 * 5 * Derived from: 6 * https://github.com/yuq/sunxi-nfc-mtd 7 * Copyright (C) 2013 Qiang Yu <yuq825@gmail.com> 8 * 9 * https://github.com/hno/Allwinner-Info 10 * Copyright (C) 2013 Henrik Nordström <Henrik Nordström> 11 * 12 * Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com> 13 * Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org> 14 */ 15 16 #include <linux/dma-mapping.h> 17 #include <linux/slab.h> 18 #include <linux/module.h> 19 #include <linux/moduleparam.h> 20 #include <linux/platform_device.h> 21 #include <linux/of.h> 22 #include <linux/of_device.h> 23 #include <linux/mtd/mtd.h> 24 #include <linux/mtd/rawnand.h> 25 #include <linux/mtd/partitions.h> 26 #include <linux/clk.h> 27 #include <linux/delay.h> 28 #include <linux/dmaengine.h> 29 #include <linux/interrupt.h> 30 #include <linux/iopoll.h> 31 #include <linux/reset.h> 32 33 #define NFC_REG_CTL 0x0000 34 #define NFC_REG_ST 0x0004 35 #define NFC_REG_INT 0x0008 36 #define NFC_REG_TIMING_CTL 0x000C 37 #define NFC_REG_TIMING_CFG 0x0010 38 #define NFC_REG_ADDR_LOW 0x0014 39 #define NFC_REG_ADDR_HIGH 0x0018 40 #define NFC_REG_SECTOR_NUM 0x001C 41 #define NFC_REG_CNT 0x0020 42 #define NFC_REG_CMD 0x0024 43 #define NFC_REG_RCMD_SET 0x0028 44 #define NFC_REG_WCMD_SET 0x002C 45 #define NFC_REG_A10_IO_DATA 0x0030 46 #define NFC_REG_A23_IO_DATA 0x0300 47 #define NFC_REG_ECC_CTL 0x0034 48 #define NFC_REG_ECC_ST 0x0038 49 #define NFC_REG_DEBUG 0x003C 50 #define NFC_REG_ECC_ERR_CNT(x) ((0x0040 + (x)) & ~0x3) 51 #define NFC_REG_USER_DATA(x) (0x0050 + ((x) * 4)) 52 #define NFC_REG_SPARE_AREA 0x00A0 53 #define NFC_REG_PAT_ID 0x00A4 54 #define NFC_REG_MDMA_ADDR 0x00C0 55 #define NFC_REG_MDMA_CNT 0x00C4 56 #define NFC_RAM0_BASE 0x0400 57 #define NFC_RAM1_BASE 0x0800 58 59 /* define bit use in NFC_CTL */ 60 #define NFC_EN BIT(0) 61 #define NFC_RESET BIT(1) 62 #define NFC_BUS_WIDTH_MSK BIT(2) 63 #define NFC_BUS_WIDTH_8 (0 << 2) 64 #define NFC_BUS_WIDTH_16 (1 << 2) 65 #define NFC_RB_SEL_MSK BIT(3) 66 #define NFC_RB_SEL(x) ((x) << 3) 67 #define NFC_CE_SEL_MSK GENMASK(26, 24) 68 #define NFC_CE_SEL(x) ((x) << 24) 69 #define NFC_CE_CTL BIT(6) 70 #define NFC_PAGE_SHIFT_MSK GENMASK(11, 8) 71 #define NFC_PAGE_SHIFT(x) (((x) < 10 ? 0 : (x) - 10) << 8) 72 #define NFC_SAM BIT(12) 73 #define NFC_RAM_METHOD BIT(14) 74 #define NFC_DMA_TYPE_NORMAL BIT(15) 75 #define NFC_DEBUG_CTL BIT(31) 76 77 /* define bit use in NFC_ST */ 78 #define NFC_RB_B2R BIT(0) 79 #define NFC_CMD_INT_FLAG BIT(1) 80 #define NFC_DMA_INT_FLAG BIT(2) 81 #define NFC_CMD_FIFO_STATUS BIT(3) 82 #define NFC_STA BIT(4) 83 #define NFC_NATCH_INT_FLAG BIT(5) 84 #define NFC_RB_STATE(x) BIT(x + 8) 85 86 /* define bit use in NFC_INT */ 87 #define NFC_B2R_INT_ENABLE BIT(0) 88 #define NFC_CMD_INT_ENABLE BIT(1) 89 #define NFC_DMA_INT_ENABLE BIT(2) 90 #define NFC_INT_MASK (NFC_B2R_INT_ENABLE | \ 91 NFC_CMD_INT_ENABLE | \ 92 NFC_DMA_INT_ENABLE) 93 94 /* define bit use in NFC_TIMING_CTL */ 95 #define NFC_TIMING_CTL_EDO BIT(8) 96 97 /* define NFC_TIMING_CFG register layout */ 98 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD) \ 99 (((tWB) & 0x3) | (((tADL) & 0x3) << 2) | \ 100 (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) | \ 101 (((tCAD) & 0x7) << 8)) 102 103 /* define bit use in NFC_CMD */ 104 #define NFC_CMD_LOW_BYTE_MSK GENMASK(7, 0) 105 #define NFC_CMD_HIGH_BYTE_MSK GENMASK(15, 8) 106 #define NFC_CMD(x) (x) 107 #define NFC_ADR_NUM_MSK GENMASK(18, 16) 108 #define NFC_ADR_NUM(x) (((x) - 1) << 16) 109 #define NFC_SEND_ADR BIT(19) 110 #define NFC_ACCESS_DIR BIT(20) 111 #define NFC_DATA_TRANS BIT(21) 112 #define NFC_SEND_CMD1 BIT(22) 113 #define NFC_WAIT_FLAG BIT(23) 114 #define NFC_SEND_CMD2 BIT(24) 115 #define NFC_SEQ BIT(25) 116 #define NFC_DATA_SWAP_METHOD BIT(26) 117 #define NFC_ROW_AUTO_INC BIT(27) 118 #define NFC_SEND_CMD3 BIT(28) 119 #define NFC_SEND_CMD4 BIT(29) 120 #define NFC_CMD_TYPE_MSK GENMASK(31, 30) 121 #define NFC_NORMAL_OP (0 << 30) 122 #define NFC_ECC_OP (1 << 30) 123 #define NFC_PAGE_OP (2U << 30) 124 125 /* define bit use in NFC_RCMD_SET */ 126 #define NFC_READ_CMD_MSK GENMASK(7, 0) 127 #define NFC_RND_READ_CMD0_MSK GENMASK(15, 8) 128 #define NFC_RND_READ_CMD1_MSK GENMASK(23, 16) 129 130 /* define bit use in NFC_WCMD_SET */ 131 #define NFC_PROGRAM_CMD_MSK GENMASK(7, 0) 132 #define NFC_RND_WRITE_CMD_MSK GENMASK(15, 8) 133 #define NFC_READ_CMD0_MSK GENMASK(23, 16) 134 #define NFC_READ_CMD1_MSK GENMASK(31, 24) 135 136 /* define bit use in NFC_ECC_CTL */ 137 #define NFC_ECC_EN BIT(0) 138 #define NFC_ECC_PIPELINE BIT(3) 139 #define NFC_ECC_EXCEPTION BIT(4) 140 #define NFC_ECC_BLOCK_SIZE_MSK BIT(5) 141 #define NFC_ECC_BLOCK_512 BIT(5) 142 #define NFC_RANDOM_EN BIT(9) 143 #define NFC_RANDOM_DIRECTION BIT(10) 144 #define NFC_ECC_MODE_MSK GENMASK(15, 12) 145 #define NFC_ECC_MODE(x) ((x) << 12) 146 #define NFC_RANDOM_SEED_MSK GENMASK(30, 16) 147 #define NFC_RANDOM_SEED(x) ((x) << 16) 148 149 /* define bit use in NFC_ECC_ST */ 150 #define NFC_ECC_ERR(x) BIT(x) 151 #define NFC_ECC_ERR_MSK GENMASK(15, 0) 152 #define NFC_ECC_PAT_FOUND(x) BIT(x + 16) 153 #define NFC_ECC_ERR_CNT(b, x) (((x) >> (((b) % 4) * 8)) & 0xff) 154 155 #define NFC_DEFAULT_TIMEOUT_MS 1000 156 157 #define NFC_SRAM_SIZE 1024 158 159 #define NFC_MAX_CS 7 160 161 /** 162 * struct sunxi_nand_chip_sel - stores information related to NAND Chip Select 163 * 164 * @cs: the NAND CS id used to communicate with a NAND Chip 165 * @rb: the Ready/Busy pin ID. -1 means no R/B pin connected to the NFC 166 */ 167 struct sunxi_nand_chip_sel { 168 u8 cs; 169 s8 rb; 170 }; 171 172 /** 173 * struct sunxi_nand_hw_ecc - stores information related to HW ECC support 174 * 175 * @mode: the sunxi ECC mode field deduced from ECC requirements 176 */ 177 struct sunxi_nand_hw_ecc { 178 int mode; 179 }; 180 181 /** 182 * struct sunxi_nand_chip - stores NAND chip device related information 183 * 184 * @node: used to store NAND chips into a list 185 * @nand: base NAND chip structure 186 * @ecc: ECC controller structure 187 * @clk_rate: clk_rate required for this NAND chip 188 * @timing_cfg: TIMING_CFG register value for this NAND chip 189 * @timing_ctl: TIMING_CTL register value for this NAND chip 190 * @nsels: number of CS lines required by the NAND chip 191 * @sels: array of CS lines descriptions 192 */ 193 struct sunxi_nand_chip { 194 struct list_head node; 195 struct nand_chip nand; 196 struct sunxi_nand_hw_ecc *ecc; 197 unsigned long clk_rate; 198 u32 timing_cfg; 199 u32 timing_ctl; 200 int nsels; 201 struct sunxi_nand_chip_sel sels[]; 202 }; 203 204 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand) 205 { 206 return container_of(nand, struct sunxi_nand_chip, nand); 207 } 208 209 /* 210 * NAND Controller capabilities structure: stores NAND controller capabilities 211 * for distinction between compatible strings. 212 * 213 * @has_mdma: Use mbus dma mode, otherwise general dma 214 * through MBUS on A23/A33 needs extra configuration. 215 * @reg_io_data: I/O data register 216 * @dma_maxburst: DMA maxburst 217 */ 218 struct sunxi_nfc_caps { 219 bool has_mdma; 220 unsigned int reg_io_data; 221 unsigned int dma_maxburst; 222 }; 223 224 /** 225 * struct sunxi_nfc - stores sunxi NAND controller information 226 * 227 * @controller: base controller structure 228 * @dev: parent device (used to print error messages) 229 * @regs: NAND controller registers 230 * @ahb_clk: NAND controller AHB clock 231 * @mod_clk: NAND controller mod clock 232 * @reset: NAND controller reset line 233 * @assigned_cs: bitmask describing already assigned CS lines 234 * @clk_rate: NAND controller current clock rate 235 * @chips: a list containing all the NAND chips attached to this NAND 236 * controller 237 * @complete: a completion object used to wait for NAND controller events 238 * @dmac: the DMA channel attached to the NAND controller 239 * @caps: NAND Controller capabilities 240 */ 241 struct sunxi_nfc { 242 struct nand_controller controller; 243 struct device *dev; 244 void __iomem *regs; 245 struct clk *ahb_clk; 246 struct clk *mod_clk; 247 struct reset_control *reset; 248 unsigned long assigned_cs; 249 unsigned long clk_rate; 250 struct list_head chips; 251 struct completion complete; 252 struct dma_chan *dmac; 253 const struct sunxi_nfc_caps *caps; 254 }; 255 256 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_controller *ctrl) 257 { 258 return container_of(ctrl, struct sunxi_nfc, controller); 259 } 260 261 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id) 262 { 263 struct sunxi_nfc *nfc = dev_id; 264 u32 st = readl(nfc->regs + NFC_REG_ST); 265 u32 ien = readl(nfc->regs + NFC_REG_INT); 266 267 if (!(ien & st)) 268 return IRQ_NONE; 269 270 if ((ien & st) == ien) 271 complete(&nfc->complete); 272 273 writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST); 274 writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT); 275 276 return IRQ_HANDLED; 277 } 278 279 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events, 280 bool use_polling, unsigned int timeout_ms) 281 { 282 int ret; 283 284 if (events & ~NFC_INT_MASK) 285 return -EINVAL; 286 287 if (!timeout_ms) 288 timeout_ms = NFC_DEFAULT_TIMEOUT_MS; 289 290 if (!use_polling) { 291 init_completion(&nfc->complete); 292 293 writel(events, nfc->regs + NFC_REG_INT); 294 295 ret = wait_for_completion_timeout(&nfc->complete, 296 msecs_to_jiffies(timeout_ms)); 297 if (!ret) 298 ret = -ETIMEDOUT; 299 else 300 ret = 0; 301 302 writel(0, nfc->regs + NFC_REG_INT); 303 } else { 304 u32 status; 305 306 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status, 307 (status & events) == events, 1, 308 timeout_ms * 1000); 309 } 310 311 writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST); 312 313 if (ret) 314 dev_err(nfc->dev, "wait interrupt timedout\n"); 315 316 return ret; 317 } 318 319 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc) 320 { 321 u32 status; 322 int ret; 323 324 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status, 325 !(status & NFC_CMD_FIFO_STATUS), 1, 326 NFC_DEFAULT_TIMEOUT_MS * 1000); 327 if (ret) 328 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n"); 329 330 return ret; 331 } 332 333 static int sunxi_nfc_rst(struct sunxi_nfc *nfc) 334 { 335 u32 ctl; 336 int ret; 337 338 writel(0, nfc->regs + NFC_REG_ECC_CTL); 339 writel(NFC_RESET, nfc->regs + NFC_REG_CTL); 340 341 ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl, 342 !(ctl & NFC_RESET), 1, 343 NFC_DEFAULT_TIMEOUT_MS * 1000); 344 if (ret) 345 dev_err(nfc->dev, "wait for NAND controller reset timedout\n"); 346 347 return ret; 348 } 349 350 static int sunxi_nfc_dma_op_prepare(struct sunxi_nfc *nfc, const void *buf, 351 int chunksize, int nchunks, 352 enum dma_data_direction ddir, 353 struct scatterlist *sg) 354 { 355 struct dma_async_tx_descriptor *dmad; 356 enum dma_transfer_direction tdir; 357 dma_cookie_t dmat; 358 int ret; 359 360 if (ddir == DMA_FROM_DEVICE) 361 tdir = DMA_DEV_TO_MEM; 362 else 363 tdir = DMA_MEM_TO_DEV; 364 365 sg_init_one(sg, buf, nchunks * chunksize); 366 ret = dma_map_sg(nfc->dev, sg, 1, ddir); 367 if (!ret) 368 return -ENOMEM; 369 370 if (!nfc->caps->has_mdma) { 371 dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK); 372 if (!dmad) { 373 ret = -EINVAL; 374 goto err_unmap_buf; 375 } 376 } 377 378 writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD, 379 nfc->regs + NFC_REG_CTL); 380 writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM); 381 writel(chunksize, nfc->regs + NFC_REG_CNT); 382 383 if (nfc->caps->has_mdma) { 384 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_DMA_TYPE_NORMAL, 385 nfc->regs + NFC_REG_CTL); 386 writel(chunksize * nchunks, nfc->regs + NFC_REG_MDMA_CNT); 387 writel(sg_dma_address(sg), nfc->regs + NFC_REG_MDMA_ADDR); 388 } else { 389 dmat = dmaengine_submit(dmad); 390 391 ret = dma_submit_error(dmat); 392 if (ret) 393 goto err_clr_dma_flag; 394 } 395 396 return 0; 397 398 err_clr_dma_flag: 399 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD, 400 nfc->regs + NFC_REG_CTL); 401 402 err_unmap_buf: 403 dma_unmap_sg(nfc->dev, sg, 1, ddir); 404 return ret; 405 } 406 407 static void sunxi_nfc_dma_op_cleanup(struct sunxi_nfc *nfc, 408 enum dma_data_direction ddir, 409 struct scatterlist *sg) 410 { 411 dma_unmap_sg(nfc->dev, sg, 1, ddir); 412 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD, 413 nfc->regs + NFC_REG_CTL); 414 } 415 416 static void sunxi_nfc_select_chip(struct nand_chip *nand, unsigned int cs) 417 { 418 struct mtd_info *mtd = nand_to_mtd(nand); 419 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 420 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 421 struct sunxi_nand_chip_sel *sel; 422 u32 ctl; 423 424 if (cs > 0 && cs >= sunxi_nand->nsels) 425 return; 426 427 ctl = readl(nfc->regs + NFC_REG_CTL) & 428 ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN); 429 430 sel = &sunxi_nand->sels[cs]; 431 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN | NFC_PAGE_SHIFT(nand->page_shift); 432 if (sel->rb >= 0) 433 ctl |= NFC_RB_SEL(sel->rb); 434 435 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA); 436 437 if (nfc->clk_rate != sunxi_nand->clk_rate) { 438 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate); 439 nfc->clk_rate = sunxi_nand->clk_rate; 440 } 441 442 writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL); 443 writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG); 444 writel(ctl, nfc->regs + NFC_REG_CTL); 445 } 446 447 static void sunxi_nfc_read_buf(struct nand_chip *nand, uint8_t *buf, int len) 448 { 449 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 450 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 451 int ret; 452 int cnt; 453 int offs = 0; 454 u32 tmp; 455 456 while (len > offs) { 457 bool poll = false; 458 459 cnt = min(len - offs, NFC_SRAM_SIZE); 460 461 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 462 if (ret) 463 break; 464 465 writel(cnt, nfc->regs + NFC_REG_CNT); 466 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD; 467 writel(tmp, nfc->regs + NFC_REG_CMD); 468 469 /* Arbitrary limit for polling mode */ 470 if (cnt < 64) 471 poll = true; 472 473 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0); 474 if (ret) 475 break; 476 477 if (buf) 478 memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE, 479 cnt); 480 offs += cnt; 481 } 482 } 483 484 static void sunxi_nfc_write_buf(struct nand_chip *nand, const uint8_t *buf, 485 int len) 486 { 487 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 488 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 489 int ret; 490 int cnt; 491 int offs = 0; 492 u32 tmp; 493 494 while (len > offs) { 495 bool poll = false; 496 497 cnt = min(len - offs, NFC_SRAM_SIZE); 498 499 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 500 if (ret) 501 break; 502 503 writel(cnt, nfc->regs + NFC_REG_CNT); 504 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt); 505 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | 506 NFC_ACCESS_DIR; 507 writel(tmp, nfc->regs + NFC_REG_CMD); 508 509 /* Arbitrary limit for polling mode */ 510 if (cnt < 64) 511 poll = true; 512 513 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0); 514 if (ret) 515 break; 516 517 offs += cnt; 518 } 519 } 520 521 /* These seed values have been extracted from Allwinner's BSP */ 522 static const u16 sunxi_nfc_randomizer_page_seeds[] = { 523 0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72, 524 0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436, 525 0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d, 526 0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130, 527 0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56, 528 0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55, 529 0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb, 530 0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17, 531 0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62, 532 0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064, 533 0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126, 534 0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e, 535 0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3, 536 0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b, 537 0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d, 538 0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db, 539 }; 540 541 /* 542 * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds 543 * have been generated using 544 * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what 545 * the randomizer engine does internally before de/scrambling OOB data. 546 * 547 * Those tables are statically defined to avoid calculating randomizer state 548 * at runtime. 549 */ 550 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = { 551 0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64, 552 0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409, 553 0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617, 554 0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d, 555 0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91, 556 0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d, 557 0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab, 558 0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8, 559 0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8, 560 0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b, 561 0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5, 562 0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a, 563 0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891, 564 0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36, 565 0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd, 566 0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0, 567 }; 568 569 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = { 570 0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6, 571 0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982, 572 0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9, 573 0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07, 574 0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e, 575 0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2, 576 0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c, 577 0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f, 578 0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc, 579 0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e, 580 0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8, 581 0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68, 582 0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d, 583 0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179, 584 0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601, 585 0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd, 586 }; 587 588 static u16 sunxi_nfc_randomizer_step(u16 state, int count) 589 { 590 state &= 0x7fff; 591 592 /* 593 * This loop is just a simple implementation of a Fibonacci LFSR using 594 * the x16 + x15 + 1 polynomial. 595 */ 596 while (count--) 597 state = ((state >> 1) | 598 (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff; 599 600 return state; 601 } 602 603 static u16 sunxi_nfc_randomizer_state(struct nand_chip *nand, int page, 604 bool ecc) 605 { 606 struct mtd_info *mtd = nand_to_mtd(nand); 607 const u16 *seeds = sunxi_nfc_randomizer_page_seeds; 608 int mod = mtd_div_by_ws(mtd->erasesize, mtd); 609 610 if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds)) 611 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds); 612 613 if (ecc) { 614 if (mtd->ecc_step_size == 512) 615 seeds = sunxi_nfc_randomizer_ecc512_seeds; 616 else 617 seeds = sunxi_nfc_randomizer_ecc1024_seeds; 618 } 619 620 return seeds[page % mod]; 621 } 622 623 static void sunxi_nfc_randomizer_config(struct nand_chip *nand, int page, 624 bool ecc) 625 { 626 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 627 u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); 628 u16 state; 629 630 if (!(nand->options & NAND_NEED_SCRAMBLING)) 631 return; 632 633 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); 634 state = sunxi_nfc_randomizer_state(nand, page, ecc); 635 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK; 636 writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL); 637 } 638 639 static void sunxi_nfc_randomizer_enable(struct nand_chip *nand) 640 { 641 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 642 643 if (!(nand->options & NAND_NEED_SCRAMBLING)) 644 return; 645 646 writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN, 647 nfc->regs + NFC_REG_ECC_CTL); 648 } 649 650 static void sunxi_nfc_randomizer_disable(struct nand_chip *nand) 651 { 652 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 653 654 if (!(nand->options & NAND_NEED_SCRAMBLING)) 655 return; 656 657 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN, 658 nfc->regs + NFC_REG_ECC_CTL); 659 } 660 661 static void sunxi_nfc_randomize_bbm(struct nand_chip *nand, int page, u8 *bbm) 662 { 663 u16 state = sunxi_nfc_randomizer_state(nand, page, true); 664 665 bbm[0] ^= state; 666 bbm[1] ^= sunxi_nfc_randomizer_step(state, 8); 667 } 668 669 static void sunxi_nfc_randomizer_write_buf(struct nand_chip *nand, 670 const uint8_t *buf, int len, 671 bool ecc, int page) 672 { 673 sunxi_nfc_randomizer_config(nand, page, ecc); 674 sunxi_nfc_randomizer_enable(nand); 675 sunxi_nfc_write_buf(nand, buf, len); 676 sunxi_nfc_randomizer_disable(nand); 677 } 678 679 static void sunxi_nfc_randomizer_read_buf(struct nand_chip *nand, uint8_t *buf, 680 int len, bool ecc, int page) 681 { 682 sunxi_nfc_randomizer_config(nand, page, ecc); 683 sunxi_nfc_randomizer_enable(nand); 684 sunxi_nfc_read_buf(nand, buf, len); 685 sunxi_nfc_randomizer_disable(nand); 686 } 687 688 static void sunxi_nfc_hw_ecc_enable(struct nand_chip *nand) 689 { 690 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 691 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 692 u32 ecc_ctl; 693 694 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); 695 ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE | 696 NFC_ECC_BLOCK_SIZE_MSK); 697 ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(sunxi_nand->ecc->mode) | 698 NFC_ECC_EXCEPTION | NFC_ECC_PIPELINE; 699 700 if (nand->ecc.size == 512) 701 ecc_ctl |= NFC_ECC_BLOCK_512; 702 703 writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL); 704 } 705 706 static void sunxi_nfc_hw_ecc_disable(struct nand_chip *nand) 707 { 708 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 709 710 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN, 711 nfc->regs + NFC_REG_ECC_CTL); 712 } 713 714 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf) 715 { 716 buf[0] = user_data; 717 buf[1] = user_data >> 8; 718 buf[2] = user_data >> 16; 719 buf[3] = user_data >> 24; 720 } 721 722 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf) 723 { 724 return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24); 725 } 726 727 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct nand_chip *nand, u8 *oob, 728 int step, bool bbm, int page) 729 { 730 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 731 732 sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)), 733 oob); 734 735 /* De-randomize the Bad Block Marker. */ 736 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) 737 sunxi_nfc_randomize_bbm(nand, page, oob); 738 } 739 740 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand, 741 const u8 *oob, int step, 742 bool bbm, int page) 743 { 744 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 745 u8 user_data[4]; 746 747 /* Randomize the Bad Block Marker. */ 748 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) { 749 memcpy(user_data, oob, sizeof(user_data)); 750 sunxi_nfc_randomize_bbm(nand, page, user_data); 751 oob = user_data; 752 } 753 754 writel(sunxi_nfc_buf_to_user_data(oob), 755 nfc->regs + NFC_REG_USER_DATA(step)); 756 } 757 758 static void sunxi_nfc_hw_ecc_update_stats(struct nand_chip *nand, 759 unsigned int *max_bitflips, int ret) 760 { 761 struct mtd_info *mtd = nand_to_mtd(nand); 762 763 if (ret < 0) { 764 mtd->ecc_stats.failed++; 765 } else { 766 mtd->ecc_stats.corrected += ret; 767 *max_bitflips = max_t(unsigned int, *max_bitflips, ret); 768 } 769 } 770 771 static int sunxi_nfc_hw_ecc_correct(struct nand_chip *nand, u8 *data, u8 *oob, 772 int step, u32 status, bool *erased) 773 { 774 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 775 struct nand_ecc_ctrl *ecc = &nand->ecc; 776 u32 tmp; 777 778 *erased = false; 779 780 if (status & NFC_ECC_ERR(step)) 781 return -EBADMSG; 782 783 if (status & NFC_ECC_PAT_FOUND(step)) { 784 u8 pattern; 785 786 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) { 787 pattern = 0x0; 788 } else { 789 pattern = 0xff; 790 *erased = true; 791 } 792 793 if (data) 794 memset(data, pattern, ecc->size); 795 796 if (oob) 797 memset(oob, pattern, ecc->bytes + 4); 798 799 return 0; 800 } 801 802 tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step)); 803 804 return NFC_ECC_ERR_CNT(step, tmp); 805 } 806 807 static int sunxi_nfc_hw_ecc_read_chunk(struct nand_chip *nand, 808 u8 *data, int data_off, 809 u8 *oob, int oob_off, 810 int *cur_off, 811 unsigned int *max_bitflips, 812 bool bbm, bool oob_required, int page) 813 { 814 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 815 struct nand_ecc_ctrl *ecc = &nand->ecc; 816 int raw_mode = 0; 817 bool erased; 818 int ret; 819 820 if (*cur_off != data_off) 821 nand_change_read_column_op(nand, data_off, NULL, 0, false); 822 823 sunxi_nfc_randomizer_read_buf(nand, NULL, ecc->size, false, page); 824 825 if (data_off + ecc->size != oob_off) 826 nand_change_read_column_op(nand, oob_off, NULL, 0, false); 827 828 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 829 if (ret) 830 return ret; 831 832 sunxi_nfc_randomizer_enable(nand); 833 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP, 834 nfc->regs + NFC_REG_CMD); 835 836 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 837 sunxi_nfc_randomizer_disable(nand); 838 if (ret) 839 return ret; 840 841 *cur_off = oob_off + ecc->bytes + 4; 842 843 ret = sunxi_nfc_hw_ecc_correct(nand, data, oob_required ? oob : NULL, 0, 844 readl(nfc->regs + NFC_REG_ECC_ST), 845 &erased); 846 if (erased) 847 return 1; 848 849 if (ret < 0) { 850 /* 851 * Re-read the data with the randomizer disabled to identify 852 * bitflips in erased pages. 853 */ 854 if (nand->options & NAND_NEED_SCRAMBLING) 855 nand_change_read_column_op(nand, data_off, data, 856 ecc->size, false); 857 else 858 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, 859 ecc->size); 860 861 nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4, 862 false); 863 864 ret = nand_check_erased_ecc_chunk(data, ecc->size, 865 oob, ecc->bytes + 4, 866 NULL, 0, ecc->strength); 867 if (ret >= 0) 868 raw_mode = 1; 869 } else { 870 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size); 871 872 if (oob_required) { 873 nand_change_read_column_op(nand, oob_off, NULL, 0, 874 false); 875 sunxi_nfc_randomizer_read_buf(nand, oob, ecc->bytes + 4, 876 true, page); 877 878 sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, 0, 879 bbm, page); 880 } 881 } 882 883 sunxi_nfc_hw_ecc_update_stats(nand, max_bitflips, ret); 884 885 return raw_mode; 886 } 887 888 static void sunxi_nfc_hw_ecc_read_extra_oob(struct nand_chip *nand, 889 u8 *oob, int *cur_off, 890 bool randomize, int page) 891 { 892 struct mtd_info *mtd = nand_to_mtd(nand); 893 struct nand_ecc_ctrl *ecc = &nand->ecc; 894 int offset = ((ecc->bytes + 4) * ecc->steps); 895 int len = mtd->oobsize - offset; 896 897 if (len <= 0) 898 return; 899 900 if (!cur_off || *cur_off != offset) 901 nand_change_read_column_op(nand, mtd->writesize, NULL, 0, 902 false); 903 904 if (!randomize) 905 sunxi_nfc_read_buf(nand, oob + offset, len); 906 else 907 sunxi_nfc_randomizer_read_buf(nand, oob + offset, len, 908 false, page); 909 910 if (cur_off) 911 *cur_off = mtd->oobsize + mtd->writesize; 912 } 913 914 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf, 915 int oob_required, int page, 916 int nchunks) 917 { 918 bool randomized = nand->options & NAND_NEED_SCRAMBLING; 919 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 920 struct mtd_info *mtd = nand_to_mtd(nand); 921 struct nand_ecc_ctrl *ecc = &nand->ecc; 922 unsigned int max_bitflips = 0; 923 int ret, i, raw_mode = 0; 924 struct scatterlist sg; 925 u32 status, wait; 926 927 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 928 if (ret) 929 return ret; 930 931 ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, nchunks, 932 DMA_FROM_DEVICE, &sg); 933 if (ret) 934 return ret; 935 936 sunxi_nfc_hw_ecc_enable(nand); 937 sunxi_nfc_randomizer_config(nand, page, false); 938 sunxi_nfc_randomizer_enable(nand); 939 940 writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) | 941 NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET); 942 943 wait = NFC_CMD_INT_FLAG; 944 945 if (nfc->caps->has_mdma) 946 wait |= NFC_DMA_INT_FLAG; 947 else 948 dma_async_issue_pending(nfc->dmac); 949 950 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS, 951 nfc->regs + NFC_REG_CMD); 952 953 ret = sunxi_nfc_wait_events(nfc, wait, false, 0); 954 if (ret && !nfc->caps->has_mdma) 955 dmaengine_terminate_all(nfc->dmac); 956 957 sunxi_nfc_randomizer_disable(nand); 958 sunxi_nfc_hw_ecc_disable(nand); 959 960 sunxi_nfc_dma_op_cleanup(nfc, DMA_FROM_DEVICE, &sg); 961 962 if (ret) 963 return ret; 964 965 status = readl(nfc->regs + NFC_REG_ECC_ST); 966 967 for (i = 0; i < nchunks; i++) { 968 int data_off = i * ecc->size; 969 int oob_off = i * (ecc->bytes + 4); 970 u8 *data = buf + data_off; 971 u8 *oob = nand->oob_poi + oob_off; 972 bool erased; 973 974 ret = sunxi_nfc_hw_ecc_correct(nand, randomized ? data : NULL, 975 oob_required ? oob : NULL, 976 i, status, &erased); 977 978 /* ECC errors are handled in the second loop. */ 979 if (ret < 0) 980 continue; 981 982 if (oob_required && !erased) { 983 /* TODO: use DMA to retrieve OOB */ 984 nand_change_read_column_op(nand, 985 mtd->writesize + oob_off, 986 oob, ecc->bytes + 4, false); 987 988 sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, i, 989 !i, page); 990 } 991 992 if (erased) 993 raw_mode = 1; 994 995 sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret); 996 } 997 998 if (status & NFC_ECC_ERR_MSK) { 999 for (i = 0; i < nchunks; i++) { 1000 int data_off = i * ecc->size; 1001 int oob_off = i * (ecc->bytes + 4); 1002 u8 *data = buf + data_off; 1003 u8 *oob = nand->oob_poi + oob_off; 1004 1005 if (!(status & NFC_ECC_ERR(i))) 1006 continue; 1007 1008 /* 1009 * Re-read the data with the randomizer disabled to 1010 * identify bitflips in erased pages. 1011 * TODO: use DMA to read page in raw mode 1012 */ 1013 if (randomized) 1014 nand_change_read_column_op(nand, data_off, 1015 data, ecc->size, 1016 false); 1017 1018 /* TODO: use DMA to retrieve OOB */ 1019 nand_change_read_column_op(nand, 1020 mtd->writesize + oob_off, 1021 oob, ecc->bytes + 4, false); 1022 1023 ret = nand_check_erased_ecc_chunk(data, ecc->size, 1024 oob, ecc->bytes + 4, 1025 NULL, 0, 1026 ecc->strength); 1027 if (ret >= 0) 1028 raw_mode = 1; 1029 1030 sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret); 1031 } 1032 } 1033 1034 if (oob_required) 1035 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi, 1036 NULL, !raw_mode, 1037 page); 1038 1039 return max_bitflips; 1040 } 1041 1042 static int sunxi_nfc_hw_ecc_write_chunk(struct nand_chip *nand, 1043 const u8 *data, int data_off, 1044 const u8 *oob, int oob_off, 1045 int *cur_off, bool bbm, 1046 int page) 1047 { 1048 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1049 struct nand_ecc_ctrl *ecc = &nand->ecc; 1050 int ret; 1051 1052 if (data_off != *cur_off) 1053 nand_change_write_column_op(nand, data_off, NULL, 0, false); 1054 1055 sunxi_nfc_randomizer_write_buf(nand, data, ecc->size, false, page); 1056 1057 if (data_off + ecc->size != oob_off) 1058 nand_change_write_column_op(nand, oob_off, NULL, 0, false); 1059 1060 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1061 if (ret) 1062 return ret; 1063 1064 sunxi_nfc_randomizer_enable(nand); 1065 sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, 0, bbm, page); 1066 1067 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | 1068 NFC_ACCESS_DIR | NFC_ECC_OP, 1069 nfc->regs + NFC_REG_CMD); 1070 1071 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); 1072 sunxi_nfc_randomizer_disable(nand); 1073 if (ret) 1074 return ret; 1075 1076 *cur_off = oob_off + ecc->bytes + 4; 1077 1078 return 0; 1079 } 1080 1081 static void sunxi_nfc_hw_ecc_write_extra_oob(struct nand_chip *nand, 1082 u8 *oob, int *cur_off, 1083 int page) 1084 { 1085 struct mtd_info *mtd = nand_to_mtd(nand); 1086 struct nand_ecc_ctrl *ecc = &nand->ecc; 1087 int offset = ((ecc->bytes + 4) * ecc->steps); 1088 int len = mtd->oobsize - offset; 1089 1090 if (len <= 0) 1091 return; 1092 1093 if (!cur_off || *cur_off != offset) 1094 nand_change_write_column_op(nand, offset + mtd->writesize, 1095 NULL, 0, false); 1096 1097 sunxi_nfc_randomizer_write_buf(nand, oob + offset, len, false, page); 1098 1099 if (cur_off) 1100 *cur_off = mtd->oobsize + mtd->writesize; 1101 } 1102 1103 static int sunxi_nfc_hw_ecc_read_page(struct nand_chip *nand, uint8_t *buf, 1104 int oob_required, int page) 1105 { 1106 struct mtd_info *mtd = nand_to_mtd(nand); 1107 struct nand_ecc_ctrl *ecc = &nand->ecc; 1108 unsigned int max_bitflips = 0; 1109 int ret, i, cur_off = 0; 1110 bool raw_mode = false; 1111 1112 sunxi_nfc_select_chip(nand, nand->cur_cs); 1113 1114 nand_read_page_op(nand, page, 0, NULL, 0); 1115 1116 sunxi_nfc_hw_ecc_enable(nand); 1117 1118 for (i = 0; i < ecc->steps; i++) { 1119 int data_off = i * ecc->size; 1120 int oob_off = i * (ecc->bytes + 4); 1121 u8 *data = buf + data_off; 1122 u8 *oob = nand->oob_poi + oob_off; 1123 1124 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off, oob, 1125 oob_off + mtd->writesize, 1126 &cur_off, &max_bitflips, 1127 !i, oob_required, page); 1128 if (ret < 0) 1129 return ret; 1130 else if (ret) 1131 raw_mode = true; 1132 } 1133 1134 if (oob_required) 1135 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi, &cur_off, 1136 !raw_mode, page); 1137 1138 sunxi_nfc_hw_ecc_disable(nand); 1139 1140 return max_bitflips; 1141 } 1142 1143 static int sunxi_nfc_hw_ecc_read_page_dma(struct nand_chip *nand, u8 *buf, 1144 int oob_required, int page) 1145 { 1146 int ret; 1147 1148 sunxi_nfc_select_chip(nand, nand->cur_cs); 1149 1150 nand_read_page_op(nand, page, 0, NULL, 0); 1151 1152 ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, oob_required, page, 1153 nand->ecc.steps); 1154 if (ret >= 0) 1155 return ret; 1156 1157 /* Fallback to PIO mode */ 1158 return sunxi_nfc_hw_ecc_read_page(nand, buf, oob_required, page); 1159 } 1160 1161 static int sunxi_nfc_hw_ecc_read_subpage(struct nand_chip *nand, 1162 u32 data_offs, u32 readlen, 1163 u8 *bufpoi, int page) 1164 { 1165 struct mtd_info *mtd = nand_to_mtd(nand); 1166 struct nand_ecc_ctrl *ecc = &nand->ecc; 1167 int ret, i, cur_off = 0; 1168 unsigned int max_bitflips = 0; 1169 1170 sunxi_nfc_select_chip(nand, nand->cur_cs); 1171 1172 nand_read_page_op(nand, page, 0, NULL, 0); 1173 1174 sunxi_nfc_hw_ecc_enable(nand); 1175 1176 for (i = data_offs / ecc->size; 1177 i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) { 1178 int data_off = i * ecc->size; 1179 int oob_off = i * (ecc->bytes + 4); 1180 u8 *data = bufpoi + data_off; 1181 u8 *oob = nand->oob_poi + oob_off; 1182 1183 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off, 1184 oob, 1185 oob_off + mtd->writesize, 1186 &cur_off, &max_bitflips, !i, 1187 false, page); 1188 if (ret < 0) 1189 return ret; 1190 } 1191 1192 sunxi_nfc_hw_ecc_disable(nand); 1193 1194 return max_bitflips; 1195 } 1196 1197 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct nand_chip *nand, 1198 u32 data_offs, u32 readlen, 1199 u8 *buf, int page) 1200 { 1201 int nchunks = DIV_ROUND_UP(data_offs + readlen, nand->ecc.size); 1202 int ret; 1203 1204 sunxi_nfc_select_chip(nand, nand->cur_cs); 1205 1206 nand_read_page_op(nand, page, 0, NULL, 0); 1207 1208 ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, false, page, nchunks); 1209 if (ret >= 0) 1210 return ret; 1211 1212 /* Fallback to PIO mode */ 1213 return sunxi_nfc_hw_ecc_read_subpage(nand, data_offs, readlen, 1214 buf, page); 1215 } 1216 1217 static int sunxi_nfc_hw_ecc_write_page(struct nand_chip *nand, 1218 const uint8_t *buf, int oob_required, 1219 int page) 1220 { 1221 struct mtd_info *mtd = nand_to_mtd(nand); 1222 struct nand_ecc_ctrl *ecc = &nand->ecc; 1223 int ret, i, cur_off = 0; 1224 1225 sunxi_nfc_select_chip(nand, nand->cur_cs); 1226 1227 nand_prog_page_begin_op(nand, page, 0, NULL, 0); 1228 1229 sunxi_nfc_hw_ecc_enable(nand); 1230 1231 for (i = 0; i < ecc->steps; i++) { 1232 int data_off = i * ecc->size; 1233 int oob_off = i * (ecc->bytes + 4); 1234 const u8 *data = buf + data_off; 1235 const u8 *oob = nand->oob_poi + oob_off; 1236 1237 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob, 1238 oob_off + mtd->writesize, 1239 &cur_off, !i, page); 1240 if (ret) 1241 return ret; 1242 } 1243 1244 if (oob_required || (nand->options & NAND_NEED_SCRAMBLING)) 1245 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi, 1246 &cur_off, page); 1247 1248 sunxi_nfc_hw_ecc_disable(nand); 1249 1250 return nand_prog_page_end_op(nand); 1251 } 1252 1253 static int sunxi_nfc_hw_ecc_write_subpage(struct nand_chip *nand, 1254 u32 data_offs, u32 data_len, 1255 const u8 *buf, int oob_required, 1256 int page) 1257 { 1258 struct mtd_info *mtd = nand_to_mtd(nand); 1259 struct nand_ecc_ctrl *ecc = &nand->ecc; 1260 int ret, i, cur_off = 0; 1261 1262 sunxi_nfc_select_chip(nand, nand->cur_cs); 1263 1264 nand_prog_page_begin_op(nand, page, 0, NULL, 0); 1265 1266 sunxi_nfc_hw_ecc_enable(nand); 1267 1268 for (i = data_offs / ecc->size; 1269 i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) { 1270 int data_off = i * ecc->size; 1271 int oob_off = i * (ecc->bytes + 4); 1272 const u8 *data = buf + data_off; 1273 const u8 *oob = nand->oob_poi + oob_off; 1274 1275 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob, 1276 oob_off + mtd->writesize, 1277 &cur_off, !i, page); 1278 if (ret) 1279 return ret; 1280 } 1281 1282 sunxi_nfc_hw_ecc_disable(nand); 1283 1284 return nand_prog_page_end_op(nand); 1285 } 1286 1287 static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *nand, 1288 const u8 *buf, 1289 int oob_required, 1290 int page) 1291 { 1292 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1293 struct nand_ecc_ctrl *ecc = &nand->ecc; 1294 struct scatterlist sg; 1295 u32 wait; 1296 int ret, i; 1297 1298 sunxi_nfc_select_chip(nand, nand->cur_cs); 1299 1300 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1301 if (ret) 1302 return ret; 1303 1304 ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, ecc->steps, 1305 DMA_TO_DEVICE, &sg); 1306 if (ret) 1307 goto pio_fallback; 1308 1309 for (i = 0; i < ecc->steps; i++) { 1310 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4)); 1311 1312 sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, i, !i, page); 1313 } 1314 1315 nand_prog_page_begin_op(nand, page, 0, NULL, 0); 1316 1317 sunxi_nfc_hw_ecc_enable(nand); 1318 sunxi_nfc_randomizer_config(nand, page, false); 1319 sunxi_nfc_randomizer_enable(nand); 1320 1321 writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG, 1322 nfc->regs + NFC_REG_WCMD_SET); 1323 1324 wait = NFC_CMD_INT_FLAG; 1325 1326 if (nfc->caps->has_mdma) 1327 wait |= NFC_DMA_INT_FLAG; 1328 else 1329 dma_async_issue_pending(nfc->dmac); 1330 1331 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | 1332 NFC_DATA_TRANS | NFC_ACCESS_DIR, 1333 nfc->regs + NFC_REG_CMD); 1334 1335 ret = sunxi_nfc_wait_events(nfc, wait, false, 0); 1336 if (ret && !nfc->caps->has_mdma) 1337 dmaengine_terminate_all(nfc->dmac); 1338 1339 sunxi_nfc_randomizer_disable(nand); 1340 sunxi_nfc_hw_ecc_disable(nand); 1341 1342 sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg); 1343 1344 if (ret) 1345 return ret; 1346 1347 if (oob_required || (nand->options & NAND_NEED_SCRAMBLING)) 1348 /* TODO: use DMA to transfer extra OOB bytes ? */ 1349 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi, 1350 NULL, page); 1351 1352 return nand_prog_page_end_op(nand); 1353 1354 pio_fallback: 1355 return sunxi_nfc_hw_ecc_write_page(nand, buf, oob_required, page); 1356 } 1357 1358 static int sunxi_nfc_hw_ecc_read_oob(struct nand_chip *nand, int page) 1359 { 1360 u8 *buf = nand_get_data_buf(nand); 1361 1362 return nand->ecc.read_page(nand, buf, 1, page); 1363 } 1364 1365 static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *nand, int page) 1366 { 1367 struct mtd_info *mtd = nand_to_mtd(nand); 1368 u8 *buf = nand_get_data_buf(nand); 1369 int ret; 1370 1371 memset(buf, 0xff, mtd->writesize); 1372 ret = nand->ecc.write_page(nand, buf, 1, page); 1373 if (ret) 1374 return ret; 1375 1376 /* Send command to program the OOB data */ 1377 return nand_prog_page_end_op(nand); 1378 } 1379 1380 static const s32 tWB_lut[] = {6, 12, 16, 20}; 1381 static const s32 tRHW_lut[] = {4, 8, 12, 20}; 1382 1383 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration, 1384 u32 clk_period) 1385 { 1386 u32 clk_cycles = DIV_ROUND_UP(duration, clk_period); 1387 int i; 1388 1389 for (i = 0; i < lut_size; i++) { 1390 if (clk_cycles <= lut[i]) 1391 return i; 1392 } 1393 1394 /* Doesn't fit */ 1395 return -EINVAL; 1396 } 1397 1398 #define sunxi_nand_lookup_timing(l, p, c) \ 1399 _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c) 1400 1401 static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline, 1402 const struct nand_interface_config *conf) 1403 { 1404 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 1405 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); 1406 const struct nand_sdr_timings *timings; 1407 u32 min_clk_period = 0; 1408 s32 tWB, tADL, tWHR, tRHW, tCAD; 1409 long real_clk_rate; 1410 1411 timings = nand_get_sdr_timings(conf); 1412 if (IS_ERR(timings)) 1413 return -ENOTSUPP; 1414 1415 /* T1 <=> tCLS */ 1416 if (timings->tCLS_min > min_clk_period) 1417 min_clk_period = timings->tCLS_min; 1418 1419 /* T2 <=> tCLH */ 1420 if (timings->tCLH_min > min_clk_period) 1421 min_clk_period = timings->tCLH_min; 1422 1423 /* T3 <=> tCS */ 1424 if (timings->tCS_min > min_clk_period) 1425 min_clk_period = timings->tCS_min; 1426 1427 /* T4 <=> tCH */ 1428 if (timings->tCH_min > min_clk_period) 1429 min_clk_period = timings->tCH_min; 1430 1431 /* T5 <=> tWP */ 1432 if (timings->tWP_min > min_clk_period) 1433 min_clk_period = timings->tWP_min; 1434 1435 /* T6 <=> tWH */ 1436 if (timings->tWH_min > min_clk_period) 1437 min_clk_period = timings->tWH_min; 1438 1439 /* T7 <=> tALS */ 1440 if (timings->tALS_min > min_clk_period) 1441 min_clk_period = timings->tALS_min; 1442 1443 /* T8 <=> tDS */ 1444 if (timings->tDS_min > min_clk_period) 1445 min_clk_period = timings->tDS_min; 1446 1447 /* T9 <=> tDH */ 1448 if (timings->tDH_min > min_clk_period) 1449 min_clk_period = timings->tDH_min; 1450 1451 /* T10 <=> tRR */ 1452 if (timings->tRR_min > (min_clk_period * 3)) 1453 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3); 1454 1455 /* T11 <=> tALH */ 1456 if (timings->tALH_min > min_clk_period) 1457 min_clk_period = timings->tALH_min; 1458 1459 /* T12 <=> tRP */ 1460 if (timings->tRP_min > min_clk_period) 1461 min_clk_period = timings->tRP_min; 1462 1463 /* T13 <=> tREH */ 1464 if (timings->tREH_min > min_clk_period) 1465 min_clk_period = timings->tREH_min; 1466 1467 /* T14 <=> tRC */ 1468 if (timings->tRC_min > (min_clk_period * 2)) 1469 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2); 1470 1471 /* T15 <=> tWC */ 1472 if (timings->tWC_min > (min_clk_period * 2)) 1473 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2); 1474 1475 /* T16 - T19 + tCAD */ 1476 if (timings->tWB_max > (min_clk_period * 20)) 1477 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20); 1478 1479 if (timings->tADL_min > (min_clk_period * 32)) 1480 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32); 1481 1482 if (timings->tWHR_min > (min_clk_period * 32)) 1483 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32); 1484 1485 if (timings->tRHW_min > (min_clk_period * 20)) 1486 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20); 1487 1488 /* 1489 * In non-EDO, tREA should be less than tRP to guarantee that the 1490 * controller does not sample the IO lines too early. Unfortunately, 1491 * the sunxi NAND controller does not allow us to have different 1492 * values for tRP and tREH (tRP = tREH = tRW / 2). 1493 * 1494 * We have 2 options to overcome this limitation: 1495 * 1496 * 1/ Extend tRC to fulfil the tREA <= tRC / 2 constraint 1497 * 2/ Use EDO mode (only works if timings->tRLOH > 0) 1498 */ 1499 if (timings->tREA_max > min_clk_period && !timings->tRLOH_min) 1500 min_clk_period = timings->tREA_max; 1501 1502 tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max, 1503 min_clk_period); 1504 if (tWB < 0) { 1505 dev_err(nfc->dev, "unsupported tWB\n"); 1506 return tWB; 1507 } 1508 1509 tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3; 1510 if (tADL > 3) { 1511 dev_err(nfc->dev, "unsupported tADL\n"); 1512 return -EINVAL; 1513 } 1514 1515 tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3; 1516 if (tWHR > 3) { 1517 dev_err(nfc->dev, "unsupported tWHR\n"); 1518 return -EINVAL; 1519 } 1520 1521 tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min, 1522 min_clk_period); 1523 if (tRHW < 0) { 1524 dev_err(nfc->dev, "unsupported tRHW\n"); 1525 return tRHW; 1526 } 1527 1528 if (csline == NAND_DATA_IFACE_CHECK_ONLY) 1529 return 0; 1530 1531 /* 1532 * TODO: according to ONFI specs this value only applies for DDR NAND, 1533 * but Allwinner seems to set this to 0x7. Mimic them for now. 1534 */ 1535 tCAD = 0x7; 1536 1537 /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */ 1538 sunxi_nand->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD); 1539 1540 /* Convert min_clk_period from picoseconds to nanoseconds */ 1541 min_clk_period = DIV_ROUND_UP(min_clk_period, 1000); 1542 1543 /* 1544 * Unlike what is stated in Allwinner datasheet, the clk_rate should 1545 * be set to (1 / min_clk_period), and not (2 / min_clk_period). 1546 * This new formula was verified with a scope and validated by 1547 * Allwinner engineers. 1548 */ 1549 sunxi_nand->clk_rate = NSEC_PER_SEC / min_clk_period; 1550 real_clk_rate = clk_round_rate(nfc->mod_clk, sunxi_nand->clk_rate); 1551 if (real_clk_rate <= 0) { 1552 dev_err(nfc->dev, "Unable to round clk %lu\n", 1553 sunxi_nand->clk_rate); 1554 return -EINVAL; 1555 } 1556 1557 sunxi_nand->timing_ctl = 0; 1558 1559 /* 1560 * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data 1561 * output cycle timings shall be used if the host drives tRC less than 1562 * 30 ns. We should also use EDO mode if tREA is bigger than tRP. 1563 */ 1564 min_clk_period = NSEC_PER_SEC / real_clk_rate; 1565 if (min_clk_period * 2 < 30 || min_clk_period * 1000 < timings->tREA_max) 1566 sunxi_nand->timing_ctl = NFC_TIMING_CTL_EDO; 1567 1568 return 0; 1569 } 1570 1571 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section, 1572 struct mtd_oob_region *oobregion) 1573 { 1574 struct nand_chip *nand = mtd_to_nand(mtd); 1575 struct nand_ecc_ctrl *ecc = &nand->ecc; 1576 1577 if (section >= ecc->steps) 1578 return -ERANGE; 1579 1580 oobregion->offset = section * (ecc->bytes + 4) + 4; 1581 oobregion->length = ecc->bytes; 1582 1583 return 0; 1584 } 1585 1586 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section, 1587 struct mtd_oob_region *oobregion) 1588 { 1589 struct nand_chip *nand = mtd_to_nand(mtd); 1590 struct nand_ecc_ctrl *ecc = &nand->ecc; 1591 1592 if (section > ecc->steps) 1593 return -ERANGE; 1594 1595 /* 1596 * The first 2 bytes are used for BB markers, hence we 1597 * only have 2 bytes available in the first user data 1598 * section. 1599 */ 1600 if (!section && ecc->engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) { 1601 oobregion->offset = 2; 1602 oobregion->length = 2; 1603 1604 return 0; 1605 } 1606 1607 oobregion->offset = section * (ecc->bytes + 4); 1608 1609 if (section < ecc->steps) 1610 oobregion->length = 4; 1611 else 1612 oobregion->offset = mtd->oobsize - oobregion->offset; 1613 1614 return 0; 1615 } 1616 1617 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = { 1618 .ecc = sunxi_nand_ooblayout_ecc, 1619 .free = sunxi_nand_ooblayout_free, 1620 }; 1621 1622 static void sunxi_nand_hw_ecc_ctrl_cleanup(struct sunxi_nand_chip *sunxi_nand) 1623 { 1624 kfree(sunxi_nand->ecc); 1625 } 1626 1627 static int sunxi_nand_hw_ecc_ctrl_init(struct nand_chip *nand, 1628 struct nand_ecc_ctrl *ecc, 1629 struct device_node *np) 1630 { 1631 static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 }; 1632 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 1633 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1634 struct mtd_info *mtd = nand_to_mtd(nand); 1635 struct nand_device *nanddev = mtd_to_nanddev(mtd); 1636 int nsectors; 1637 int ret; 1638 int i; 1639 1640 if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) { 1641 int bytes; 1642 1643 ecc->size = 1024; 1644 nsectors = mtd->writesize / ecc->size; 1645 1646 /* Reserve 2 bytes for the BBM */ 1647 bytes = (mtd->oobsize - 2) / nsectors; 1648 1649 /* 4 non-ECC bytes are added before each ECC bytes section */ 1650 bytes -= 4; 1651 1652 /* and bytes has to be even. */ 1653 if (bytes % 2) 1654 bytes--; 1655 1656 ecc->strength = bytes * 8 / fls(8 * ecc->size); 1657 1658 for (i = 0; i < ARRAY_SIZE(strengths); i++) { 1659 if (strengths[i] > ecc->strength) 1660 break; 1661 } 1662 1663 if (!i) 1664 ecc->strength = 0; 1665 else 1666 ecc->strength = strengths[i - 1]; 1667 } 1668 1669 if (ecc->size != 512 && ecc->size != 1024) 1670 return -EINVAL; 1671 1672 sunxi_nand->ecc = kzalloc(sizeof(*sunxi_nand->ecc), GFP_KERNEL); 1673 if (!sunxi_nand->ecc) 1674 return -ENOMEM; 1675 1676 /* Prefer 1k ECC chunk over 512 ones */ 1677 if (ecc->size == 512 && mtd->writesize > 512) { 1678 ecc->size = 1024; 1679 ecc->strength *= 2; 1680 } 1681 1682 /* Add ECC info retrieval from DT */ 1683 for (i = 0; i < ARRAY_SIZE(strengths); i++) { 1684 if (ecc->strength <= strengths[i]) { 1685 /* 1686 * Update ecc->strength value with the actual strength 1687 * that will be used by the ECC engine. 1688 */ 1689 ecc->strength = strengths[i]; 1690 break; 1691 } 1692 } 1693 1694 if (i >= ARRAY_SIZE(strengths)) { 1695 dev_err(nfc->dev, "unsupported strength\n"); 1696 ret = -ENOTSUPP; 1697 goto err; 1698 } 1699 1700 sunxi_nand->ecc->mode = i; 1701 1702 /* HW ECC always request ECC bytes for 1024 bytes blocks */ 1703 ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8); 1704 1705 /* HW ECC always work with even numbers of ECC bytes */ 1706 ecc->bytes = ALIGN(ecc->bytes, 2); 1707 1708 nsectors = mtd->writesize / ecc->size; 1709 1710 if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) { 1711 ret = -EINVAL; 1712 goto err; 1713 } 1714 1715 ecc->read_oob = sunxi_nfc_hw_ecc_read_oob; 1716 ecc->write_oob = sunxi_nfc_hw_ecc_write_oob; 1717 mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops); 1718 1719 if (nfc->dmac || nfc->caps->has_mdma) { 1720 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma; 1721 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma; 1722 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma; 1723 nand->options |= NAND_USES_DMA; 1724 } else { 1725 ecc->read_page = sunxi_nfc_hw_ecc_read_page; 1726 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage; 1727 ecc->write_page = sunxi_nfc_hw_ecc_write_page; 1728 } 1729 1730 /* TODO: support DMA for raw accesses and subpage write */ 1731 ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage; 1732 ecc->read_oob_raw = nand_read_oob_std; 1733 ecc->write_oob_raw = nand_write_oob_std; 1734 1735 return 0; 1736 1737 err: 1738 kfree(sunxi_nand->ecc); 1739 1740 return ret; 1741 } 1742 1743 static void sunxi_nand_ecc_cleanup(struct sunxi_nand_chip *sunxi_nand) 1744 { 1745 struct nand_ecc_ctrl *ecc = &sunxi_nand->nand.ecc; 1746 1747 switch (ecc->engine_type) { 1748 case NAND_ECC_ENGINE_TYPE_ON_HOST: 1749 sunxi_nand_hw_ecc_ctrl_cleanup(sunxi_nand); 1750 break; 1751 case NAND_ECC_ENGINE_TYPE_NONE: 1752 default: 1753 break; 1754 } 1755 } 1756 1757 static int sunxi_nand_attach_chip(struct nand_chip *nand) 1758 { 1759 const struct nand_ecc_props *requirements = 1760 nanddev_get_ecc_requirements(&nand->base); 1761 struct nand_ecc_ctrl *ecc = &nand->ecc; 1762 struct device_node *np = nand_get_flash_node(nand); 1763 int ret; 1764 1765 if (nand->bbt_options & NAND_BBT_USE_FLASH) 1766 nand->bbt_options |= NAND_BBT_NO_OOB; 1767 1768 if (nand->options & NAND_NEED_SCRAMBLING) 1769 nand->options |= NAND_NO_SUBPAGE_WRITE; 1770 1771 nand->options |= NAND_SUBPAGE_READ; 1772 1773 if (!ecc->size) { 1774 ecc->size = requirements->step_size; 1775 ecc->strength = requirements->strength; 1776 } 1777 1778 if (!ecc->size || !ecc->strength) 1779 return -EINVAL; 1780 1781 switch (ecc->engine_type) { 1782 case NAND_ECC_ENGINE_TYPE_ON_HOST: 1783 ret = sunxi_nand_hw_ecc_ctrl_init(nand, ecc, np); 1784 if (ret) 1785 return ret; 1786 break; 1787 case NAND_ECC_ENGINE_TYPE_NONE: 1788 case NAND_ECC_ENGINE_TYPE_SOFT: 1789 break; 1790 default: 1791 return -EINVAL; 1792 } 1793 1794 return 0; 1795 } 1796 1797 static int sunxi_nfc_exec_subop(struct nand_chip *nand, 1798 const struct nand_subop *subop) 1799 { 1800 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); 1801 u32 cmd = 0, extcmd = 0, cnt = 0, addrs[2] = { }; 1802 unsigned int i, j, remaining, start; 1803 void *inbuf = NULL; 1804 int ret; 1805 1806 for (i = 0; i < subop->ninstrs; i++) { 1807 const struct nand_op_instr *instr = &subop->instrs[i]; 1808 1809 switch (instr->type) { 1810 case NAND_OP_CMD_INSTR: 1811 if (cmd & NFC_SEND_CMD1) { 1812 if (WARN_ON(cmd & NFC_SEND_CMD2)) 1813 return -EINVAL; 1814 1815 cmd |= NFC_SEND_CMD2; 1816 extcmd |= instr->ctx.cmd.opcode; 1817 } else { 1818 cmd |= NFC_SEND_CMD1 | 1819 NFC_CMD(instr->ctx.cmd.opcode); 1820 } 1821 break; 1822 1823 case NAND_OP_ADDR_INSTR: 1824 remaining = nand_subop_get_num_addr_cyc(subop, i); 1825 start = nand_subop_get_addr_start_off(subop, i); 1826 for (j = 0; j < 8 && j + start < remaining; j++) { 1827 u32 addr = instr->ctx.addr.addrs[j + start]; 1828 1829 addrs[j / 4] |= addr << (j % 4) * 8; 1830 } 1831 1832 if (j) 1833 cmd |= NFC_SEND_ADR | NFC_ADR_NUM(j); 1834 1835 break; 1836 1837 case NAND_OP_DATA_IN_INSTR: 1838 case NAND_OP_DATA_OUT_INSTR: 1839 start = nand_subop_get_data_start_off(subop, i); 1840 remaining = nand_subop_get_data_len(subop, i); 1841 cnt = min_t(u32, remaining, NFC_SRAM_SIZE); 1842 cmd |= NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD; 1843 1844 if (instr->type == NAND_OP_DATA_OUT_INSTR) { 1845 cmd |= NFC_ACCESS_DIR; 1846 memcpy_toio(nfc->regs + NFC_RAM0_BASE, 1847 instr->ctx.data.buf.out + start, 1848 cnt); 1849 } else { 1850 inbuf = instr->ctx.data.buf.in + start; 1851 } 1852 1853 break; 1854 1855 case NAND_OP_WAITRDY_INSTR: 1856 cmd |= NFC_WAIT_FLAG; 1857 break; 1858 } 1859 } 1860 1861 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); 1862 if (ret) 1863 return ret; 1864 1865 if (cmd & NFC_SEND_ADR) { 1866 writel(addrs[0], nfc->regs + NFC_REG_ADDR_LOW); 1867 writel(addrs[1], nfc->regs + NFC_REG_ADDR_HIGH); 1868 } 1869 1870 if (cmd & NFC_SEND_CMD2) 1871 writel(extcmd, 1872 nfc->regs + 1873 (cmd & NFC_ACCESS_DIR ? 1874 NFC_REG_WCMD_SET : NFC_REG_RCMD_SET)); 1875 1876 if (cmd & NFC_DATA_TRANS) 1877 writel(cnt, nfc->regs + NFC_REG_CNT); 1878 1879 writel(cmd, nfc->regs + NFC_REG_CMD); 1880 1881 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, 1882 !(cmd & NFC_WAIT_FLAG) && cnt < 64, 1883 0); 1884 if (ret) 1885 return ret; 1886 1887 if (inbuf) 1888 memcpy_fromio(inbuf, nfc->regs + NFC_RAM0_BASE, cnt); 1889 1890 return 0; 1891 } 1892 1893 static int sunxi_nfc_soft_waitrdy(struct nand_chip *nand, 1894 const struct nand_subop *subop) 1895 { 1896 return nand_soft_waitrdy(nand, 1897 subop->instrs[0].ctx.waitrdy.timeout_ms); 1898 } 1899 1900 static const struct nand_op_parser sunxi_nfc_op_parser = NAND_OP_PARSER( 1901 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop, 1902 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1903 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8), 1904 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1905 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true), 1906 NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)), 1907 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop, 1908 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1909 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8), 1910 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024), 1911 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1912 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)), 1913 ); 1914 1915 static const struct nand_op_parser sunxi_nfc_norb_op_parser = NAND_OP_PARSER( 1916 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop, 1917 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1918 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8), 1919 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1920 NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)), 1921 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop, 1922 NAND_OP_PARSER_PAT_CMD_ELEM(true), 1923 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8), 1924 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024), 1925 NAND_OP_PARSER_PAT_CMD_ELEM(true)), 1926 NAND_OP_PARSER_PATTERN(sunxi_nfc_soft_waitrdy, 1927 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)), 1928 ); 1929 1930 static int sunxi_nfc_exec_op(struct nand_chip *nand, 1931 const struct nand_operation *op, bool check_only) 1932 { 1933 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); 1934 const struct nand_op_parser *parser; 1935 1936 if (!check_only) 1937 sunxi_nfc_select_chip(nand, op->cs); 1938 1939 if (sunxi_nand->sels[op->cs].rb >= 0) 1940 parser = &sunxi_nfc_op_parser; 1941 else 1942 parser = &sunxi_nfc_norb_op_parser; 1943 1944 return nand_op_parser_exec_op(nand, parser, op, check_only); 1945 } 1946 1947 static const struct nand_controller_ops sunxi_nand_controller_ops = { 1948 .attach_chip = sunxi_nand_attach_chip, 1949 .setup_interface = sunxi_nfc_setup_interface, 1950 .exec_op = sunxi_nfc_exec_op, 1951 }; 1952 1953 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc, 1954 struct device_node *np) 1955 { 1956 struct sunxi_nand_chip *sunxi_nand; 1957 struct mtd_info *mtd; 1958 struct nand_chip *nand; 1959 int nsels; 1960 int ret; 1961 int i; 1962 u32 tmp; 1963 1964 if (!of_get_property(np, "reg", &nsels)) 1965 return -EINVAL; 1966 1967 nsels /= sizeof(u32); 1968 if (!nsels) { 1969 dev_err(dev, "invalid reg property size\n"); 1970 return -EINVAL; 1971 } 1972 1973 sunxi_nand = devm_kzalloc(dev, struct_size(sunxi_nand, sels, nsels), 1974 GFP_KERNEL); 1975 if (!sunxi_nand) { 1976 dev_err(dev, "could not allocate chip\n"); 1977 return -ENOMEM; 1978 } 1979 1980 sunxi_nand->nsels = nsels; 1981 1982 for (i = 0; i < nsels; i++) { 1983 ret = of_property_read_u32_index(np, "reg", i, &tmp); 1984 if (ret) { 1985 dev_err(dev, "could not retrieve reg property: %d\n", 1986 ret); 1987 return ret; 1988 } 1989 1990 if (tmp > NFC_MAX_CS) { 1991 dev_err(dev, 1992 "invalid reg value: %u (max CS = 7)\n", 1993 tmp); 1994 return -EINVAL; 1995 } 1996 1997 if (test_and_set_bit(tmp, &nfc->assigned_cs)) { 1998 dev_err(dev, "CS %d already assigned\n", tmp); 1999 return -EINVAL; 2000 } 2001 2002 sunxi_nand->sels[i].cs = tmp; 2003 2004 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) && 2005 tmp < 2) 2006 sunxi_nand->sels[i].rb = tmp; 2007 else 2008 sunxi_nand->sels[i].rb = -1; 2009 } 2010 2011 nand = &sunxi_nand->nand; 2012 /* Default tR value specified in the ONFI spec (chapter 4.15.1) */ 2013 nand->controller = &nfc->controller; 2014 nand->controller->ops = &sunxi_nand_controller_ops; 2015 2016 /* 2017 * Set the ECC mode to the default value in case nothing is specified 2018 * in the DT. 2019 */ 2020 nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; 2021 nand_set_flash_node(nand, np); 2022 2023 mtd = nand_to_mtd(nand); 2024 mtd->dev.parent = dev; 2025 2026 ret = nand_scan(nand, nsels); 2027 if (ret) 2028 return ret; 2029 2030 ret = mtd_device_register(mtd, NULL, 0); 2031 if (ret) { 2032 dev_err(dev, "failed to register mtd device: %d\n", ret); 2033 nand_cleanup(nand); 2034 return ret; 2035 } 2036 2037 list_add_tail(&sunxi_nand->node, &nfc->chips); 2038 2039 return 0; 2040 } 2041 2042 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc) 2043 { 2044 struct device_node *np = dev->of_node; 2045 struct device_node *nand_np; 2046 int nchips = of_get_child_count(np); 2047 int ret; 2048 2049 if (nchips > 8) { 2050 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips); 2051 return -EINVAL; 2052 } 2053 2054 for_each_child_of_node(np, nand_np) { 2055 ret = sunxi_nand_chip_init(dev, nfc, nand_np); 2056 if (ret) { 2057 of_node_put(nand_np); 2058 return ret; 2059 } 2060 } 2061 2062 return 0; 2063 } 2064 2065 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc) 2066 { 2067 struct sunxi_nand_chip *sunxi_nand; 2068 struct nand_chip *chip; 2069 int ret; 2070 2071 while (!list_empty(&nfc->chips)) { 2072 sunxi_nand = list_first_entry(&nfc->chips, 2073 struct sunxi_nand_chip, 2074 node); 2075 chip = &sunxi_nand->nand; 2076 ret = mtd_device_unregister(nand_to_mtd(chip)); 2077 WARN_ON(ret); 2078 nand_cleanup(chip); 2079 sunxi_nand_ecc_cleanup(sunxi_nand); 2080 list_del(&sunxi_nand->node); 2081 } 2082 } 2083 2084 static int sunxi_nfc_dma_init(struct sunxi_nfc *nfc, struct resource *r) 2085 { 2086 int ret; 2087 2088 if (nfc->caps->has_mdma) 2089 return 0; 2090 2091 nfc->dmac = dma_request_chan(nfc->dev, "rxtx"); 2092 if (IS_ERR(nfc->dmac)) { 2093 ret = PTR_ERR(nfc->dmac); 2094 if (ret == -EPROBE_DEFER) 2095 return ret; 2096 2097 /* Ignore errors to fall back to PIO mode */ 2098 dev_warn(nfc->dev, "failed to request rxtx DMA channel: %d\n", ret); 2099 nfc->dmac = NULL; 2100 } else { 2101 struct dma_slave_config dmac_cfg = { }; 2102 2103 dmac_cfg.src_addr = r->start + nfc->caps->reg_io_data; 2104 dmac_cfg.dst_addr = dmac_cfg.src_addr; 2105 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 2106 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width; 2107 dmac_cfg.src_maxburst = nfc->caps->dma_maxburst; 2108 dmac_cfg.dst_maxburst = nfc->caps->dma_maxburst; 2109 dmaengine_slave_config(nfc->dmac, &dmac_cfg); 2110 } 2111 return 0; 2112 } 2113 2114 static int sunxi_nfc_probe(struct platform_device *pdev) 2115 { 2116 struct device *dev = &pdev->dev; 2117 struct resource *r; 2118 struct sunxi_nfc *nfc; 2119 int irq; 2120 int ret; 2121 2122 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL); 2123 if (!nfc) 2124 return -ENOMEM; 2125 2126 nfc->dev = dev; 2127 nand_controller_init(&nfc->controller); 2128 INIT_LIST_HEAD(&nfc->chips); 2129 2130 r = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2131 nfc->regs = devm_ioremap_resource(dev, r); 2132 if (IS_ERR(nfc->regs)) 2133 return PTR_ERR(nfc->regs); 2134 2135 irq = platform_get_irq(pdev, 0); 2136 if (irq < 0) 2137 return irq; 2138 2139 nfc->ahb_clk = devm_clk_get(dev, "ahb"); 2140 if (IS_ERR(nfc->ahb_clk)) { 2141 dev_err(dev, "failed to retrieve ahb clk\n"); 2142 return PTR_ERR(nfc->ahb_clk); 2143 } 2144 2145 ret = clk_prepare_enable(nfc->ahb_clk); 2146 if (ret) 2147 return ret; 2148 2149 nfc->mod_clk = devm_clk_get(dev, "mod"); 2150 if (IS_ERR(nfc->mod_clk)) { 2151 dev_err(dev, "failed to retrieve mod clk\n"); 2152 ret = PTR_ERR(nfc->mod_clk); 2153 goto out_ahb_clk_unprepare; 2154 } 2155 2156 ret = clk_prepare_enable(nfc->mod_clk); 2157 if (ret) 2158 goto out_ahb_clk_unprepare; 2159 2160 nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb"); 2161 if (IS_ERR(nfc->reset)) { 2162 ret = PTR_ERR(nfc->reset); 2163 goto out_mod_clk_unprepare; 2164 } 2165 2166 ret = reset_control_deassert(nfc->reset); 2167 if (ret) { 2168 dev_err(dev, "reset err %d\n", ret); 2169 goto out_mod_clk_unprepare; 2170 } 2171 2172 nfc->caps = of_device_get_match_data(&pdev->dev); 2173 if (!nfc->caps) { 2174 ret = -EINVAL; 2175 goto out_ahb_reset_reassert; 2176 } 2177 2178 ret = sunxi_nfc_rst(nfc); 2179 if (ret) 2180 goto out_ahb_reset_reassert; 2181 2182 writel(0, nfc->regs + NFC_REG_INT); 2183 ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt, 2184 0, "sunxi-nand", nfc); 2185 if (ret) 2186 goto out_ahb_reset_reassert; 2187 2188 ret = sunxi_nfc_dma_init(nfc, r); 2189 2190 if (ret) 2191 goto out_ahb_reset_reassert; 2192 2193 platform_set_drvdata(pdev, nfc); 2194 2195 ret = sunxi_nand_chips_init(dev, nfc); 2196 if (ret) { 2197 dev_err(dev, "failed to init nand chips\n"); 2198 goto out_release_dmac; 2199 } 2200 2201 return 0; 2202 2203 out_release_dmac: 2204 if (nfc->dmac) 2205 dma_release_channel(nfc->dmac); 2206 out_ahb_reset_reassert: 2207 reset_control_assert(nfc->reset); 2208 out_mod_clk_unprepare: 2209 clk_disable_unprepare(nfc->mod_clk); 2210 out_ahb_clk_unprepare: 2211 clk_disable_unprepare(nfc->ahb_clk); 2212 2213 return ret; 2214 } 2215 2216 static int sunxi_nfc_remove(struct platform_device *pdev) 2217 { 2218 struct sunxi_nfc *nfc = platform_get_drvdata(pdev); 2219 2220 sunxi_nand_chips_cleanup(nfc); 2221 2222 reset_control_assert(nfc->reset); 2223 2224 if (nfc->dmac) 2225 dma_release_channel(nfc->dmac); 2226 clk_disable_unprepare(nfc->mod_clk); 2227 clk_disable_unprepare(nfc->ahb_clk); 2228 2229 return 0; 2230 } 2231 2232 static const struct sunxi_nfc_caps sunxi_nfc_a10_caps = { 2233 .reg_io_data = NFC_REG_A10_IO_DATA, 2234 .dma_maxburst = 4, 2235 }; 2236 2237 static const struct sunxi_nfc_caps sunxi_nfc_a23_caps = { 2238 .has_mdma = true, 2239 .reg_io_data = NFC_REG_A23_IO_DATA, 2240 .dma_maxburst = 8, 2241 }; 2242 2243 static const struct of_device_id sunxi_nfc_ids[] = { 2244 { 2245 .compatible = "allwinner,sun4i-a10-nand", 2246 .data = &sunxi_nfc_a10_caps, 2247 }, 2248 { 2249 .compatible = "allwinner,sun8i-a23-nand-controller", 2250 .data = &sunxi_nfc_a23_caps, 2251 }, 2252 { /* sentinel */ } 2253 }; 2254 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids); 2255 2256 static struct platform_driver sunxi_nfc_driver = { 2257 .driver = { 2258 .name = "sunxi_nand", 2259 .of_match_table = sunxi_nfc_ids, 2260 }, 2261 .probe = sunxi_nfc_probe, 2262 .remove = sunxi_nfc_remove, 2263 }; 2264 module_platform_driver(sunxi_nfc_driver); 2265 2266 MODULE_LICENSE("GPL"); 2267 MODULE_AUTHOR("Boris BREZILLON"); 2268 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver"); 2269 MODULE_ALIAS("platform:sunxi_nand"); 2270