1 /* 2 * Copyright © 2004 Texas Instruments, Jian Zhang <jzhang@ti.com> 3 * Copyright © 2004 Micron Technology Inc. 4 * Copyright © 2004 David Brownell 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <linux/platform_device.h> 12 #include <linux/dmaengine.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/delay.h> 15 #include <linux/gpio/consumer.h> 16 #include <linux/module.h> 17 #include <linux/interrupt.h> 18 #include <linux/jiffies.h> 19 #include <linux/sched.h> 20 #include <linux/mtd/mtd.h> 21 #include <linux/mtd/rawnand.h> 22 #include <linux/mtd/partitions.h> 23 #include <linux/omap-dma.h> 24 #include <linux/io.h> 25 #include <linux/slab.h> 26 #include <linux/of.h> 27 #include <linux/of_device.h> 28 29 #include <linux/mtd/nand_bch.h> 30 #include <linux/platform_data/elm.h> 31 32 #include <linux/omap-gpmc.h> 33 #include <linux/platform_data/mtd-nand-omap2.h> 34 35 #define DRIVER_NAME "omap2-nand" 36 #define OMAP_NAND_TIMEOUT_MS 5000 37 38 #define NAND_Ecc_P1e (1 << 0) 39 #define NAND_Ecc_P2e (1 << 1) 40 #define NAND_Ecc_P4e (1 << 2) 41 #define NAND_Ecc_P8e (1 << 3) 42 #define NAND_Ecc_P16e (1 << 4) 43 #define NAND_Ecc_P32e (1 << 5) 44 #define NAND_Ecc_P64e (1 << 6) 45 #define NAND_Ecc_P128e (1 << 7) 46 #define NAND_Ecc_P256e (1 << 8) 47 #define NAND_Ecc_P512e (1 << 9) 48 #define NAND_Ecc_P1024e (1 << 10) 49 #define NAND_Ecc_P2048e (1 << 11) 50 51 #define NAND_Ecc_P1o (1 << 16) 52 #define NAND_Ecc_P2o (1 << 17) 53 #define NAND_Ecc_P4o (1 << 18) 54 #define NAND_Ecc_P8o (1 << 19) 55 #define NAND_Ecc_P16o (1 << 20) 56 #define NAND_Ecc_P32o (1 << 21) 57 #define NAND_Ecc_P64o (1 << 22) 58 #define NAND_Ecc_P128o (1 << 23) 59 #define NAND_Ecc_P256o (1 << 24) 60 #define NAND_Ecc_P512o (1 << 25) 61 #define NAND_Ecc_P1024o (1 << 26) 62 #define NAND_Ecc_P2048o (1 << 27) 63 64 #define TF(value) (value ? 1 : 0) 65 66 #define P2048e(a) (TF(a & NAND_Ecc_P2048e) << 0) 67 #define P2048o(a) (TF(a & NAND_Ecc_P2048o) << 1) 68 #define P1e(a) (TF(a & NAND_Ecc_P1e) << 2) 69 #define P1o(a) (TF(a & NAND_Ecc_P1o) << 3) 70 #define P2e(a) (TF(a & NAND_Ecc_P2e) << 4) 71 #define P2o(a) (TF(a & NAND_Ecc_P2o) << 5) 72 #define P4e(a) (TF(a & NAND_Ecc_P4e) << 6) 73 #define P4o(a) (TF(a & NAND_Ecc_P4o) << 7) 74 75 #define P8e(a) (TF(a & NAND_Ecc_P8e) << 0) 76 #define P8o(a) (TF(a & NAND_Ecc_P8o) << 1) 77 #define P16e(a) (TF(a & NAND_Ecc_P16e) << 2) 78 #define P16o(a) (TF(a & NAND_Ecc_P16o) << 3) 79 #define P32e(a) (TF(a & NAND_Ecc_P32e) << 4) 80 #define P32o(a) (TF(a & NAND_Ecc_P32o) << 5) 81 #define P64e(a) (TF(a & NAND_Ecc_P64e) << 6) 82 #define P64o(a) (TF(a & NAND_Ecc_P64o) << 7) 83 84 #define P128e(a) (TF(a & NAND_Ecc_P128e) << 0) 85 #define P128o(a) (TF(a & NAND_Ecc_P128o) << 1) 86 #define P256e(a) (TF(a & NAND_Ecc_P256e) << 2) 87 #define P256o(a) (TF(a & NAND_Ecc_P256o) << 3) 88 #define P512e(a) (TF(a & NAND_Ecc_P512e) << 4) 89 #define P512o(a) (TF(a & NAND_Ecc_P512o) << 5) 90 #define P1024e(a) (TF(a & NAND_Ecc_P1024e) << 6) 91 #define P1024o(a) (TF(a & NAND_Ecc_P1024o) << 7) 92 93 #define P8e_s(a) (TF(a & NAND_Ecc_P8e) << 0) 94 #define P8o_s(a) (TF(a & NAND_Ecc_P8o) << 1) 95 #define P16e_s(a) (TF(a & NAND_Ecc_P16e) << 2) 96 #define P16o_s(a) (TF(a & NAND_Ecc_P16o) << 3) 97 #define P1e_s(a) (TF(a & NAND_Ecc_P1e) << 4) 98 #define P1o_s(a) (TF(a & NAND_Ecc_P1o) << 5) 99 #define P2e_s(a) (TF(a & NAND_Ecc_P2e) << 6) 100 #define P2o_s(a) (TF(a & NAND_Ecc_P2o) << 7) 101 102 #define P4e_s(a) (TF(a & NAND_Ecc_P4e) << 0) 103 #define P4o_s(a) (TF(a & NAND_Ecc_P4o) << 1) 104 105 #define PREFETCH_CONFIG1_CS_SHIFT 24 106 #define ECC_CONFIG_CS_SHIFT 1 107 #define CS_MASK 0x7 108 #define ENABLE_PREFETCH (0x1 << 7) 109 #define DMA_MPU_MODE_SHIFT 2 110 #define ECCSIZE0_SHIFT 12 111 #define ECCSIZE1_SHIFT 22 112 #define ECC1RESULTSIZE 0x1 113 #define ECCCLEAR 0x100 114 #define ECC1 0x1 115 #define PREFETCH_FIFOTHRESHOLD_MAX 0x40 116 #define PREFETCH_FIFOTHRESHOLD(val) ((val) << 8) 117 #define PREFETCH_STATUS_COUNT(val) (val & 0x00003fff) 118 #define PREFETCH_STATUS_FIFO_CNT(val) ((val >> 24) & 0x7F) 119 #define STATUS_BUFF_EMPTY 0x00000001 120 121 #define SECTOR_BYTES 512 122 /* 4 bit padding to make byte aligned, 56 = 52 + 4 */ 123 #define BCH4_BIT_PAD 4 124 125 /* GPMC ecc engine settings for read */ 126 #define BCH_WRAPMODE_1 1 /* BCH wrap mode 1 */ 127 #define BCH8R_ECC_SIZE0 0x1a /* ecc_size0 = 26 */ 128 #define BCH8R_ECC_SIZE1 0x2 /* ecc_size1 = 2 */ 129 #define BCH4R_ECC_SIZE0 0xd /* ecc_size0 = 13 */ 130 #define BCH4R_ECC_SIZE1 0x3 /* ecc_size1 = 3 */ 131 132 /* GPMC ecc engine settings for write */ 133 #define BCH_WRAPMODE_6 6 /* BCH wrap mode 6 */ 134 #define BCH_ECC_SIZE0 0x0 /* ecc_size0 = 0, no oob protection */ 135 #define BCH_ECC_SIZE1 0x20 /* ecc_size1 = 32 */ 136 137 #define BADBLOCK_MARKER_LENGTH 2 138 139 static u_char bch16_vector[] = {0xf5, 0x24, 0x1c, 0xd0, 0x61, 0xb3, 0xf1, 0x55, 140 0x2e, 0x2c, 0x86, 0xa3, 0xed, 0x36, 0x1b, 0x78, 141 0x48, 0x76, 0xa9, 0x3b, 0x97, 0xd1, 0x7a, 0x93, 142 0x07, 0x0e}; 143 static u_char bch8_vector[] = {0xf3, 0xdb, 0x14, 0x16, 0x8b, 0xd2, 0xbe, 0xcc, 144 0xac, 0x6b, 0xff, 0x99, 0x7b}; 145 static u_char bch4_vector[] = {0x00, 0x6b, 0x31, 0xdd, 0x41, 0xbc, 0x10}; 146 147 struct omap_nand_info { 148 struct nand_chip nand; 149 struct platform_device *pdev; 150 151 int gpmc_cs; 152 bool dev_ready; 153 enum nand_io xfer_type; 154 int devsize; 155 enum omap_ecc ecc_opt; 156 struct device_node *elm_of_node; 157 158 unsigned long phys_base; 159 struct completion comp; 160 struct dma_chan *dma; 161 int gpmc_irq_fifo; 162 int gpmc_irq_count; 163 enum { 164 OMAP_NAND_IO_READ = 0, /* read */ 165 OMAP_NAND_IO_WRITE, /* write */ 166 } iomode; 167 u_char *buf; 168 int buf_len; 169 /* Interface to GPMC */ 170 struct gpmc_nand_regs reg; 171 struct gpmc_nand_ops *ops; 172 bool flash_bbt; 173 /* fields specific for BCHx_HW ECC scheme */ 174 struct device *elm_dev; 175 /* NAND ready gpio */ 176 struct gpio_desc *ready_gpiod; 177 }; 178 179 static inline struct omap_nand_info *mtd_to_omap(struct mtd_info *mtd) 180 { 181 return container_of(mtd_to_nand(mtd), struct omap_nand_info, nand); 182 } 183 184 /** 185 * omap_prefetch_enable - configures and starts prefetch transfer 186 * @cs: cs (chip select) number 187 * @fifo_th: fifo threshold to be used for read/ write 188 * @dma_mode: dma mode enable (1) or disable (0) 189 * @u32_count: number of bytes to be transferred 190 * @is_write: prefetch read(0) or write post(1) mode 191 */ 192 static int omap_prefetch_enable(int cs, int fifo_th, int dma_mode, 193 unsigned int u32_count, int is_write, struct omap_nand_info *info) 194 { 195 u32 val; 196 197 if (fifo_th > PREFETCH_FIFOTHRESHOLD_MAX) 198 return -1; 199 200 if (readl(info->reg.gpmc_prefetch_control)) 201 return -EBUSY; 202 203 /* Set the amount of bytes to be prefetched */ 204 writel(u32_count, info->reg.gpmc_prefetch_config2); 205 206 /* Set dma/mpu mode, the prefetch read / post write and 207 * enable the engine. Set which cs is has requested for. 208 */ 209 val = ((cs << PREFETCH_CONFIG1_CS_SHIFT) | 210 PREFETCH_FIFOTHRESHOLD(fifo_th) | ENABLE_PREFETCH | 211 (dma_mode << DMA_MPU_MODE_SHIFT) | (is_write & 0x1)); 212 writel(val, info->reg.gpmc_prefetch_config1); 213 214 /* Start the prefetch engine */ 215 writel(0x1, info->reg.gpmc_prefetch_control); 216 217 return 0; 218 } 219 220 /** 221 * omap_prefetch_reset - disables and stops the prefetch engine 222 */ 223 static int omap_prefetch_reset(int cs, struct omap_nand_info *info) 224 { 225 u32 config1; 226 227 /* check if the same module/cs is trying to reset */ 228 config1 = readl(info->reg.gpmc_prefetch_config1); 229 if (((config1 >> PREFETCH_CONFIG1_CS_SHIFT) & CS_MASK) != cs) 230 return -EINVAL; 231 232 /* Stop the PFPW engine */ 233 writel(0x0, info->reg.gpmc_prefetch_control); 234 235 /* Reset/disable the PFPW engine */ 236 writel(0x0, info->reg.gpmc_prefetch_config1); 237 238 return 0; 239 } 240 241 /** 242 * omap_hwcontrol - hardware specific access to control-lines 243 * @mtd: MTD device structure 244 * @cmd: command to device 245 * @ctrl: 246 * NAND_NCE: bit 0 -> don't care 247 * NAND_CLE: bit 1 -> Command Latch 248 * NAND_ALE: bit 2 -> Address Latch 249 * 250 * NOTE: boards may use different bits for these!! 251 */ 252 static void omap_hwcontrol(struct mtd_info *mtd, int cmd, unsigned int ctrl) 253 { 254 struct omap_nand_info *info = mtd_to_omap(mtd); 255 256 if (cmd != NAND_CMD_NONE) { 257 if (ctrl & NAND_CLE) 258 writeb(cmd, info->reg.gpmc_nand_command); 259 260 else if (ctrl & NAND_ALE) 261 writeb(cmd, info->reg.gpmc_nand_address); 262 263 else /* NAND_NCE */ 264 writeb(cmd, info->reg.gpmc_nand_data); 265 } 266 } 267 268 /** 269 * omap_read_buf8 - read data from NAND controller into buffer 270 * @mtd: MTD device structure 271 * @buf: buffer to store date 272 * @len: number of bytes to read 273 */ 274 static void omap_read_buf8(struct mtd_info *mtd, u_char *buf, int len) 275 { 276 struct nand_chip *nand = mtd_to_nand(mtd); 277 278 ioread8_rep(nand->IO_ADDR_R, buf, len); 279 } 280 281 /** 282 * omap_write_buf8 - write buffer to NAND controller 283 * @mtd: MTD device structure 284 * @buf: data buffer 285 * @len: number of bytes to write 286 */ 287 static void omap_write_buf8(struct mtd_info *mtd, const u_char *buf, int len) 288 { 289 struct omap_nand_info *info = mtd_to_omap(mtd); 290 u_char *p = (u_char *)buf; 291 bool status; 292 293 while (len--) { 294 iowrite8(*p++, info->nand.IO_ADDR_W); 295 /* wait until buffer is available for write */ 296 do { 297 status = info->ops->nand_writebuffer_empty(); 298 } while (!status); 299 } 300 } 301 302 /** 303 * omap_read_buf16 - read data from NAND controller into buffer 304 * @mtd: MTD device structure 305 * @buf: buffer to store date 306 * @len: number of bytes to read 307 */ 308 static void omap_read_buf16(struct mtd_info *mtd, u_char *buf, int len) 309 { 310 struct nand_chip *nand = mtd_to_nand(mtd); 311 312 ioread16_rep(nand->IO_ADDR_R, buf, len / 2); 313 } 314 315 /** 316 * omap_write_buf16 - write buffer to NAND controller 317 * @mtd: MTD device structure 318 * @buf: data buffer 319 * @len: number of bytes to write 320 */ 321 static void omap_write_buf16(struct mtd_info *mtd, const u_char * buf, int len) 322 { 323 struct omap_nand_info *info = mtd_to_omap(mtd); 324 u16 *p = (u16 *) buf; 325 bool status; 326 /* FIXME try bursts of writesw() or DMA ... */ 327 len >>= 1; 328 329 while (len--) { 330 iowrite16(*p++, info->nand.IO_ADDR_W); 331 /* wait until buffer is available for write */ 332 do { 333 status = info->ops->nand_writebuffer_empty(); 334 } while (!status); 335 } 336 } 337 338 /** 339 * omap_read_buf_pref - read data from NAND controller into buffer 340 * @mtd: MTD device structure 341 * @buf: buffer to store date 342 * @len: number of bytes to read 343 */ 344 static void omap_read_buf_pref(struct mtd_info *mtd, u_char *buf, int len) 345 { 346 struct omap_nand_info *info = mtd_to_omap(mtd); 347 uint32_t r_count = 0; 348 int ret = 0; 349 u32 *p = (u32 *)buf; 350 351 /* take care of subpage reads */ 352 if (len % 4) { 353 if (info->nand.options & NAND_BUSWIDTH_16) 354 omap_read_buf16(mtd, buf, len % 4); 355 else 356 omap_read_buf8(mtd, buf, len % 4); 357 p = (u32 *) (buf + len % 4); 358 len -= len % 4; 359 } 360 361 /* configure and start prefetch transfer */ 362 ret = omap_prefetch_enable(info->gpmc_cs, 363 PREFETCH_FIFOTHRESHOLD_MAX, 0x0, len, 0x0, info); 364 if (ret) { 365 /* PFPW engine is busy, use cpu copy method */ 366 if (info->nand.options & NAND_BUSWIDTH_16) 367 omap_read_buf16(mtd, (u_char *)p, len); 368 else 369 omap_read_buf8(mtd, (u_char *)p, len); 370 } else { 371 do { 372 r_count = readl(info->reg.gpmc_prefetch_status); 373 r_count = PREFETCH_STATUS_FIFO_CNT(r_count); 374 r_count = r_count >> 2; 375 ioread32_rep(info->nand.IO_ADDR_R, p, r_count); 376 p += r_count; 377 len -= r_count << 2; 378 } while (len); 379 /* disable and stop the PFPW engine */ 380 omap_prefetch_reset(info->gpmc_cs, info); 381 } 382 } 383 384 /** 385 * omap_write_buf_pref - write buffer to NAND controller 386 * @mtd: MTD device structure 387 * @buf: data buffer 388 * @len: number of bytes to write 389 */ 390 static void omap_write_buf_pref(struct mtd_info *mtd, 391 const u_char *buf, int len) 392 { 393 struct omap_nand_info *info = mtd_to_omap(mtd); 394 uint32_t w_count = 0; 395 int i = 0, ret = 0; 396 u16 *p = (u16 *)buf; 397 unsigned long tim, limit; 398 u32 val; 399 400 /* take care of subpage writes */ 401 if (len % 2 != 0) { 402 writeb(*buf, info->nand.IO_ADDR_W); 403 p = (u16 *)(buf + 1); 404 len--; 405 } 406 407 /* configure and start prefetch transfer */ 408 ret = omap_prefetch_enable(info->gpmc_cs, 409 PREFETCH_FIFOTHRESHOLD_MAX, 0x0, len, 0x1, info); 410 if (ret) { 411 /* PFPW engine is busy, use cpu copy method */ 412 if (info->nand.options & NAND_BUSWIDTH_16) 413 omap_write_buf16(mtd, (u_char *)p, len); 414 else 415 omap_write_buf8(mtd, (u_char *)p, len); 416 } else { 417 while (len) { 418 w_count = readl(info->reg.gpmc_prefetch_status); 419 w_count = PREFETCH_STATUS_FIFO_CNT(w_count); 420 w_count = w_count >> 1; 421 for (i = 0; (i < w_count) && len; i++, len -= 2) 422 iowrite16(*p++, info->nand.IO_ADDR_W); 423 } 424 /* wait for data to flushed-out before reset the prefetch */ 425 tim = 0; 426 limit = (loops_per_jiffy * 427 msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS)); 428 do { 429 cpu_relax(); 430 val = readl(info->reg.gpmc_prefetch_status); 431 val = PREFETCH_STATUS_COUNT(val); 432 } while (val && (tim++ < limit)); 433 434 /* disable and stop the PFPW engine */ 435 omap_prefetch_reset(info->gpmc_cs, info); 436 } 437 } 438 439 /* 440 * omap_nand_dma_callback: callback on the completion of dma transfer 441 * @data: pointer to completion data structure 442 */ 443 static void omap_nand_dma_callback(void *data) 444 { 445 complete((struct completion *) data); 446 } 447 448 /* 449 * omap_nand_dma_transfer: configure and start dma transfer 450 * @mtd: MTD device structure 451 * @addr: virtual address in RAM of source/destination 452 * @len: number of data bytes to be transferred 453 * @is_write: flag for read/write operation 454 */ 455 static inline int omap_nand_dma_transfer(struct mtd_info *mtd, void *addr, 456 unsigned int len, int is_write) 457 { 458 struct omap_nand_info *info = mtd_to_omap(mtd); 459 struct dma_async_tx_descriptor *tx; 460 enum dma_data_direction dir = is_write ? DMA_TO_DEVICE : 461 DMA_FROM_DEVICE; 462 struct scatterlist sg; 463 unsigned long tim, limit; 464 unsigned n; 465 int ret; 466 u32 val; 467 468 if (!virt_addr_valid(addr)) 469 goto out_copy; 470 471 sg_init_one(&sg, addr, len); 472 n = dma_map_sg(info->dma->device->dev, &sg, 1, dir); 473 if (n == 0) { 474 dev_err(&info->pdev->dev, 475 "Couldn't DMA map a %d byte buffer\n", len); 476 goto out_copy; 477 } 478 479 tx = dmaengine_prep_slave_sg(info->dma, &sg, n, 480 is_write ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM, 481 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 482 if (!tx) 483 goto out_copy_unmap; 484 485 tx->callback = omap_nand_dma_callback; 486 tx->callback_param = &info->comp; 487 dmaengine_submit(tx); 488 489 init_completion(&info->comp); 490 491 /* setup and start DMA using dma_addr */ 492 dma_async_issue_pending(info->dma); 493 494 /* configure and start prefetch transfer */ 495 ret = omap_prefetch_enable(info->gpmc_cs, 496 PREFETCH_FIFOTHRESHOLD_MAX, 0x1, len, is_write, info); 497 if (ret) 498 /* PFPW engine is busy, use cpu copy method */ 499 goto out_copy_unmap; 500 501 wait_for_completion(&info->comp); 502 tim = 0; 503 limit = (loops_per_jiffy * msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS)); 504 505 do { 506 cpu_relax(); 507 val = readl(info->reg.gpmc_prefetch_status); 508 val = PREFETCH_STATUS_COUNT(val); 509 } while (val && (tim++ < limit)); 510 511 /* disable and stop the PFPW engine */ 512 omap_prefetch_reset(info->gpmc_cs, info); 513 514 dma_unmap_sg(info->dma->device->dev, &sg, 1, dir); 515 return 0; 516 517 out_copy_unmap: 518 dma_unmap_sg(info->dma->device->dev, &sg, 1, dir); 519 out_copy: 520 if (info->nand.options & NAND_BUSWIDTH_16) 521 is_write == 0 ? omap_read_buf16(mtd, (u_char *) addr, len) 522 : omap_write_buf16(mtd, (u_char *) addr, len); 523 else 524 is_write == 0 ? omap_read_buf8(mtd, (u_char *) addr, len) 525 : omap_write_buf8(mtd, (u_char *) addr, len); 526 return 0; 527 } 528 529 /** 530 * omap_read_buf_dma_pref - read data from NAND controller into buffer 531 * @mtd: MTD device structure 532 * @buf: buffer to store date 533 * @len: number of bytes to read 534 */ 535 static void omap_read_buf_dma_pref(struct mtd_info *mtd, u_char *buf, int len) 536 { 537 if (len <= mtd->oobsize) 538 omap_read_buf_pref(mtd, buf, len); 539 else 540 /* start transfer in DMA mode */ 541 omap_nand_dma_transfer(mtd, buf, len, 0x0); 542 } 543 544 /** 545 * omap_write_buf_dma_pref - write buffer to NAND controller 546 * @mtd: MTD device structure 547 * @buf: data buffer 548 * @len: number of bytes to write 549 */ 550 static void omap_write_buf_dma_pref(struct mtd_info *mtd, 551 const u_char *buf, int len) 552 { 553 if (len <= mtd->oobsize) 554 omap_write_buf_pref(mtd, buf, len); 555 else 556 /* start transfer in DMA mode */ 557 omap_nand_dma_transfer(mtd, (u_char *) buf, len, 0x1); 558 } 559 560 /* 561 * omap_nand_irq - GPMC irq handler 562 * @this_irq: gpmc irq number 563 * @dev: omap_nand_info structure pointer is passed here 564 */ 565 static irqreturn_t omap_nand_irq(int this_irq, void *dev) 566 { 567 struct omap_nand_info *info = (struct omap_nand_info *) dev; 568 u32 bytes; 569 570 bytes = readl(info->reg.gpmc_prefetch_status); 571 bytes = PREFETCH_STATUS_FIFO_CNT(bytes); 572 bytes = bytes & 0xFFFC; /* io in multiple of 4 bytes */ 573 if (info->iomode == OMAP_NAND_IO_WRITE) { /* checks for write io */ 574 if (this_irq == info->gpmc_irq_count) 575 goto done; 576 577 if (info->buf_len && (info->buf_len < bytes)) 578 bytes = info->buf_len; 579 else if (!info->buf_len) 580 bytes = 0; 581 iowrite32_rep(info->nand.IO_ADDR_W, 582 (u32 *)info->buf, bytes >> 2); 583 info->buf = info->buf + bytes; 584 info->buf_len -= bytes; 585 586 } else { 587 ioread32_rep(info->nand.IO_ADDR_R, 588 (u32 *)info->buf, bytes >> 2); 589 info->buf = info->buf + bytes; 590 591 if (this_irq == info->gpmc_irq_count) 592 goto done; 593 } 594 595 return IRQ_HANDLED; 596 597 done: 598 complete(&info->comp); 599 600 disable_irq_nosync(info->gpmc_irq_fifo); 601 disable_irq_nosync(info->gpmc_irq_count); 602 603 return IRQ_HANDLED; 604 } 605 606 /* 607 * omap_read_buf_irq_pref - read data from NAND controller into buffer 608 * @mtd: MTD device structure 609 * @buf: buffer to store date 610 * @len: number of bytes to read 611 */ 612 static void omap_read_buf_irq_pref(struct mtd_info *mtd, u_char *buf, int len) 613 { 614 struct omap_nand_info *info = mtd_to_omap(mtd); 615 int ret = 0; 616 617 if (len <= mtd->oobsize) { 618 omap_read_buf_pref(mtd, buf, len); 619 return; 620 } 621 622 info->iomode = OMAP_NAND_IO_READ; 623 info->buf = buf; 624 init_completion(&info->comp); 625 626 /* configure and start prefetch transfer */ 627 ret = omap_prefetch_enable(info->gpmc_cs, 628 PREFETCH_FIFOTHRESHOLD_MAX/2, 0x0, len, 0x0, info); 629 if (ret) 630 /* PFPW engine is busy, use cpu copy method */ 631 goto out_copy; 632 633 info->buf_len = len; 634 635 enable_irq(info->gpmc_irq_count); 636 enable_irq(info->gpmc_irq_fifo); 637 638 /* waiting for read to complete */ 639 wait_for_completion(&info->comp); 640 641 /* disable and stop the PFPW engine */ 642 omap_prefetch_reset(info->gpmc_cs, info); 643 return; 644 645 out_copy: 646 if (info->nand.options & NAND_BUSWIDTH_16) 647 omap_read_buf16(mtd, buf, len); 648 else 649 omap_read_buf8(mtd, buf, len); 650 } 651 652 /* 653 * omap_write_buf_irq_pref - write buffer to NAND controller 654 * @mtd: MTD device structure 655 * @buf: data buffer 656 * @len: number of bytes to write 657 */ 658 static void omap_write_buf_irq_pref(struct mtd_info *mtd, 659 const u_char *buf, int len) 660 { 661 struct omap_nand_info *info = mtd_to_omap(mtd); 662 int ret = 0; 663 unsigned long tim, limit; 664 u32 val; 665 666 if (len <= mtd->oobsize) { 667 omap_write_buf_pref(mtd, buf, len); 668 return; 669 } 670 671 info->iomode = OMAP_NAND_IO_WRITE; 672 info->buf = (u_char *) buf; 673 init_completion(&info->comp); 674 675 /* configure and start prefetch transfer : size=24 */ 676 ret = omap_prefetch_enable(info->gpmc_cs, 677 (PREFETCH_FIFOTHRESHOLD_MAX * 3) / 8, 0x0, len, 0x1, info); 678 if (ret) 679 /* PFPW engine is busy, use cpu copy method */ 680 goto out_copy; 681 682 info->buf_len = len; 683 684 enable_irq(info->gpmc_irq_count); 685 enable_irq(info->gpmc_irq_fifo); 686 687 /* waiting for write to complete */ 688 wait_for_completion(&info->comp); 689 690 /* wait for data to flushed-out before reset the prefetch */ 691 tim = 0; 692 limit = (loops_per_jiffy * msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS)); 693 do { 694 val = readl(info->reg.gpmc_prefetch_status); 695 val = PREFETCH_STATUS_COUNT(val); 696 cpu_relax(); 697 } while (val && (tim++ < limit)); 698 699 /* disable and stop the PFPW engine */ 700 omap_prefetch_reset(info->gpmc_cs, info); 701 return; 702 703 out_copy: 704 if (info->nand.options & NAND_BUSWIDTH_16) 705 omap_write_buf16(mtd, buf, len); 706 else 707 omap_write_buf8(mtd, buf, len); 708 } 709 710 /** 711 * gen_true_ecc - This function will generate true ECC value 712 * @ecc_buf: buffer to store ecc code 713 * 714 * This generated true ECC value can be used when correcting 715 * data read from NAND flash memory core 716 */ 717 static void gen_true_ecc(u8 *ecc_buf) 718 { 719 u32 tmp = ecc_buf[0] | (ecc_buf[1] << 16) | 720 ((ecc_buf[2] & 0xF0) << 20) | ((ecc_buf[2] & 0x0F) << 8); 721 722 ecc_buf[0] = ~(P64o(tmp) | P64e(tmp) | P32o(tmp) | P32e(tmp) | 723 P16o(tmp) | P16e(tmp) | P8o(tmp) | P8e(tmp)); 724 ecc_buf[1] = ~(P1024o(tmp) | P1024e(tmp) | P512o(tmp) | P512e(tmp) | 725 P256o(tmp) | P256e(tmp) | P128o(tmp) | P128e(tmp)); 726 ecc_buf[2] = ~(P4o(tmp) | P4e(tmp) | P2o(tmp) | P2e(tmp) | P1o(tmp) | 727 P1e(tmp) | P2048o(tmp) | P2048e(tmp)); 728 } 729 730 /** 731 * omap_compare_ecc - Detect (2 bits) and correct (1 bit) error in data 732 * @ecc_data1: ecc code from nand spare area 733 * @ecc_data2: ecc code from hardware register obtained from hardware ecc 734 * @page_data: page data 735 * 736 * This function compares two ECC's and indicates if there is an error. 737 * If the error can be corrected it will be corrected to the buffer. 738 * If there is no error, %0 is returned. If there is an error but it 739 * was corrected, %1 is returned. Otherwise, %-1 is returned. 740 */ 741 static int omap_compare_ecc(u8 *ecc_data1, /* read from NAND memory */ 742 u8 *ecc_data2, /* read from register */ 743 u8 *page_data) 744 { 745 uint i; 746 u8 tmp0_bit[8], tmp1_bit[8], tmp2_bit[8]; 747 u8 comp0_bit[8], comp1_bit[8], comp2_bit[8]; 748 u8 ecc_bit[24]; 749 u8 ecc_sum = 0; 750 u8 find_bit = 0; 751 uint find_byte = 0; 752 int isEccFF; 753 754 isEccFF = ((*(u32 *)ecc_data1 & 0xFFFFFF) == 0xFFFFFF); 755 756 gen_true_ecc(ecc_data1); 757 gen_true_ecc(ecc_data2); 758 759 for (i = 0; i <= 2; i++) { 760 *(ecc_data1 + i) = ~(*(ecc_data1 + i)); 761 *(ecc_data2 + i) = ~(*(ecc_data2 + i)); 762 } 763 764 for (i = 0; i < 8; i++) { 765 tmp0_bit[i] = *ecc_data1 % 2; 766 *ecc_data1 = *ecc_data1 / 2; 767 } 768 769 for (i = 0; i < 8; i++) { 770 tmp1_bit[i] = *(ecc_data1 + 1) % 2; 771 *(ecc_data1 + 1) = *(ecc_data1 + 1) / 2; 772 } 773 774 for (i = 0; i < 8; i++) { 775 tmp2_bit[i] = *(ecc_data1 + 2) % 2; 776 *(ecc_data1 + 2) = *(ecc_data1 + 2) / 2; 777 } 778 779 for (i = 0; i < 8; i++) { 780 comp0_bit[i] = *ecc_data2 % 2; 781 *ecc_data2 = *ecc_data2 / 2; 782 } 783 784 for (i = 0; i < 8; i++) { 785 comp1_bit[i] = *(ecc_data2 + 1) % 2; 786 *(ecc_data2 + 1) = *(ecc_data2 + 1) / 2; 787 } 788 789 for (i = 0; i < 8; i++) { 790 comp2_bit[i] = *(ecc_data2 + 2) % 2; 791 *(ecc_data2 + 2) = *(ecc_data2 + 2) / 2; 792 } 793 794 for (i = 0; i < 6; i++) 795 ecc_bit[i] = tmp2_bit[i + 2] ^ comp2_bit[i + 2]; 796 797 for (i = 0; i < 8; i++) 798 ecc_bit[i + 6] = tmp0_bit[i] ^ comp0_bit[i]; 799 800 for (i = 0; i < 8; i++) 801 ecc_bit[i + 14] = tmp1_bit[i] ^ comp1_bit[i]; 802 803 ecc_bit[22] = tmp2_bit[0] ^ comp2_bit[0]; 804 ecc_bit[23] = tmp2_bit[1] ^ comp2_bit[1]; 805 806 for (i = 0; i < 24; i++) 807 ecc_sum += ecc_bit[i]; 808 809 switch (ecc_sum) { 810 case 0: 811 /* Not reached because this function is not called if 812 * ECC values are equal 813 */ 814 return 0; 815 816 case 1: 817 /* Uncorrectable error */ 818 pr_debug("ECC UNCORRECTED_ERROR 1\n"); 819 return -EBADMSG; 820 821 case 11: 822 /* UN-Correctable error */ 823 pr_debug("ECC UNCORRECTED_ERROR B\n"); 824 return -EBADMSG; 825 826 case 12: 827 /* Correctable error */ 828 find_byte = (ecc_bit[23] << 8) + 829 (ecc_bit[21] << 7) + 830 (ecc_bit[19] << 6) + 831 (ecc_bit[17] << 5) + 832 (ecc_bit[15] << 4) + 833 (ecc_bit[13] << 3) + 834 (ecc_bit[11] << 2) + 835 (ecc_bit[9] << 1) + 836 ecc_bit[7]; 837 838 find_bit = (ecc_bit[5] << 2) + (ecc_bit[3] << 1) + ecc_bit[1]; 839 840 pr_debug("Correcting single bit ECC error at offset: " 841 "%d, bit: %d\n", find_byte, find_bit); 842 843 page_data[find_byte] ^= (1 << find_bit); 844 845 return 1; 846 default: 847 if (isEccFF) { 848 if (ecc_data2[0] == 0 && 849 ecc_data2[1] == 0 && 850 ecc_data2[2] == 0) 851 return 0; 852 } 853 pr_debug("UNCORRECTED_ERROR default\n"); 854 return -EBADMSG; 855 } 856 } 857 858 /** 859 * omap_correct_data - Compares the ECC read with HW generated ECC 860 * @mtd: MTD device structure 861 * @dat: page data 862 * @read_ecc: ecc read from nand flash 863 * @calc_ecc: ecc read from HW ECC registers 864 * 865 * Compares the ecc read from nand spare area with ECC registers values 866 * and if ECC's mismatched, it will call 'omap_compare_ecc' for error 867 * detection and correction. If there are no errors, %0 is returned. If 868 * there were errors and all of the errors were corrected, the number of 869 * corrected errors is returned. If uncorrectable errors exist, %-1 is 870 * returned. 871 */ 872 static int omap_correct_data(struct mtd_info *mtd, u_char *dat, 873 u_char *read_ecc, u_char *calc_ecc) 874 { 875 struct omap_nand_info *info = mtd_to_omap(mtd); 876 int blockCnt = 0, i = 0, ret = 0; 877 int stat = 0; 878 879 /* Ex NAND_ECC_HW12_2048 */ 880 if ((info->nand.ecc.mode == NAND_ECC_HW) && 881 (info->nand.ecc.size == 2048)) 882 blockCnt = 4; 883 else 884 blockCnt = 1; 885 886 for (i = 0; i < blockCnt; i++) { 887 if (memcmp(read_ecc, calc_ecc, 3) != 0) { 888 ret = omap_compare_ecc(read_ecc, calc_ecc, dat); 889 if (ret < 0) 890 return ret; 891 /* keep track of the number of corrected errors */ 892 stat += ret; 893 } 894 read_ecc += 3; 895 calc_ecc += 3; 896 dat += 512; 897 } 898 return stat; 899 } 900 901 /** 902 * omap_calcuate_ecc - Generate non-inverted ECC bytes. 903 * @mtd: MTD device structure 904 * @dat: The pointer to data on which ecc is computed 905 * @ecc_code: The ecc_code buffer 906 * 907 * Using noninverted ECC can be considered ugly since writing a blank 908 * page ie. padding will clear the ECC bytes. This is no problem as long 909 * nobody is trying to write data on the seemingly unused page. Reading 910 * an erased page will produce an ECC mismatch between generated and read 911 * ECC bytes that has to be dealt with separately. 912 */ 913 static int omap_calculate_ecc(struct mtd_info *mtd, const u_char *dat, 914 u_char *ecc_code) 915 { 916 struct omap_nand_info *info = mtd_to_omap(mtd); 917 u32 val; 918 919 val = readl(info->reg.gpmc_ecc_config); 920 if (((val >> ECC_CONFIG_CS_SHIFT) & CS_MASK) != info->gpmc_cs) 921 return -EINVAL; 922 923 /* read ecc result */ 924 val = readl(info->reg.gpmc_ecc1_result); 925 *ecc_code++ = val; /* P128e, ..., P1e */ 926 *ecc_code++ = val >> 16; /* P128o, ..., P1o */ 927 /* P2048o, P1024o, P512o, P256o, P2048e, P1024e, P512e, P256e */ 928 *ecc_code++ = ((val >> 8) & 0x0f) | ((val >> 20) & 0xf0); 929 930 return 0; 931 } 932 933 /** 934 * omap_enable_hwecc - This function enables the hardware ecc functionality 935 * @mtd: MTD device structure 936 * @mode: Read/Write mode 937 */ 938 static void omap_enable_hwecc(struct mtd_info *mtd, int mode) 939 { 940 struct omap_nand_info *info = mtd_to_omap(mtd); 941 struct nand_chip *chip = mtd_to_nand(mtd); 942 unsigned int dev_width = (chip->options & NAND_BUSWIDTH_16) ? 1 : 0; 943 u32 val; 944 945 /* clear ecc and enable bits */ 946 val = ECCCLEAR | ECC1; 947 writel(val, info->reg.gpmc_ecc_control); 948 949 /* program ecc and result sizes */ 950 val = ((((info->nand.ecc.size >> 1) - 1) << ECCSIZE1_SHIFT) | 951 ECC1RESULTSIZE); 952 writel(val, info->reg.gpmc_ecc_size_config); 953 954 switch (mode) { 955 case NAND_ECC_READ: 956 case NAND_ECC_WRITE: 957 writel(ECCCLEAR | ECC1, info->reg.gpmc_ecc_control); 958 break; 959 case NAND_ECC_READSYN: 960 writel(ECCCLEAR, info->reg.gpmc_ecc_control); 961 break; 962 default: 963 dev_info(&info->pdev->dev, 964 "error: unrecognized Mode[%d]!\n", mode); 965 break; 966 } 967 968 /* (ECC 16 or 8 bit col) | ( CS ) | ECC Enable */ 969 val = (dev_width << 7) | (info->gpmc_cs << 1) | (0x1); 970 writel(val, info->reg.gpmc_ecc_config); 971 } 972 973 /** 974 * omap_wait - wait until the command is done 975 * @mtd: MTD device structure 976 * @chip: NAND Chip structure 977 * 978 * Wait function is called during Program and erase operations and 979 * the way it is called from MTD layer, we should wait till the NAND 980 * chip is ready after the programming/erase operation has completed. 981 * 982 * Erase can take up to 400ms and program up to 20ms according to 983 * general NAND and SmartMedia specs 984 */ 985 static int omap_wait(struct mtd_info *mtd, struct nand_chip *chip) 986 { 987 struct nand_chip *this = mtd_to_nand(mtd); 988 struct omap_nand_info *info = mtd_to_omap(mtd); 989 unsigned long timeo = jiffies; 990 int status, state = this->state; 991 992 if (state == FL_ERASING) 993 timeo += msecs_to_jiffies(400); 994 else 995 timeo += msecs_to_jiffies(20); 996 997 writeb(NAND_CMD_STATUS & 0xFF, info->reg.gpmc_nand_command); 998 while (time_before(jiffies, timeo)) { 999 status = readb(info->reg.gpmc_nand_data); 1000 if (status & NAND_STATUS_READY) 1001 break; 1002 cond_resched(); 1003 } 1004 1005 status = readb(info->reg.gpmc_nand_data); 1006 return status; 1007 } 1008 1009 /** 1010 * omap_dev_ready - checks the NAND Ready GPIO line 1011 * @mtd: MTD device structure 1012 * 1013 * Returns true if ready and false if busy. 1014 */ 1015 static int omap_dev_ready(struct mtd_info *mtd) 1016 { 1017 struct omap_nand_info *info = mtd_to_omap(mtd); 1018 1019 return gpiod_get_value(info->ready_gpiod); 1020 } 1021 1022 /** 1023 * omap_enable_hwecc_bch - Program GPMC to perform BCH ECC calculation 1024 * @mtd: MTD device structure 1025 * @mode: Read/Write mode 1026 * 1027 * When using BCH with SW correction (i.e. no ELM), sector size is set 1028 * to 512 bytes and we use BCH_WRAPMODE_6 wrapping mode 1029 * for both reading and writing with: 1030 * eccsize0 = 0 (no additional protected byte in spare area) 1031 * eccsize1 = 32 (skip 32 nibbles = 16 bytes per sector in spare area) 1032 */ 1033 static void __maybe_unused omap_enable_hwecc_bch(struct mtd_info *mtd, int mode) 1034 { 1035 unsigned int bch_type; 1036 unsigned int dev_width, nsectors; 1037 struct omap_nand_info *info = mtd_to_omap(mtd); 1038 enum omap_ecc ecc_opt = info->ecc_opt; 1039 struct nand_chip *chip = mtd_to_nand(mtd); 1040 u32 val, wr_mode; 1041 unsigned int ecc_size1, ecc_size0; 1042 1043 /* GPMC configurations for calculating ECC */ 1044 switch (ecc_opt) { 1045 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: 1046 bch_type = 0; 1047 nsectors = 1; 1048 wr_mode = BCH_WRAPMODE_6; 1049 ecc_size0 = BCH_ECC_SIZE0; 1050 ecc_size1 = BCH_ECC_SIZE1; 1051 break; 1052 case OMAP_ECC_BCH4_CODE_HW: 1053 bch_type = 0; 1054 nsectors = chip->ecc.steps; 1055 if (mode == NAND_ECC_READ) { 1056 wr_mode = BCH_WRAPMODE_1; 1057 ecc_size0 = BCH4R_ECC_SIZE0; 1058 ecc_size1 = BCH4R_ECC_SIZE1; 1059 } else { 1060 wr_mode = BCH_WRAPMODE_6; 1061 ecc_size0 = BCH_ECC_SIZE0; 1062 ecc_size1 = BCH_ECC_SIZE1; 1063 } 1064 break; 1065 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: 1066 bch_type = 1; 1067 nsectors = 1; 1068 wr_mode = BCH_WRAPMODE_6; 1069 ecc_size0 = BCH_ECC_SIZE0; 1070 ecc_size1 = BCH_ECC_SIZE1; 1071 break; 1072 case OMAP_ECC_BCH8_CODE_HW: 1073 bch_type = 1; 1074 nsectors = chip->ecc.steps; 1075 if (mode == NAND_ECC_READ) { 1076 wr_mode = BCH_WRAPMODE_1; 1077 ecc_size0 = BCH8R_ECC_SIZE0; 1078 ecc_size1 = BCH8R_ECC_SIZE1; 1079 } else { 1080 wr_mode = BCH_WRAPMODE_6; 1081 ecc_size0 = BCH_ECC_SIZE0; 1082 ecc_size1 = BCH_ECC_SIZE1; 1083 } 1084 break; 1085 case OMAP_ECC_BCH16_CODE_HW: 1086 bch_type = 0x2; 1087 nsectors = chip->ecc.steps; 1088 if (mode == NAND_ECC_READ) { 1089 wr_mode = 0x01; 1090 ecc_size0 = 52; /* ECC bits in nibbles per sector */ 1091 ecc_size1 = 0; /* non-ECC bits in nibbles per sector */ 1092 } else { 1093 wr_mode = 0x01; 1094 ecc_size0 = 0; /* extra bits in nibbles per sector */ 1095 ecc_size1 = 52; /* OOB bits in nibbles per sector */ 1096 } 1097 break; 1098 default: 1099 return; 1100 } 1101 1102 writel(ECC1, info->reg.gpmc_ecc_control); 1103 1104 /* Configure ecc size for BCH */ 1105 val = (ecc_size1 << ECCSIZE1_SHIFT) | (ecc_size0 << ECCSIZE0_SHIFT); 1106 writel(val, info->reg.gpmc_ecc_size_config); 1107 1108 dev_width = (chip->options & NAND_BUSWIDTH_16) ? 1 : 0; 1109 1110 /* BCH configuration */ 1111 val = ((1 << 16) | /* enable BCH */ 1112 (bch_type << 12) | /* BCH4/BCH8/BCH16 */ 1113 (wr_mode << 8) | /* wrap mode */ 1114 (dev_width << 7) | /* bus width */ 1115 (((nsectors-1) & 0x7) << 4) | /* number of sectors */ 1116 (info->gpmc_cs << 1) | /* ECC CS */ 1117 (0x1)); /* enable ECC */ 1118 1119 writel(val, info->reg.gpmc_ecc_config); 1120 1121 /* Clear ecc and enable bits */ 1122 writel(ECCCLEAR | ECC1, info->reg.gpmc_ecc_control); 1123 } 1124 1125 static u8 bch4_polynomial[] = {0x28, 0x13, 0xcc, 0x39, 0x96, 0xac, 0x7f}; 1126 static u8 bch8_polynomial[] = {0xef, 0x51, 0x2e, 0x09, 0xed, 0x93, 0x9a, 0xc2, 1127 0x97, 0x79, 0xe5, 0x24, 0xb5}; 1128 1129 /** 1130 * _omap_calculate_ecc_bch - Generate ECC bytes for one sector 1131 * @mtd: MTD device structure 1132 * @dat: The pointer to data on which ecc is computed 1133 * @ecc_code: The ecc_code buffer 1134 * @i: The sector number (for a multi sector page) 1135 * 1136 * Support calculating of BCH4/8/16 ECC vectors for one sector 1137 * within a page. Sector number is in @i. 1138 */ 1139 static int _omap_calculate_ecc_bch(struct mtd_info *mtd, 1140 const u_char *dat, u_char *ecc_calc, int i) 1141 { 1142 struct omap_nand_info *info = mtd_to_omap(mtd); 1143 int eccbytes = info->nand.ecc.bytes; 1144 struct gpmc_nand_regs *gpmc_regs = &info->reg; 1145 u8 *ecc_code; 1146 unsigned long bch_val1, bch_val2, bch_val3, bch_val4; 1147 u32 val; 1148 int j; 1149 1150 ecc_code = ecc_calc; 1151 switch (info->ecc_opt) { 1152 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: 1153 case OMAP_ECC_BCH8_CODE_HW: 1154 bch_val1 = readl(gpmc_regs->gpmc_bch_result0[i]); 1155 bch_val2 = readl(gpmc_regs->gpmc_bch_result1[i]); 1156 bch_val3 = readl(gpmc_regs->gpmc_bch_result2[i]); 1157 bch_val4 = readl(gpmc_regs->gpmc_bch_result3[i]); 1158 *ecc_code++ = (bch_val4 & 0xFF); 1159 *ecc_code++ = ((bch_val3 >> 24) & 0xFF); 1160 *ecc_code++ = ((bch_val3 >> 16) & 0xFF); 1161 *ecc_code++ = ((bch_val3 >> 8) & 0xFF); 1162 *ecc_code++ = (bch_val3 & 0xFF); 1163 *ecc_code++ = ((bch_val2 >> 24) & 0xFF); 1164 *ecc_code++ = ((bch_val2 >> 16) & 0xFF); 1165 *ecc_code++ = ((bch_val2 >> 8) & 0xFF); 1166 *ecc_code++ = (bch_val2 & 0xFF); 1167 *ecc_code++ = ((bch_val1 >> 24) & 0xFF); 1168 *ecc_code++ = ((bch_val1 >> 16) & 0xFF); 1169 *ecc_code++ = ((bch_val1 >> 8) & 0xFF); 1170 *ecc_code++ = (bch_val1 & 0xFF); 1171 break; 1172 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: 1173 case OMAP_ECC_BCH4_CODE_HW: 1174 bch_val1 = readl(gpmc_regs->gpmc_bch_result0[i]); 1175 bch_val2 = readl(gpmc_regs->gpmc_bch_result1[i]); 1176 *ecc_code++ = ((bch_val2 >> 12) & 0xFF); 1177 *ecc_code++ = ((bch_val2 >> 4) & 0xFF); 1178 *ecc_code++ = ((bch_val2 & 0xF) << 4) | 1179 ((bch_val1 >> 28) & 0xF); 1180 *ecc_code++ = ((bch_val1 >> 20) & 0xFF); 1181 *ecc_code++ = ((bch_val1 >> 12) & 0xFF); 1182 *ecc_code++ = ((bch_val1 >> 4) & 0xFF); 1183 *ecc_code++ = ((bch_val1 & 0xF) << 4); 1184 break; 1185 case OMAP_ECC_BCH16_CODE_HW: 1186 val = readl(gpmc_regs->gpmc_bch_result6[i]); 1187 ecc_code[0] = ((val >> 8) & 0xFF); 1188 ecc_code[1] = ((val >> 0) & 0xFF); 1189 val = readl(gpmc_regs->gpmc_bch_result5[i]); 1190 ecc_code[2] = ((val >> 24) & 0xFF); 1191 ecc_code[3] = ((val >> 16) & 0xFF); 1192 ecc_code[4] = ((val >> 8) & 0xFF); 1193 ecc_code[5] = ((val >> 0) & 0xFF); 1194 val = readl(gpmc_regs->gpmc_bch_result4[i]); 1195 ecc_code[6] = ((val >> 24) & 0xFF); 1196 ecc_code[7] = ((val >> 16) & 0xFF); 1197 ecc_code[8] = ((val >> 8) & 0xFF); 1198 ecc_code[9] = ((val >> 0) & 0xFF); 1199 val = readl(gpmc_regs->gpmc_bch_result3[i]); 1200 ecc_code[10] = ((val >> 24) & 0xFF); 1201 ecc_code[11] = ((val >> 16) & 0xFF); 1202 ecc_code[12] = ((val >> 8) & 0xFF); 1203 ecc_code[13] = ((val >> 0) & 0xFF); 1204 val = readl(gpmc_regs->gpmc_bch_result2[i]); 1205 ecc_code[14] = ((val >> 24) & 0xFF); 1206 ecc_code[15] = ((val >> 16) & 0xFF); 1207 ecc_code[16] = ((val >> 8) & 0xFF); 1208 ecc_code[17] = ((val >> 0) & 0xFF); 1209 val = readl(gpmc_regs->gpmc_bch_result1[i]); 1210 ecc_code[18] = ((val >> 24) & 0xFF); 1211 ecc_code[19] = ((val >> 16) & 0xFF); 1212 ecc_code[20] = ((val >> 8) & 0xFF); 1213 ecc_code[21] = ((val >> 0) & 0xFF); 1214 val = readl(gpmc_regs->gpmc_bch_result0[i]); 1215 ecc_code[22] = ((val >> 24) & 0xFF); 1216 ecc_code[23] = ((val >> 16) & 0xFF); 1217 ecc_code[24] = ((val >> 8) & 0xFF); 1218 ecc_code[25] = ((val >> 0) & 0xFF); 1219 break; 1220 default: 1221 return -EINVAL; 1222 } 1223 1224 /* ECC scheme specific syndrome customizations */ 1225 switch (info->ecc_opt) { 1226 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: 1227 /* Add constant polynomial to remainder, so that 1228 * ECC of blank pages results in 0x0 on reading back 1229 */ 1230 for (j = 0; j < eccbytes; j++) 1231 ecc_calc[j] ^= bch4_polynomial[j]; 1232 break; 1233 case OMAP_ECC_BCH4_CODE_HW: 1234 /* Set 8th ECC byte as 0x0 for ROM compatibility */ 1235 ecc_calc[eccbytes - 1] = 0x0; 1236 break; 1237 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: 1238 /* Add constant polynomial to remainder, so that 1239 * ECC of blank pages results in 0x0 on reading back 1240 */ 1241 for (j = 0; j < eccbytes; j++) 1242 ecc_calc[j] ^= bch8_polynomial[j]; 1243 break; 1244 case OMAP_ECC_BCH8_CODE_HW: 1245 /* Set 14th ECC byte as 0x0 for ROM compatibility */ 1246 ecc_calc[eccbytes - 1] = 0x0; 1247 break; 1248 case OMAP_ECC_BCH16_CODE_HW: 1249 break; 1250 default: 1251 return -EINVAL; 1252 } 1253 1254 return 0; 1255 } 1256 1257 /** 1258 * omap_calculate_ecc_bch_sw - ECC generator for sector for SW based correction 1259 * @mtd: MTD device structure 1260 * @dat: The pointer to data on which ecc is computed 1261 * @ecc_code: The ecc_code buffer 1262 * 1263 * Support calculating of BCH4/8/16 ECC vectors for one sector. This is used 1264 * when SW based correction is required as ECC is required for one sector 1265 * at a time. 1266 */ 1267 static int omap_calculate_ecc_bch_sw(struct mtd_info *mtd, 1268 const u_char *dat, u_char *ecc_calc) 1269 { 1270 return _omap_calculate_ecc_bch(mtd, dat, ecc_calc, 0); 1271 } 1272 1273 /** 1274 * omap_calculate_ecc_bch_multi - Generate ECC for multiple sectors 1275 * @mtd: MTD device structure 1276 * @dat: The pointer to data on which ecc is computed 1277 * @ecc_code: The ecc_code buffer 1278 * 1279 * Support calculating of BCH4/8/16 ecc vectors for the entire page in one go. 1280 */ 1281 static int omap_calculate_ecc_bch_multi(struct mtd_info *mtd, 1282 const u_char *dat, u_char *ecc_calc) 1283 { 1284 struct omap_nand_info *info = mtd_to_omap(mtd); 1285 int eccbytes = info->nand.ecc.bytes; 1286 unsigned long nsectors; 1287 int i, ret; 1288 1289 nsectors = ((readl(info->reg.gpmc_ecc_config) >> 4) & 0x7) + 1; 1290 for (i = 0; i < nsectors; i++) { 1291 ret = _omap_calculate_ecc_bch(mtd, dat, ecc_calc, i); 1292 if (ret) 1293 return ret; 1294 1295 ecc_calc += eccbytes; 1296 } 1297 1298 return 0; 1299 } 1300 1301 /** 1302 * erased_sector_bitflips - count bit flips 1303 * @data: data sector buffer 1304 * @oob: oob buffer 1305 * @info: omap_nand_info 1306 * 1307 * Check the bit flips in erased page falls below correctable level. 1308 * If falls below, report the page as erased with correctable bit 1309 * flip, else report as uncorrectable page. 1310 */ 1311 static int erased_sector_bitflips(u_char *data, u_char *oob, 1312 struct omap_nand_info *info) 1313 { 1314 int flip_bits = 0, i; 1315 1316 for (i = 0; i < info->nand.ecc.size; i++) { 1317 flip_bits += hweight8(~data[i]); 1318 if (flip_bits > info->nand.ecc.strength) 1319 return 0; 1320 } 1321 1322 for (i = 0; i < info->nand.ecc.bytes - 1; i++) { 1323 flip_bits += hweight8(~oob[i]); 1324 if (flip_bits > info->nand.ecc.strength) 1325 return 0; 1326 } 1327 1328 /* 1329 * Bit flips falls in correctable level. 1330 * Fill data area with 0xFF 1331 */ 1332 if (flip_bits) { 1333 memset(data, 0xFF, info->nand.ecc.size); 1334 memset(oob, 0xFF, info->nand.ecc.bytes); 1335 } 1336 1337 return flip_bits; 1338 } 1339 1340 /** 1341 * omap_elm_correct_data - corrects page data area in case error reported 1342 * @mtd: MTD device structure 1343 * @data: page data 1344 * @read_ecc: ecc read from nand flash 1345 * @calc_ecc: ecc read from HW ECC registers 1346 * 1347 * Calculated ecc vector reported as zero in case of non-error pages. 1348 * In case of non-zero ecc vector, first filter out erased-pages, and 1349 * then process data via ELM to detect bit-flips. 1350 */ 1351 static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data, 1352 u_char *read_ecc, u_char *calc_ecc) 1353 { 1354 struct omap_nand_info *info = mtd_to_omap(mtd); 1355 struct nand_ecc_ctrl *ecc = &info->nand.ecc; 1356 int eccsteps = info->nand.ecc.steps; 1357 int i , j, stat = 0; 1358 int eccflag, actual_eccbytes; 1359 struct elm_errorvec err_vec[ERROR_VECTOR_MAX]; 1360 u_char *ecc_vec = calc_ecc; 1361 u_char *spare_ecc = read_ecc; 1362 u_char *erased_ecc_vec; 1363 u_char *buf; 1364 int bitflip_count; 1365 bool is_error_reported = false; 1366 u32 bit_pos, byte_pos, error_max, pos; 1367 int err; 1368 1369 switch (info->ecc_opt) { 1370 case OMAP_ECC_BCH4_CODE_HW: 1371 /* omit 7th ECC byte reserved for ROM code compatibility */ 1372 actual_eccbytes = ecc->bytes - 1; 1373 erased_ecc_vec = bch4_vector; 1374 break; 1375 case OMAP_ECC_BCH8_CODE_HW: 1376 /* omit 14th ECC byte reserved for ROM code compatibility */ 1377 actual_eccbytes = ecc->bytes - 1; 1378 erased_ecc_vec = bch8_vector; 1379 break; 1380 case OMAP_ECC_BCH16_CODE_HW: 1381 actual_eccbytes = ecc->bytes; 1382 erased_ecc_vec = bch16_vector; 1383 break; 1384 default: 1385 dev_err(&info->pdev->dev, "invalid driver configuration\n"); 1386 return -EINVAL; 1387 } 1388 1389 /* Initialize elm error vector to zero */ 1390 memset(err_vec, 0, sizeof(err_vec)); 1391 1392 for (i = 0; i < eccsteps ; i++) { 1393 eccflag = 0; /* initialize eccflag */ 1394 1395 /* 1396 * Check any error reported, 1397 * In case of error, non zero ecc reported. 1398 */ 1399 for (j = 0; j < actual_eccbytes; j++) { 1400 if (calc_ecc[j] != 0) { 1401 eccflag = 1; /* non zero ecc, error present */ 1402 break; 1403 } 1404 } 1405 1406 if (eccflag == 1) { 1407 if (memcmp(calc_ecc, erased_ecc_vec, 1408 actual_eccbytes) == 0) { 1409 /* 1410 * calc_ecc[] matches pattern for ECC(all 0xff) 1411 * so this is definitely an erased-page 1412 */ 1413 } else { 1414 buf = &data[info->nand.ecc.size * i]; 1415 /* 1416 * count number of 0-bits in read_buf. 1417 * This check can be removed once a similar 1418 * check is introduced in generic NAND driver 1419 */ 1420 bitflip_count = erased_sector_bitflips( 1421 buf, read_ecc, info); 1422 if (bitflip_count) { 1423 /* 1424 * number of 0-bits within ECC limits 1425 * So this may be an erased-page 1426 */ 1427 stat += bitflip_count; 1428 } else { 1429 /* 1430 * Too many 0-bits. It may be a 1431 * - programmed-page, OR 1432 * - erased-page with many bit-flips 1433 * So this page requires check by ELM 1434 */ 1435 err_vec[i].error_reported = true; 1436 is_error_reported = true; 1437 } 1438 } 1439 } 1440 1441 /* Update the ecc vector */ 1442 calc_ecc += ecc->bytes; 1443 read_ecc += ecc->bytes; 1444 } 1445 1446 /* Check if any error reported */ 1447 if (!is_error_reported) 1448 return stat; 1449 1450 /* Decode BCH error using ELM module */ 1451 elm_decode_bch_error_page(info->elm_dev, ecc_vec, err_vec); 1452 1453 err = 0; 1454 for (i = 0; i < eccsteps; i++) { 1455 if (err_vec[i].error_uncorrectable) { 1456 dev_err(&info->pdev->dev, 1457 "uncorrectable bit-flips found\n"); 1458 err = -EBADMSG; 1459 } else if (err_vec[i].error_reported) { 1460 for (j = 0; j < err_vec[i].error_count; j++) { 1461 switch (info->ecc_opt) { 1462 case OMAP_ECC_BCH4_CODE_HW: 1463 /* Add 4 bits to take care of padding */ 1464 pos = err_vec[i].error_loc[j] + 1465 BCH4_BIT_PAD; 1466 break; 1467 case OMAP_ECC_BCH8_CODE_HW: 1468 case OMAP_ECC_BCH16_CODE_HW: 1469 pos = err_vec[i].error_loc[j]; 1470 break; 1471 default: 1472 return -EINVAL; 1473 } 1474 error_max = (ecc->size + actual_eccbytes) * 8; 1475 /* Calculate bit position of error */ 1476 bit_pos = pos % 8; 1477 1478 /* Calculate byte position of error */ 1479 byte_pos = (error_max - pos - 1) / 8; 1480 1481 if (pos < error_max) { 1482 if (byte_pos < 512) { 1483 pr_debug("bitflip@dat[%d]=%x\n", 1484 byte_pos, data[byte_pos]); 1485 data[byte_pos] ^= 1 << bit_pos; 1486 } else { 1487 pr_debug("bitflip@oob[%d]=%x\n", 1488 (byte_pos - 512), 1489 spare_ecc[byte_pos - 512]); 1490 spare_ecc[byte_pos - 512] ^= 1491 1 << bit_pos; 1492 } 1493 } else { 1494 dev_err(&info->pdev->dev, 1495 "invalid bit-flip @ %d:%d\n", 1496 byte_pos, bit_pos); 1497 err = -EBADMSG; 1498 } 1499 } 1500 } 1501 1502 /* Update number of correctable errors */ 1503 stat += err_vec[i].error_count; 1504 1505 /* Update page data with sector size */ 1506 data += ecc->size; 1507 spare_ecc += ecc->bytes; 1508 } 1509 1510 return (err) ? err : stat; 1511 } 1512 1513 /** 1514 * omap_write_page_bch - BCH ecc based write page function for entire page 1515 * @mtd: mtd info structure 1516 * @chip: nand chip info structure 1517 * @buf: data buffer 1518 * @oob_required: must write chip->oob_poi to OOB 1519 * @page: page 1520 * 1521 * Custom write page method evolved to support multi sector writing in one shot 1522 */ 1523 static int omap_write_page_bch(struct mtd_info *mtd, struct nand_chip *chip, 1524 const uint8_t *buf, int oob_required, int page) 1525 { 1526 int ret; 1527 uint8_t *ecc_calc = chip->ecc.calc_buf; 1528 1529 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1530 1531 /* Enable GPMC ecc engine */ 1532 chip->ecc.hwctl(mtd, NAND_ECC_WRITE); 1533 1534 /* Write data */ 1535 chip->write_buf(mtd, buf, mtd->writesize); 1536 1537 /* Update ecc vector from GPMC result registers */ 1538 omap_calculate_ecc_bch_multi(mtd, buf, &ecc_calc[0]); 1539 1540 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0, 1541 chip->ecc.total); 1542 if (ret) 1543 return ret; 1544 1545 /* Write ecc vector to OOB area */ 1546 chip->write_buf(mtd, chip->oob_poi, mtd->oobsize); 1547 1548 return nand_prog_page_end_op(chip); 1549 } 1550 1551 /** 1552 * omap_write_subpage_bch - BCH hardware ECC based subpage write 1553 * @mtd: mtd info structure 1554 * @chip: nand chip info structure 1555 * @offset: column address of subpage within the page 1556 * @data_len: data length 1557 * @buf: data buffer 1558 * @oob_required: must write chip->oob_poi to OOB 1559 * @page: page number to write 1560 * 1561 * OMAP optimized subpage write method. 1562 */ 1563 static int omap_write_subpage_bch(struct mtd_info *mtd, 1564 struct nand_chip *chip, u32 offset, 1565 u32 data_len, const u8 *buf, 1566 int oob_required, int page) 1567 { 1568 u8 *ecc_calc = chip->ecc.calc_buf; 1569 int ecc_size = chip->ecc.size; 1570 int ecc_bytes = chip->ecc.bytes; 1571 int ecc_steps = chip->ecc.steps; 1572 u32 start_step = offset / ecc_size; 1573 u32 end_step = (offset + data_len - 1) / ecc_size; 1574 int step, ret = 0; 1575 1576 /* 1577 * Write entire page at one go as it would be optimal 1578 * as ECC is calculated by hardware. 1579 * ECC is calculated for all subpages but we choose 1580 * only what we want. 1581 */ 1582 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1583 1584 /* Enable GPMC ECC engine */ 1585 chip->ecc.hwctl(mtd, NAND_ECC_WRITE); 1586 1587 /* Write data */ 1588 chip->write_buf(mtd, buf, mtd->writesize); 1589 1590 for (step = 0; step < ecc_steps; step++) { 1591 /* mask ECC of un-touched subpages by padding 0xFF */ 1592 if (step < start_step || step > end_step) 1593 memset(ecc_calc, 0xff, ecc_bytes); 1594 else 1595 ret = _omap_calculate_ecc_bch(mtd, buf, ecc_calc, step); 1596 1597 if (ret) 1598 return ret; 1599 1600 buf += ecc_size; 1601 ecc_calc += ecc_bytes; 1602 } 1603 1604 /* copy calculated ECC for whole page to chip->buffer->oob */ 1605 /* this include masked-value(0xFF) for unwritten subpages */ 1606 ecc_calc = chip->ecc.calc_buf; 1607 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0, 1608 chip->ecc.total); 1609 if (ret) 1610 return ret; 1611 1612 /* write OOB buffer to NAND device */ 1613 chip->write_buf(mtd, chip->oob_poi, mtd->oobsize); 1614 1615 return nand_prog_page_end_op(chip); 1616 } 1617 1618 /** 1619 * omap_read_page_bch - BCH ecc based page read function for entire page 1620 * @mtd: mtd info structure 1621 * @chip: nand chip info structure 1622 * @buf: buffer to store read data 1623 * @oob_required: caller requires OOB data read to chip->oob_poi 1624 * @page: page number to read 1625 * 1626 * For BCH ecc scheme, GPMC used for syndrome calculation and ELM module 1627 * used for error correction. 1628 * Custom method evolved to support ELM error correction & multi sector 1629 * reading. On reading page data area is read along with OOB data with 1630 * ecc engine enabled. ecc vector updated after read of OOB data. 1631 * For non error pages ecc vector reported as zero. 1632 */ 1633 static int omap_read_page_bch(struct mtd_info *mtd, struct nand_chip *chip, 1634 uint8_t *buf, int oob_required, int page) 1635 { 1636 uint8_t *ecc_calc = chip->ecc.calc_buf; 1637 uint8_t *ecc_code = chip->ecc.code_buf; 1638 int stat, ret; 1639 unsigned int max_bitflips = 0; 1640 1641 nand_read_page_op(chip, page, 0, NULL, 0); 1642 1643 /* Enable GPMC ecc engine */ 1644 chip->ecc.hwctl(mtd, NAND_ECC_READ); 1645 1646 /* Read data */ 1647 chip->read_buf(mtd, buf, mtd->writesize); 1648 1649 /* Read oob bytes */ 1650 nand_change_read_column_op(chip, 1651 mtd->writesize + BADBLOCK_MARKER_LENGTH, 1652 chip->oob_poi + BADBLOCK_MARKER_LENGTH, 1653 chip->ecc.total, false); 1654 1655 /* Calculate ecc bytes */ 1656 omap_calculate_ecc_bch_multi(mtd, buf, ecc_calc); 1657 1658 ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0, 1659 chip->ecc.total); 1660 if (ret) 1661 return ret; 1662 1663 stat = chip->ecc.correct(mtd, buf, ecc_code, ecc_calc); 1664 1665 if (stat < 0) { 1666 mtd->ecc_stats.failed++; 1667 } else { 1668 mtd->ecc_stats.corrected += stat; 1669 max_bitflips = max_t(unsigned int, max_bitflips, stat); 1670 } 1671 1672 return max_bitflips; 1673 } 1674 1675 /** 1676 * is_elm_present - checks for presence of ELM module by scanning DT nodes 1677 * @omap_nand_info: NAND device structure containing platform data 1678 */ 1679 static bool is_elm_present(struct omap_nand_info *info, 1680 struct device_node *elm_node) 1681 { 1682 struct platform_device *pdev; 1683 1684 /* check whether elm-id is passed via DT */ 1685 if (!elm_node) { 1686 dev_err(&info->pdev->dev, "ELM devicetree node not found\n"); 1687 return false; 1688 } 1689 pdev = of_find_device_by_node(elm_node); 1690 /* check whether ELM device is registered */ 1691 if (!pdev) { 1692 dev_err(&info->pdev->dev, "ELM device not found\n"); 1693 return false; 1694 } 1695 /* ELM module available, now configure it */ 1696 info->elm_dev = &pdev->dev; 1697 return true; 1698 } 1699 1700 static bool omap2_nand_ecc_check(struct omap_nand_info *info) 1701 { 1702 bool ecc_needs_bch, ecc_needs_omap_bch, ecc_needs_elm; 1703 1704 switch (info->ecc_opt) { 1705 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: 1706 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: 1707 ecc_needs_omap_bch = false; 1708 ecc_needs_bch = true; 1709 ecc_needs_elm = false; 1710 break; 1711 case OMAP_ECC_BCH4_CODE_HW: 1712 case OMAP_ECC_BCH8_CODE_HW: 1713 case OMAP_ECC_BCH16_CODE_HW: 1714 ecc_needs_omap_bch = true; 1715 ecc_needs_bch = false; 1716 ecc_needs_elm = true; 1717 break; 1718 default: 1719 ecc_needs_omap_bch = false; 1720 ecc_needs_bch = false; 1721 ecc_needs_elm = false; 1722 break; 1723 } 1724 1725 if (ecc_needs_bch && !IS_ENABLED(CONFIG_MTD_NAND_ECC_BCH)) { 1726 dev_err(&info->pdev->dev, 1727 "CONFIG_MTD_NAND_ECC_BCH not enabled\n"); 1728 return false; 1729 } 1730 if (ecc_needs_omap_bch && !IS_ENABLED(CONFIG_MTD_NAND_OMAP_BCH)) { 1731 dev_err(&info->pdev->dev, 1732 "CONFIG_MTD_NAND_OMAP_BCH not enabled\n"); 1733 return false; 1734 } 1735 if (ecc_needs_elm && !is_elm_present(info, info->elm_of_node)) { 1736 dev_err(&info->pdev->dev, "ELM not available\n"); 1737 return false; 1738 } 1739 1740 return true; 1741 } 1742 1743 static const char * const nand_xfer_types[] = { 1744 [NAND_OMAP_PREFETCH_POLLED] = "prefetch-polled", 1745 [NAND_OMAP_POLLED] = "polled", 1746 [NAND_OMAP_PREFETCH_DMA] = "prefetch-dma", 1747 [NAND_OMAP_PREFETCH_IRQ] = "prefetch-irq", 1748 }; 1749 1750 static int omap_get_dt_info(struct device *dev, struct omap_nand_info *info) 1751 { 1752 struct device_node *child = dev->of_node; 1753 int i; 1754 const char *s; 1755 u32 cs; 1756 1757 if (of_property_read_u32(child, "reg", &cs) < 0) { 1758 dev_err(dev, "reg not found in DT\n"); 1759 return -EINVAL; 1760 } 1761 1762 info->gpmc_cs = cs; 1763 1764 /* detect availability of ELM module. Won't be present pre-OMAP4 */ 1765 info->elm_of_node = of_parse_phandle(child, "ti,elm-id", 0); 1766 if (!info->elm_of_node) { 1767 info->elm_of_node = of_parse_phandle(child, "elm_id", 0); 1768 if (!info->elm_of_node) 1769 dev_dbg(dev, "ti,elm-id not in DT\n"); 1770 } 1771 1772 /* select ecc-scheme for NAND */ 1773 if (of_property_read_string(child, "ti,nand-ecc-opt", &s)) { 1774 dev_err(dev, "ti,nand-ecc-opt not found\n"); 1775 return -EINVAL; 1776 } 1777 1778 if (!strcmp(s, "sw")) { 1779 info->ecc_opt = OMAP_ECC_HAM1_CODE_SW; 1780 } else if (!strcmp(s, "ham1") || 1781 !strcmp(s, "hw") || !strcmp(s, "hw-romcode")) { 1782 info->ecc_opt = OMAP_ECC_HAM1_CODE_HW; 1783 } else if (!strcmp(s, "bch4")) { 1784 if (info->elm_of_node) 1785 info->ecc_opt = OMAP_ECC_BCH4_CODE_HW; 1786 else 1787 info->ecc_opt = OMAP_ECC_BCH4_CODE_HW_DETECTION_SW; 1788 } else if (!strcmp(s, "bch8")) { 1789 if (info->elm_of_node) 1790 info->ecc_opt = OMAP_ECC_BCH8_CODE_HW; 1791 else 1792 info->ecc_opt = OMAP_ECC_BCH8_CODE_HW_DETECTION_SW; 1793 } else if (!strcmp(s, "bch16")) { 1794 info->ecc_opt = OMAP_ECC_BCH16_CODE_HW; 1795 } else { 1796 dev_err(dev, "unrecognized value for ti,nand-ecc-opt\n"); 1797 return -EINVAL; 1798 } 1799 1800 /* select data transfer mode */ 1801 if (!of_property_read_string(child, "ti,nand-xfer-type", &s)) { 1802 for (i = 0; i < ARRAY_SIZE(nand_xfer_types); i++) { 1803 if (!strcasecmp(s, nand_xfer_types[i])) { 1804 info->xfer_type = i; 1805 return 0; 1806 } 1807 } 1808 1809 dev_err(dev, "unrecognized value for ti,nand-xfer-type\n"); 1810 return -EINVAL; 1811 } 1812 1813 return 0; 1814 } 1815 1816 static int omap_ooblayout_ecc(struct mtd_info *mtd, int section, 1817 struct mtd_oob_region *oobregion) 1818 { 1819 struct omap_nand_info *info = mtd_to_omap(mtd); 1820 struct nand_chip *chip = &info->nand; 1821 int off = BADBLOCK_MARKER_LENGTH; 1822 1823 if (info->ecc_opt == OMAP_ECC_HAM1_CODE_HW && 1824 !(chip->options & NAND_BUSWIDTH_16)) 1825 off = 1; 1826 1827 if (section) 1828 return -ERANGE; 1829 1830 oobregion->offset = off; 1831 oobregion->length = chip->ecc.total; 1832 1833 return 0; 1834 } 1835 1836 static int omap_ooblayout_free(struct mtd_info *mtd, int section, 1837 struct mtd_oob_region *oobregion) 1838 { 1839 struct omap_nand_info *info = mtd_to_omap(mtd); 1840 struct nand_chip *chip = &info->nand; 1841 int off = BADBLOCK_MARKER_LENGTH; 1842 1843 if (info->ecc_opt == OMAP_ECC_HAM1_CODE_HW && 1844 !(chip->options & NAND_BUSWIDTH_16)) 1845 off = 1; 1846 1847 if (section) 1848 return -ERANGE; 1849 1850 off += chip->ecc.total; 1851 if (off >= mtd->oobsize) 1852 return -ERANGE; 1853 1854 oobregion->offset = off; 1855 oobregion->length = mtd->oobsize - off; 1856 1857 return 0; 1858 } 1859 1860 static const struct mtd_ooblayout_ops omap_ooblayout_ops = { 1861 .ecc = omap_ooblayout_ecc, 1862 .free = omap_ooblayout_free, 1863 }; 1864 1865 static int omap_sw_ooblayout_ecc(struct mtd_info *mtd, int section, 1866 struct mtd_oob_region *oobregion) 1867 { 1868 struct nand_chip *chip = mtd_to_nand(mtd); 1869 int off = BADBLOCK_MARKER_LENGTH; 1870 1871 if (section >= chip->ecc.steps) 1872 return -ERANGE; 1873 1874 /* 1875 * When SW correction is employed, one OMAP specific marker byte is 1876 * reserved after each ECC step. 1877 */ 1878 oobregion->offset = off + (section * (chip->ecc.bytes + 1)); 1879 oobregion->length = chip->ecc.bytes; 1880 1881 return 0; 1882 } 1883 1884 static int omap_sw_ooblayout_free(struct mtd_info *mtd, int section, 1885 struct mtd_oob_region *oobregion) 1886 { 1887 struct nand_chip *chip = mtd_to_nand(mtd); 1888 int off = BADBLOCK_MARKER_LENGTH; 1889 1890 if (section) 1891 return -ERANGE; 1892 1893 /* 1894 * When SW correction is employed, one OMAP specific marker byte is 1895 * reserved after each ECC step. 1896 */ 1897 off += ((chip->ecc.bytes + 1) * chip->ecc.steps); 1898 if (off >= mtd->oobsize) 1899 return -ERANGE; 1900 1901 oobregion->offset = off; 1902 oobregion->length = mtd->oobsize - off; 1903 1904 return 0; 1905 } 1906 1907 static const struct mtd_ooblayout_ops omap_sw_ooblayout_ops = { 1908 .ecc = omap_sw_ooblayout_ecc, 1909 .free = omap_sw_ooblayout_free, 1910 }; 1911 1912 static int omap_nand_attach_chip(struct nand_chip *chip) 1913 { 1914 struct mtd_info *mtd = nand_to_mtd(chip); 1915 struct omap_nand_info *info = mtd_to_omap(mtd); 1916 struct device *dev = &info->pdev->dev; 1917 int min_oobbytes = BADBLOCK_MARKER_LENGTH; 1918 int oobbytes_per_step; 1919 dma_cap_mask_t mask; 1920 int err; 1921 1922 if (chip->bbt_options & NAND_BBT_USE_FLASH) 1923 chip->bbt_options |= NAND_BBT_NO_OOB; 1924 else 1925 chip->options |= NAND_SKIP_BBTSCAN; 1926 1927 /* Re-populate low-level callbacks based on xfer modes */ 1928 switch (info->xfer_type) { 1929 case NAND_OMAP_PREFETCH_POLLED: 1930 chip->read_buf = omap_read_buf_pref; 1931 chip->write_buf = omap_write_buf_pref; 1932 break; 1933 1934 case NAND_OMAP_POLLED: 1935 /* Use nand_base defaults for {read,write}_buf */ 1936 break; 1937 1938 case NAND_OMAP_PREFETCH_DMA: 1939 dma_cap_zero(mask); 1940 dma_cap_set(DMA_SLAVE, mask); 1941 info->dma = dma_request_chan(dev, "rxtx"); 1942 1943 if (IS_ERR(info->dma)) { 1944 dev_err(dev, "DMA engine request failed\n"); 1945 return PTR_ERR(info->dma); 1946 } else { 1947 struct dma_slave_config cfg; 1948 1949 memset(&cfg, 0, sizeof(cfg)); 1950 cfg.src_addr = info->phys_base; 1951 cfg.dst_addr = info->phys_base; 1952 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 1953 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 1954 cfg.src_maxburst = 16; 1955 cfg.dst_maxburst = 16; 1956 err = dmaengine_slave_config(info->dma, &cfg); 1957 if (err) { 1958 dev_err(dev, 1959 "DMA engine slave config failed: %d\n", 1960 err); 1961 return err; 1962 } 1963 chip->read_buf = omap_read_buf_dma_pref; 1964 chip->write_buf = omap_write_buf_dma_pref; 1965 } 1966 break; 1967 1968 case NAND_OMAP_PREFETCH_IRQ: 1969 info->gpmc_irq_fifo = platform_get_irq(info->pdev, 0); 1970 if (info->gpmc_irq_fifo <= 0) { 1971 dev_err(dev, "Error getting fifo IRQ\n"); 1972 return -ENODEV; 1973 } 1974 err = devm_request_irq(dev, info->gpmc_irq_fifo, 1975 omap_nand_irq, IRQF_SHARED, 1976 "gpmc-nand-fifo", info); 1977 if (err) { 1978 dev_err(dev, "Requesting IRQ %d, error %d\n", 1979 info->gpmc_irq_fifo, err); 1980 info->gpmc_irq_fifo = 0; 1981 return err; 1982 } 1983 1984 info->gpmc_irq_count = platform_get_irq(info->pdev, 1); 1985 if (info->gpmc_irq_count <= 0) { 1986 dev_err(dev, "Error getting IRQ count\n"); 1987 return -ENODEV; 1988 } 1989 err = devm_request_irq(dev, info->gpmc_irq_count, 1990 omap_nand_irq, IRQF_SHARED, 1991 "gpmc-nand-count", info); 1992 if (err) { 1993 dev_err(dev, "Requesting IRQ %d, error %d\n", 1994 info->gpmc_irq_count, err); 1995 info->gpmc_irq_count = 0; 1996 return err; 1997 } 1998 1999 chip->read_buf = omap_read_buf_irq_pref; 2000 chip->write_buf = omap_write_buf_irq_pref; 2001 2002 break; 2003 2004 default: 2005 dev_err(dev, "xfer_type %d not supported!\n", info->xfer_type); 2006 return -EINVAL; 2007 } 2008 2009 if (!omap2_nand_ecc_check(info)) 2010 return -EINVAL; 2011 2012 /* 2013 * Bail out earlier to let NAND_ECC_SOFT code create its own 2014 * ooblayout instead of using ours. 2015 */ 2016 if (info->ecc_opt == OMAP_ECC_HAM1_CODE_SW) { 2017 chip->ecc.mode = NAND_ECC_SOFT; 2018 chip->ecc.algo = NAND_ECC_HAMMING; 2019 return 0; 2020 } 2021 2022 /* Populate MTD interface based on ECC scheme */ 2023 switch (info->ecc_opt) { 2024 case OMAP_ECC_HAM1_CODE_HW: 2025 dev_info(dev, "nand: using OMAP_ECC_HAM1_CODE_HW\n"); 2026 chip->ecc.mode = NAND_ECC_HW; 2027 chip->ecc.bytes = 3; 2028 chip->ecc.size = 512; 2029 chip->ecc.strength = 1; 2030 chip->ecc.calculate = omap_calculate_ecc; 2031 chip->ecc.hwctl = omap_enable_hwecc; 2032 chip->ecc.correct = omap_correct_data; 2033 mtd_set_ooblayout(mtd, &omap_ooblayout_ops); 2034 oobbytes_per_step = chip->ecc.bytes; 2035 2036 if (!(chip->options & NAND_BUSWIDTH_16)) 2037 min_oobbytes = 1; 2038 2039 break; 2040 2041 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: 2042 pr_info("nand: using OMAP_ECC_BCH4_CODE_HW_DETECTION_SW\n"); 2043 chip->ecc.mode = NAND_ECC_HW; 2044 chip->ecc.size = 512; 2045 chip->ecc.bytes = 7; 2046 chip->ecc.strength = 4; 2047 chip->ecc.hwctl = omap_enable_hwecc_bch; 2048 chip->ecc.correct = nand_bch_correct_data; 2049 chip->ecc.calculate = omap_calculate_ecc_bch_sw; 2050 mtd_set_ooblayout(mtd, &omap_sw_ooblayout_ops); 2051 /* Reserve one byte for the OMAP marker */ 2052 oobbytes_per_step = chip->ecc.bytes + 1; 2053 /* Software BCH library is used for locating errors */ 2054 chip->ecc.priv = nand_bch_init(mtd); 2055 if (!chip->ecc.priv) { 2056 dev_err(dev, "Unable to use BCH library\n"); 2057 return -EINVAL; 2058 } 2059 break; 2060 2061 case OMAP_ECC_BCH4_CODE_HW: 2062 pr_info("nand: using OMAP_ECC_BCH4_CODE_HW ECC scheme\n"); 2063 chip->ecc.mode = NAND_ECC_HW; 2064 chip->ecc.size = 512; 2065 /* 14th bit is kept reserved for ROM-code compatibility */ 2066 chip->ecc.bytes = 7 + 1; 2067 chip->ecc.strength = 4; 2068 chip->ecc.hwctl = omap_enable_hwecc_bch; 2069 chip->ecc.correct = omap_elm_correct_data; 2070 chip->ecc.read_page = omap_read_page_bch; 2071 chip->ecc.write_page = omap_write_page_bch; 2072 chip->ecc.write_subpage = omap_write_subpage_bch; 2073 mtd_set_ooblayout(mtd, &omap_ooblayout_ops); 2074 oobbytes_per_step = chip->ecc.bytes; 2075 2076 err = elm_config(info->elm_dev, BCH4_ECC, 2077 mtd->writesize / chip->ecc.size, 2078 chip->ecc.size, chip->ecc.bytes); 2079 if (err < 0) 2080 return err; 2081 break; 2082 2083 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: 2084 pr_info("nand: using OMAP_ECC_BCH8_CODE_HW_DETECTION_SW\n"); 2085 chip->ecc.mode = NAND_ECC_HW; 2086 chip->ecc.size = 512; 2087 chip->ecc.bytes = 13; 2088 chip->ecc.strength = 8; 2089 chip->ecc.hwctl = omap_enable_hwecc_bch; 2090 chip->ecc.correct = nand_bch_correct_data; 2091 chip->ecc.calculate = omap_calculate_ecc_bch_sw; 2092 mtd_set_ooblayout(mtd, &omap_sw_ooblayout_ops); 2093 /* Reserve one byte for the OMAP marker */ 2094 oobbytes_per_step = chip->ecc.bytes + 1; 2095 /* Software BCH library is used for locating errors */ 2096 chip->ecc.priv = nand_bch_init(mtd); 2097 if (!chip->ecc.priv) { 2098 dev_err(dev, "unable to use BCH library\n"); 2099 return -EINVAL; 2100 } 2101 break; 2102 2103 case OMAP_ECC_BCH8_CODE_HW: 2104 pr_info("nand: using OMAP_ECC_BCH8_CODE_HW ECC scheme\n"); 2105 chip->ecc.mode = NAND_ECC_HW; 2106 chip->ecc.size = 512; 2107 /* 14th bit is kept reserved for ROM-code compatibility */ 2108 chip->ecc.bytes = 13 + 1; 2109 chip->ecc.strength = 8; 2110 chip->ecc.hwctl = omap_enable_hwecc_bch; 2111 chip->ecc.correct = omap_elm_correct_data; 2112 chip->ecc.read_page = omap_read_page_bch; 2113 chip->ecc.write_page = omap_write_page_bch; 2114 chip->ecc.write_subpage = omap_write_subpage_bch; 2115 mtd_set_ooblayout(mtd, &omap_ooblayout_ops); 2116 oobbytes_per_step = chip->ecc.bytes; 2117 2118 err = elm_config(info->elm_dev, BCH8_ECC, 2119 mtd->writesize / chip->ecc.size, 2120 chip->ecc.size, chip->ecc.bytes); 2121 if (err < 0) 2122 return err; 2123 2124 break; 2125 2126 case OMAP_ECC_BCH16_CODE_HW: 2127 pr_info("Using OMAP_ECC_BCH16_CODE_HW ECC scheme\n"); 2128 chip->ecc.mode = NAND_ECC_HW; 2129 chip->ecc.size = 512; 2130 chip->ecc.bytes = 26; 2131 chip->ecc.strength = 16; 2132 chip->ecc.hwctl = omap_enable_hwecc_bch; 2133 chip->ecc.correct = omap_elm_correct_data; 2134 chip->ecc.read_page = omap_read_page_bch; 2135 chip->ecc.write_page = omap_write_page_bch; 2136 chip->ecc.write_subpage = omap_write_subpage_bch; 2137 mtd_set_ooblayout(mtd, &omap_ooblayout_ops); 2138 oobbytes_per_step = chip->ecc.bytes; 2139 2140 err = elm_config(info->elm_dev, BCH16_ECC, 2141 mtd->writesize / chip->ecc.size, 2142 chip->ecc.size, chip->ecc.bytes); 2143 if (err < 0) 2144 return err; 2145 2146 break; 2147 default: 2148 dev_err(dev, "Invalid or unsupported ECC scheme\n"); 2149 return -EINVAL; 2150 } 2151 2152 /* Check if NAND device's OOB is enough to store ECC signatures */ 2153 min_oobbytes += (oobbytes_per_step * 2154 (mtd->writesize / chip->ecc.size)); 2155 if (mtd->oobsize < min_oobbytes) { 2156 dev_err(dev, 2157 "Not enough OOB bytes: required = %d, available=%d\n", 2158 min_oobbytes, mtd->oobsize); 2159 return -EINVAL; 2160 } 2161 2162 return 0; 2163 } 2164 2165 static const struct nand_controller_ops omap_nand_controller_ops = { 2166 .attach_chip = omap_nand_attach_chip, 2167 }; 2168 2169 /* Shared among all NAND instances to synchronize access to the ECC Engine */ 2170 static struct nand_controller omap_gpmc_controller = { 2171 .lock = __SPIN_LOCK_UNLOCKED(omap_gpmc_controller.lock), 2172 .wq = __WAIT_QUEUE_HEAD_INITIALIZER(omap_gpmc_controller.wq), 2173 .ops = &omap_nand_controller_ops, 2174 }; 2175 2176 static int omap_nand_probe(struct platform_device *pdev) 2177 { 2178 struct omap_nand_info *info; 2179 struct mtd_info *mtd; 2180 struct nand_chip *nand_chip; 2181 int err; 2182 struct resource *res; 2183 struct device *dev = &pdev->dev; 2184 2185 info = devm_kzalloc(&pdev->dev, sizeof(struct omap_nand_info), 2186 GFP_KERNEL); 2187 if (!info) 2188 return -ENOMEM; 2189 2190 info->pdev = pdev; 2191 2192 err = omap_get_dt_info(dev, info); 2193 if (err) 2194 return err; 2195 2196 info->ops = gpmc_omap_get_nand_ops(&info->reg, info->gpmc_cs); 2197 if (!info->ops) { 2198 dev_err(&pdev->dev, "Failed to get GPMC->NAND interface\n"); 2199 return -ENODEV; 2200 } 2201 2202 nand_chip = &info->nand; 2203 mtd = nand_to_mtd(nand_chip); 2204 mtd->dev.parent = &pdev->dev; 2205 nand_chip->ecc.priv = NULL; 2206 nand_set_flash_node(nand_chip, dev->of_node); 2207 2208 if (!mtd->name) { 2209 mtd->name = devm_kasprintf(&pdev->dev, GFP_KERNEL, 2210 "omap2-nand.%d", info->gpmc_cs); 2211 if (!mtd->name) { 2212 dev_err(&pdev->dev, "Failed to set MTD name\n"); 2213 return -ENOMEM; 2214 } 2215 } 2216 2217 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2218 nand_chip->IO_ADDR_R = devm_ioremap_resource(&pdev->dev, res); 2219 if (IS_ERR(nand_chip->IO_ADDR_R)) 2220 return PTR_ERR(nand_chip->IO_ADDR_R); 2221 2222 info->phys_base = res->start; 2223 2224 nand_chip->controller = &omap_gpmc_controller; 2225 2226 nand_chip->IO_ADDR_W = nand_chip->IO_ADDR_R; 2227 nand_chip->cmd_ctrl = omap_hwcontrol; 2228 2229 info->ready_gpiod = devm_gpiod_get_optional(&pdev->dev, "rb", 2230 GPIOD_IN); 2231 if (IS_ERR(info->ready_gpiod)) { 2232 dev_err(dev, "failed to get ready gpio\n"); 2233 return PTR_ERR(info->ready_gpiod); 2234 } 2235 2236 /* 2237 * If RDY/BSY line is connected to OMAP then use the omap ready 2238 * function and the generic nand_wait function which reads the status 2239 * register after monitoring the RDY/BSY line. Otherwise use a standard 2240 * chip delay which is slightly more than tR (AC Timing) of the NAND 2241 * device and read status register until you get a failure or success 2242 */ 2243 if (info->ready_gpiod) { 2244 nand_chip->dev_ready = omap_dev_ready; 2245 nand_chip->chip_delay = 0; 2246 } else { 2247 nand_chip->waitfunc = omap_wait; 2248 nand_chip->chip_delay = 50; 2249 } 2250 2251 if (info->flash_bbt) 2252 nand_chip->bbt_options |= NAND_BBT_USE_FLASH; 2253 2254 /* scan NAND device connected to chip controller */ 2255 nand_chip->options |= info->devsize & NAND_BUSWIDTH_16; 2256 2257 err = nand_scan(mtd, 1); 2258 if (err) 2259 goto return_error; 2260 2261 err = mtd_device_register(mtd, NULL, 0); 2262 if (err) 2263 goto cleanup_nand; 2264 2265 platform_set_drvdata(pdev, mtd); 2266 2267 return 0; 2268 2269 cleanup_nand: 2270 nand_cleanup(nand_chip); 2271 2272 return_error: 2273 if (!IS_ERR_OR_NULL(info->dma)) 2274 dma_release_channel(info->dma); 2275 if (nand_chip->ecc.priv) { 2276 nand_bch_free(nand_chip->ecc.priv); 2277 nand_chip->ecc.priv = NULL; 2278 } 2279 return err; 2280 } 2281 2282 static int omap_nand_remove(struct platform_device *pdev) 2283 { 2284 struct mtd_info *mtd = platform_get_drvdata(pdev); 2285 struct nand_chip *nand_chip = mtd_to_nand(mtd); 2286 struct omap_nand_info *info = mtd_to_omap(mtd); 2287 if (nand_chip->ecc.priv) { 2288 nand_bch_free(nand_chip->ecc.priv); 2289 nand_chip->ecc.priv = NULL; 2290 } 2291 if (info->dma) 2292 dma_release_channel(info->dma); 2293 nand_release(mtd); 2294 return 0; 2295 } 2296 2297 static const struct of_device_id omap_nand_ids[] = { 2298 { .compatible = "ti,omap2-nand", }, 2299 {}, 2300 }; 2301 MODULE_DEVICE_TABLE(of, omap_nand_ids); 2302 2303 static struct platform_driver omap_nand_driver = { 2304 .probe = omap_nand_probe, 2305 .remove = omap_nand_remove, 2306 .driver = { 2307 .name = DRIVER_NAME, 2308 .of_match_table = of_match_ptr(omap_nand_ids), 2309 }, 2310 }; 2311 2312 module_platform_driver(omap_nand_driver); 2313 2314 MODULE_ALIAS("platform:" DRIVER_NAME); 2315 MODULE_LICENSE("GPL"); 2316 MODULE_DESCRIPTION("Glue layer for NAND flash on TI OMAP boards"); 2317