1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright © 2004 Texas Instruments, Jian Zhang <jzhang@ti.com> 4 * Copyright © 2004 Micron Technology Inc. 5 * Copyright © 2004 David Brownell 6 */ 7 8 #include <linux/platform_device.h> 9 #include <linux/dmaengine.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/delay.h> 12 #include <linux/gpio/consumer.h> 13 #include <linux/module.h> 14 #include <linux/interrupt.h> 15 #include <linux/jiffies.h> 16 #include <linux/sched.h> 17 #include <linux/mtd/mtd.h> 18 #include <linux/mtd/rawnand.h> 19 #include <linux/mtd/partitions.h> 20 #include <linux/omap-dma.h> 21 #include <linux/io.h> 22 #include <linux/slab.h> 23 #include <linux/of.h> 24 #include <linux/of_device.h> 25 26 #include <linux/mtd/nand_bch.h> 27 #include <linux/platform_data/elm.h> 28 29 #include <linux/omap-gpmc.h> 30 #include <linux/platform_data/mtd-nand-omap2.h> 31 32 #define DRIVER_NAME "omap2-nand" 33 #define OMAP_NAND_TIMEOUT_MS 5000 34 35 #define NAND_Ecc_P1e (1 << 0) 36 #define NAND_Ecc_P2e (1 << 1) 37 #define NAND_Ecc_P4e (1 << 2) 38 #define NAND_Ecc_P8e (1 << 3) 39 #define NAND_Ecc_P16e (1 << 4) 40 #define NAND_Ecc_P32e (1 << 5) 41 #define NAND_Ecc_P64e (1 << 6) 42 #define NAND_Ecc_P128e (1 << 7) 43 #define NAND_Ecc_P256e (1 << 8) 44 #define NAND_Ecc_P512e (1 << 9) 45 #define NAND_Ecc_P1024e (1 << 10) 46 #define NAND_Ecc_P2048e (1 << 11) 47 48 #define NAND_Ecc_P1o (1 << 16) 49 #define NAND_Ecc_P2o (1 << 17) 50 #define NAND_Ecc_P4o (1 << 18) 51 #define NAND_Ecc_P8o (1 << 19) 52 #define NAND_Ecc_P16o (1 << 20) 53 #define NAND_Ecc_P32o (1 << 21) 54 #define NAND_Ecc_P64o (1 << 22) 55 #define NAND_Ecc_P128o (1 << 23) 56 #define NAND_Ecc_P256o (1 << 24) 57 #define NAND_Ecc_P512o (1 << 25) 58 #define NAND_Ecc_P1024o (1 << 26) 59 #define NAND_Ecc_P2048o (1 << 27) 60 61 #define TF(value) (value ? 1 : 0) 62 63 #define P2048e(a) (TF(a & NAND_Ecc_P2048e) << 0) 64 #define P2048o(a) (TF(a & NAND_Ecc_P2048o) << 1) 65 #define P1e(a) (TF(a & NAND_Ecc_P1e) << 2) 66 #define P1o(a) (TF(a & NAND_Ecc_P1o) << 3) 67 #define P2e(a) (TF(a & NAND_Ecc_P2e) << 4) 68 #define P2o(a) (TF(a & NAND_Ecc_P2o) << 5) 69 #define P4e(a) (TF(a & NAND_Ecc_P4e) << 6) 70 #define P4o(a) (TF(a & NAND_Ecc_P4o) << 7) 71 72 #define P8e(a) (TF(a & NAND_Ecc_P8e) << 0) 73 #define P8o(a) (TF(a & NAND_Ecc_P8o) << 1) 74 #define P16e(a) (TF(a & NAND_Ecc_P16e) << 2) 75 #define P16o(a) (TF(a & NAND_Ecc_P16o) << 3) 76 #define P32e(a) (TF(a & NAND_Ecc_P32e) << 4) 77 #define P32o(a) (TF(a & NAND_Ecc_P32o) << 5) 78 #define P64e(a) (TF(a & NAND_Ecc_P64e) << 6) 79 #define P64o(a) (TF(a & NAND_Ecc_P64o) << 7) 80 81 #define P128e(a) (TF(a & NAND_Ecc_P128e) << 0) 82 #define P128o(a) (TF(a & NAND_Ecc_P128o) << 1) 83 #define P256e(a) (TF(a & NAND_Ecc_P256e) << 2) 84 #define P256o(a) (TF(a & NAND_Ecc_P256o) << 3) 85 #define P512e(a) (TF(a & NAND_Ecc_P512e) << 4) 86 #define P512o(a) (TF(a & NAND_Ecc_P512o) << 5) 87 #define P1024e(a) (TF(a & NAND_Ecc_P1024e) << 6) 88 #define P1024o(a) (TF(a & NAND_Ecc_P1024o) << 7) 89 90 #define P8e_s(a) (TF(a & NAND_Ecc_P8e) << 0) 91 #define P8o_s(a) (TF(a & NAND_Ecc_P8o) << 1) 92 #define P16e_s(a) (TF(a & NAND_Ecc_P16e) << 2) 93 #define P16o_s(a) (TF(a & NAND_Ecc_P16o) << 3) 94 #define P1e_s(a) (TF(a & NAND_Ecc_P1e) << 4) 95 #define P1o_s(a) (TF(a & NAND_Ecc_P1o) << 5) 96 #define P2e_s(a) (TF(a & NAND_Ecc_P2e) << 6) 97 #define P2o_s(a) (TF(a & NAND_Ecc_P2o) << 7) 98 99 #define P4e_s(a) (TF(a & NAND_Ecc_P4e) << 0) 100 #define P4o_s(a) (TF(a & NAND_Ecc_P4o) << 1) 101 102 #define PREFETCH_CONFIG1_CS_SHIFT 24 103 #define ECC_CONFIG_CS_SHIFT 1 104 #define CS_MASK 0x7 105 #define ENABLE_PREFETCH (0x1 << 7) 106 #define DMA_MPU_MODE_SHIFT 2 107 #define ECCSIZE0_SHIFT 12 108 #define ECCSIZE1_SHIFT 22 109 #define ECC1RESULTSIZE 0x1 110 #define ECCCLEAR 0x100 111 #define ECC1 0x1 112 #define PREFETCH_FIFOTHRESHOLD_MAX 0x40 113 #define PREFETCH_FIFOTHRESHOLD(val) ((val) << 8) 114 #define PREFETCH_STATUS_COUNT(val) (val & 0x00003fff) 115 #define PREFETCH_STATUS_FIFO_CNT(val) ((val >> 24) & 0x7F) 116 #define STATUS_BUFF_EMPTY 0x00000001 117 118 #define SECTOR_BYTES 512 119 /* 4 bit padding to make byte aligned, 56 = 52 + 4 */ 120 #define BCH4_BIT_PAD 4 121 122 /* GPMC ecc engine settings for read */ 123 #define BCH_WRAPMODE_1 1 /* BCH wrap mode 1 */ 124 #define BCH8R_ECC_SIZE0 0x1a /* ecc_size0 = 26 */ 125 #define BCH8R_ECC_SIZE1 0x2 /* ecc_size1 = 2 */ 126 #define BCH4R_ECC_SIZE0 0xd /* ecc_size0 = 13 */ 127 #define BCH4R_ECC_SIZE1 0x3 /* ecc_size1 = 3 */ 128 129 /* GPMC ecc engine settings for write */ 130 #define BCH_WRAPMODE_6 6 /* BCH wrap mode 6 */ 131 #define BCH_ECC_SIZE0 0x0 /* ecc_size0 = 0, no oob protection */ 132 #define BCH_ECC_SIZE1 0x20 /* ecc_size1 = 32 */ 133 134 #define BADBLOCK_MARKER_LENGTH 2 135 136 static u_char bch16_vector[] = {0xf5, 0x24, 0x1c, 0xd0, 0x61, 0xb3, 0xf1, 0x55, 137 0x2e, 0x2c, 0x86, 0xa3, 0xed, 0x36, 0x1b, 0x78, 138 0x48, 0x76, 0xa9, 0x3b, 0x97, 0xd1, 0x7a, 0x93, 139 0x07, 0x0e}; 140 static u_char bch8_vector[] = {0xf3, 0xdb, 0x14, 0x16, 0x8b, 0xd2, 0xbe, 0xcc, 141 0xac, 0x6b, 0xff, 0x99, 0x7b}; 142 static u_char bch4_vector[] = {0x00, 0x6b, 0x31, 0xdd, 0x41, 0xbc, 0x10}; 143 144 struct omap_nand_info { 145 struct nand_chip nand; 146 struct platform_device *pdev; 147 148 int gpmc_cs; 149 bool dev_ready; 150 enum nand_io xfer_type; 151 int devsize; 152 enum omap_ecc ecc_opt; 153 struct device_node *elm_of_node; 154 155 unsigned long phys_base; 156 struct completion comp; 157 struct dma_chan *dma; 158 int gpmc_irq_fifo; 159 int gpmc_irq_count; 160 enum { 161 OMAP_NAND_IO_READ = 0, /* read */ 162 OMAP_NAND_IO_WRITE, /* write */ 163 } iomode; 164 u_char *buf; 165 int buf_len; 166 /* Interface to GPMC */ 167 struct gpmc_nand_regs reg; 168 struct gpmc_nand_ops *ops; 169 bool flash_bbt; 170 /* fields specific for BCHx_HW ECC scheme */ 171 struct device *elm_dev; 172 /* NAND ready gpio */ 173 struct gpio_desc *ready_gpiod; 174 }; 175 176 static inline struct omap_nand_info *mtd_to_omap(struct mtd_info *mtd) 177 { 178 return container_of(mtd_to_nand(mtd), struct omap_nand_info, nand); 179 } 180 181 /** 182 * omap_prefetch_enable - configures and starts prefetch transfer 183 * @cs: cs (chip select) number 184 * @fifo_th: fifo threshold to be used for read/ write 185 * @dma_mode: dma mode enable (1) or disable (0) 186 * @u32_count: number of bytes to be transferred 187 * @is_write: prefetch read(0) or write post(1) mode 188 */ 189 static int omap_prefetch_enable(int cs, int fifo_th, int dma_mode, 190 unsigned int u32_count, int is_write, struct omap_nand_info *info) 191 { 192 u32 val; 193 194 if (fifo_th > PREFETCH_FIFOTHRESHOLD_MAX) 195 return -1; 196 197 if (readl(info->reg.gpmc_prefetch_control)) 198 return -EBUSY; 199 200 /* Set the amount of bytes to be prefetched */ 201 writel(u32_count, info->reg.gpmc_prefetch_config2); 202 203 /* Set dma/mpu mode, the prefetch read / post write and 204 * enable the engine. Set which cs is has requested for. 205 */ 206 val = ((cs << PREFETCH_CONFIG1_CS_SHIFT) | 207 PREFETCH_FIFOTHRESHOLD(fifo_th) | ENABLE_PREFETCH | 208 (dma_mode << DMA_MPU_MODE_SHIFT) | (is_write & 0x1)); 209 writel(val, info->reg.gpmc_prefetch_config1); 210 211 /* Start the prefetch engine */ 212 writel(0x1, info->reg.gpmc_prefetch_control); 213 214 return 0; 215 } 216 217 /** 218 * omap_prefetch_reset - disables and stops the prefetch engine 219 */ 220 static int omap_prefetch_reset(int cs, struct omap_nand_info *info) 221 { 222 u32 config1; 223 224 /* check if the same module/cs is trying to reset */ 225 config1 = readl(info->reg.gpmc_prefetch_config1); 226 if (((config1 >> PREFETCH_CONFIG1_CS_SHIFT) & CS_MASK) != cs) 227 return -EINVAL; 228 229 /* Stop the PFPW engine */ 230 writel(0x0, info->reg.gpmc_prefetch_control); 231 232 /* Reset/disable the PFPW engine */ 233 writel(0x0, info->reg.gpmc_prefetch_config1); 234 235 return 0; 236 } 237 238 /** 239 * omap_hwcontrol - hardware specific access to control-lines 240 * @chip: NAND chip object 241 * @cmd: command to device 242 * @ctrl: 243 * NAND_NCE: bit 0 -> don't care 244 * NAND_CLE: bit 1 -> Command Latch 245 * NAND_ALE: bit 2 -> Address Latch 246 * 247 * NOTE: boards may use different bits for these!! 248 */ 249 static void omap_hwcontrol(struct nand_chip *chip, int cmd, unsigned int ctrl) 250 { 251 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 252 253 if (cmd != NAND_CMD_NONE) { 254 if (ctrl & NAND_CLE) 255 writeb(cmd, info->reg.gpmc_nand_command); 256 257 else if (ctrl & NAND_ALE) 258 writeb(cmd, info->reg.gpmc_nand_address); 259 260 else /* NAND_NCE */ 261 writeb(cmd, info->reg.gpmc_nand_data); 262 } 263 } 264 265 /** 266 * omap_read_buf8 - read data from NAND controller into buffer 267 * @mtd: MTD device structure 268 * @buf: buffer to store date 269 * @len: number of bytes to read 270 */ 271 static void omap_read_buf8(struct mtd_info *mtd, u_char *buf, int len) 272 { 273 struct nand_chip *nand = mtd_to_nand(mtd); 274 275 ioread8_rep(nand->legacy.IO_ADDR_R, buf, len); 276 } 277 278 /** 279 * omap_write_buf8 - write buffer to NAND controller 280 * @mtd: MTD device structure 281 * @buf: data buffer 282 * @len: number of bytes to write 283 */ 284 static void omap_write_buf8(struct mtd_info *mtd, const u_char *buf, int len) 285 { 286 struct omap_nand_info *info = mtd_to_omap(mtd); 287 u_char *p = (u_char *)buf; 288 bool status; 289 290 while (len--) { 291 iowrite8(*p++, info->nand.legacy.IO_ADDR_W); 292 /* wait until buffer is available for write */ 293 do { 294 status = info->ops->nand_writebuffer_empty(); 295 } while (!status); 296 } 297 } 298 299 /** 300 * omap_read_buf16 - read data from NAND controller into buffer 301 * @mtd: MTD device structure 302 * @buf: buffer to store date 303 * @len: number of bytes to read 304 */ 305 static void omap_read_buf16(struct mtd_info *mtd, u_char *buf, int len) 306 { 307 struct nand_chip *nand = mtd_to_nand(mtd); 308 309 ioread16_rep(nand->legacy.IO_ADDR_R, buf, len / 2); 310 } 311 312 /** 313 * omap_write_buf16 - write buffer to NAND controller 314 * @mtd: MTD device structure 315 * @buf: data buffer 316 * @len: number of bytes to write 317 */ 318 static void omap_write_buf16(struct mtd_info *mtd, const u_char * buf, int len) 319 { 320 struct omap_nand_info *info = mtd_to_omap(mtd); 321 u16 *p = (u16 *) buf; 322 bool status; 323 /* FIXME try bursts of writesw() or DMA ... */ 324 len >>= 1; 325 326 while (len--) { 327 iowrite16(*p++, info->nand.legacy.IO_ADDR_W); 328 /* wait until buffer is available for write */ 329 do { 330 status = info->ops->nand_writebuffer_empty(); 331 } while (!status); 332 } 333 } 334 335 /** 336 * omap_read_buf_pref - read data from NAND controller into buffer 337 * @chip: NAND chip object 338 * @buf: buffer to store date 339 * @len: number of bytes to read 340 */ 341 static void omap_read_buf_pref(struct nand_chip *chip, u_char *buf, int len) 342 { 343 struct mtd_info *mtd = nand_to_mtd(chip); 344 struct omap_nand_info *info = mtd_to_omap(mtd); 345 uint32_t r_count = 0; 346 int ret = 0; 347 u32 *p = (u32 *)buf; 348 349 /* take care of subpage reads */ 350 if (len % 4) { 351 if (info->nand.options & NAND_BUSWIDTH_16) 352 omap_read_buf16(mtd, buf, len % 4); 353 else 354 omap_read_buf8(mtd, buf, len % 4); 355 p = (u32 *) (buf + len % 4); 356 len -= len % 4; 357 } 358 359 /* configure and start prefetch transfer */ 360 ret = omap_prefetch_enable(info->gpmc_cs, 361 PREFETCH_FIFOTHRESHOLD_MAX, 0x0, len, 0x0, info); 362 if (ret) { 363 /* PFPW engine is busy, use cpu copy method */ 364 if (info->nand.options & NAND_BUSWIDTH_16) 365 omap_read_buf16(mtd, (u_char *)p, len); 366 else 367 omap_read_buf8(mtd, (u_char *)p, len); 368 } else { 369 do { 370 r_count = readl(info->reg.gpmc_prefetch_status); 371 r_count = PREFETCH_STATUS_FIFO_CNT(r_count); 372 r_count = r_count >> 2; 373 ioread32_rep(info->nand.legacy.IO_ADDR_R, p, r_count); 374 p += r_count; 375 len -= r_count << 2; 376 } while (len); 377 /* disable and stop the PFPW engine */ 378 omap_prefetch_reset(info->gpmc_cs, info); 379 } 380 } 381 382 /** 383 * omap_write_buf_pref - write buffer to NAND controller 384 * @chip: NAND chip object 385 * @buf: data buffer 386 * @len: number of bytes to write 387 */ 388 static void omap_write_buf_pref(struct nand_chip *chip, const u_char *buf, 389 int len) 390 { 391 struct mtd_info *mtd = nand_to_mtd(chip); 392 struct omap_nand_info *info = mtd_to_omap(mtd); 393 uint32_t w_count = 0; 394 int i = 0, ret = 0; 395 u16 *p = (u16 *)buf; 396 unsigned long tim, limit; 397 u32 val; 398 399 /* take care of subpage writes */ 400 if (len % 2 != 0) { 401 writeb(*buf, info->nand.legacy.IO_ADDR_W); 402 p = (u16 *)(buf + 1); 403 len--; 404 } 405 406 /* configure and start prefetch transfer */ 407 ret = omap_prefetch_enable(info->gpmc_cs, 408 PREFETCH_FIFOTHRESHOLD_MAX, 0x0, len, 0x1, info); 409 if (ret) { 410 /* PFPW engine is busy, use cpu copy method */ 411 if (info->nand.options & NAND_BUSWIDTH_16) 412 omap_write_buf16(mtd, (u_char *)p, len); 413 else 414 omap_write_buf8(mtd, (u_char *)p, len); 415 } else { 416 while (len) { 417 w_count = readl(info->reg.gpmc_prefetch_status); 418 w_count = PREFETCH_STATUS_FIFO_CNT(w_count); 419 w_count = w_count >> 1; 420 for (i = 0; (i < w_count) && len; i++, len -= 2) 421 iowrite16(*p++, info->nand.legacy.IO_ADDR_W); 422 } 423 /* wait for data to flushed-out before reset the prefetch */ 424 tim = 0; 425 limit = (loops_per_jiffy * 426 msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS)); 427 do { 428 cpu_relax(); 429 val = readl(info->reg.gpmc_prefetch_status); 430 val = PREFETCH_STATUS_COUNT(val); 431 } while (val && (tim++ < limit)); 432 433 /* disable and stop the PFPW engine */ 434 omap_prefetch_reset(info->gpmc_cs, info); 435 } 436 } 437 438 /* 439 * omap_nand_dma_callback: callback on the completion of dma transfer 440 * @data: pointer to completion data structure 441 */ 442 static void omap_nand_dma_callback(void *data) 443 { 444 complete((struct completion *) data); 445 } 446 447 /* 448 * omap_nand_dma_transfer: configure and start dma transfer 449 * @mtd: MTD device structure 450 * @addr: virtual address in RAM of source/destination 451 * @len: number of data bytes to be transferred 452 * @is_write: flag for read/write operation 453 */ 454 static inline int omap_nand_dma_transfer(struct mtd_info *mtd, void *addr, 455 unsigned int len, int is_write) 456 { 457 struct omap_nand_info *info = mtd_to_omap(mtd); 458 struct dma_async_tx_descriptor *tx; 459 enum dma_data_direction dir = is_write ? DMA_TO_DEVICE : 460 DMA_FROM_DEVICE; 461 struct scatterlist sg; 462 unsigned long tim, limit; 463 unsigned n; 464 int ret; 465 u32 val; 466 467 if (!virt_addr_valid(addr)) 468 goto out_copy; 469 470 sg_init_one(&sg, addr, len); 471 n = dma_map_sg(info->dma->device->dev, &sg, 1, dir); 472 if (n == 0) { 473 dev_err(&info->pdev->dev, 474 "Couldn't DMA map a %d byte buffer\n", len); 475 goto out_copy; 476 } 477 478 tx = dmaengine_prep_slave_sg(info->dma, &sg, n, 479 is_write ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM, 480 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 481 if (!tx) 482 goto out_copy_unmap; 483 484 tx->callback = omap_nand_dma_callback; 485 tx->callback_param = &info->comp; 486 dmaengine_submit(tx); 487 488 init_completion(&info->comp); 489 490 /* setup and start DMA using dma_addr */ 491 dma_async_issue_pending(info->dma); 492 493 /* configure and start prefetch transfer */ 494 ret = omap_prefetch_enable(info->gpmc_cs, 495 PREFETCH_FIFOTHRESHOLD_MAX, 0x1, len, is_write, info); 496 if (ret) 497 /* PFPW engine is busy, use cpu copy method */ 498 goto out_copy_unmap; 499 500 wait_for_completion(&info->comp); 501 tim = 0; 502 limit = (loops_per_jiffy * msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS)); 503 504 do { 505 cpu_relax(); 506 val = readl(info->reg.gpmc_prefetch_status); 507 val = PREFETCH_STATUS_COUNT(val); 508 } while (val && (tim++ < limit)); 509 510 /* disable and stop the PFPW engine */ 511 omap_prefetch_reset(info->gpmc_cs, info); 512 513 dma_unmap_sg(info->dma->device->dev, &sg, 1, dir); 514 return 0; 515 516 out_copy_unmap: 517 dma_unmap_sg(info->dma->device->dev, &sg, 1, dir); 518 out_copy: 519 if (info->nand.options & NAND_BUSWIDTH_16) 520 is_write == 0 ? omap_read_buf16(mtd, (u_char *) addr, len) 521 : omap_write_buf16(mtd, (u_char *) addr, len); 522 else 523 is_write == 0 ? omap_read_buf8(mtd, (u_char *) addr, len) 524 : omap_write_buf8(mtd, (u_char *) addr, len); 525 return 0; 526 } 527 528 /** 529 * omap_read_buf_dma_pref - read data from NAND controller into buffer 530 * @chip: NAND chip object 531 * @buf: buffer to store date 532 * @len: number of bytes to read 533 */ 534 static void omap_read_buf_dma_pref(struct nand_chip *chip, u_char *buf, 535 int len) 536 { 537 struct mtd_info *mtd = nand_to_mtd(chip); 538 539 if (len <= mtd->oobsize) 540 omap_read_buf_pref(chip, buf, len); 541 else 542 /* start transfer in DMA mode */ 543 omap_nand_dma_transfer(mtd, buf, len, 0x0); 544 } 545 546 /** 547 * omap_write_buf_dma_pref - write buffer to NAND controller 548 * @chip: NAND chip object 549 * @buf: data buffer 550 * @len: number of bytes to write 551 */ 552 static void omap_write_buf_dma_pref(struct nand_chip *chip, const u_char *buf, 553 int len) 554 { 555 struct mtd_info *mtd = nand_to_mtd(chip); 556 557 if (len <= mtd->oobsize) 558 omap_write_buf_pref(chip, buf, len); 559 else 560 /* start transfer in DMA mode */ 561 omap_nand_dma_transfer(mtd, (u_char *)buf, len, 0x1); 562 } 563 564 /* 565 * omap_nand_irq - GPMC irq handler 566 * @this_irq: gpmc irq number 567 * @dev: omap_nand_info structure pointer is passed here 568 */ 569 static irqreturn_t omap_nand_irq(int this_irq, void *dev) 570 { 571 struct omap_nand_info *info = (struct omap_nand_info *) dev; 572 u32 bytes; 573 574 bytes = readl(info->reg.gpmc_prefetch_status); 575 bytes = PREFETCH_STATUS_FIFO_CNT(bytes); 576 bytes = bytes & 0xFFFC; /* io in multiple of 4 bytes */ 577 if (info->iomode == OMAP_NAND_IO_WRITE) { /* checks for write io */ 578 if (this_irq == info->gpmc_irq_count) 579 goto done; 580 581 if (info->buf_len && (info->buf_len < bytes)) 582 bytes = info->buf_len; 583 else if (!info->buf_len) 584 bytes = 0; 585 iowrite32_rep(info->nand.legacy.IO_ADDR_W, (u32 *)info->buf, 586 bytes >> 2); 587 info->buf = info->buf + bytes; 588 info->buf_len -= bytes; 589 590 } else { 591 ioread32_rep(info->nand.legacy.IO_ADDR_R, (u32 *)info->buf, 592 bytes >> 2); 593 info->buf = info->buf + bytes; 594 595 if (this_irq == info->gpmc_irq_count) 596 goto done; 597 } 598 599 return IRQ_HANDLED; 600 601 done: 602 complete(&info->comp); 603 604 disable_irq_nosync(info->gpmc_irq_fifo); 605 disable_irq_nosync(info->gpmc_irq_count); 606 607 return IRQ_HANDLED; 608 } 609 610 /* 611 * omap_read_buf_irq_pref - read data from NAND controller into buffer 612 * @chip: NAND chip object 613 * @buf: buffer to store date 614 * @len: number of bytes to read 615 */ 616 static void omap_read_buf_irq_pref(struct nand_chip *chip, u_char *buf, 617 int len) 618 { 619 struct mtd_info *mtd = nand_to_mtd(chip); 620 struct omap_nand_info *info = mtd_to_omap(mtd); 621 int ret = 0; 622 623 if (len <= mtd->oobsize) { 624 omap_read_buf_pref(chip, buf, len); 625 return; 626 } 627 628 info->iomode = OMAP_NAND_IO_READ; 629 info->buf = buf; 630 init_completion(&info->comp); 631 632 /* configure and start prefetch transfer */ 633 ret = omap_prefetch_enable(info->gpmc_cs, 634 PREFETCH_FIFOTHRESHOLD_MAX/2, 0x0, len, 0x0, info); 635 if (ret) 636 /* PFPW engine is busy, use cpu copy method */ 637 goto out_copy; 638 639 info->buf_len = len; 640 641 enable_irq(info->gpmc_irq_count); 642 enable_irq(info->gpmc_irq_fifo); 643 644 /* waiting for read to complete */ 645 wait_for_completion(&info->comp); 646 647 /* disable and stop the PFPW engine */ 648 omap_prefetch_reset(info->gpmc_cs, info); 649 return; 650 651 out_copy: 652 if (info->nand.options & NAND_BUSWIDTH_16) 653 omap_read_buf16(mtd, buf, len); 654 else 655 omap_read_buf8(mtd, buf, len); 656 } 657 658 /* 659 * omap_write_buf_irq_pref - write buffer to NAND controller 660 * @chip: NAND chip object 661 * @buf: data buffer 662 * @len: number of bytes to write 663 */ 664 static void omap_write_buf_irq_pref(struct nand_chip *chip, const u_char *buf, 665 int len) 666 { 667 struct mtd_info *mtd = nand_to_mtd(chip); 668 struct omap_nand_info *info = mtd_to_omap(mtd); 669 int ret = 0; 670 unsigned long tim, limit; 671 u32 val; 672 673 if (len <= mtd->oobsize) { 674 omap_write_buf_pref(chip, buf, len); 675 return; 676 } 677 678 info->iomode = OMAP_NAND_IO_WRITE; 679 info->buf = (u_char *) buf; 680 init_completion(&info->comp); 681 682 /* configure and start prefetch transfer : size=24 */ 683 ret = omap_prefetch_enable(info->gpmc_cs, 684 (PREFETCH_FIFOTHRESHOLD_MAX * 3) / 8, 0x0, len, 0x1, info); 685 if (ret) 686 /* PFPW engine is busy, use cpu copy method */ 687 goto out_copy; 688 689 info->buf_len = len; 690 691 enable_irq(info->gpmc_irq_count); 692 enable_irq(info->gpmc_irq_fifo); 693 694 /* waiting for write to complete */ 695 wait_for_completion(&info->comp); 696 697 /* wait for data to flushed-out before reset the prefetch */ 698 tim = 0; 699 limit = (loops_per_jiffy * msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS)); 700 do { 701 val = readl(info->reg.gpmc_prefetch_status); 702 val = PREFETCH_STATUS_COUNT(val); 703 cpu_relax(); 704 } while (val && (tim++ < limit)); 705 706 /* disable and stop the PFPW engine */ 707 omap_prefetch_reset(info->gpmc_cs, info); 708 return; 709 710 out_copy: 711 if (info->nand.options & NAND_BUSWIDTH_16) 712 omap_write_buf16(mtd, buf, len); 713 else 714 omap_write_buf8(mtd, buf, len); 715 } 716 717 /** 718 * gen_true_ecc - This function will generate true ECC value 719 * @ecc_buf: buffer to store ecc code 720 * 721 * This generated true ECC value can be used when correcting 722 * data read from NAND flash memory core 723 */ 724 static void gen_true_ecc(u8 *ecc_buf) 725 { 726 u32 tmp = ecc_buf[0] | (ecc_buf[1] << 16) | 727 ((ecc_buf[2] & 0xF0) << 20) | ((ecc_buf[2] & 0x0F) << 8); 728 729 ecc_buf[0] = ~(P64o(tmp) | P64e(tmp) | P32o(tmp) | P32e(tmp) | 730 P16o(tmp) | P16e(tmp) | P8o(tmp) | P8e(tmp)); 731 ecc_buf[1] = ~(P1024o(tmp) | P1024e(tmp) | P512o(tmp) | P512e(tmp) | 732 P256o(tmp) | P256e(tmp) | P128o(tmp) | P128e(tmp)); 733 ecc_buf[2] = ~(P4o(tmp) | P4e(tmp) | P2o(tmp) | P2e(tmp) | P1o(tmp) | 734 P1e(tmp) | P2048o(tmp) | P2048e(tmp)); 735 } 736 737 /** 738 * omap_compare_ecc - Detect (2 bits) and correct (1 bit) error in data 739 * @ecc_data1: ecc code from nand spare area 740 * @ecc_data2: ecc code from hardware register obtained from hardware ecc 741 * @page_data: page data 742 * 743 * This function compares two ECC's and indicates if there is an error. 744 * If the error can be corrected it will be corrected to the buffer. 745 * If there is no error, %0 is returned. If there is an error but it 746 * was corrected, %1 is returned. Otherwise, %-1 is returned. 747 */ 748 static int omap_compare_ecc(u8 *ecc_data1, /* read from NAND memory */ 749 u8 *ecc_data2, /* read from register */ 750 u8 *page_data) 751 { 752 uint i; 753 u8 tmp0_bit[8], tmp1_bit[8], tmp2_bit[8]; 754 u8 comp0_bit[8], comp1_bit[8], comp2_bit[8]; 755 u8 ecc_bit[24]; 756 u8 ecc_sum = 0; 757 u8 find_bit = 0; 758 uint find_byte = 0; 759 int isEccFF; 760 761 isEccFF = ((*(u32 *)ecc_data1 & 0xFFFFFF) == 0xFFFFFF); 762 763 gen_true_ecc(ecc_data1); 764 gen_true_ecc(ecc_data2); 765 766 for (i = 0; i <= 2; i++) { 767 *(ecc_data1 + i) = ~(*(ecc_data1 + i)); 768 *(ecc_data2 + i) = ~(*(ecc_data2 + i)); 769 } 770 771 for (i = 0; i < 8; i++) { 772 tmp0_bit[i] = *ecc_data1 % 2; 773 *ecc_data1 = *ecc_data1 / 2; 774 } 775 776 for (i = 0; i < 8; i++) { 777 tmp1_bit[i] = *(ecc_data1 + 1) % 2; 778 *(ecc_data1 + 1) = *(ecc_data1 + 1) / 2; 779 } 780 781 for (i = 0; i < 8; i++) { 782 tmp2_bit[i] = *(ecc_data1 + 2) % 2; 783 *(ecc_data1 + 2) = *(ecc_data1 + 2) / 2; 784 } 785 786 for (i = 0; i < 8; i++) { 787 comp0_bit[i] = *ecc_data2 % 2; 788 *ecc_data2 = *ecc_data2 / 2; 789 } 790 791 for (i = 0; i < 8; i++) { 792 comp1_bit[i] = *(ecc_data2 + 1) % 2; 793 *(ecc_data2 + 1) = *(ecc_data2 + 1) / 2; 794 } 795 796 for (i = 0; i < 8; i++) { 797 comp2_bit[i] = *(ecc_data2 + 2) % 2; 798 *(ecc_data2 + 2) = *(ecc_data2 + 2) / 2; 799 } 800 801 for (i = 0; i < 6; i++) 802 ecc_bit[i] = tmp2_bit[i + 2] ^ comp2_bit[i + 2]; 803 804 for (i = 0; i < 8; i++) 805 ecc_bit[i + 6] = tmp0_bit[i] ^ comp0_bit[i]; 806 807 for (i = 0; i < 8; i++) 808 ecc_bit[i + 14] = tmp1_bit[i] ^ comp1_bit[i]; 809 810 ecc_bit[22] = tmp2_bit[0] ^ comp2_bit[0]; 811 ecc_bit[23] = tmp2_bit[1] ^ comp2_bit[1]; 812 813 for (i = 0; i < 24; i++) 814 ecc_sum += ecc_bit[i]; 815 816 switch (ecc_sum) { 817 case 0: 818 /* Not reached because this function is not called if 819 * ECC values are equal 820 */ 821 return 0; 822 823 case 1: 824 /* Uncorrectable error */ 825 pr_debug("ECC UNCORRECTED_ERROR 1\n"); 826 return -EBADMSG; 827 828 case 11: 829 /* UN-Correctable error */ 830 pr_debug("ECC UNCORRECTED_ERROR B\n"); 831 return -EBADMSG; 832 833 case 12: 834 /* Correctable error */ 835 find_byte = (ecc_bit[23] << 8) + 836 (ecc_bit[21] << 7) + 837 (ecc_bit[19] << 6) + 838 (ecc_bit[17] << 5) + 839 (ecc_bit[15] << 4) + 840 (ecc_bit[13] << 3) + 841 (ecc_bit[11] << 2) + 842 (ecc_bit[9] << 1) + 843 ecc_bit[7]; 844 845 find_bit = (ecc_bit[5] << 2) + (ecc_bit[3] << 1) + ecc_bit[1]; 846 847 pr_debug("Correcting single bit ECC error at offset: " 848 "%d, bit: %d\n", find_byte, find_bit); 849 850 page_data[find_byte] ^= (1 << find_bit); 851 852 return 1; 853 default: 854 if (isEccFF) { 855 if (ecc_data2[0] == 0 && 856 ecc_data2[1] == 0 && 857 ecc_data2[2] == 0) 858 return 0; 859 } 860 pr_debug("UNCORRECTED_ERROR default\n"); 861 return -EBADMSG; 862 } 863 } 864 865 /** 866 * omap_correct_data - Compares the ECC read with HW generated ECC 867 * @chip: NAND chip object 868 * @dat: page data 869 * @read_ecc: ecc read from nand flash 870 * @calc_ecc: ecc read from HW ECC registers 871 * 872 * Compares the ecc read from nand spare area with ECC registers values 873 * and if ECC's mismatched, it will call 'omap_compare_ecc' for error 874 * detection and correction. If there are no errors, %0 is returned. If 875 * there were errors and all of the errors were corrected, the number of 876 * corrected errors is returned. If uncorrectable errors exist, %-1 is 877 * returned. 878 */ 879 static int omap_correct_data(struct nand_chip *chip, u_char *dat, 880 u_char *read_ecc, u_char *calc_ecc) 881 { 882 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 883 int blockCnt = 0, i = 0, ret = 0; 884 int stat = 0; 885 886 /* Ex NAND_ECC_HW12_2048 */ 887 if ((info->nand.ecc.mode == NAND_ECC_HW) && 888 (info->nand.ecc.size == 2048)) 889 blockCnt = 4; 890 else 891 blockCnt = 1; 892 893 for (i = 0; i < blockCnt; i++) { 894 if (memcmp(read_ecc, calc_ecc, 3) != 0) { 895 ret = omap_compare_ecc(read_ecc, calc_ecc, dat); 896 if (ret < 0) 897 return ret; 898 /* keep track of the number of corrected errors */ 899 stat += ret; 900 } 901 read_ecc += 3; 902 calc_ecc += 3; 903 dat += 512; 904 } 905 return stat; 906 } 907 908 /** 909 * omap_calcuate_ecc - Generate non-inverted ECC bytes. 910 * @chip: NAND chip object 911 * @dat: The pointer to data on which ecc is computed 912 * @ecc_code: The ecc_code buffer 913 * 914 * Using noninverted ECC can be considered ugly since writing a blank 915 * page ie. padding will clear the ECC bytes. This is no problem as long 916 * nobody is trying to write data on the seemingly unused page. Reading 917 * an erased page will produce an ECC mismatch between generated and read 918 * ECC bytes that has to be dealt with separately. 919 */ 920 static int omap_calculate_ecc(struct nand_chip *chip, const u_char *dat, 921 u_char *ecc_code) 922 { 923 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 924 u32 val; 925 926 val = readl(info->reg.gpmc_ecc_config); 927 if (((val >> ECC_CONFIG_CS_SHIFT) & CS_MASK) != info->gpmc_cs) 928 return -EINVAL; 929 930 /* read ecc result */ 931 val = readl(info->reg.gpmc_ecc1_result); 932 *ecc_code++ = val; /* P128e, ..., P1e */ 933 *ecc_code++ = val >> 16; /* P128o, ..., P1o */ 934 /* P2048o, P1024o, P512o, P256o, P2048e, P1024e, P512e, P256e */ 935 *ecc_code++ = ((val >> 8) & 0x0f) | ((val >> 20) & 0xf0); 936 937 return 0; 938 } 939 940 /** 941 * omap_enable_hwecc - This function enables the hardware ecc functionality 942 * @mtd: MTD device structure 943 * @mode: Read/Write mode 944 */ 945 static void omap_enable_hwecc(struct nand_chip *chip, int mode) 946 { 947 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 948 unsigned int dev_width = (chip->options & NAND_BUSWIDTH_16) ? 1 : 0; 949 u32 val; 950 951 /* clear ecc and enable bits */ 952 val = ECCCLEAR | ECC1; 953 writel(val, info->reg.gpmc_ecc_control); 954 955 /* program ecc and result sizes */ 956 val = ((((info->nand.ecc.size >> 1) - 1) << ECCSIZE1_SHIFT) | 957 ECC1RESULTSIZE); 958 writel(val, info->reg.gpmc_ecc_size_config); 959 960 switch (mode) { 961 case NAND_ECC_READ: 962 case NAND_ECC_WRITE: 963 writel(ECCCLEAR | ECC1, info->reg.gpmc_ecc_control); 964 break; 965 case NAND_ECC_READSYN: 966 writel(ECCCLEAR, info->reg.gpmc_ecc_control); 967 break; 968 default: 969 dev_info(&info->pdev->dev, 970 "error: unrecognized Mode[%d]!\n", mode); 971 break; 972 } 973 974 /* (ECC 16 or 8 bit col) | ( CS ) | ECC Enable */ 975 val = (dev_width << 7) | (info->gpmc_cs << 1) | (0x1); 976 writel(val, info->reg.gpmc_ecc_config); 977 } 978 979 /** 980 * omap_wait - wait until the command is done 981 * @this: NAND Chip structure 982 * 983 * Wait function is called during Program and erase operations and 984 * the way it is called from MTD layer, we should wait till the NAND 985 * chip is ready after the programming/erase operation has completed. 986 * 987 * Erase can take up to 400ms and program up to 20ms according to 988 * general NAND and SmartMedia specs 989 */ 990 static int omap_wait(struct nand_chip *this) 991 { 992 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(this)); 993 unsigned long timeo = jiffies; 994 int status; 995 996 timeo += msecs_to_jiffies(400); 997 998 writeb(NAND_CMD_STATUS & 0xFF, info->reg.gpmc_nand_command); 999 while (time_before(jiffies, timeo)) { 1000 status = readb(info->reg.gpmc_nand_data); 1001 if (status & NAND_STATUS_READY) 1002 break; 1003 cond_resched(); 1004 } 1005 1006 status = readb(info->reg.gpmc_nand_data); 1007 return status; 1008 } 1009 1010 /** 1011 * omap_dev_ready - checks the NAND Ready GPIO line 1012 * @mtd: MTD device structure 1013 * 1014 * Returns true if ready and false if busy. 1015 */ 1016 static int omap_dev_ready(struct nand_chip *chip) 1017 { 1018 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 1019 1020 return gpiod_get_value(info->ready_gpiod); 1021 } 1022 1023 /** 1024 * omap_enable_hwecc_bch - Program GPMC to perform BCH ECC calculation 1025 * @mtd: MTD device structure 1026 * @mode: Read/Write mode 1027 * 1028 * When using BCH with SW correction (i.e. no ELM), sector size is set 1029 * to 512 bytes and we use BCH_WRAPMODE_6 wrapping mode 1030 * for both reading and writing with: 1031 * eccsize0 = 0 (no additional protected byte in spare area) 1032 * eccsize1 = 32 (skip 32 nibbles = 16 bytes per sector in spare area) 1033 */ 1034 static void __maybe_unused omap_enable_hwecc_bch(struct nand_chip *chip, 1035 int mode) 1036 { 1037 unsigned int bch_type; 1038 unsigned int dev_width, nsectors; 1039 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 1040 enum omap_ecc ecc_opt = info->ecc_opt; 1041 u32 val, wr_mode; 1042 unsigned int ecc_size1, ecc_size0; 1043 1044 /* GPMC configurations for calculating ECC */ 1045 switch (ecc_opt) { 1046 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: 1047 bch_type = 0; 1048 nsectors = 1; 1049 wr_mode = BCH_WRAPMODE_6; 1050 ecc_size0 = BCH_ECC_SIZE0; 1051 ecc_size1 = BCH_ECC_SIZE1; 1052 break; 1053 case OMAP_ECC_BCH4_CODE_HW: 1054 bch_type = 0; 1055 nsectors = chip->ecc.steps; 1056 if (mode == NAND_ECC_READ) { 1057 wr_mode = BCH_WRAPMODE_1; 1058 ecc_size0 = BCH4R_ECC_SIZE0; 1059 ecc_size1 = BCH4R_ECC_SIZE1; 1060 } else { 1061 wr_mode = BCH_WRAPMODE_6; 1062 ecc_size0 = BCH_ECC_SIZE0; 1063 ecc_size1 = BCH_ECC_SIZE1; 1064 } 1065 break; 1066 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: 1067 bch_type = 1; 1068 nsectors = 1; 1069 wr_mode = BCH_WRAPMODE_6; 1070 ecc_size0 = BCH_ECC_SIZE0; 1071 ecc_size1 = BCH_ECC_SIZE1; 1072 break; 1073 case OMAP_ECC_BCH8_CODE_HW: 1074 bch_type = 1; 1075 nsectors = chip->ecc.steps; 1076 if (mode == NAND_ECC_READ) { 1077 wr_mode = BCH_WRAPMODE_1; 1078 ecc_size0 = BCH8R_ECC_SIZE0; 1079 ecc_size1 = BCH8R_ECC_SIZE1; 1080 } else { 1081 wr_mode = BCH_WRAPMODE_6; 1082 ecc_size0 = BCH_ECC_SIZE0; 1083 ecc_size1 = BCH_ECC_SIZE1; 1084 } 1085 break; 1086 case OMAP_ECC_BCH16_CODE_HW: 1087 bch_type = 0x2; 1088 nsectors = chip->ecc.steps; 1089 if (mode == NAND_ECC_READ) { 1090 wr_mode = 0x01; 1091 ecc_size0 = 52; /* ECC bits in nibbles per sector */ 1092 ecc_size1 = 0; /* non-ECC bits in nibbles per sector */ 1093 } else { 1094 wr_mode = 0x01; 1095 ecc_size0 = 0; /* extra bits in nibbles per sector */ 1096 ecc_size1 = 52; /* OOB bits in nibbles per sector */ 1097 } 1098 break; 1099 default: 1100 return; 1101 } 1102 1103 writel(ECC1, info->reg.gpmc_ecc_control); 1104 1105 /* Configure ecc size for BCH */ 1106 val = (ecc_size1 << ECCSIZE1_SHIFT) | (ecc_size0 << ECCSIZE0_SHIFT); 1107 writel(val, info->reg.gpmc_ecc_size_config); 1108 1109 dev_width = (chip->options & NAND_BUSWIDTH_16) ? 1 : 0; 1110 1111 /* BCH configuration */ 1112 val = ((1 << 16) | /* enable BCH */ 1113 (bch_type << 12) | /* BCH4/BCH8/BCH16 */ 1114 (wr_mode << 8) | /* wrap mode */ 1115 (dev_width << 7) | /* bus width */ 1116 (((nsectors-1) & 0x7) << 4) | /* number of sectors */ 1117 (info->gpmc_cs << 1) | /* ECC CS */ 1118 (0x1)); /* enable ECC */ 1119 1120 writel(val, info->reg.gpmc_ecc_config); 1121 1122 /* Clear ecc and enable bits */ 1123 writel(ECCCLEAR | ECC1, info->reg.gpmc_ecc_control); 1124 } 1125 1126 static u8 bch4_polynomial[] = {0x28, 0x13, 0xcc, 0x39, 0x96, 0xac, 0x7f}; 1127 static u8 bch8_polynomial[] = {0xef, 0x51, 0x2e, 0x09, 0xed, 0x93, 0x9a, 0xc2, 1128 0x97, 0x79, 0xe5, 0x24, 0xb5}; 1129 1130 /** 1131 * _omap_calculate_ecc_bch - Generate ECC bytes for one sector 1132 * @mtd: MTD device structure 1133 * @dat: The pointer to data on which ecc is computed 1134 * @ecc_code: The ecc_code buffer 1135 * @i: The sector number (for a multi sector page) 1136 * 1137 * Support calculating of BCH4/8/16 ECC vectors for one sector 1138 * within a page. Sector number is in @i. 1139 */ 1140 static int _omap_calculate_ecc_bch(struct mtd_info *mtd, 1141 const u_char *dat, u_char *ecc_calc, int i) 1142 { 1143 struct omap_nand_info *info = mtd_to_omap(mtd); 1144 int eccbytes = info->nand.ecc.bytes; 1145 struct gpmc_nand_regs *gpmc_regs = &info->reg; 1146 u8 *ecc_code; 1147 unsigned long bch_val1, bch_val2, bch_val3, bch_val4; 1148 u32 val; 1149 int j; 1150 1151 ecc_code = ecc_calc; 1152 switch (info->ecc_opt) { 1153 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: 1154 case OMAP_ECC_BCH8_CODE_HW: 1155 bch_val1 = readl(gpmc_regs->gpmc_bch_result0[i]); 1156 bch_val2 = readl(gpmc_regs->gpmc_bch_result1[i]); 1157 bch_val3 = readl(gpmc_regs->gpmc_bch_result2[i]); 1158 bch_val4 = readl(gpmc_regs->gpmc_bch_result3[i]); 1159 *ecc_code++ = (bch_val4 & 0xFF); 1160 *ecc_code++ = ((bch_val3 >> 24) & 0xFF); 1161 *ecc_code++ = ((bch_val3 >> 16) & 0xFF); 1162 *ecc_code++ = ((bch_val3 >> 8) & 0xFF); 1163 *ecc_code++ = (bch_val3 & 0xFF); 1164 *ecc_code++ = ((bch_val2 >> 24) & 0xFF); 1165 *ecc_code++ = ((bch_val2 >> 16) & 0xFF); 1166 *ecc_code++ = ((bch_val2 >> 8) & 0xFF); 1167 *ecc_code++ = (bch_val2 & 0xFF); 1168 *ecc_code++ = ((bch_val1 >> 24) & 0xFF); 1169 *ecc_code++ = ((bch_val1 >> 16) & 0xFF); 1170 *ecc_code++ = ((bch_val1 >> 8) & 0xFF); 1171 *ecc_code++ = (bch_val1 & 0xFF); 1172 break; 1173 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: 1174 case OMAP_ECC_BCH4_CODE_HW: 1175 bch_val1 = readl(gpmc_regs->gpmc_bch_result0[i]); 1176 bch_val2 = readl(gpmc_regs->gpmc_bch_result1[i]); 1177 *ecc_code++ = ((bch_val2 >> 12) & 0xFF); 1178 *ecc_code++ = ((bch_val2 >> 4) & 0xFF); 1179 *ecc_code++ = ((bch_val2 & 0xF) << 4) | 1180 ((bch_val1 >> 28) & 0xF); 1181 *ecc_code++ = ((bch_val1 >> 20) & 0xFF); 1182 *ecc_code++ = ((bch_val1 >> 12) & 0xFF); 1183 *ecc_code++ = ((bch_val1 >> 4) & 0xFF); 1184 *ecc_code++ = ((bch_val1 & 0xF) << 4); 1185 break; 1186 case OMAP_ECC_BCH16_CODE_HW: 1187 val = readl(gpmc_regs->gpmc_bch_result6[i]); 1188 ecc_code[0] = ((val >> 8) & 0xFF); 1189 ecc_code[1] = ((val >> 0) & 0xFF); 1190 val = readl(gpmc_regs->gpmc_bch_result5[i]); 1191 ecc_code[2] = ((val >> 24) & 0xFF); 1192 ecc_code[3] = ((val >> 16) & 0xFF); 1193 ecc_code[4] = ((val >> 8) & 0xFF); 1194 ecc_code[5] = ((val >> 0) & 0xFF); 1195 val = readl(gpmc_regs->gpmc_bch_result4[i]); 1196 ecc_code[6] = ((val >> 24) & 0xFF); 1197 ecc_code[7] = ((val >> 16) & 0xFF); 1198 ecc_code[8] = ((val >> 8) & 0xFF); 1199 ecc_code[9] = ((val >> 0) & 0xFF); 1200 val = readl(gpmc_regs->gpmc_bch_result3[i]); 1201 ecc_code[10] = ((val >> 24) & 0xFF); 1202 ecc_code[11] = ((val >> 16) & 0xFF); 1203 ecc_code[12] = ((val >> 8) & 0xFF); 1204 ecc_code[13] = ((val >> 0) & 0xFF); 1205 val = readl(gpmc_regs->gpmc_bch_result2[i]); 1206 ecc_code[14] = ((val >> 24) & 0xFF); 1207 ecc_code[15] = ((val >> 16) & 0xFF); 1208 ecc_code[16] = ((val >> 8) & 0xFF); 1209 ecc_code[17] = ((val >> 0) & 0xFF); 1210 val = readl(gpmc_regs->gpmc_bch_result1[i]); 1211 ecc_code[18] = ((val >> 24) & 0xFF); 1212 ecc_code[19] = ((val >> 16) & 0xFF); 1213 ecc_code[20] = ((val >> 8) & 0xFF); 1214 ecc_code[21] = ((val >> 0) & 0xFF); 1215 val = readl(gpmc_regs->gpmc_bch_result0[i]); 1216 ecc_code[22] = ((val >> 24) & 0xFF); 1217 ecc_code[23] = ((val >> 16) & 0xFF); 1218 ecc_code[24] = ((val >> 8) & 0xFF); 1219 ecc_code[25] = ((val >> 0) & 0xFF); 1220 break; 1221 default: 1222 return -EINVAL; 1223 } 1224 1225 /* ECC scheme specific syndrome customizations */ 1226 switch (info->ecc_opt) { 1227 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: 1228 /* Add constant polynomial to remainder, so that 1229 * ECC of blank pages results in 0x0 on reading back 1230 */ 1231 for (j = 0; j < eccbytes; j++) 1232 ecc_calc[j] ^= bch4_polynomial[j]; 1233 break; 1234 case OMAP_ECC_BCH4_CODE_HW: 1235 /* Set 8th ECC byte as 0x0 for ROM compatibility */ 1236 ecc_calc[eccbytes - 1] = 0x0; 1237 break; 1238 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: 1239 /* Add constant polynomial to remainder, so that 1240 * ECC of blank pages results in 0x0 on reading back 1241 */ 1242 for (j = 0; j < eccbytes; j++) 1243 ecc_calc[j] ^= bch8_polynomial[j]; 1244 break; 1245 case OMAP_ECC_BCH8_CODE_HW: 1246 /* Set 14th ECC byte as 0x0 for ROM compatibility */ 1247 ecc_calc[eccbytes - 1] = 0x0; 1248 break; 1249 case OMAP_ECC_BCH16_CODE_HW: 1250 break; 1251 default: 1252 return -EINVAL; 1253 } 1254 1255 return 0; 1256 } 1257 1258 /** 1259 * omap_calculate_ecc_bch_sw - ECC generator for sector for SW based correction 1260 * @chip: NAND chip object 1261 * @dat: The pointer to data on which ecc is computed 1262 * @ecc_code: The ecc_code buffer 1263 * 1264 * Support calculating of BCH4/8/16 ECC vectors for one sector. This is used 1265 * when SW based correction is required as ECC is required for one sector 1266 * at a time. 1267 */ 1268 static int omap_calculate_ecc_bch_sw(struct nand_chip *chip, 1269 const u_char *dat, u_char *ecc_calc) 1270 { 1271 return _omap_calculate_ecc_bch(nand_to_mtd(chip), dat, ecc_calc, 0); 1272 } 1273 1274 /** 1275 * omap_calculate_ecc_bch_multi - Generate ECC for multiple sectors 1276 * @mtd: MTD device structure 1277 * @dat: The pointer to data on which ecc is computed 1278 * @ecc_code: The ecc_code buffer 1279 * 1280 * Support calculating of BCH4/8/16 ecc vectors for the entire page in one go. 1281 */ 1282 static int omap_calculate_ecc_bch_multi(struct mtd_info *mtd, 1283 const u_char *dat, u_char *ecc_calc) 1284 { 1285 struct omap_nand_info *info = mtd_to_omap(mtd); 1286 int eccbytes = info->nand.ecc.bytes; 1287 unsigned long nsectors; 1288 int i, ret; 1289 1290 nsectors = ((readl(info->reg.gpmc_ecc_config) >> 4) & 0x7) + 1; 1291 for (i = 0; i < nsectors; i++) { 1292 ret = _omap_calculate_ecc_bch(mtd, dat, ecc_calc, i); 1293 if (ret) 1294 return ret; 1295 1296 ecc_calc += eccbytes; 1297 } 1298 1299 return 0; 1300 } 1301 1302 /** 1303 * erased_sector_bitflips - count bit flips 1304 * @data: data sector buffer 1305 * @oob: oob buffer 1306 * @info: omap_nand_info 1307 * 1308 * Check the bit flips in erased page falls below correctable level. 1309 * If falls below, report the page as erased with correctable bit 1310 * flip, else report as uncorrectable page. 1311 */ 1312 static int erased_sector_bitflips(u_char *data, u_char *oob, 1313 struct omap_nand_info *info) 1314 { 1315 int flip_bits = 0, i; 1316 1317 for (i = 0; i < info->nand.ecc.size; i++) { 1318 flip_bits += hweight8(~data[i]); 1319 if (flip_bits > info->nand.ecc.strength) 1320 return 0; 1321 } 1322 1323 for (i = 0; i < info->nand.ecc.bytes - 1; i++) { 1324 flip_bits += hweight8(~oob[i]); 1325 if (flip_bits > info->nand.ecc.strength) 1326 return 0; 1327 } 1328 1329 /* 1330 * Bit flips falls in correctable level. 1331 * Fill data area with 0xFF 1332 */ 1333 if (flip_bits) { 1334 memset(data, 0xFF, info->nand.ecc.size); 1335 memset(oob, 0xFF, info->nand.ecc.bytes); 1336 } 1337 1338 return flip_bits; 1339 } 1340 1341 /** 1342 * omap_elm_correct_data - corrects page data area in case error reported 1343 * @chip: NAND chip object 1344 * @data: page data 1345 * @read_ecc: ecc read from nand flash 1346 * @calc_ecc: ecc read from HW ECC registers 1347 * 1348 * Calculated ecc vector reported as zero in case of non-error pages. 1349 * In case of non-zero ecc vector, first filter out erased-pages, and 1350 * then process data via ELM to detect bit-flips. 1351 */ 1352 static int omap_elm_correct_data(struct nand_chip *chip, u_char *data, 1353 u_char *read_ecc, u_char *calc_ecc) 1354 { 1355 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 1356 struct nand_ecc_ctrl *ecc = &info->nand.ecc; 1357 int eccsteps = info->nand.ecc.steps; 1358 int i , j, stat = 0; 1359 int eccflag, actual_eccbytes; 1360 struct elm_errorvec err_vec[ERROR_VECTOR_MAX]; 1361 u_char *ecc_vec = calc_ecc; 1362 u_char *spare_ecc = read_ecc; 1363 u_char *erased_ecc_vec; 1364 u_char *buf; 1365 int bitflip_count; 1366 bool is_error_reported = false; 1367 u32 bit_pos, byte_pos, error_max, pos; 1368 int err; 1369 1370 switch (info->ecc_opt) { 1371 case OMAP_ECC_BCH4_CODE_HW: 1372 /* omit 7th ECC byte reserved for ROM code compatibility */ 1373 actual_eccbytes = ecc->bytes - 1; 1374 erased_ecc_vec = bch4_vector; 1375 break; 1376 case OMAP_ECC_BCH8_CODE_HW: 1377 /* omit 14th ECC byte reserved for ROM code compatibility */ 1378 actual_eccbytes = ecc->bytes - 1; 1379 erased_ecc_vec = bch8_vector; 1380 break; 1381 case OMAP_ECC_BCH16_CODE_HW: 1382 actual_eccbytes = ecc->bytes; 1383 erased_ecc_vec = bch16_vector; 1384 break; 1385 default: 1386 dev_err(&info->pdev->dev, "invalid driver configuration\n"); 1387 return -EINVAL; 1388 } 1389 1390 /* Initialize elm error vector to zero */ 1391 memset(err_vec, 0, sizeof(err_vec)); 1392 1393 for (i = 0; i < eccsteps ; i++) { 1394 eccflag = 0; /* initialize eccflag */ 1395 1396 /* 1397 * Check any error reported, 1398 * In case of error, non zero ecc reported. 1399 */ 1400 for (j = 0; j < actual_eccbytes; j++) { 1401 if (calc_ecc[j] != 0) { 1402 eccflag = 1; /* non zero ecc, error present */ 1403 break; 1404 } 1405 } 1406 1407 if (eccflag == 1) { 1408 if (memcmp(calc_ecc, erased_ecc_vec, 1409 actual_eccbytes) == 0) { 1410 /* 1411 * calc_ecc[] matches pattern for ECC(all 0xff) 1412 * so this is definitely an erased-page 1413 */ 1414 } else { 1415 buf = &data[info->nand.ecc.size * i]; 1416 /* 1417 * count number of 0-bits in read_buf. 1418 * This check can be removed once a similar 1419 * check is introduced in generic NAND driver 1420 */ 1421 bitflip_count = erased_sector_bitflips( 1422 buf, read_ecc, info); 1423 if (bitflip_count) { 1424 /* 1425 * number of 0-bits within ECC limits 1426 * So this may be an erased-page 1427 */ 1428 stat += bitflip_count; 1429 } else { 1430 /* 1431 * Too many 0-bits. It may be a 1432 * - programmed-page, OR 1433 * - erased-page with many bit-flips 1434 * So this page requires check by ELM 1435 */ 1436 err_vec[i].error_reported = true; 1437 is_error_reported = true; 1438 } 1439 } 1440 } 1441 1442 /* Update the ecc vector */ 1443 calc_ecc += ecc->bytes; 1444 read_ecc += ecc->bytes; 1445 } 1446 1447 /* Check if any error reported */ 1448 if (!is_error_reported) 1449 return stat; 1450 1451 /* Decode BCH error using ELM module */ 1452 elm_decode_bch_error_page(info->elm_dev, ecc_vec, err_vec); 1453 1454 err = 0; 1455 for (i = 0; i < eccsteps; i++) { 1456 if (err_vec[i].error_uncorrectable) { 1457 dev_err(&info->pdev->dev, 1458 "uncorrectable bit-flips found\n"); 1459 err = -EBADMSG; 1460 } else if (err_vec[i].error_reported) { 1461 for (j = 0; j < err_vec[i].error_count; j++) { 1462 switch (info->ecc_opt) { 1463 case OMAP_ECC_BCH4_CODE_HW: 1464 /* Add 4 bits to take care of padding */ 1465 pos = err_vec[i].error_loc[j] + 1466 BCH4_BIT_PAD; 1467 break; 1468 case OMAP_ECC_BCH8_CODE_HW: 1469 case OMAP_ECC_BCH16_CODE_HW: 1470 pos = err_vec[i].error_loc[j]; 1471 break; 1472 default: 1473 return -EINVAL; 1474 } 1475 error_max = (ecc->size + actual_eccbytes) * 8; 1476 /* Calculate bit position of error */ 1477 bit_pos = pos % 8; 1478 1479 /* Calculate byte position of error */ 1480 byte_pos = (error_max - pos - 1) / 8; 1481 1482 if (pos < error_max) { 1483 if (byte_pos < 512) { 1484 pr_debug("bitflip@dat[%d]=%x\n", 1485 byte_pos, data[byte_pos]); 1486 data[byte_pos] ^= 1 << bit_pos; 1487 } else { 1488 pr_debug("bitflip@oob[%d]=%x\n", 1489 (byte_pos - 512), 1490 spare_ecc[byte_pos - 512]); 1491 spare_ecc[byte_pos - 512] ^= 1492 1 << bit_pos; 1493 } 1494 } else { 1495 dev_err(&info->pdev->dev, 1496 "invalid bit-flip @ %d:%d\n", 1497 byte_pos, bit_pos); 1498 err = -EBADMSG; 1499 } 1500 } 1501 } 1502 1503 /* Update number of correctable errors */ 1504 stat = max_t(unsigned int, stat, err_vec[i].error_count); 1505 1506 /* Update page data with sector size */ 1507 data += ecc->size; 1508 spare_ecc += ecc->bytes; 1509 } 1510 1511 return (err) ? err : stat; 1512 } 1513 1514 /** 1515 * omap_write_page_bch - BCH ecc based write page function for entire page 1516 * @chip: nand chip info structure 1517 * @buf: data buffer 1518 * @oob_required: must write chip->oob_poi to OOB 1519 * @page: page 1520 * 1521 * Custom write page method evolved to support multi sector writing in one shot 1522 */ 1523 static int omap_write_page_bch(struct nand_chip *chip, const uint8_t *buf, 1524 int oob_required, int page) 1525 { 1526 struct mtd_info *mtd = nand_to_mtd(chip); 1527 int ret; 1528 uint8_t *ecc_calc = chip->ecc.calc_buf; 1529 1530 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1531 1532 /* Enable GPMC ecc engine */ 1533 chip->ecc.hwctl(chip, NAND_ECC_WRITE); 1534 1535 /* Write data */ 1536 chip->legacy.write_buf(chip, buf, mtd->writesize); 1537 1538 /* Update ecc vector from GPMC result registers */ 1539 omap_calculate_ecc_bch_multi(mtd, buf, &ecc_calc[0]); 1540 1541 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0, 1542 chip->ecc.total); 1543 if (ret) 1544 return ret; 1545 1546 /* Write ecc vector to OOB area */ 1547 chip->legacy.write_buf(chip, chip->oob_poi, mtd->oobsize); 1548 1549 return nand_prog_page_end_op(chip); 1550 } 1551 1552 /** 1553 * omap_write_subpage_bch - BCH hardware ECC based subpage write 1554 * @chip: nand chip info structure 1555 * @offset: column address of subpage within the page 1556 * @data_len: data length 1557 * @buf: data buffer 1558 * @oob_required: must write chip->oob_poi to OOB 1559 * @page: page number to write 1560 * 1561 * OMAP optimized subpage write method. 1562 */ 1563 static int omap_write_subpage_bch(struct nand_chip *chip, u32 offset, 1564 u32 data_len, const u8 *buf, 1565 int oob_required, int page) 1566 { 1567 struct mtd_info *mtd = nand_to_mtd(chip); 1568 u8 *ecc_calc = chip->ecc.calc_buf; 1569 int ecc_size = chip->ecc.size; 1570 int ecc_bytes = chip->ecc.bytes; 1571 int ecc_steps = chip->ecc.steps; 1572 u32 start_step = offset / ecc_size; 1573 u32 end_step = (offset + data_len - 1) / ecc_size; 1574 int step, ret = 0; 1575 1576 /* 1577 * Write entire page at one go as it would be optimal 1578 * as ECC is calculated by hardware. 1579 * ECC is calculated for all subpages but we choose 1580 * only what we want. 1581 */ 1582 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1583 1584 /* Enable GPMC ECC engine */ 1585 chip->ecc.hwctl(chip, NAND_ECC_WRITE); 1586 1587 /* Write data */ 1588 chip->legacy.write_buf(chip, buf, mtd->writesize); 1589 1590 for (step = 0; step < ecc_steps; step++) { 1591 /* mask ECC of un-touched subpages by padding 0xFF */ 1592 if (step < start_step || step > end_step) 1593 memset(ecc_calc, 0xff, ecc_bytes); 1594 else 1595 ret = _omap_calculate_ecc_bch(mtd, buf, ecc_calc, step); 1596 1597 if (ret) 1598 return ret; 1599 1600 buf += ecc_size; 1601 ecc_calc += ecc_bytes; 1602 } 1603 1604 /* copy calculated ECC for whole page to chip->buffer->oob */ 1605 /* this include masked-value(0xFF) for unwritten subpages */ 1606 ecc_calc = chip->ecc.calc_buf; 1607 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0, 1608 chip->ecc.total); 1609 if (ret) 1610 return ret; 1611 1612 /* write OOB buffer to NAND device */ 1613 chip->legacy.write_buf(chip, chip->oob_poi, mtd->oobsize); 1614 1615 return nand_prog_page_end_op(chip); 1616 } 1617 1618 /** 1619 * omap_read_page_bch - BCH ecc based page read function for entire page 1620 * @chip: nand chip info structure 1621 * @buf: buffer to store read data 1622 * @oob_required: caller requires OOB data read to chip->oob_poi 1623 * @page: page number to read 1624 * 1625 * For BCH ecc scheme, GPMC used for syndrome calculation and ELM module 1626 * used for error correction. 1627 * Custom method evolved to support ELM error correction & multi sector 1628 * reading. On reading page data area is read along with OOB data with 1629 * ecc engine enabled. ecc vector updated after read of OOB data. 1630 * For non error pages ecc vector reported as zero. 1631 */ 1632 static int omap_read_page_bch(struct nand_chip *chip, uint8_t *buf, 1633 int oob_required, int page) 1634 { 1635 struct mtd_info *mtd = nand_to_mtd(chip); 1636 uint8_t *ecc_calc = chip->ecc.calc_buf; 1637 uint8_t *ecc_code = chip->ecc.code_buf; 1638 int stat, ret; 1639 unsigned int max_bitflips = 0; 1640 1641 nand_read_page_op(chip, page, 0, NULL, 0); 1642 1643 /* Enable GPMC ecc engine */ 1644 chip->ecc.hwctl(chip, NAND_ECC_READ); 1645 1646 /* Read data */ 1647 chip->legacy.read_buf(chip, buf, mtd->writesize); 1648 1649 /* Read oob bytes */ 1650 nand_change_read_column_op(chip, 1651 mtd->writesize + BADBLOCK_MARKER_LENGTH, 1652 chip->oob_poi + BADBLOCK_MARKER_LENGTH, 1653 chip->ecc.total, false); 1654 1655 /* Calculate ecc bytes */ 1656 omap_calculate_ecc_bch_multi(mtd, buf, ecc_calc); 1657 1658 ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0, 1659 chip->ecc.total); 1660 if (ret) 1661 return ret; 1662 1663 stat = chip->ecc.correct(chip, buf, ecc_code, ecc_calc); 1664 1665 if (stat < 0) { 1666 mtd->ecc_stats.failed++; 1667 } else { 1668 mtd->ecc_stats.corrected += stat; 1669 max_bitflips = max_t(unsigned int, max_bitflips, stat); 1670 } 1671 1672 return max_bitflips; 1673 } 1674 1675 /** 1676 * is_elm_present - checks for presence of ELM module by scanning DT nodes 1677 * @omap_nand_info: NAND device structure containing platform data 1678 */ 1679 static bool is_elm_present(struct omap_nand_info *info, 1680 struct device_node *elm_node) 1681 { 1682 struct platform_device *pdev; 1683 1684 /* check whether elm-id is passed via DT */ 1685 if (!elm_node) { 1686 dev_err(&info->pdev->dev, "ELM devicetree node not found\n"); 1687 return false; 1688 } 1689 pdev = of_find_device_by_node(elm_node); 1690 /* check whether ELM device is registered */ 1691 if (!pdev) { 1692 dev_err(&info->pdev->dev, "ELM device not found\n"); 1693 return false; 1694 } 1695 /* ELM module available, now configure it */ 1696 info->elm_dev = &pdev->dev; 1697 return true; 1698 } 1699 1700 static bool omap2_nand_ecc_check(struct omap_nand_info *info) 1701 { 1702 bool ecc_needs_bch, ecc_needs_omap_bch, ecc_needs_elm; 1703 1704 switch (info->ecc_opt) { 1705 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: 1706 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: 1707 ecc_needs_omap_bch = false; 1708 ecc_needs_bch = true; 1709 ecc_needs_elm = false; 1710 break; 1711 case OMAP_ECC_BCH4_CODE_HW: 1712 case OMAP_ECC_BCH8_CODE_HW: 1713 case OMAP_ECC_BCH16_CODE_HW: 1714 ecc_needs_omap_bch = true; 1715 ecc_needs_bch = false; 1716 ecc_needs_elm = true; 1717 break; 1718 default: 1719 ecc_needs_omap_bch = false; 1720 ecc_needs_bch = false; 1721 ecc_needs_elm = false; 1722 break; 1723 } 1724 1725 if (ecc_needs_bch && !IS_ENABLED(CONFIG_MTD_NAND_ECC_SW_BCH)) { 1726 dev_err(&info->pdev->dev, 1727 "CONFIG_MTD_NAND_ECC_SW_BCH not enabled\n"); 1728 return false; 1729 } 1730 if (ecc_needs_omap_bch && !IS_ENABLED(CONFIG_MTD_NAND_OMAP_BCH)) { 1731 dev_err(&info->pdev->dev, 1732 "CONFIG_MTD_NAND_OMAP_BCH not enabled\n"); 1733 return false; 1734 } 1735 if (ecc_needs_elm && !is_elm_present(info, info->elm_of_node)) { 1736 dev_err(&info->pdev->dev, "ELM not available\n"); 1737 return false; 1738 } 1739 1740 return true; 1741 } 1742 1743 static const char * const nand_xfer_types[] = { 1744 [NAND_OMAP_PREFETCH_POLLED] = "prefetch-polled", 1745 [NAND_OMAP_POLLED] = "polled", 1746 [NAND_OMAP_PREFETCH_DMA] = "prefetch-dma", 1747 [NAND_OMAP_PREFETCH_IRQ] = "prefetch-irq", 1748 }; 1749 1750 static int omap_get_dt_info(struct device *dev, struct omap_nand_info *info) 1751 { 1752 struct device_node *child = dev->of_node; 1753 int i; 1754 const char *s; 1755 u32 cs; 1756 1757 if (of_property_read_u32(child, "reg", &cs) < 0) { 1758 dev_err(dev, "reg not found in DT\n"); 1759 return -EINVAL; 1760 } 1761 1762 info->gpmc_cs = cs; 1763 1764 /* detect availability of ELM module. Won't be present pre-OMAP4 */ 1765 info->elm_of_node = of_parse_phandle(child, "ti,elm-id", 0); 1766 if (!info->elm_of_node) { 1767 info->elm_of_node = of_parse_phandle(child, "elm_id", 0); 1768 if (!info->elm_of_node) 1769 dev_dbg(dev, "ti,elm-id not in DT\n"); 1770 } 1771 1772 /* select ecc-scheme for NAND */ 1773 if (of_property_read_string(child, "ti,nand-ecc-opt", &s)) { 1774 dev_err(dev, "ti,nand-ecc-opt not found\n"); 1775 return -EINVAL; 1776 } 1777 1778 if (!strcmp(s, "sw")) { 1779 info->ecc_opt = OMAP_ECC_HAM1_CODE_SW; 1780 } else if (!strcmp(s, "ham1") || 1781 !strcmp(s, "hw") || !strcmp(s, "hw-romcode")) { 1782 info->ecc_opt = OMAP_ECC_HAM1_CODE_HW; 1783 } else if (!strcmp(s, "bch4")) { 1784 if (info->elm_of_node) 1785 info->ecc_opt = OMAP_ECC_BCH4_CODE_HW; 1786 else 1787 info->ecc_opt = OMAP_ECC_BCH4_CODE_HW_DETECTION_SW; 1788 } else if (!strcmp(s, "bch8")) { 1789 if (info->elm_of_node) 1790 info->ecc_opt = OMAP_ECC_BCH8_CODE_HW; 1791 else 1792 info->ecc_opt = OMAP_ECC_BCH8_CODE_HW_DETECTION_SW; 1793 } else if (!strcmp(s, "bch16")) { 1794 info->ecc_opt = OMAP_ECC_BCH16_CODE_HW; 1795 } else { 1796 dev_err(dev, "unrecognized value for ti,nand-ecc-opt\n"); 1797 return -EINVAL; 1798 } 1799 1800 /* select data transfer mode */ 1801 if (!of_property_read_string(child, "ti,nand-xfer-type", &s)) { 1802 for (i = 0; i < ARRAY_SIZE(nand_xfer_types); i++) { 1803 if (!strcasecmp(s, nand_xfer_types[i])) { 1804 info->xfer_type = i; 1805 return 0; 1806 } 1807 } 1808 1809 dev_err(dev, "unrecognized value for ti,nand-xfer-type\n"); 1810 return -EINVAL; 1811 } 1812 1813 return 0; 1814 } 1815 1816 static int omap_ooblayout_ecc(struct mtd_info *mtd, int section, 1817 struct mtd_oob_region *oobregion) 1818 { 1819 struct omap_nand_info *info = mtd_to_omap(mtd); 1820 struct nand_chip *chip = &info->nand; 1821 int off = BADBLOCK_MARKER_LENGTH; 1822 1823 if (info->ecc_opt == OMAP_ECC_HAM1_CODE_HW && 1824 !(chip->options & NAND_BUSWIDTH_16)) 1825 off = 1; 1826 1827 if (section) 1828 return -ERANGE; 1829 1830 oobregion->offset = off; 1831 oobregion->length = chip->ecc.total; 1832 1833 return 0; 1834 } 1835 1836 static int omap_ooblayout_free(struct mtd_info *mtd, int section, 1837 struct mtd_oob_region *oobregion) 1838 { 1839 struct omap_nand_info *info = mtd_to_omap(mtd); 1840 struct nand_chip *chip = &info->nand; 1841 int off = BADBLOCK_MARKER_LENGTH; 1842 1843 if (info->ecc_opt == OMAP_ECC_HAM1_CODE_HW && 1844 !(chip->options & NAND_BUSWIDTH_16)) 1845 off = 1; 1846 1847 if (section) 1848 return -ERANGE; 1849 1850 off += chip->ecc.total; 1851 if (off >= mtd->oobsize) 1852 return -ERANGE; 1853 1854 oobregion->offset = off; 1855 oobregion->length = mtd->oobsize - off; 1856 1857 return 0; 1858 } 1859 1860 static const struct mtd_ooblayout_ops omap_ooblayout_ops = { 1861 .ecc = omap_ooblayout_ecc, 1862 .free = omap_ooblayout_free, 1863 }; 1864 1865 static int omap_sw_ooblayout_ecc(struct mtd_info *mtd, int section, 1866 struct mtd_oob_region *oobregion) 1867 { 1868 struct nand_chip *chip = mtd_to_nand(mtd); 1869 int off = BADBLOCK_MARKER_LENGTH; 1870 1871 if (section >= chip->ecc.steps) 1872 return -ERANGE; 1873 1874 /* 1875 * When SW correction is employed, one OMAP specific marker byte is 1876 * reserved after each ECC step. 1877 */ 1878 oobregion->offset = off + (section * (chip->ecc.bytes + 1)); 1879 oobregion->length = chip->ecc.bytes; 1880 1881 return 0; 1882 } 1883 1884 static int omap_sw_ooblayout_free(struct mtd_info *mtd, int section, 1885 struct mtd_oob_region *oobregion) 1886 { 1887 struct nand_chip *chip = mtd_to_nand(mtd); 1888 int off = BADBLOCK_MARKER_LENGTH; 1889 1890 if (section) 1891 return -ERANGE; 1892 1893 /* 1894 * When SW correction is employed, one OMAP specific marker byte is 1895 * reserved after each ECC step. 1896 */ 1897 off += ((chip->ecc.bytes + 1) * chip->ecc.steps); 1898 if (off >= mtd->oobsize) 1899 return -ERANGE; 1900 1901 oobregion->offset = off; 1902 oobregion->length = mtd->oobsize - off; 1903 1904 return 0; 1905 } 1906 1907 static const struct mtd_ooblayout_ops omap_sw_ooblayout_ops = { 1908 .ecc = omap_sw_ooblayout_ecc, 1909 .free = omap_sw_ooblayout_free, 1910 }; 1911 1912 static int omap_nand_attach_chip(struct nand_chip *chip) 1913 { 1914 struct mtd_info *mtd = nand_to_mtd(chip); 1915 struct omap_nand_info *info = mtd_to_omap(mtd); 1916 struct device *dev = &info->pdev->dev; 1917 int min_oobbytes = BADBLOCK_MARKER_LENGTH; 1918 int oobbytes_per_step; 1919 dma_cap_mask_t mask; 1920 int err; 1921 1922 if (chip->bbt_options & NAND_BBT_USE_FLASH) 1923 chip->bbt_options |= NAND_BBT_NO_OOB; 1924 else 1925 chip->options |= NAND_SKIP_BBTSCAN; 1926 1927 /* Re-populate low-level callbacks based on xfer modes */ 1928 switch (info->xfer_type) { 1929 case NAND_OMAP_PREFETCH_POLLED: 1930 chip->legacy.read_buf = omap_read_buf_pref; 1931 chip->legacy.write_buf = omap_write_buf_pref; 1932 break; 1933 1934 case NAND_OMAP_POLLED: 1935 /* Use nand_base defaults for {read,write}_buf */ 1936 break; 1937 1938 case NAND_OMAP_PREFETCH_DMA: 1939 dma_cap_zero(mask); 1940 dma_cap_set(DMA_SLAVE, mask); 1941 info->dma = dma_request_chan(dev->parent, "rxtx"); 1942 1943 if (IS_ERR(info->dma)) { 1944 dev_err(dev, "DMA engine request failed\n"); 1945 return PTR_ERR(info->dma); 1946 } else { 1947 struct dma_slave_config cfg; 1948 1949 memset(&cfg, 0, sizeof(cfg)); 1950 cfg.src_addr = info->phys_base; 1951 cfg.dst_addr = info->phys_base; 1952 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 1953 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 1954 cfg.src_maxburst = 16; 1955 cfg.dst_maxburst = 16; 1956 err = dmaengine_slave_config(info->dma, &cfg); 1957 if (err) { 1958 dev_err(dev, 1959 "DMA engine slave config failed: %d\n", 1960 err); 1961 return err; 1962 } 1963 chip->legacy.read_buf = omap_read_buf_dma_pref; 1964 chip->legacy.write_buf = omap_write_buf_dma_pref; 1965 } 1966 break; 1967 1968 case NAND_OMAP_PREFETCH_IRQ: 1969 info->gpmc_irq_fifo = platform_get_irq(info->pdev, 0); 1970 if (info->gpmc_irq_fifo <= 0) 1971 return -ENODEV; 1972 err = devm_request_irq(dev, info->gpmc_irq_fifo, 1973 omap_nand_irq, IRQF_SHARED, 1974 "gpmc-nand-fifo", info); 1975 if (err) { 1976 dev_err(dev, "Requesting IRQ %d, error %d\n", 1977 info->gpmc_irq_fifo, err); 1978 info->gpmc_irq_fifo = 0; 1979 return err; 1980 } 1981 1982 info->gpmc_irq_count = platform_get_irq(info->pdev, 1); 1983 if (info->gpmc_irq_count <= 0) 1984 return -ENODEV; 1985 err = devm_request_irq(dev, info->gpmc_irq_count, 1986 omap_nand_irq, IRQF_SHARED, 1987 "gpmc-nand-count", info); 1988 if (err) { 1989 dev_err(dev, "Requesting IRQ %d, error %d\n", 1990 info->gpmc_irq_count, err); 1991 info->gpmc_irq_count = 0; 1992 return err; 1993 } 1994 1995 chip->legacy.read_buf = omap_read_buf_irq_pref; 1996 chip->legacy.write_buf = omap_write_buf_irq_pref; 1997 1998 break; 1999 2000 default: 2001 dev_err(dev, "xfer_type %d not supported!\n", info->xfer_type); 2002 return -EINVAL; 2003 } 2004 2005 if (!omap2_nand_ecc_check(info)) 2006 return -EINVAL; 2007 2008 /* 2009 * Bail out earlier to let NAND_ECC_SOFT code create its own 2010 * ooblayout instead of using ours. 2011 */ 2012 if (info->ecc_opt == OMAP_ECC_HAM1_CODE_SW) { 2013 chip->ecc.mode = NAND_ECC_SOFT; 2014 chip->ecc.algo = NAND_ECC_HAMMING; 2015 return 0; 2016 } 2017 2018 /* Populate MTD interface based on ECC scheme */ 2019 switch (info->ecc_opt) { 2020 case OMAP_ECC_HAM1_CODE_HW: 2021 dev_info(dev, "nand: using OMAP_ECC_HAM1_CODE_HW\n"); 2022 chip->ecc.mode = NAND_ECC_HW; 2023 chip->ecc.bytes = 3; 2024 chip->ecc.size = 512; 2025 chip->ecc.strength = 1; 2026 chip->ecc.calculate = omap_calculate_ecc; 2027 chip->ecc.hwctl = omap_enable_hwecc; 2028 chip->ecc.correct = omap_correct_data; 2029 mtd_set_ooblayout(mtd, &omap_ooblayout_ops); 2030 oobbytes_per_step = chip->ecc.bytes; 2031 2032 if (!(chip->options & NAND_BUSWIDTH_16)) 2033 min_oobbytes = 1; 2034 2035 break; 2036 2037 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: 2038 pr_info("nand: using OMAP_ECC_BCH4_CODE_HW_DETECTION_SW\n"); 2039 chip->ecc.mode = NAND_ECC_HW; 2040 chip->ecc.size = 512; 2041 chip->ecc.bytes = 7; 2042 chip->ecc.strength = 4; 2043 chip->ecc.hwctl = omap_enable_hwecc_bch; 2044 chip->ecc.correct = nand_bch_correct_data; 2045 chip->ecc.calculate = omap_calculate_ecc_bch_sw; 2046 mtd_set_ooblayout(mtd, &omap_sw_ooblayout_ops); 2047 /* Reserve one byte for the OMAP marker */ 2048 oobbytes_per_step = chip->ecc.bytes + 1; 2049 /* Software BCH library is used for locating errors */ 2050 chip->ecc.priv = nand_bch_init(mtd); 2051 if (!chip->ecc.priv) { 2052 dev_err(dev, "Unable to use BCH library\n"); 2053 return -EINVAL; 2054 } 2055 break; 2056 2057 case OMAP_ECC_BCH4_CODE_HW: 2058 pr_info("nand: using OMAP_ECC_BCH4_CODE_HW ECC scheme\n"); 2059 chip->ecc.mode = NAND_ECC_HW; 2060 chip->ecc.size = 512; 2061 /* 14th bit is kept reserved for ROM-code compatibility */ 2062 chip->ecc.bytes = 7 + 1; 2063 chip->ecc.strength = 4; 2064 chip->ecc.hwctl = omap_enable_hwecc_bch; 2065 chip->ecc.correct = omap_elm_correct_data; 2066 chip->ecc.read_page = omap_read_page_bch; 2067 chip->ecc.write_page = omap_write_page_bch; 2068 chip->ecc.write_subpage = omap_write_subpage_bch; 2069 mtd_set_ooblayout(mtd, &omap_ooblayout_ops); 2070 oobbytes_per_step = chip->ecc.bytes; 2071 2072 err = elm_config(info->elm_dev, BCH4_ECC, 2073 mtd->writesize / chip->ecc.size, 2074 chip->ecc.size, chip->ecc.bytes); 2075 if (err < 0) 2076 return err; 2077 break; 2078 2079 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: 2080 pr_info("nand: using OMAP_ECC_BCH8_CODE_HW_DETECTION_SW\n"); 2081 chip->ecc.mode = NAND_ECC_HW; 2082 chip->ecc.size = 512; 2083 chip->ecc.bytes = 13; 2084 chip->ecc.strength = 8; 2085 chip->ecc.hwctl = omap_enable_hwecc_bch; 2086 chip->ecc.correct = nand_bch_correct_data; 2087 chip->ecc.calculate = omap_calculate_ecc_bch_sw; 2088 mtd_set_ooblayout(mtd, &omap_sw_ooblayout_ops); 2089 /* Reserve one byte for the OMAP marker */ 2090 oobbytes_per_step = chip->ecc.bytes + 1; 2091 /* Software BCH library is used for locating errors */ 2092 chip->ecc.priv = nand_bch_init(mtd); 2093 if (!chip->ecc.priv) { 2094 dev_err(dev, "unable to use BCH library\n"); 2095 return -EINVAL; 2096 } 2097 break; 2098 2099 case OMAP_ECC_BCH8_CODE_HW: 2100 pr_info("nand: using OMAP_ECC_BCH8_CODE_HW ECC scheme\n"); 2101 chip->ecc.mode = NAND_ECC_HW; 2102 chip->ecc.size = 512; 2103 /* 14th bit is kept reserved for ROM-code compatibility */ 2104 chip->ecc.bytes = 13 + 1; 2105 chip->ecc.strength = 8; 2106 chip->ecc.hwctl = omap_enable_hwecc_bch; 2107 chip->ecc.correct = omap_elm_correct_data; 2108 chip->ecc.read_page = omap_read_page_bch; 2109 chip->ecc.write_page = omap_write_page_bch; 2110 chip->ecc.write_subpage = omap_write_subpage_bch; 2111 mtd_set_ooblayout(mtd, &omap_ooblayout_ops); 2112 oobbytes_per_step = chip->ecc.bytes; 2113 2114 err = elm_config(info->elm_dev, BCH8_ECC, 2115 mtd->writesize / chip->ecc.size, 2116 chip->ecc.size, chip->ecc.bytes); 2117 if (err < 0) 2118 return err; 2119 2120 break; 2121 2122 case OMAP_ECC_BCH16_CODE_HW: 2123 pr_info("Using OMAP_ECC_BCH16_CODE_HW ECC scheme\n"); 2124 chip->ecc.mode = NAND_ECC_HW; 2125 chip->ecc.size = 512; 2126 chip->ecc.bytes = 26; 2127 chip->ecc.strength = 16; 2128 chip->ecc.hwctl = omap_enable_hwecc_bch; 2129 chip->ecc.correct = omap_elm_correct_data; 2130 chip->ecc.read_page = omap_read_page_bch; 2131 chip->ecc.write_page = omap_write_page_bch; 2132 chip->ecc.write_subpage = omap_write_subpage_bch; 2133 mtd_set_ooblayout(mtd, &omap_ooblayout_ops); 2134 oobbytes_per_step = chip->ecc.bytes; 2135 2136 err = elm_config(info->elm_dev, BCH16_ECC, 2137 mtd->writesize / chip->ecc.size, 2138 chip->ecc.size, chip->ecc.bytes); 2139 if (err < 0) 2140 return err; 2141 2142 break; 2143 default: 2144 dev_err(dev, "Invalid or unsupported ECC scheme\n"); 2145 return -EINVAL; 2146 } 2147 2148 /* Check if NAND device's OOB is enough to store ECC signatures */ 2149 min_oobbytes += (oobbytes_per_step * 2150 (mtd->writesize / chip->ecc.size)); 2151 if (mtd->oobsize < min_oobbytes) { 2152 dev_err(dev, 2153 "Not enough OOB bytes: required = %d, available=%d\n", 2154 min_oobbytes, mtd->oobsize); 2155 return -EINVAL; 2156 } 2157 2158 return 0; 2159 } 2160 2161 static const struct nand_controller_ops omap_nand_controller_ops = { 2162 .attach_chip = omap_nand_attach_chip, 2163 }; 2164 2165 /* Shared among all NAND instances to synchronize access to the ECC Engine */ 2166 static struct nand_controller omap_gpmc_controller; 2167 static bool omap_gpmc_controller_initialized; 2168 2169 static int omap_nand_probe(struct platform_device *pdev) 2170 { 2171 struct omap_nand_info *info; 2172 struct mtd_info *mtd; 2173 struct nand_chip *nand_chip; 2174 int err; 2175 struct resource *res; 2176 struct device *dev = &pdev->dev; 2177 2178 info = devm_kzalloc(&pdev->dev, sizeof(struct omap_nand_info), 2179 GFP_KERNEL); 2180 if (!info) 2181 return -ENOMEM; 2182 2183 info->pdev = pdev; 2184 2185 err = omap_get_dt_info(dev, info); 2186 if (err) 2187 return err; 2188 2189 info->ops = gpmc_omap_get_nand_ops(&info->reg, info->gpmc_cs); 2190 if (!info->ops) { 2191 dev_err(&pdev->dev, "Failed to get GPMC->NAND interface\n"); 2192 return -ENODEV; 2193 } 2194 2195 nand_chip = &info->nand; 2196 mtd = nand_to_mtd(nand_chip); 2197 mtd->dev.parent = &pdev->dev; 2198 nand_chip->ecc.priv = NULL; 2199 nand_set_flash_node(nand_chip, dev->of_node); 2200 2201 if (!mtd->name) { 2202 mtd->name = devm_kasprintf(&pdev->dev, GFP_KERNEL, 2203 "omap2-nand.%d", info->gpmc_cs); 2204 if (!mtd->name) { 2205 dev_err(&pdev->dev, "Failed to set MTD name\n"); 2206 return -ENOMEM; 2207 } 2208 } 2209 2210 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2211 nand_chip->legacy.IO_ADDR_R = devm_ioremap_resource(&pdev->dev, res); 2212 if (IS_ERR(nand_chip->legacy.IO_ADDR_R)) 2213 return PTR_ERR(nand_chip->legacy.IO_ADDR_R); 2214 2215 info->phys_base = res->start; 2216 2217 if (!omap_gpmc_controller_initialized) { 2218 omap_gpmc_controller.ops = &omap_nand_controller_ops; 2219 nand_controller_init(&omap_gpmc_controller); 2220 omap_gpmc_controller_initialized = true; 2221 } 2222 2223 nand_chip->controller = &omap_gpmc_controller; 2224 2225 nand_chip->legacy.IO_ADDR_W = nand_chip->legacy.IO_ADDR_R; 2226 nand_chip->legacy.cmd_ctrl = omap_hwcontrol; 2227 2228 info->ready_gpiod = devm_gpiod_get_optional(&pdev->dev, "rb", 2229 GPIOD_IN); 2230 if (IS_ERR(info->ready_gpiod)) { 2231 dev_err(dev, "failed to get ready gpio\n"); 2232 return PTR_ERR(info->ready_gpiod); 2233 } 2234 2235 /* 2236 * If RDY/BSY line is connected to OMAP then use the omap ready 2237 * function and the generic nand_wait function which reads the status 2238 * register after monitoring the RDY/BSY line. Otherwise use a standard 2239 * chip delay which is slightly more than tR (AC Timing) of the NAND 2240 * device and read status register until you get a failure or success 2241 */ 2242 if (info->ready_gpiod) { 2243 nand_chip->legacy.dev_ready = omap_dev_ready; 2244 nand_chip->legacy.chip_delay = 0; 2245 } else { 2246 nand_chip->legacy.waitfunc = omap_wait; 2247 nand_chip->legacy.chip_delay = 50; 2248 } 2249 2250 if (info->flash_bbt) 2251 nand_chip->bbt_options |= NAND_BBT_USE_FLASH; 2252 2253 /* scan NAND device connected to chip controller */ 2254 nand_chip->options |= info->devsize & NAND_BUSWIDTH_16; 2255 2256 err = nand_scan(nand_chip, 1); 2257 if (err) 2258 goto return_error; 2259 2260 err = mtd_device_register(mtd, NULL, 0); 2261 if (err) 2262 goto cleanup_nand; 2263 2264 platform_set_drvdata(pdev, mtd); 2265 2266 return 0; 2267 2268 cleanup_nand: 2269 nand_cleanup(nand_chip); 2270 2271 return_error: 2272 if (!IS_ERR_OR_NULL(info->dma)) 2273 dma_release_channel(info->dma); 2274 if (nand_chip->ecc.priv) { 2275 nand_bch_free(nand_chip->ecc.priv); 2276 nand_chip->ecc.priv = NULL; 2277 } 2278 return err; 2279 } 2280 2281 static int omap_nand_remove(struct platform_device *pdev) 2282 { 2283 struct mtd_info *mtd = platform_get_drvdata(pdev); 2284 struct nand_chip *nand_chip = mtd_to_nand(mtd); 2285 struct omap_nand_info *info = mtd_to_omap(mtd); 2286 if (nand_chip->ecc.priv) { 2287 nand_bch_free(nand_chip->ecc.priv); 2288 nand_chip->ecc.priv = NULL; 2289 } 2290 if (info->dma) 2291 dma_release_channel(info->dma); 2292 nand_release(nand_chip); 2293 return 0; 2294 } 2295 2296 static const struct of_device_id omap_nand_ids[] = { 2297 { .compatible = "ti,omap2-nand", }, 2298 {}, 2299 }; 2300 MODULE_DEVICE_TABLE(of, omap_nand_ids); 2301 2302 static struct platform_driver omap_nand_driver = { 2303 .probe = omap_nand_probe, 2304 .remove = omap_nand_remove, 2305 .driver = { 2306 .name = DRIVER_NAME, 2307 .of_match_table = of_match_ptr(omap_nand_ids), 2308 }, 2309 }; 2310 2311 module_platform_driver(omap_nand_driver); 2312 2313 MODULE_ALIAS("platform:" DRIVER_NAME); 2314 MODULE_LICENSE("GPL"); 2315 MODULE_DESCRIPTION("Glue layer for NAND flash on TI OMAP boards"); 2316