1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright © 2004 Texas Instruments, Jian Zhang <jzhang@ti.com> 4 * Copyright © 2004 Micron Technology Inc. 5 * Copyright © 2004 David Brownell 6 */ 7 8 #include <linux/platform_device.h> 9 #include <linux/dmaengine.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/delay.h> 12 #include <linux/gpio/consumer.h> 13 #include <linux/module.h> 14 #include <linux/interrupt.h> 15 #include <linux/jiffies.h> 16 #include <linux/sched.h> 17 #include <linux/mtd/mtd.h> 18 #include <linux/mtd/rawnand.h> 19 #include <linux/mtd/partitions.h> 20 #include <linux/omap-dma.h> 21 #include <linux/io.h> 22 #include <linux/slab.h> 23 #include <linux/of.h> 24 #include <linux/of_device.h> 25 26 #include <linux/platform_data/elm.h> 27 28 #include <linux/omap-gpmc.h> 29 #include <linux/platform_data/mtd-nand-omap2.h> 30 31 #define DRIVER_NAME "omap2-nand" 32 #define OMAP_NAND_TIMEOUT_MS 5000 33 34 #define NAND_Ecc_P1e (1 << 0) 35 #define NAND_Ecc_P2e (1 << 1) 36 #define NAND_Ecc_P4e (1 << 2) 37 #define NAND_Ecc_P8e (1 << 3) 38 #define NAND_Ecc_P16e (1 << 4) 39 #define NAND_Ecc_P32e (1 << 5) 40 #define NAND_Ecc_P64e (1 << 6) 41 #define NAND_Ecc_P128e (1 << 7) 42 #define NAND_Ecc_P256e (1 << 8) 43 #define NAND_Ecc_P512e (1 << 9) 44 #define NAND_Ecc_P1024e (1 << 10) 45 #define NAND_Ecc_P2048e (1 << 11) 46 47 #define NAND_Ecc_P1o (1 << 16) 48 #define NAND_Ecc_P2o (1 << 17) 49 #define NAND_Ecc_P4o (1 << 18) 50 #define NAND_Ecc_P8o (1 << 19) 51 #define NAND_Ecc_P16o (1 << 20) 52 #define NAND_Ecc_P32o (1 << 21) 53 #define NAND_Ecc_P64o (1 << 22) 54 #define NAND_Ecc_P128o (1 << 23) 55 #define NAND_Ecc_P256o (1 << 24) 56 #define NAND_Ecc_P512o (1 << 25) 57 #define NAND_Ecc_P1024o (1 << 26) 58 #define NAND_Ecc_P2048o (1 << 27) 59 60 #define TF(value) (value ? 1 : 0) 61 62 #define P2048e(a) (TF(a & NAND_Ecc_P2048e) << 0) 63 #define P2048o(a) (TF(a & NAND_Ecc_P2048o) << 1) 64 #define P1e(a) (TF(a & NAND_Ecc_P1e) << 2) 65 #define P1o(a) (TF(a & NAND_Ecc_P1o) << 3) 66 #define P2e(a) (TF(a & NAND_Ecc_P2e) << 4) 67 #define P2o(a) (TF(a & NAND_Ecc_P2o) << 5) 68 #define P4e(a) (TF(a & NAND_Ecc_P4e) << 6) 69 #define P4o(a) (TF(a & NAND_Ecc_P4o) << 7) 70 71 #define P8e(a) (TF(a & NAND_Ecc_P8e) << 0) 72 #define P8o(a) (TF(a & NAND_Ecc_P8o) << 1) 73 #define P16e(a) (TF(a & NAND_Ecc_P16e) << 2) 74 #define P16o(a) (TF(a & NAND_Ecc_P16o) << 3) 75 #define P32e(a) (TF(a & NAND_Ecc_P32e) << 4) 76 #define P32o(a) (TF(a & NAND_Ecc_P32o) << 5) 77 #define P64e(a) (TF(a & NAND_Ecc_P64e) << 6) 78 #define P64o(a) (TF(a & NAND_Ecc_P64o) << 7) 79 80 #define P128e(a) (TF(a & NAND_Ecc_P128e) << 0) 81 #define P128o(a) (TF(a & NAND_Ecc_P128o) << 1) 82 #define P256e(a) (TF(a & NAND_Ecc_P256e) << 2) 83 #define P256o(a) (TF(a & NAND_Ecc_P256o) << 3) 84 #define P512e(a) (TF(a & NAND_Ecc_P512e) << 4) 85 #define P512o(a) (TF(a & NAND_Ecc_P512o) << 5) 86 #define P1024e(a) (TF(a & NAND_Ecc_P1024e) << 6) 87 #define P1024o(a) (TF(a & NAND_Ecc_P1024o) << 7) 88 89 #define P8e_s(a) (TF(a & NAND_Ecc_P8e) << 0) 90 #define P8o_s(a) (TF(a & NAND_Ecc_P8o) << 1) 91 #define P16e_s(a) (TF(a & NAND_Ecc_P16e) << 2) 92 #define P16o_s(a) (TF(a & NAND_Ecc_P16o) << 3) 93 #define P1e_s(a) (TF(a & NAND_Ecc_P1e) << 4) 94 #define P1o_s(a) (TF(a & NAND_Ecc_P1o) << 5) 95 #define P2e_s(a) (TF(a & NAND_Ecc_P2e) << 6) 96 #define P2o_s(a) (TF(a & NAND_Ecc_P2o) << 7) 97 98 #define P4e_s(a) (TF(a & NAND_Ecc_P4e) << 0) 99 #define P4o_s(a) (TF(a & NAND_Ecc_P4o) << 1) 100 101 #define PREFETCH_CONFIG1_CS_SHIFT 24 102 #define ECC_CONFIG_CS_SHIFT 1 103 #define CS_MASK 0x7 104 #define ENABLE_PREFETCH (0x1 << 7) 105 #define DMA_MPU_MODE_SHIFT 2 106 #define ECCSIZE0_SHIFT 12 107 #define ECCSIZE1_SHIFT 22 108 #define ECC1RESULTSIZE 0x1 109 #define ECCCLEAR 0x100 110 #define ECC1 0x1 111 #define PREFETCH_FIFOTHRESHOLD_MAX 0x40 112 #define PREFETCH_FIFOTHRESHOLD(val) ((val) << 8) 113 #define PREFETCH_STATUS_COUNT(val) (val & 0x00003fff) 114 #define PREFETCH_STATUS_FIFO_CNT(val) ((val >> 24) & 0x7F) 115 #define STATUS_BUFF_EMPTY 0x00000001 116 117 #define SECTOR_BYTES 512 118 /* 4 bit padding to make byte aligned, 56 = 52 + 4 */ 119 #define BCH4_BIT_PAD 4 120 121 /* GPMC ecc engine settings for read */ 122 #define BCH_WRAPMODE_1 1 /* BCH wrap mode 1 */ 123 #define BCH8R_ECC_SIZE0 0x1a /* ecc_size0 = 26 */ 124 #define BCH8R_ECC_SIZE1 0x2 /* ecc_size1 = 2 */ 125 #define BCH4R_ECC_SIZE0 0xd /* ecc_size0 = 13 */ 126 #define BCH4R_ECC_SIZE1 0x3 /* ecc_size1 = 3 */ 127 128 /* GPMC ecc engine settings for write */ 129 #define BCH_WRAPMODE_6 6 /* BCH wrap mode 6 */ 130 #define BCH_ECC_SIZE0 0x0 /* ecc_size0 = 0, no oob protection */ 131 #define BCH_ECC_SIZE1 0x20 /* ecc_size1 = 32 */ 132 133 #define BADBLOCK_MARKER_LENGTH 2 134 135 static u_char bch16_vector[] = {0xf5, 0x24, 0x1c, 0xd0, 0x61, 0xb3, 0xf1, 0x55, 136 0x2e, 0x2c, 0x86, 0xa3, 0xed, 0x36, 0x1b, 0x78, 137 0x48, 0x76, 0xa9, 0x3b, 0x97, 0xd1, 0x7a, 0x93, 138 0x07, 0x0e}; 139 static u_char bch8_vector[] = {0xf3, 0xdb, 0x14, 0x16, 0x8b, 0xd2, 0xbe, 0xcc, 140 0xac, 0x6b, 0xff, 0x99, 0x7b}; 141 static u_char bch4_vector[] = {0x00, 0x6b, 0x31, 0xdd, 0x41, 0xbc, 0x10}; 142 143 struct omap_nand_info { 144 struct nand_chip nand; 145 struct platform_device *pdev; 146 147 int gpmc_cs; 148 bool dev_ready; 149 enum nand_io xfer_type; 150 int devsize; 151 enum omap_ecc ecc_opt; 152 struct device_node *elm_of_node; 153 154 unsigned long phys_base; 155 struct completion comp; 156 struct dma_chan *dma; 157 int gpmc_irq_fifo; 158 int gpmc_irq_count; 159 enum { 160 OMAP_NAND_IO_READ = 0, /* read */ 161 OMAP_NAND_IO_WRITE, /* write */ 162 } iomode; 163 u_char *buf; 164 int buf_len; 165 /* Interface to GPMC */ 166 struct gpmc_nand_regs reg; 167 struct gpmc_nand_ops *ops; 168 bool flash_bbt; 169 /* fields specific for BCHx_HW ECC scheme */ 170 struct device *elm_dev; 171 /* NAND ready gpio */ 172 struct gpio_desc *ready_gpiod; 173 }; 174 175 static inline struct omap_nand_info *mtd_to_omap(struct mtd_info *mtd) 176 { 177 return container_of(mtd_to_nand(mtd), struct omap_nand_info, nand); 178 } 179 180 /** 181 * omap_prefetch_enable - configures and starts prefetch transfer 182 * @cs: cs (chip select) number 183 * @fifo_th: fifo threshold to be used for read/ write 184 * @dma_mode: dma mode enable (1) or disable (0) 185 * @u32_count: number of bytes to be transferred 186 * @is_write: prefetch read(0) or write post(1) mode 187 * @info: NAND device structure containing platform data 188 */ 189 static int omap_prefetch_enable(int cs, int fifo_th, int dma_mode, 190 unsigned int u32_count, int is_write, struct omap_nand_info *info) 191 { 192 u32 val; 193 194 if (fifo_th > PREFETCH_FIFOTHRESHOLD_MAX) 195 return -1; 196 197 if (readl(info->reg.gpmc_prefetch_control)) 198 return -EBUSY; 199 200 /* Set the amount of bytes to be prefetched */ 201 writel(u32_count, info->reg.gpmc_prefetch_config2); 202 203 /* Set dma/mpu mode, the prefetch read / post write and 204 * enable the engine. Set which cs is has requested for. 205 */ 206 val = ((cs << PREFETCH_CONFIG1_CS_SHIFT) | 207 PREFETCH_FIFOTHRESHOLD(fifo_th) | ENABLE_PREFETCH | 208 (dma_mode << DMA_MPU_MODE_SHIFT) | (is_write & 0x1)); 209 writel(val, info->reg.gpmc_prefetch_config1); 210 211 /* Start the prefetch engine */ 212 writel(0x1, info->reg.gpmc_prefetch_control); 213 214 return 0; 215 } 216 217 /* 218 * omap_prefetch_reset - disables and stops the prefetch engine 219 */ 220 static int omap_prefetch_reset(int cs, struct omap_nand_info *info) 221 { 222 u32 config1; 223 224 /* check if the same module/cs is trying to reset */ 225 config1 = readl(info->reg.gpmc_prefetch_config1); 226 if (((config1 >> PREFETCH_CONFIG1_CS_SHIFT) & CS_MASK) != cs) 227 return -EINVAL; 228 229 /* Stop the PFPW engine */ 230 writel(0x0, info->reg.gpmc_prefetch_control); 231 232 /* Reset/disable the PFPW engine */ 233 writel(0x0, info->reg.gpmc_prefetch_config1); 234 235 return 0; 236 } 237 238 /** 239 * omap_hwcontrol - hardware specific access to control-lines 240 * @chip: NAND chip object 241 * @cmd: command to device 242 * @ctrl: 243 * NAND_NCE: bit 0 -> don't care 244 * NAND_CLE: bit 1 -> Command Latch 245 * NAND_ALE: bit 2 -> Address Latch 246 * 247 * NOTE: boards may use different bits for these!! 248 */ 249 static void omap_hwcontrol(struct nand_chip *chip, int cmd, unsigned int ctrl) 250 { 251 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 252 253 if (cmd != NAND_CMD_NONE) { 254 if (ctrl & NAND_CLE) 255 writeb(cmd, info->reg.gpmc_nand_command); 256 257 else if (ctrl & NAND_ALE) 258 writeb(cmd, info->reg.gpmc_nand_address); 259 260 else /* NAND_NCE */ 261 writeb(cmd, info->reg.gpmc_nand_data); 262 } 263 } 264 265 /** 266 * omap_read_buf8 - read data from NAND controller into buffer 267 * @mtd: MTD device structure 268 * @buf: buffer to store date 269 * @len: number of bytes to read 270 */ 271 static void omap_read_buf8(struct mtd_info *mtd, u_char *buf, int len) 272 { 273 struct nand_chip *nand = mtd_to_nand(mtd); 274 275 ioread8_rep(nand->legacy.IO_ADDR_R, buf, len); 276 } 277 278 /** 279 * omap_write_buf8 - write buffer to NAND controller 280 * @mtd: MTD device structure 281 * @buf: data buffer 282 * @len: number of bytes to write 283 */ 284 static void omap_write_buf8(struct mtd_info *mtd, const u_char *buf, int len) 285 { 286 struct omap_nand_info *info = mtd_to_omap(mtd); 287 u_char *p = (u_char *)buf; 288 bool status; 289 290 while (len--) { 291 iowrite8(*p++, info->nand.legacy.IO_ADDR_W); 292 /* wait until buffer is available for write */ 293 do { 294 status = info->ops->nand_writebuffer_empty(); 295 } while (!status); 296 } 297 } 298 299 /** 300 * omap_read_buf16 - read data from NAND controller into buffer 301 * @mtd: MTD device structure 302 * @buf: buffer to store date 303 * @len: number of bytes to read 304 */ 305 static void omap_read_buf16(struct mtd_info *mtd, u_char *buf, int len) 306 { 307 struct nand_chip *nand = mtd_to_nand(mtd); 308 309 ioread16_rep(nand->legacy.IO_ADDR_R, buf, len / 2); 310 } 311 312 /** 313 * omap_write_buf16 - write buffer to NAND controller 314 * @mtd: MTD device structure 315 * @buf: data buffer 316 * @len: number of bytes to write 317 */ 318 static void omap_write_buf16(struct mtd_info *mtd, const u_char * buf, int len) 319 { 320 struct omap_nand_info *info = mtd_to_omap(mtd); 321 u16 *p = (u16 *) buf; 322 bool status; 323 /* FIXME try bursts of writesw() or DMA ... */ 324 len >>= 1; 325 326 while (len--) { 327 iowrite16(*p++, info->nand.legacy.IO_ADDR_W); 328 /* wait until buffer is available for write */ 329 do { 330 status = info->ops->nand_writebuffer_empty(); 331 } while (!status); 332 } 333 } 334 335 /** 336 * omap_read_buf_pref - read data from NAND controller into buffer 337 * @chip: NAND chip object 338 * @buf: buffer to store date 339 * @len: number of bytes to read 340 */ 341 static void omap_read_buf_pref(struct nand_chip *chip, u_char *buf, int len) 342 { 343 struct mtd_info *mtd = nand_to_mtd(chip); 344 struct omap_nand_info *info = mtd_to_omap(mtd); 345 uint32_t r_count = 0; 346 int ret = 0; 347 u32 *p = (u32 *)buf; 348 349 /* take care of subpage reads */ 350 if (len % 4) { 351 if (info->nand.options & NAND_BUSWIDTH_16) 352 omap_read_buf16(mtd, buf, len % 4); 353 else 354 omap_read_buf8(mtd, buf, len % 4); 355 p = (u32 *) (buf + len % 4); 356 len -= len % 4; 357 } 358 359 /* configure and start prefetch transfer */ 360 ret = omap_prefetch_enable(info->gpmc_cs, 361 PREFETCH_FIFOTHRESHOLD_MAX, 0x0, len, 0x0, info); 362 if (ret) { 363 /* PFPW engine is busy, use cpu copy method */ 364 if (info->nand.options & NAND_BUSWIDTH_16) 365 omap_read_buf16(mtd, (u_char *)p, len); 366 else 367 omap_read_buf8(mtd, (u_char *)p, len); 368 } else { 369 do { 370 r_count = readl(info->reg.gpmc_prefetch_status); 371 r_count = PREFETCH_STATUS_FIFO_CNT(r_count); 372 r_count = r_count >> 2; 373 ioread32_rep(info->nand.legacy.IO_ADDR_R, p, r_count); 374 p += r_count; 375 len -= r_count << 2; 376 } while (len); 377 /* disable and stop the PFPW engine */ 378 omap_prefetch_reset(info->gpmc_cs, info); 379 } 380 } 381 382 /** 383 * omap_write_buf_pref - write buffer to NAND controller 384 * @chip: NAND chip object 385 * @buf: data buffer 386 * @len: number of bytes to write 387 */ 388 static void omap_write_buf_pref(struct nand_chip *chip, const u_char *buf, 389 int len) 390 { 391 struct mtd_info *mtd = nand_to_mtd(chip); 392 struct omap_nand_info *info = mtd_to_omap(mtd); 393 uint32_t w_count = 0; 394 int i = 0, ret = 0; 395 u16 *p = (u16 *)buf; 396 unsigned long tim, limit; 397 u32 val; 398 399 /* take care of subpage writes */ 400 if (len % 2 != 0) { 401 writeb(*buf, info->nand.legacy.IO_ADDR_W); 402 p = (u16 *)(buf + 1); 403 len--; 404 } 405 406 /* configure and start prefetch transfer */ 407 ret = omap_prefetch_enable(info->gpmc_cs, 408 PREFETCH_FIFOTHRESHOLD_MAX, 0x0, len, 0x1, info); 409 if (ret) { 410 /* PFPW engine is busy, use cpu copy method */ 411 if (info->nand.options & NAND_BUSWIDTH_16) 412 omap_write_buf16(mtd, (u_char *)p, len); 413 else 414 omap_write_buf8(mtd, (u_char *)p, len); 415 } else { 416 while (len) { 417 w_count = readl(info->reg.gpmc_prefetch_status); 418 w_count = PREFETCH_STATUS_FIFO_CNT(w_count); 419 w_count = w_count >> 1; 420 for (i = 0; (i < w_count) && len; i++, len -= 2) 421 iowrite16(*p++, info->nand.legacy.IO_ADDR_W); 422 } 423 /* wait for data to flushed-out before reset the prefetch */ 424 tim = 0; 425 limit = (loops_per_jiffy * 426 msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS)); 427 do { 428 cpu_relax(); 429 val = readl(info->reg.gpmc_prefetch_status); 430 val = PREFETCH_STATUS_COUNT(val); 431 } while (val && (tim++ < limit)); 432 433 /* disable and stop the PFPW engine */ 434 omap_prefetch_reset(info->gpmc_cs, info); 435 } 436 } 437 438 /* 439 * omap_nand_dma_callback: callback on the completion of dma transfer 440 * @data: pointer to completion data structure 441 */ 442 static void omap_nand_dma_callback(void *data) 443 { 444 complete((struct completion *) data); 445 } 446 447 /* 448 * omap_nand_dma_transfer: configure and start dma transfer 449 * @mtd: MTD device structure 450 * @addr: virtual address in RAM of source/destination 451 * @len: number of data bytes to be transferred 452 * @is_write: flag for read/write operation 453 */ 454 static inline int omap_nand_dma_transfer(struct mtd_info *mtd, void *addr, 455 unsigned int len, int is_write) 456 { 457 struct omap_nand_info *info = mtd_to_omap(mtd); 458 struct dma_async_tx_descriptor *tx; 459 enum dma_data_direction dir = is_write ? DMA_TO_DEVICE : 460 DMA_FROM_DEVICE; 461 struct scatterlist sg; 462 unsigned long tim, limit; 463 unsigned n; 464 int ret; 465 u32 val; 466 467 if (!virt_addr_valid(addr)) 468 goto out_copy; 469 470 sg_init_one(&sg, addr, len); 471 n = dma_map_sg(info->dma->device->dev, &sg, 1, dir); 472 if (n == 0) { 473 dev_err(&info->pdev->dev, 474 "Couldn't DMA map a %d byte buffer\n", len); 475 goto out_copy; 476 } 477 478 tx = dmaengine_prep_slave_sg(info->dma, &sg, n, 479 is_write ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM, 480 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 481 if (!tx) 482 goto out_copy_unmap; 483 484 tx->callback = omap_nand_dma_callback; 485 tx->callback_param = &info->comp; 486 dmaengine_submit(tx); 487 488 init_completion(&info->comp); 489 490 /* setup and start DMA using dma_addr */ 491 dma_async_issue_pending(info->dma); 492 493 /* configure and start prefetch transfer */ 494 ret = omap_prefetch_enable(info->gpmc_cs, 495 PREFETCH_FIFOTHRESHOLD_MAX, 0x1, len, is_write, info); 496 if (ret) 497 /* PFPW engine is busy, use cpu copy method */ 498 goto out_copy_unmap; 499 500 wait_for_completion(&info->comp); 501 tim = 0; 502 limit = (loops_per_jiffy * msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS)); 503 504 do { 505 cpu_relax(); 506 val = readl(info->reg.gpmc_prefetch_status); 507 val = PREFETCH_STATUS_COUNT(val); 508 } while (val && (tim++ < limit)); 509 510 /* disable and stop the PFPW engine */ 511 omap_prefetch_reset(info->gpmc_cs, info); 512 513 dma_unmap_sg(info->dma->device->dev, &sg, 1, dir); 514 return 0; 515 516 out_copy_unmap: 517 dma_unmap_sg(info->dma->device->dev, &sg, 1, dir); 518 out_copy: 519 if (info->nand.options & NAND_BUSWIDTH_16) 520 is_write == 0 ? omap_read_buf16(mtd, (u_char *) addr, len) 521 : omap_write_buf16(mtd, (u_char *) addr, len); 522 else 523 is_write == 0 ? omap_read_buf8(mtd, (u_char *) addr, len) 524 : omap_write_buf8(mtd, (u_char *) addr, len); 525 return 0; 526 } 527 528 /** 529 * omap_read_buf_dma_pref - read data from NAND controller into buffer 530 * @chip: NAND chip object 531 * @buf: buffer to store date 532 * @len: number of bytes to read 533 */ 534 static void omap_read_buf_dma_pref(struct nand_chip *chip, u_char *buf, 535 int len) 536 { 537 struct mtd_info *mtd = nand_to_mtd(chip); 538 539 if (len <= mtd->oobsize) 540 omap_read_buf_pref(chip, buf, len); 541 else 542 /* start transfer in DMA mode */ 543 omap_nand_dma_transfer(mtd, buf, len, 0x0); 544 } 545 546 /** 547 * omap_write_buf_dma_pref - write buffer to NAND controller 548 * @chip: NAND chip object 549 * @buf: data buffer 550 * @len: number of bytes to write 551 */ 552 static void omap_write_buf_dma_pref(struct nand_chip *chip, const u_char *buf, 553 int len) 554 { 555 struct mtd_info *mtd = nand_to_mtd(chip); 556 557 if (len <= mtd->oobsize) 558 omap_write_buf_pref(chip, buf, len); 559 else 560 /* start transfer in DMA mode */ 561 omap_nand_dma_transfer(mtd, (u_char *)buf, len, 0x1); 562 } 563 564 /* 565 * omap_nand_irq - GPMC irq handler 566 * @this_irq: gpmc irq number 567 * @dev: omap_nand_info structure pointer is passed here 568 */ 569 static irqreturn_t omap_nand_irq(int this_irq, void *dev) 570 { 571 struct omap_nand_info *info = (struct omap_nand_info *) dev; 572 u32 bytes; 573 574 bytes = readl(info->reg.gpmc_prefetch_status); 575 bytes = PREFETCH_STATUS_FIFO_CNT(bytes); 576 bytes = bytes & 0xFFFC; /* io in multiple of 4 bytes */ 577 if (info->iomode == OMAP_NAND_IO_WRITE) { /* checks for write io */ 578 if (this_irq == info->gpmc_irq_count) 579 goto done; 580 581 if (info->buf_len && (info->buf_len < bytes)) 582 bytes = info->buf_len; 583 else if (!info->buf_len) 584 bytes = 0; 585 iowrite32_rep(info->nand.legacy.IO_ADDR_W, (u32 *)info->buf, 586 bytes >> 2); 587 info->buf = info->buf + bytes; 588 info->buf_len -= bytes; 589 590 } else { 591 ioread32_rep(info->nand.legacy.IO_ADDR_R, (u32 *)info->buf, 592 bytes >> 2); 593 info->buf = info->buf + bytes; 594 595 if (this_irq == info->gpmc_irq_count) 596 goto done; 597 } 598 599 return IRQ_HANDLED; 600 601 done: 602 complete(&info->comp); 603 604 disable_irq_nosync(info->gpmc_irq_fifo); 605 disable_irq_nosync(info->gpmc_irq_count); 606 607 return IRQ_HANDLED; 608 } 609 610 /* 611 * omap_read_buf_irq_pref - read data from NAND controller into buffer 612 * @chip: NAND chip object 613 * @buf: buffer to store date 614 * @len: number of bytes to read 615 */ 616 static void omap_read_buf_irq_pref(struct nand_chip *chip, u_char *buf, 617 int len) 618 { 619 struct mtd_info *mtd = nand_to_mtd(chip); 620 struct omap_nand_info *info = mtd_to_omap(mtd); 621 int ret = 0; 622 623 if (len <= mtd->oobsize) { 624 omap_read_buf_pref(chip, buf, len); 625 return; 626 } 627 628 info->iomode = OMAP_NAND_IO_READ; 629 info->buf = buf; 630 init_completion(&info->comp); 631 632 /* configure and start prefetch transfer */ 633 ret = omap_prefetch_enable(info->gpmc_cs, 634 PREFETCH_FIFOTHRESHOLD_MAX/2, 0x0, len, 0x0, info); 635 if (ret) 636 /* PFPW engine is busy, use cpu copy method */ 637 goto out_copy; 638 639 info->buf_len = len; 640 641 enable_irq(info->gpmc_irq_count); 642 enable_irq(info->gpmc_irq_fifo); 643 644 /* waiting for read to complete */ 645 wait_for_completion(&info->comp); 646 647 /* disable and stop the PFPW engine */ 648 omap_prefetch_reset(info->gpmc_cs, info); 649 return; 650 651 out_copy: 652 if (info->nand.options & NAND_BUSWIDTH_16) 653 omap_read_buf16(mtd, buf, len); 654 else 655 omap_read_buf8(mtd, buf, len); 656 } 657 658 /* 659 * omap_write_buf_irq_pref - write buffer to NAND controller 660 * @chip: NAND chip object 661 * @buf: data buffer 662 * @len: number of bytes to write 663 */ 664 static void omap_write_buf_irq_pref(struct nand_chip *chip, const u_char *buf, 665 int len) 666 { 667 struct mtd_info *mtd = nand_to_mtd(chip); 668 struct omap_nand_info *info = mtd_to_omap(mtd); 669 int ret = 0; 670 unsigned long tim, limit; 671 u32 val; 672 673 if (len <= mtd->oobsize) { 674 omap_write_buf_pref(chip, buf, len); 675 return; 676 } 677 678 info->iomode = OMAP_NAND_IO_WRITE; 679 info->buf = (u_char *) buf; 680 init_completion(&info->comp); 681 682 /* configure and start prefetch transfer : size=24 */ 683 ret = omap_prefetch_enable(info->gpmc_cs, 684 (PREFETCH_FIFOTHRESHOLD_MAX * 3) / 8, 0x0, len, 0x1, info); 685 if (ret) 686 /* PFPW engine is busy, use cpu copy method */ 687 goto out_copy; 688 689 info->buf_len = len; 690 691 enable_irq(info->gpmc_irq_count); 692 enable_irq(info->gpmc_irq_fifo); 693 694 /* waiting for write to complete */ 695 wait_for_completion(&info->comp); 696 697 /* wait for data to flushed-out before reset the prefetch */ 698 tim = 0; 699 limit = (loops_per_jiffy * msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS)); 700 do { 701 val = readl(info->reg.gpmc_prefetch_status); 702 val = PREFETCH_STATUS_COUNT(val); 703 cpu_relax(); 704 } while (val && (tim++ < limit)); 705 706 /* disable and stop the PFPW engine */ 707 omap_prefetch_reset(info->gpmc_cs, info); 708 return; 709 710 out_copy: 711 if (info->nand.options & NAND_BUSWIDTH_16) 712 omap_write_buf16(mtd, buf, len); 713 else 714 omap_write_buf8(mtd, buf, len); 715 } 716 717 /** 718 * gen_true_ecc - This function will generate true ECC value 719 * @ecc_buf: buffer to store ecc code 720 * 721 * This generated true ECC value can be used when correcting 722 * data read from NAND flash memory core 723 */ 724 static void gen_true_ecc(u8 *ecc_buf) 725 { 726 u32 tmp = ecc_buf[0] | (ecc_buf[1] << 16) | 727 ((ecc_buf[2] & 0xF0) << 20) | ((ecc_buf[2] & 0x0F) << 8); 728 729 ecc_buf[0] = ~(P64o(tmp) | P64e(tmp) | P32o(tmp) | P32e(tmp) | 730 P16o(tmp) | P16e(tmp) | P8o(tmp) | P8e(tmp)); 731 ecc_buf[1] = ~(P1024o(tmp) | P1024e(tmp) | P512o(tmp) | P512e(tmp) | 732 P256o(tmp) | P256e(tmp) | P128o(tmp) | P128e(tmp)); 733 ecc_buf[2] = ~(P4o(tmp) | P4e(tmp) | P2o(tmp) | P2e(tmp) | P1o(tmp) | 734 P1e(tmp) | P2048o(tmp) | P2048e(tmp)); 735 } 736 737 /** 738 * omap_compare_ecc - Detect (2 bits) and correct (1 bit) error in data 739 * @ecc_data1: ecc code from nand spare area 740 * @ecc_data2: ecc code from hardware register obtained from hardware ecc 741 * @page_data: page data 742 * 743 * This function compares two ECC's and indicates if there is an error. 744 * If the error can be corrected it will be corrected to the buffer. 745 * If there is no error, %0 is returned. If there is an error but it 746 * was corrected, %1 is returned. Otherwise, %-1 is returned. 747 */ 748 static int omap_compare_ecc(u8 *ecc_data1, /* read from NAND memory */ 749 u8 *ecc_data2, /* read from register */ 750 u8 *page_data) 751 { 752 uint i; 753 u8 tmp0_bit[8], tmp1_bit[8], tmp2_bit[8]; 754 u8 comp0_bit[8], comp1_bit[8], comp2_bit[8]; 755 u8 ecc_bit[24]; 756 u8 ecc_sum = 0; 757 u8 find_bit = 0; 758 uint find_byte = 0; 759 int isEccFF; 760 761 isEccFF = ((*(u32 *)ecc_data1 & 0xFFFFFF) == 0xFFFFFF); 762 763 gen_true_ecc(ecc_data1); 764 gen_true_ecc(ecc_data2); 765 766 for (i = 0; i <= 2; i++) { 767 *(ecc_data1 + i) = ~(*(ecc_data1 + i)); 768 *(ecc_data2 + i) = ~(*(ecc_data2 + i)); 769 } 770 771 for (i = 0; i < 8; i++) { 772 tmp0_bit[i] = *ecc_data1 % 2; 773 *ecc_data1 = *ecc_data1 / 2; 774 } 775 776 for (i = 0; i < 8; i++) { 777 tmp1_bit[i] = *(ecc_data1 + 1) % 2; 778 *(ecc_data1 + 1) = *(ecc_data1 + 1) / 2; 779 } 780 781 for (i = 0; i < 8; i++) { 782 tmp2_bit[i] = *(ecc_data1 + 2) % 2; 783 *(ecc_data1 + 2) = *(ecc_data1 + 2) / 2; 784 } 785 786 for (i = 0; i < 8; i++) { 787 comp0_bit[i] = *ecc_data2 % 2; 788 *ecc_data2 = *ecc_data2 / 2; 789 } 790 791 for (i = 0; i < 8; i++) { 792 comp1_bit[i] = *(ecc_data2 + 1) % 2; 793 *(ecc_data2 + 1) = *(ecc_data2 + 1) / 2; 794 } 795 796 for (i = 0; i < 8; i++) { 797 comp2_bit[i] = *(ecc_data2 + 2) % 2; 798 *(ecc_data2 + 2) = *(ecc_data2 + 2) / 2; 799 } 800 801 for (i = 0; i < 6; i++) 802 ecc_bit[i] = tmp2_bit[i + 2] ^ comp2_bit[i + 2]; 803 804 for (i = 0; i < 8; i++) 805 ecc_bit[i + 6] = tmp0_bit[i] ^ comp0_bit[i]; 806 807 for (i = 0; i < 8; i++) 808 ecc_bit[i + 14] = tmp1_bit[i] ^ comp1_bit[i]; 809 810 ecc_bit[22] = tmp2_bit[0] ^ comp2_bit[0]; 811 ecc_bit[23] = tmp2_bit[1] ^ comp2_bit[1]; 812 813 for (i = 0; i < 24; i++) 814 ecc_sum += ecc_bit[i]; 815 816 switch (ecc_sum) { 817 case 0: 818 /* Not reached because this function is not called if 819 * ECC values are equal 820 */ 821 return 0; 822 823 case 1: 824 /* Uncorrectable error */ 825 pr_debug("ECC UNCORRECTED_ERROR 1\n"); 826 return -EBADMSG; 827 828 case 11: 829 /* UN-Correctable error */ 830 pr_debug("ECC UNCORRECTED_ERROR B\n"); 831 return -EBADMSG; 832 833 case 12: 834 /* Correctable error */ 835 find_byte = (ecc_bit[23] << 8) + 836 (ecc_bit[21] << 7) + 837 (ecc_bit[19] << 6) + 838 (ecc_bit[17] << 5) + 839 (ecc_bit[15] << 4) + 840 (ecc_bit[13] << 3) + 841 (ecc_bit[11] << 2) + 842 (ecc_bit[9] << 1) + 843 ecc_bit[7]; 844 845 find_bit = (ecc_bit[5] << 2) + (ecc_bit[3] << 1) + ecc_bit[1]; 846 847 pr_debug("Correcting single bit ECC error at offset: " 848 "%d, bit: %d\n", find_byte, find_bit); 849 850 page_data[find_byte] ^= (1 << find_bit); 851 852 return 1; 853 default: 854 if (isEccFF) { 855 if (ecc_data2[0] == 0 && 856 ecc_data2[1] == 0 && 857 ecc_data2[2] == 0) 858 return 0; 859 } 860 pr_debug("UNCORRECTED_ERROR default\n"); 861 return -EBADMSG; 862 } 863 } 864 865 /** 866 * omap_correct_data - Compares the ECC read with HW generated ECC 867 * @chip: NAND chip object 868 * @dat: page data 869 * @read_ecc: ecc read from nand flash 870 * @calc_ecc: ecc read from HW ECC registers 871 * 872 * Compares the ecc read from nand spare area with ECC registers values 873 * and if ECC's mismatched, it will call 'omap_compare_ecc' for error 874 * detection and correction. If there are no errors, %0 is returned. If 875 * there were errors and all of the errors were corrected, the number of 876 * corrected errors is returned. If uncorrectable errors exist, %-1 is 877 * returned. 878 */ 879 static int omap_correct_data(struct nand_chip *chip, u_char *dat, 880 u_char *read_ecc, u_char *calc_ecc) 881 { 882 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 883 int blockCnt = 0, i = 0, ret = 0; 884 int stat = 0; 885 886 /* Ex NAND_ECC_HW12_2048 */ 887 if (info->nand.ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST && 888 info->nand.ecc.size == 2048) 889 blockCnt = 4; 890 else 891 blockCnt = 1; 892 893 for (i = 0; i < blockCnt; i++) { 894 if (memcmp(read_ecc, calc_ecc, 3) != 0) { 895 ret = omap_compare_ecc(read_ecc, calc_ecc, dat); 896 if (ret < 0) 897 return ret; 898 /* keep track of the number of corrected errors */ 899 stat += ret; 900 } 901 read_ecc += 3; 902 calc_ecc += 3; 903 dat += 512; 904 } 905 return stat; 906 } 907 908 /** 909 * omap_calcuate_ecc - Generate non-inverted ECC bytes. 910 * @chip: NAND chip object 911 * @dat: The pointer to data on which ecc is computed 912 * @ecc_code: The ecc_code buffer 913 * 914 * Using noninverted ECC can be considered ugly since writing a blank 915 * page ie. padding will clear the ECC bytes. This is no problem as long 916 * nobody is trying to write data on the seemingly unused page. Reading 917 * an erased page will produce an ECC mismatch between generated and read 918 * ECC bytes that has to be dealt with separately. 919 */ 920 static int omap_calculate_ecc(struct nand_chip *chip, const u_char *dat, 921 u_char *ecc_code) 922 { 923 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 924 u32 val; 925 926 val = readl(info->reg.gpmc_ecc_config); 927 if (((val >> ECC_CONFIG_CS_SHIFT) & CS_MASK) != info->gpmc_cs) 928 return -EINVAL; 929 930 /* read ecc result */ 931 val = readl(info->reg.gpmc_ecc1_result); 932 *ecc_code++ = val; /* P128e, ..., P1e */ 933 *ecc_code++ = val >> 16; /* P128o, ..., P1o */ 934 /* P2048o, P1024o, P512o, P256o, P2048e, P1024e, P512e, P256e */ 935 *ecc_code++ = ((val >> 8) & 0x0f) | ((val >> 20) & 0xf0); 936 937 return 0; 938 } 939 940 /** 941 * omap_enable_hwecc - This function enables the hardware ecc functionality 942 * @chip: NAND chip object 943 * @mode: Read/Write mode 944 */ 945 static void omap_enable_hwecc(struct nand_chip *chip, int mode) 946 { 947 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 948 unsigned int dev_width = (chip->options & NAND_BUSWIDTH_16) ? 1 : 0; 949 u32 val; 950 951 /* clear ecc and enable bits */ 952 val = ECCCLEAR | ECC1; 953 writel(val, info->reg.gpmc_ecc_control); 954 955 /* program ecc and result sizes */ 956 val = ((((info->nand.ecc.size >> 1) - 1) << ECCSIZE1_SHIFT) | 957 ECC1RESULTSIZE); 958 writel(val, info->reg.gpmc_ecc_size_config); 959 960 switch (mode) { 961 case NAND_ECC_READ: 962 case NAND_ECC_WRITE: 963 writel(ECCCLEAR | ECC1, info->reg.gpmc_ecc_control); 964 break; 965 case NAND_ECC_READSYN: 966 writel(ECCCLEAR, info->reg.gpmc_ecc_control); 967 break; 968 default: 969 dev_info(&info->pdev->dev, 970 "error: unrecognized Mode[%d]!\n", mode); 971 break; 972 } 973 974 /* (ECC 16 or 8 bit col) | ( CS ) | ECC Enable */ 975 val = (dev_width << 7) | (info->gpmc_cs << 1) | (0x1); 976 writel(val, info->reg.gpmc_ecc_config); 977 } 978 979 /** 980 * omap_wait - wait until the command is done 981 * @this: NAND Chip structure 982 * 983 * Wait function is called during Program and erase operations and 984 * the way it is called from MTD layer, we should wait till the NAND 985 * chip is ready after the programming/erase operation has completed. 986 * 987 * Erase can take up to 400ms and program up to 20ms according to 988 * general NAND and SmartMedia specs 989 */ 990 static int omap_wait(struct nand_chip *this) 991 { 992 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(this)); 993 unsigned long timeo = jiffies; 994 int status; 995 996 timeo += msecs_to_jiffies(400); 997 998 writeb(NAND_CMD_STATUS & 0xFF, info->reg.gpmc_nand_command); 999 while (time_before(jiffies, timeo)) { 1000 status = readb(info->reg.gpmc_nand_data); 1001 if (status & NAND_STATUS_READY) 1002 break; 1003 cond_resched(); 1004 } 1005 1006 status = readb(info->reg.gpmc_nand_data); 1007 return status; 1008 } 1009 1010 /** 1011 * omap_dev_ready - checks the NAND Ready GPIO line 1012 * @chip: NAND chip object 1013 * 1014 * Returns true if ready and false if busy. 1015 */ 1016 static int omap_dev_ready(struct nand_chip *chip) 1017 { 1018 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 1019 1020 return gpiod_get_value(info->ready_gpiod); 1021 } 1022 1023 /** 1024 * omap_enable_hwecc_bch - Program GPMC to perform BCH ECC calculation 1025 * @chip: NAND chip object 1026 * @mode: Read/Write mode 1027 * 1028 * When using BCH with SW correction (i.e. no ELM), sector size is set 1029 * to 512 bytes and we use BCH_WRAPMODE_6 wrapping mode 1030 * for both reading and writing with: 1031 * eccsize0 = 0 (no additional protected byte in spare area) 1032 * eccsize1 = 32 (skip 32 nibbles = 16 bytes per sector in spare area) 1033 */ 1034 static void __maybe_unused omap_enable_hwecc_bch(struct nand_chip *chip, 1035 int mode) 1036 { 1037 unsigned int bch_type; 1038 unsigned int dev_width, nsectors; 1039 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 1040 enum omap_ecc ecc_opt = info->ecc_opt; 1041 u32 val, wr_mode; 1042 unsigned int ecc_size1, ecc_size0; 1043 1044 /* GPMC configurations for calculating ECC */ 1045 switch (ecc_opt) { 1046 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: 1047 bch_type = 0; 1048 nsectors = 1; 1049 wr_mode = BCH_WRAPMODE_6; 1050 ecc_size0 = BCH_ECC_SIZE0; 1051 ecc_size1 = BCH_ECC_SIZE1; 1052 break; 1053 case OMAP_ECC_BCH4_CODE_HW: 1054 bch_type = 0; 1055 nsectors = chip->ecc.steps; 1056 if (mode == NAND_ECC_READ) { 1057 wr_mode = BCH_WRAPMODE_1; 1058 ecc_size0 = BCH4R_ECC_SIZE0; 1059 ecc_size1 = BCH4R_ECC_SIZE1; 1060 } else { 1061 wr_mode = BCH_WRAPMODE_6; 1062 ecc_size0 = BCH_ECC_SIZE0; 1063 ecc_size1 = BCH_ECC_SIZE1; 1064 } 1065 break; 1066 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: 1067 bch_type = 1; 1068 nsectors = 1; 1069 wr_mode = BCH_WRAPMODE_6; 1070 ecc_size0 = BCH_ECC_SIZE0; 1071 ecc_size1 = BCH_ECC_SIZE1; 1072 break; 1073 case OMAP_ECC_BCH8_CODE_HW: 1074 bch_type = 1; 1075 nsectors = chip->ecc.steps; 1076 if (mode == NAND_ECC_READ) { 1077 wr_mode = BCH_WRAPMODE_1; 1078 ecc_size0 = BCH8R_ECC_SIZE0; 1079 ecc_size1 = BCH8R_ECC_SIZE1; 1080 } else { 1081 wr_mode = BCH_WRAPMODE_6; 1082 ecc_size0 = BCH_ECC_SIZE0; 1083 ecc_size1 = BCH_ECC_SIZE1; 1084 } 1085 break; 1086 case OMAP_ECC_BCH16_CODE_HW: 1087 bch_type = 0x2; 1088 nsectors = chip->ecc.steps; 1089 if (mode == NAND_ECC_READ) { 1090 wr_mode = 0x01; 1091 ecc_size0 = 52; /* ECC bits in nibbles per sector */ 1092 ecc_size1 = 0; /* non-ECC bits in nibbles per sector */ 1093 } else { 1094 wr_mode = 0x01; 1095 ecc_size0 = 0; /* extra bits in nibbles per sector */ 1096 ecc_size1 = 52; /* OOB bits in nibbles per sector */ 1097 } 1098 break; 1099 default: 1100 return; 1101 } 1102 1103 writel(ECC1, info->reg.gpmc_ecc_control); 1104 1105 /* Configure ecc size for BCH */ 1106 val = (ecc_size1 << ECCSIZE1_SHIFT) | (ecc_size0 << ECCSIZE0_SHIFT); 1107 writel(val, info->reg.gpmc_ecc_size_config); 1108 1109 dev_width = (chip->options & NAND_BUSWIDTH_16) ? 1 : 0; 1110 1111 /* BCH configuration */ 1112 val = ((1 << 16) | /* enable BCH */ 1113 (bch_type << 12) | /* BCH4/BCH8/BCH16 */ 1114 (wr_mode << 8) | /* wrap mode */ 1115 (dev_width << 7) | /* bus width */ 1116 (((nsectors-1) & 0x7) << 4) | /* number of sectors */ 1117 (info->gpmc_cs << 1) | /* ECC CS */ 1118 (0x1)); /* enable ECC */ 1119 1120 writel(val, info->reg.gpmc_ecc_config); 1121 1122 /* Clear ecc and enable bits */ 1123 writel(ECCCLEAR | ECC1, info->reg.gpmc_ecc_control); 1124 } 1125 1126 static u8 bch4_polynomial[] = {0x28, 0x13, 0xcc, 0x39, 0x96, 0xac, 0x7f}; 1127 static u8 bch8_polynomial[] = {0xef, 0x51, 0x2e, 0x09, 0xed, 0x93, 0x9a, 0xc2, 1128 0x97, 0x79, 0xe5, 0x24, 0xb5}; 1129 1130 /** 1131 * _omap_calculate_ecc_bch - Generate ECC bytes for one sector 1132 * @mtd: MTD device structure 1133 * @dat: The pointer to data on which ecc is computed 1134 * @ecc_calc: The ecc_code buffer 1135 * @i: The sector number (for a multi sector page) 1136 * 1137 * Support calculating of BCH4/8/16 ECC vectors for one sector 1138 * within a page. Sector number is in @i. 1139 */ 1140 static int _omap_calculate_ecc_bch(struct mtd_info *mtd, 1141 const u_char *dat, u_char *ecc_calc, int i) 1142 { 1143 struct omap_nand_info *info = mtd_to_omap(mtd); 1144 int eccbytes = info->nand.ecc.bytes; 1145 struct gpmc_nand_regs *gpmc_regs = &info->reg; 1146 u8 *ecc_code; 1147 unsigned long bch_val1, bch_val2, bch_val3, bch_val4; 1148 u32 val; 1149 int j; 1150 1151 ecc_code = ecc_calc; 1152 switch (info->ecc_opt) { 1153 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: 1154 case OMAP_ECC_BCH8_CODE_HW: 1155 bch_val1 = readl(gpmc_regs->gpmc_bch_result0[i]); 1156 bch_val2 = readl(gpmc_regs->gpmc_bch_result1[i]); 1157 bch_val3 = readl(gpmc_regs->gpmc_bch_result2[i]); 1158 bch_val4 = readl(gpmc_regs->gpmc_bch_result3[i]); 1159 *ecc_code++ = (bch_val4 & 0xFF); 1160 *ecc_code++ = ((bch_val3 >> 24) & 0xFF); 1161 *ecc_code++ = ((bch_val3 >> 16) & 0xFF); 1162 *ecc_code++ = ((bch_val3 >> 8) & 0xFF); 1163 *ecc_code++ = (bch_val3 & 0xFF); 1164 *ecc_code++ = ((bch_val2 >> 24) & 0xFF); 1165 *ecc_code++ = ((bch_val2 >> 16) & 0xFF); 1166 *ecc_code++ = ((bch_val2 >> 8) & 0xFF); 1167 *ecc_code++ = (bch_val2 & 0xFF); 1168 *ecc_code++ = ((bch_val1 >> 24) & 0xFF); 1169 *ecc_code++ = ((bch_val1 >> 16) & 0xFF); 1170 *ecc_code++ = ((bch_val1 >> 8) & 0xFF); 1171 *ecc_code++ = (bch_val1 & 0xFF); 1172 break; 1173 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: 1174 case OMAP_ECC_BCH4_CODE_HW: 1175 bch_val1 = readl(gpmc_regs->gpmc_bch_result0[i]); 1176 bch_val2 = readl(gpmc_regs->gpmc_bch_result1[i]); 1177 *ecc_code++ = ((bch_val2 >> 12) & 0xFF); 1178 *ecc_code++ = ((bch_val2 >> 4) & 0xFF); 1179 *ecc_code++ = ((bch_val2 & 0xF) << 4) | 1180 ((bch_val1 >> 28) & 0xF); 1181 *ecc_code++ = ((bch_val1 >> 20) & 0xFF); 1182 *ecc_code++ = ((bch_val1 >> 12) & 0xFF); 1183 *ecc_code++ = ((bch_val1 >> 4) & 0xFF); 1184 *ecc_code++ = ((bch_val1 & 0xF) << 4); 1185 break; 1186 case OMAP_ECC_BCH16_CODE_HW: 1187 val = readl(gpmc_regs->gpmc_bch_result6[i]); 1188 ecc_code[0] = ((val >> 8) & 0xFF); 1189 ecc_code[1] = ((val >> 0) & 0xFF); 1190 val = readl(gpmc_regs->gpmc_bch_result5[i]); 1191 ecc_code[2] = ((val >> 24) & 0xFF); 1192 ecc_code[3] = ((val >> 16) & 0xFF); 1193 ecc_code[4] = ((val >> 8) & 0xFF); 1194 ecc_code[5] = ((val >> 0) & 0xFF); 1195 val = readl(gpmc_regs->gpmc_bch_result4[i]); 1196 ecc_code[6] = ((val >> 24) & 0xFF); 1197 ecc_code[7] = ((val >> 16) & 0xFF); 1198 ecc_code[8] = ((val >> 8) & 0xFF); 1199 ecc_code[9] = ((val >> 0) & 0xFF); 1200 val = readl(gpmc_regs->gpmc_bch_result3[i]); 1201 ecc_code[10] = ((val >> 24) & 0xFF); 1202 ecc_code[11] = ((val >> 16) & 0xFF); 1203 ecc_code[12] = ((val >> 8) & 0xFF); 1204 ecc_code[13] = ((val >> 0) & 0xFF); 1205 val = readl(gpmc_regs->gpmc_bch_result2[i]); 1206 ecc_code[14] = ((val >> 24) & 0xFF); 1207 ecc_code[15] = ((val >> 16) & 0xFF); 1208 ecc_code[16] = ((val >> 8) & 0xFF); 1209 ecc_code[17] = ((val >> 0) & 0xFF); 1210 val = readl(gpmc_regs->gpmc_bch_result1[i]); 1211 ecc_code[18] = ((val >> 24) & 0xFF); 1212 ecc_code[19] = ((val >> 16) & 0xFF); 1213 ecc_code[20] = ((val >> 8) & 0xFF); 1214 ecc_code[21] = ((val >> 0) & 0xFF); 1215 val = readl(gpmc_regs->gpmc_bch_result0[i]); 1216 ecc_code[22] = ((val >> 24) & 0xFF); 1217 ecc_code[23] = ((val >> 16) & 0xFF); 1218 ecc_code[24] = ((val >> 8) & 0xFF); 1219 ecc_code[25] = ((val >> 0) & 0xFF); 1220 break; 1221 default: 1222 return -EINVAL; 1223 } 1224 1225 /* ECC scheme specific syndrome customizations */ 1226 switch (info->ecc_opt) { 1227 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: 1228 /* Add constant polynomial to remainder, so that 1229 * ECC of blank pages results in 0x0 on reading back 1230 */ 1231 for (j = 0; j < eccbytes; j++) 1232 ecc_calc[j] ^= bch4_polynomial[j]; 1233 break; 1234 case OMAP_ECC_BCH4_CODE_HW: 1235 /* Set 8th ECC byte as 0x0 for ROM compatibility */ 1236 ecc_calc[eccbytes - 1] = 0x0; 1237 break; 1238 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: 1239 /* Add constant polynomial to remainder, so that 1240 * ECC of blank pages results in 0x0 on reading back 1241 */ 1242 for (j = 0; j < eccbytes; j++) 1243 ecc_calc[j] ^= bch8_polynomial[j]; 1244 break; 1245 case OMAP_ECC_BCH8_CODE_HW: 1246 /* Set 14th ECC byte as 0x0 for ROM compatibility */ 1247 ecc_calc[eccbytes - 1] = 0x0; 1248 break; 1249 case OMAP_ECC_BCH16_CODE_HW: 1250 break; 1251 default: 1252 return -EINVAL; 1253 } 1254 1255 return 0; 1256 } 1257 1258 /** 1259 * omap_calculate_ecc_bch_sw - ECC generator for sector for SW based correction 1260 * @chip: NAND chip object 1261 * @dat: The pointer to data on which ecc is computed 1262 * @ecc_calc: Buffer storing the calculated ECC bytes 1263 * 1264 * Support calculating of BCH4/8/16 ECC vectors for one sector. This is used 1265 * when SW based correction is required as ECC is required for one sector 1266 * at a time. 1267 */ 1268 static int omap_calculate_ecc_bch_sw(struct nand_chip *chip, 1269 const u_char *dat, u_char *ecc_calc) 1270 { 1271 return _omap_calculate_ecc_bch(nand_to_mtd(chip), dat, ecc_calc, 0); 1272 } 1273 1274 /** 1275 * omap_calculate_ecc_bch_multi - Generate ECC for multiple sectors 1276 * @mtd: MTD device structure 1277 * @dat: The pointer to data on which ecc is computed 1278 * @ecc_calc: Buffer storing the calculated ECC bytes 1279 * 1280 * Support calculating of BCH4/8/16 ecc vectors for the entire page in one go. 1281 */ 1282 static int omap_calculate_ecc_bch_multi(struct mtd_info *mtd, 1283 const u_char *dat, u_char *ecc_calc) 1284 { 1285 struct omap_nand_info *info = mtd_to_omap(mtd); 1286 int eccbytes = info->nand.ecc.bytes; 1287 unsigned long nsectors; 1288 int i, ret; 1289 1290 nsectors = ((readl(info->reg.gpmc_ecc_config) >> 4) & 0x7) + 1; 1291 for (i = 0; i < nsectors; i++) { 1292 ret = _omap_calculate_ecc_bch(mtd, dat, ecc_calc, i); 1293 if (ret) 1294 return ret; 1295 1296 ecc_calc += eccbytes; 1297 } 1298 1299 return 0; 1300 } 1301 1302 /** 1303 * erased_sector_bitflips - count bit flips 1304 * @data: data sector buffer 1305 * @oob: oob buffer 1306 * @info: omap_nand_info 1307 * 1308 * Check the bit flips in erased page falls below correctable level. 1309 * If falls below, report the page as erased with correctable bit 1310 * flip, else report as uncorrectable page. 1311 */ 1312 static int erased_sector_bitflips(u_char *data, u_char *oob, 1313 struct omap_nand_info *info) 1314 { 1315 int flip_bits = 0, i; 1316 1317 for (i = 0; i < info->nand.ecc.size; i++) { 1318 flip_bits += hweight8(~data[i]); 1319 if (flip_bits > info->nand.ecc.strength) 1320 return 0; 1321 } 1322 1323 for (i = 0; i < info->nand.ecc.bytes - 1; i++) { 1324 flip_bits += hweight8(~oob[i]); 1325 if (flip_bits > info->nand.ecc.strength) 1326 return 0; 1327 } 1328 1329 /* 1330 * Bit flips falls in correctable level. 1331 * Fill data area with 0xFF 1332 */ 1333 if (flip_bits) { 1334 memset(data, 0xFF, info->nand.ecc.size); 1335 memset(oob, 0xFF, info->nand.ecc.bytes); 1336 } 1337 1338 return flip_bits; 1339 } 1340 1341 /** 1342 * omap_elm_correct_data - corrects page data area in case error reported 1343 * @chip: NAND chip object 1344 * @data: page data 1345 * @read_ecc: ecc read from nand flash 1346 * @calc_ecc: ecc read from HW ECC registers 1347 * 1348 * Calculated ecc vector reported as zero in case of non-error pages. 1349 * In case of non-zero ecc vector, first filter out erased-pages, and 1350 * then process data via ELM to detect bit-flips. 1351 */ 1352 static int omap_elm_correct_data(struct nand_chip *chip, u_char *data, 1353 u_char *read_ecc, u_char *calc_ecc) 1354 { 1355 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 1356 struct nand_ecc_ctrl *ecc = &info->nand.ecc; 1357 int eccsteps = info->nand.ecc.steps; 1358 int i , j, stat = 0; 1359 int eccflag, actual_eccbytes; 1360 struct elm_errorvec err_vec[ERROR_VECTOR_MAX]; 1361 u_char *ecc_vec = calc_ecc; 1362 u_char *spare_ecc = read_ecc; 1363 u_char *erased_ecc_vec; 1364 u_char *buf; 1365 int bitflip_count; 1366 bool is_error_reported = false; 1367 u32 bit_pos, byte_pos, error_max, pos; 1368 int err; 1369 1370 switch (info->ecc_opt) { 1371 case OMAP_ECC_BCH4_CODE_HW: 1372 /* omit 7th ECC byte reserved for ROM code compatibility */ 1373 actual_eccbytes = ecc->bytes - 1; 1374 erased_ecc_vec = bch4_vector; 1375 break; 1376 case OMAP_ECC_BCH8_CODE_HW: 1377 /* omit 14th ECC byte reserved for ROM code compatibility */ 1378 actual_eccbytes = ecc->bytes - 1; 1379 erased_ecc_vec = bch8_vector; 1380 break; 1381 case OMAP_ECC_BCH16_CODE_HW: 1382 actual_eccbytes = ecc->bytes; 1383 erased_ecc_vec = bch16_vector; 1384 break; 1385 default: 1386 dev_err(&info->pdev->dev, "invalid driver configuration\n"); 1387 return -EINVAL; 1388 } 1389 1390 /* Initialize elm error vector to zero */ 1391 memset(err_vec, 0, sizeof(err_vec)); 1392 1393 for (i = 0; i < eccsteps ; i++) { 1394 eccflag = 0; /* initialize eccflag */ 1395 1396 /* 1397 * Check any error reported, 1398 * In case of error, non zero ecc reported. 1399 */ 1400 for (j = 0; j < actual_eccbytes; j++) { 1401 if (calc_ecc[j] != 0) { 1402 eccflag = 1; /* non zero ecc, error present */ 1403 break; 1404 } 1405 } 1406 1407 if (eccflag == 1) { 1408 if (memcmp(calc_ecc, erased_ecc_vec, 1409 actual_eccbytes) == 0) { 1410 /* 1411 * calc_ecc[] matches pattern for ECC(all 0xff) 1412 * so this is definitely an erased-page 1413 */ 1414 } else { 1415 buf = &data[info->nand.ecc.size * i]; 1416 /* 1417 * count number of 0-bits in read_buf. 1418 * This check can be removed once a similar 1419 * check is introduced in generic NAND driver 1420 */ 1421 bitflip_count = erased_sector_bitflips( 1422 buf, read_ecc, info); 1423 if (bitflip_count) { 1424 /* 1425 * number of 0-bits within ECC limits 1426 * So this may be an erased-page 1427 */ 1428 stat += bitflip_count; 1429 } else { 1430 /* 1431 * Too many 0-bits. It may be a 1432 * - programmed-page, OR 1433 * - erased-page with many bit-flips 1434 * So this page requires check by ELM 1435 */ 1436 err_vec[i].error_reported = true; 1437 is_error_reported = true; 1438 } 1439 } 1440 } 1441 1442 /* Update the ecc vector */ 1443 calc_ecc += ecc->bytes; 1444 read_ecc += ecc->bytes; 1445 } 1446 1447 /* Check if any error reported */ 1448 if (!is_error_reported) 1449 return stat; 1450 1451 /* Decode BCH error using ELM module */ 1452 elm_decode_bch_error_page(info->elm_dev, ecc_vec, err_vec); 1453 1454 err = 0; 1455 for (i = 0; i < eccsteps; i++) { 1456 if (err_vec[i].error_uncorrectable) { 1457 dev_err(&info->pdev->dev, 1458 "uncorrectable bit-flips found\n"); 1459 err = -EBADMSG; 1460 } else if (err_vec[i].error_reported) { 1461 for (j = 0; j < err_vec[i].error_count; j++) { 1462 switch (info->ecc_opt) { 1463 case OMAP_ECC_BCH4_CODE_HW: 1464 /* Add 4 bits to take care of padding */ 1465 pos = err_vec[i].error_loc[j] + 1466 BCH4_BIT_PAD; 1467 break; 1468 case OMAP_ECC_BCH8_CODE_HW: 1469 case OMAP_ECC_BCH16_CODE_HW: 1470 pos = err_vec[i].error_loc[j]; 1471 break; 1472 default: 1473 return -EINVAL; 1474 } 1475 error_max = (ecc->size + actual_eccbytes) * 8; 1476 /* Calculate bit position of error */ 1477 bit_pos = pos % 8; 1478 1479 /* Calculate byte position of error */ 1480 byte_pos = (error_max - pos - 1) / 8; 1481 1482 if (pos < error_max) { 1483 if (byte_pos < 512) { 1484 pr_debug("bitflip@dat[%d]=%x\n", 1485 byte_pos, data[byte_pos]); 1486 data[byte_pos] ^= 1 << bit_pos; 1487 } else { 1488 pr_debug("bitflip@oob[%d]=%x\n", 1489 (byte_pos - 512), 1490 spare_ecc[byte_pos - 512]); 1491 spare_ecc[byte_pos - 512] ^= 1492 1 << bit_pos; 1493 } 1494 } else { 1495 dev_err(&info->pdev->dev, 1496 "invalid bit-flip @ %d:%d\n", 1497 byte_pos, bit_pos); 1498 err = -EBADMSG; 1499 } 1500 } 1501 } 1502 1503 /* Update number of correctable errors */ 1504 stat = max_t(unsigned int, stat, err_vec[i].error_count); 1505 1506 /* Update page data with sector size */ 1507 data += ecc->size; 1508 spare_ecc += ecc->bytes; 1509 } 1510 1511 return (err) ? err : stat; 1512 } 1513 1514 /** 1515 * omap_write_page_bch - BCH ecc based write page function for entire page 1516 * @chip: nand chip info structure 1517 * @buf: data buffer 1518 * @oob_required: must write chip->oob_poi to OOB 1519 * @page: page 1520 * 1521 * Custom write page method evolved to support multi sector writing in one shot 1522 */ 1523 static int omap_write_page_bch(struct nand_chip *chip, const uint8_t *buf, 1524 int oob_required, int page) 1525 { 1526 struct mtd_info *mtd = nand_to_mtd(chip); 1527 int ret; 1528 uint8_t *ecc_calc = chip->ecc.calc_buf; 1529 1530 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1531 1532 /* Enable GPMC ecc engine */ 1533 chip->ecc.hwctl(chip, NAND_ECC_WRITE); 1534 1535 /* Write data */ 1536 chip->legacy.write_buf(chip, buf, mtd->writesize); 1537 1538 /* Update ecc vector from GPMC result registers */ 1539 omap_calculate_ecc_bch_multi(mtd, buf, &ecc_calc[0]); 1540 1541 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0, 1542 chip->ecc.total); 1543 if (ret) 1544 return ret; 1545 1546 /* Write ecc vector to OOB area */ 1547 chip->legacy.write_buf(chip, chip->oob_poi, mtd->oobsize); 1548 1549 return nand_prog_page_end_op(chip); 1550 } 1551 1552 /** 1553 * omap_write_subpage_bch - BCH hardware ECC based subpage write 1554 * @chip: nand chip info structure 1555 * @offset: column address of subpage within the page 1556 * @data_len: data length 1557 * @buf: data buffer 1558 * @oob_required: must write chip->oob_poi to OOB 1559 * @page: page number to write 1560 * 1561 * OMAP optimized subpage write method. 1562 */ 1563 static int omap_write_subpage_bch(struct nand_chip *chip, u32 offset, 1564 u32 data_len, const u8 *buf, 1565 int oob_required, int page) 1566 { 1567 struct mtd_info *mtd = nand_to_mtd(chip); 1568 u8 *ecc_calc = chip->ecc.calc_buf; 1569 int ecc_size = chip->ecc.size; 1570 int ecc_bytes = chip->ecc.bytes; 1571 int ecc_steps = chip->ecc.steps; 1572 u32 start_step = offset / ecc_size; 1573 u32 end_step = (offset + data_len - 1) / ecc_size; 1574 int step, ret = 0; 1575 1576 /* 1577 * Write entire page at one go as it would be optimal 1578 * as ECC is calculated by hardware. 1579 * ECC is calculated for all subpages but we choose 1580 * only what we want. 1581 */ 1582 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1583 1584 /* Enable GPMC ECC engine */ 1585 chip->ecc.hwctl(chip, NAND_ECC_WRITE); 1586 1587 /* Write data */ 1588 chip->legacy.write_buf(chip, buf, mtd->writesize); 1589 1590 for (step = 0; step < ecc_steps; step++) { 1591 /* mask ECC of un-touched subpages by padding 0xFF */ 1592 if (step < start_step || step > end_step) 1593 memset(ecc_calc, 0xff, ecc_bytes); 1594 else 1595 ret = _omap_calculate_ecc_bch(mtd, buf, ecc_calc, step); 1596 1597 if (ret) 1598 return ret; 1599 1600 buf += ecc_size; 1601 ecc_calc += ecc_bytes; 1602 } 1603 1604 /* copy calculated ECC for whole page to chip->buffer->oob */ 1605 /* this include masked-value(0xFF) for unwritten subpages */ 1606 ecc_calc = chip->ecc.calc_buf; 1607 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0, 1608 chip->ecc.total); 1609 if (ret) 1610 return ret; 1611 1612 /* write OOB buffer to NAND device */ 1613 chip->legacy.write_buf(chip, chip->oob_poi, mtd->oobsize); 1614 1615 return nand_prog_page_end_op(chip); 1616 } 1617 1618 /** 1619 * omap_read_page_bch - BCH ecc based page read function for entire page 1620 * @chip: nand chip info structure 1621 * @buf: buffer to store read data 1622 * @oob_required: caller requires OOB data read to chip->oob_poi 1623 * @page: page number to read 1624 * 1625 * For BCH ecc scheme, GPMC used for syndrome calculation and ELM module 1626 * used for error correction. 1627 * Custom method evolved to support ELM error correction & multi sector 1628 * reading. On reading page data area is read along with OOB data with 1629 * ecc engine enabled. ecc vector updated after read of OOB data. 1630 * For non error pages ecc vector reported as zero. 1631 */ 1632 static int omap_read_page_bch(struct nand_chip *chip, uint8_t *buf, 1633 int oob_required, int page) 1634 { 1635 struct mtd_info *mtd = nand_to_mtd(chip); 1636 uint8_t *ecc_calc = chip->ecc.calc_buf; 1637 uint8_t *ecc_code = chip->ecc.code_buf; 1638 int stat, ret; 1639 unsigned int max_bitflips = 0; 1640 1641 nand_read_page_op(chip, page, 0, NULL, 0); 1642 1643 /* Enable GPMC ecc engine */ 1644 chip->ecc.hwctl(chip, NAND_ECC_READ); 1645 1646 /* Read data */ 1647 chip->legacy.read_buf(chip, buf, mtd->writesize); 1648 1649 /* Read oob bytes */ 1650 nand_change_read_column_op(chip, 1651 mtd->writesize + BADBLOCK_MARKER_LENGTH, 1652 chip->oob_poi + BADBLOCK_MARKER_LENGTH, 1653 chip->ecc.total, false); 1654 1655 /* Calculate ecc bytes */ 1656 omap_calculate_ecc_bch_multi(mtd, buf, ecc_calc); 1657 1658 ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0, 1659 chip->ecc.total); 1660 if (ret) 1661 return ret; 1662 1663 stat = chip->ecc.correct(chip, buf, ecc_code, ecc_calc); 1664 1665 if (stat < 0) { 1666 mtd->ecc_stats.failed++; 1667 } else { 1668 mtd->ecc_stats.corrected += stat; 1669 max_bitflips = max_t(unsigned int, max_bitflips, stat); 1670 } 1671 1672 return max_bitflips; 1673 } 1674 1675 /** 1676 * is_elm_present - checks for presence of ELM module by scanning DT nodes 1677 * @info: NAND device structure containing platform data 1678 * @elm_node: ELM's DT node 1679 */ 1680 static bool is_elm_present(struct omap_nand_info *info, 1681 struct device_node *elm_node) 1682 { 1683 struct platform_device *pdev; 1684 1685 /* check whether elm-id is passed via DT */ 1686 if (!elm_node) { 1687 dev_err(&info->pdev->dev, "ELM devicetree node not found\n"); 1688 return false; 1689 } 1690 pdev = of_find_device_by_node(elm_node); 1691 /* check whether ELM device is registered */ 1692 if (!pdev) { 1693 dev_err(&info->pdev->dev, "ELM device not found\n"); 1694 return false; 1695 } 1696 /* ELM module available, now configure it */ 1697 info->elm_dev = &pdev->dev; 1698 return true; 1699 } 1700 1701 static bool omap2_nand_ecc_check(struct omap_nand_info *info) 1702 { 1703 bool ecc_needs_bch, ecc_needs_omap_bch, ecc_needs_elm; 1704 1705 switch (info->ecc_opt) { 1706 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: 1707 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: 1708 ecc_needs_omap_bch = false; 1709 ecc_needs_bch = true; 1710 ecc_needs_elm = false; 1711 break; 1712 case OMAP_ECC_BCH4_CODE_HW: 1713 case OMAP_ECC_BCH8_CODE_HW: 1714 case OMAP_ECC_BCH16_CODE_HW: 1715 ecc_needs_omap_bch = true; 1716 ecc_needs_bch = false; 1717 ecc_needs_elm = true; 1718 break; 1719 default: 1720 ecc_needs_omap_bch = false; 1721 ecc_needs_bch = false; 1722 ecc_needs_elm = false; 1723 break; 1724 } 1725 1726 if (ecc_needs_bch && !IS_ENABLED(CONFIG_MTD_NAND_ECC_SW_BCH)) { 1727 dev_err(&info->pdev->dev, 1728 "CONFIG_MTD_NAND_ECC_SW_BCH not enabled\n"); 1729 return false; 1730 } 1731 if (ecc_needs_omap_bch && !IS_ENABLED(CONFIG_MTD_NAND_OMAP_BCH)) { 1732 dev_err(&info->pdev->dev, 1733 "CONFIG_MTD_NAND_OMAP_BCH not enabled\n"); 1734 return false; 1735 } 1736 if (ecc_needs_elm && !is_elm_present(info, info->elm_of_node)) { 1737 dev_err(&info->pdev->dev, "ELM not available\n"); 1738 return false; 1739 } 1740 1741 return true; 1742 } 1743 1744 static const char * const nand_xfer_types[] = { 1745 [NAND_OMAP_PREFETCH_POLLED] = "prefetch-polled", 1746 [NAND_OMAP_POLLED] = "polled", 1747 [NAND_OMAP_PREFETCH_DMA] = "prefetch-dma", 1748 [NAND_OMAP_PREFETCH_IRQ] = "prefetch-irq", 1749 }; 1750 1751 static int omap_get_dt_info(struct device *dev, struct omap_nand_info *info) 1752 { 1753 struct device_node *child = dev->of_node; 1754 int i; 1755 const char *s; 1756 u32 cs; 1757 1758 if (of_property_read_u32(child, "reg", &cs) < 0) { 1759 dev_err(dev, "reg not found in DT\n"); 1760 return -EINVAL; 1761 } 1762 1763 info->gpmc_cs = cs; 1764 1765 /* detect availability of ELM module. Won't be present pre-OMAP4 */ 1766 info->elm_of_node = of_parse_phandle(child, "ti,elm-id", 0); 1767 if (!info->elm_of_node) { 1768 info->elm_of_node = of_parse_phandle(child, "elm_id", 0); 1769 if (!info->elm_of_node) 1770 dev_dbg(dev, "ti,elm-id not in DT\n"); 1771 } 1772 1773 /* select ecc-scheme for NAND */ 1774 if (of_property_read_string(child, "ti,nand-ecc-opt", &s)) { 1775 dev_err(dev, "ti,nand-ecc-opt not found\n"); 1776 return -EINVAL; 1777 } 1778 1779 if (!strcmp(s, "sw")) { 1780 info->ecc_opt = OMAP_ECC_HAM1_CODE_SW; 1781 } else if (!strcmp(s, "ham1") || 1782 !strcmp(s, "hw") || !strcmp(s, "hw-romcode")) { 1783 info->ecc_opt = OMAP_ECC_HAM1_CODE_HW; 1784 } else if (!strcmp(s, "bch4")) { 1785 if (info->elm_of_node) 1786 info->ecc_opt = OMAP_ECC_BCH4_CODE_HW; 1787 else 1788 info->ecc_opt = OMAP_ECC_BCH4_CODE_HW_DETECTION_SW; 1789 } else if (!strcmp(s, "bch8")) { 1790 if (info->elm_of_node) 1791 info->ecc_opt = OMAP_ECC_BCH8_CODE_HW; 1792 else 1793 info->ecc_opt = OMAP_ECC_BCH8_CODE_HW_DETECTION_SW; 1794 } else if (!strcmp(s, "bch16")) { 1795 info->ecc_opt = OMAP_ECC_BCH16_CODE_HW; 1796 } else { 1797 dev_err(dev, "unrecognized value for ti,nand-ecc-opt\n"); 1798 return -EINVAL; 1799 } 1800 1801 /* select data transfer mode */ 1802 if (!of_property_read_string(child, "ti,nand-xfer-type", &s)) { 1803 for (i = 0; i < ARRAY_SIZE(nand_xfer_types); i++) { 1804 if (!strcasecmp(s, nand_xfer_types[i])) { 1805 info->xfer_type = i; 1806 return 0; 1807 } 1808 } 1809 1810 dev_err(dev, "unrecognized value for ti,nand-xfer-type\n"); 1811 return -EINVAL; 1812 } 1813 1814 return 0; 1815 } 1816 1817 static int omap_ooblayout_ecc(struct mtd_info *mtd, int section, 1818 struct mtd_oob_region *oobregion) 1819 { 1820 struct omap_nand_info *info = mtd_to_omap(mtd); 1821 struct nand_chip *chip = &info->nand; 1822 int off = BADBLOCK_MARKER_LENGTH; 1823 1824 if (info->ecc_opt == OMAP_ECC_HAM1_CODE_HW && 1825 !(chip->options & NAND_BUSWIDTH_16)) 1826 off = 1; 1827 1828 if (section) 1829 return -ERANGE; 1830 1831 oobregion->offset = off; 1832 oobregion->length = chip->ecc.total; 1833 1834 return 0; 1835 } 1836 1837 static int omap_ooblayout_free(struct mtd_info *mtd, int section, 1838 struct mtd_oob_region *oobregion) 1839 { 1840 struct omap_nand_info *info = mtd_to_omap(mtd); 1841 struct nand_chip *chip = &info->nand; 1842 int off = BADBLOCK_MARKER_LENGTH; 1843 1844 if (info->ecc_opt == OMAP_ECC_HAM1_CODE_HW && 1845 !(chip->options & NAND_BUSWIDTH_16)) 1846 off = 1; 1847 1848 if (section) 1849 return -ERANGE; 1850 1851 off += chip->ecc.total; 1852 if (off >= mtd->oobsize) 1853 return -ERANGE; 1854 1855 oobregion->offset = off; 1856 oobregion->length = mtd->oobsize - off; 1857 1858 return 0; 1859 } 1860 1861 static const struct mtd_ooblayout_ops omap_ooblayout_ops = { 1862 .ecc = omap_ooblayout_ecc, 1863 .free = omap_ooblayout_free, 1864 }; 1865 1866 static int omap_sw_ooblayout_ecc(struct mtd_info *mtd, int section, 1867 struct mtd_oob_region *oobregion) 1868 { 1869 struct nand_chip *chip = mtd_to_nand(mtd); 1870 int off = BADBLOCK_MARKER_LENGTH; 1871 1872 if (section >= chip->ecc.steps) 1873 return -ERANGE; 1874 1875 /* 1876 * When SW correction is employed, one OMAP specific marker byte is 1877 * reserved after each ECC step. 1878 */ 1879 oobregion->offset = off + (section * (chip->ecc.bytes + 1)); 1880 oobregion->length = chip->ecc.bytes; 1881 1882 return 0; 1883 } 1884 1885 static int omap_sw_ooblayout_free(struct mtd_info *mtd, int section, 1886 struct mtd_oob_region *oobregion) 1887 { 1888 struct nand_chip *chip = mtd_to_nand(mtd); 1889 int off = BADBLOCK_MARKER_LENGTH; 1890 1891 if (section) 1892 return -ERANGE; 1893 1894 /* 1895 * When SW correction is employed, one OMAP specific marker byte is 1896 * reserved after each ECC step. 1897 */ 1898 off += ((chip->ecc.bytes + 1) * chip->ecc.steps); 1899 if (off >= mtd->oobsize) 1900 return -ERANGE; 1901 1902 oobregion->offset = off; 1903 oobregion->length = mtd->oobsize - off; 1904 1905 return 0; 1906 } 1907 1908 static const struct mtd_ooblayout_ops omap_sw_ooblayout_ops = { 1909 .ecc = omap_sw_ooblayout_ecc, 1910 .free = omap_sw_ooblayout_free, 1911 }; 1912 1913 static int omap_nand_attach_chip(struct nand_chip *chip) 1914 { 1915 struct mtd_info *mtd = nand_to_mtd(chip); 1916 struct omap_nand_info *info = mtd_to_omap(mtd); 1917 struct device *dev = &info->pdev->dev; 1918 int min_oobbytes = BADBLOCK_MARKER_LENGTH; 1919 int oobbytes_per_step; 1920 dma_cap_mask_t mask; 1921 int err; 1922 1923 if (chip->bbt_options & NAND_BBT_USE_FLASH) 1924 chip->bbt_options |= NAND_BBT_NO_OOB; 1925 else 1926 chip->options |= NAND_SKIP_BBTSCAN; 1927 1928 /* Re-populate low-level callbacks based on xfer modes */ 1929 switch (info->xfer_type) { 1930 case NAND_OMAP_PREFETCH_POLLED: 1931 chip->legacy.read_buf = omap_read_buf_pref; 1932 chip->legacy.write_buf = omap_write_buf_pref; 1933 break; 1934 1935 case NAND_OMAP_POLLED: 1936 /* Use nand_base defaults for {read,write}_buf */ 1937 break; 1938 1939 case NAND_OMAP_PREFETCH_DMA: 1940 dma_cap_zero(mask); 1941 dma_cap_set(DMA_SLAVE, mask); 1942 info->dma = dma_request_chan(dev->parent, "rxtx"); 1943 1944 if (IS_ERR(info->dma)) { 1945 dev_err(dev, "DMA engine request failed\n"); 1946 return PTR_ERR(info->dma); 1947 } else { 1948 struct dma_slave_config cfg; 1949 1950 memset(&cfg, 0, sizeof(cfg)); 1951 cfg.src_addr = info->phys_base; 1952 cfg.dst_addr = info->phys_base; 1953 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 1954 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 1955 cfg.src_maxburst = 16; 1956 cfg.dst_maxburst = 16; 1957 err = dmaengine_slave_config(info->dma, &cfg); 1958 if (err) { 1959 dev_err(dev, 1960 "DMA engine slave config failed: %d\n", 1961 err); 1962 return err; 1963 } 1964 chip->legacy.read_buf = omap_read_buf_dma_pref; 1965 chip->legacy.write_buf = omap_write_buf_dma_pref; 1966 } 1967 break; 1968 1969 case NAND_OMAP_PREFETCH_IRQ: 1970 info->gpmc_irq_fifo = platform_get_irq(info->pdev, 0); 1971 if (info->gpmc_irq_fifo <= 0) 1972 return -ENODEV; 1973 err = devm_request_irq(dev, info->gpmc_irq_fifo, 1974 omap_nand_irq, IRQF_SHARED, 1975 "gpmc-nand-fifo", info); 1976 if (err) { 1977 dev_err(dev, "Requesting IRQ %d, error %d\n", 1978 info->gpmc_irq_fifo, err); 1979 info->gpmc_irq_fifo = 0; 1980 return err; 1981 } 1982 1983 info->gpmc_irq_count = platform_get_irq(info->pdev, 1); 1984 if (info->gpmc_irq_count <= 0) 1985 return -ENODEV; 1986 err = devm_request_irq(dev, info->gpmc_irq_count, 1987 omap_nand_irq, IRQF_SHARED, 1988 "gpmc-nand-count", info); 1989 if (err) { 1990 dev_err(dev, "Requesting IRQ %d, error %d\n", 1991 info->gpmc_irq_count, err); 1992 info->gpmc_irq_count = 0; 1993 return err; 1994 } 1995 1996 chip->legacy.read_buf = omap_read_buf_irq_pref; 1997 chip->legacy.write_buf = omap_write_buf_irq_pref; 1998 1999 break; 2000 2001 default: 2002 dev_err(dev, "xfer_type %d not supported!\n", info->xfer_type); 2003 return -EINVAL; 2004 } 2005 2006 if (!omap2_nand_ecc_check(info)) 2007 return -EINVAL; 2008 2009 /* 2010 * Bail out earlier to let NAND_ECC_ENGINE_TYPE_SOFT code create its own 2011 * ooblayout instead of using ours. 2012 */ 2013 if (info->ecc_opt == OMAP_ECC_HAM1_CODE_SW) { 2014 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; 2015 chip->ecc.algo = NAND_ECC_ALGO_HAMMING; 2016 return 0; 2017 } 2018 2019 /* Populate MTD interface based on ECC scheme */ 2020 switch (info->ecc_opt) { 2021 case OMAP_ECC_HAM1_CODE_HW: 2022 dev_info(dev, "nand: using OMAP_ECC_HAM1_CODE_HW\n"); 2023 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; 2024 chip->ecc.bytes = 3; 2025 chip->ecc.size = 512; 2026 chip->ecc.strength = 1; 2027 chip->ecc.calculate = omap_calculate_ecc; 2028 chip->ecc.hwctl = omap_enable_hwecc; 2029 chip->ecc.correct = omap_correct_data; 2030 mtd_set_ooblayout(mtd, &omap_ooblayout_ops); 2031 oobbytes_per_step = chip->ecc.bytes; 2032 2033 if (!(chip->options & NAND_BUSWIDTH_16)) 2034 min_oobbytes = 1; 2035 2036 break; 2037 2038 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: 2039 pr_info("nand: using OMAP_ECC_BCH4_CODE_HW_DETECTION_SW\n"); 2040 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; 2041 chip->ecc.size = 512; 2042 chip->ecc.bytes = 7; 2043 chip->ecc.strength = 4; 2044 chip->ecc.hwctl = omap_enable_hwecc_bch; 2045 chip->ecc.correct = rawnand_sw_bch_correct; 2046 chip->ecc.calculate = omap_calculate_ecc_bch_sw; 2047 mtd_set_ooblayout(mtd, &omap_sw_ooblayout_ops); 2048 /* Reserve one byte for the OMAP marker */ 2049 oobbytes_per_step = chip->ecc.bytes + 1; 2050 /* Software BCH library is used for locating errors */ 2051 err = rawnand_sw_bch_init(chip); 2052 if (err) { 2053 dev_err(dev, "Unable to use BCH library\n"); 2054 return err; 2055 } 2056 break; 2057 2058 case OMAP_ECC_BCH4_CODE_HW: 2059 pr_info("nand: using OMAP_ECC_BCH4_CODE_HW ECC scheme\n"); 2060 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; 2061 chip->ecc.size = 512; 2062 /* 14th bit is kept reserved for ROM-code compatibility */ 2063 chip->ecc.bytes = 7 + 1; 2064 chip->ecc.strength = 4; 2065 chip->ecc.hwctl = omap_enable_hwecc_bch; 2066 chip->ecc.correct = omap_elm_correct_data; 2067 chip->ecc.read_page = omap_read_page_bch; 2068 chip->ecc.write_page = omap_write_page_bch; 2069 chip->ecc.write_subpage = omap_write_subpage_bch; 2070 mtd_set_ooblayout(mtd, &omap_ooblayout_ops); 2071 oobbytes_per_step = chip->ecc.bytes; 2072 2073 err = elm_config(info->elm_dev, BCH4_ECC, 2074 mtd->writesize / chip->ecc.size, 2075 chip->ecc.size, chip->ecc.bytes); 2076 if (err < 0) 2077 return err; 2078 break; 2079 2080 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: 2081 pr_info("nand: using OMAP_ECC_BCH8_CODE_HW_DETECTION_SW\n"); 2082 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; 2083 chip->ecc.size = 512; 2084 chip->ecc.bytes = 13; 2085 chip->ecc.strength = 8; 2086 chip->ecc.hwctl = omap_enable_hwecc_bch; 2087 chip->ecc.correct = rawnand_sw_bch_correct; 2088 chip->ecc.calculate = omap_calculate_ecc_bch_sw; 2089 mtd_set_ooblayout(mtd, &omap_sw_ooblayout_ops); 2090 /* Reserve one byte for the OMAP marker */ 2091 oobbytes_per_step = chip->ecc.bytes + 1; 2092 /* Software BCH library is used for locating errors */ 2093 err = rawnand_sw_bch_init(chip); 2094 if (err) { 2095 dev_err(dev, "unable to use BCH library\n"); 2096 return err; 2097 } 2098 break; 2099 2100 case OMAP_ECC_BCH8_CODE_HW: 2101 pr_info("nand: using OMAP_ECC_BCH8_CODE_HW ECC scheme\n"); 2102 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; 2103 chip->ecc.size = 512; 2104 /* 14th bit is kept reserved for ROM-code compatibility */ 2105 chip->ecc.bytes = 13 + 1; 2106 chip->ecc.strength = 8; 2107 chip->ecc.hwctl = omap_enable_hwecc_bch; 2108 chip->ecc.correct = omap_elm_correct_data; 2109 chip->ecc.read_page = omap_read_page_bch; 2110 chip->ecc.write_page = omap_write_page_bch; 2111 chip->ecc.write_subpage = omap_write_subpage_bch; 2112 mtd_set_ooblayout(mtd, &omap_ooblayout_ops); 2113 oobbytes_per_step = chip->ecc.bytes; 2114 2115 err = elm_config(info->elm_dev, BCH8_ECC, 2116 mtd->writesize / chip->ecc.size, 2117 chip->ecc.size, chip->ecc.bytes); 2118 if (err < 0) 2119 return err; 2120 2121 break; 2122 2123 case OMAP_ECC_BCH16_CODE_HW: 2124 pr_info("Using OMAP_ECC_BCH16_CODE_HW ECC scheme\n"); 2125 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; 2126 chip->ecc.size = 512; 2127 chip->ecc.bytes = 26; 2128 chip->ecc.strength = 16; 2129 chip->ecc.hwctl = omap_enable_hwecc_bch; 2130 chip->ecc.correct = omap_elm_correct_data; 2131 chip->ecc.read_page = omap_read_page_bch; 2132 chip->ecc.write_page = omap_write_page_bch; 2133 chip->ecc.write_subpage = omap_write_subpage_bch; 2134 mtd_set_ooblayout(mtd, &omap_ooblayout_ops); 2135 oobbytes_per_step = chip->ecc.bytes; 2136 2137 err = elm_config(info->elm_dev, BCH16_ECC, 2138 mtd->writesize / chip->ecc.size, 2139 chip->ecc.size, chip->ecc.bytes); 2140 if (err < 0) 2141 return err; 2142 2143 break; 2144 default: 2145 dev_err(dev, "Invalid or unsupported ECC scheme\n"); 2146 return -EINVAL; 2147 } 2148 2149 /* Check if NAND device's OOB is enough to store ECC signatures */ 2150 min_oobbytes += (oobbytes_per_step * 2151 (mtd->writesize / chip->ecc.size)); 2152 if (mtd->oobsize < min_oobbytes) { 2153 dev_err(dev, 2154 "Not enough OOB bytes: required = %d, available=%d\n", 2155 min_oobbytes, mtd->oobsize); 2156 return -EINVAL; 2157 } 2158 2159 return 0; 2160 } 2161 2162 static const struct nand_controller_ops omap_nand_controller_ops = { 2163 .attach_chip = omap_nand_attach_chip, 2164 }; 2165 2166 /* Shared among all NAND instances to synchronize access to the ECC Engine */ 2167 static struct nand_controller omap_gpmc_controller; 2168 static bool omap_gpmc_controller_initialized; 2169 2170 static int omap_nand_probe(struct platform_device *pdev) 2171 { 2172 struct omap_nand_info *info; 2173 struct mtd_info *mtd; 2174 struct nand_chip *nand_chip; 2175 int err; 2176 struct resource *res; 2177 struct device *dev = &pdev->dev; 2178 2179 info = devm_kzalloc(&pdev->dev, sizeof(struct omap_nand_info), 2180 GFP_KERNEL); 2181 if (!info) 2182 return -ENOMEM; 2183 2184 info->pdev = pdev; 2185 2186 err = omap_get_dt_info(dev, info); 2187 if (err) 2188 return err; 2189 2190 info->ops = gpmc_omap_get_nand_ops(&info->reg, info->gpmc_cs); 2191 if (!info->ops) { 2192 dev_err(&pdev->dev, "Failed to get GPMC->NAND interface\n"); 2193 return -ENODEV; 2194 } 2195 2196 nand_chip = &info->nand; 2197 mtd = nand_to_mtd(nand_chip); 2198 mtd->dev.parent = &pdev->dev; 2199 nand_set_flash_node(nand_chip, dev->of_node); 2200 2201 if (!mtd->name) { 2202 mtd->name = devm_kasprintf(&pdev->dev, GFP_KERNEL, 2203 "omap2-nand.%d", info->gpmc_cs); 2204 if (!mtd->name) { 2205 dev_err(&pdev->dev, "Failed to set MTD name\n"); 2206 return -ENOMEM; 2207 } 2208 } 2209 2210 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2211 nand_chip->legacy.IO_ADDR_R = devm_ioremap_resource(&pdev->dev, res); 2212 if (IS_ERR(nand_chip->legacy.IO_ADDR_R)) 2213 return PTR_ERR(nand_chip->legacy.IO_ADDR_R); 2214 2215 info->phys_base = res->start; 2216 2217 if (!omap_gpmc_controller_initialized) { 2218 omap_gpmc_controller.ops = &omap_nand_controller_ops; 2219 nand_controller_init(&omap_gpmc_controller); 2220 omap_gpmc_controller_initialized = true; 2221 } 2222 2223 nand_chip->controller = &omap_gpmc_controller; 2224 2225 nand_chip->legacy.IO_ADDR_W = nand_chip->legacy.IO_ADDR_R; 2226 nand_chip->legacy.cmd_ctrl = omap_hwcontrol; 2227 2228 info->ready_gpiod = devm_gpiod_get_optional(&pdev->dev, "rb", 2229 GPIOD_IN); 2230 if (IS_ERR(info->ready_gpiod)) { 2231 dev_err(dev, "failed to get ready gpio\n"); 2232 return PTR_ERR(info->ready_gpiod); 2233 } 2234 2235 /* 2236 * If RDY/BSY line is connected to OMAP then use the omap ready 2237 * function and the generic nand_wait function which reads the status 2238 * register after monitoring the RDY/BSY line. Otherwise use a standard 2239 * chip delay which is slightly more than tR (AC Timing) of the NAND 2240 * device and read status register until you get a failure or success 2241 */ 2242 if (info->ready_gpiod) { 2243 nand_chip->legacy.dev_ready = omap_dev_ready; 2244 nand_chip->legacy.chip_delay = 0; 2245 } else { 2246 nand_chip->legacy.waitfunc = omap_wait; 2247 nand_chip->legacy.chip_delay = 50; 2248 } 2249 2250 if (info->flash_bbt) 2251 nand_chip->bbt_options |= NAND_BBT_USE_FLASH; 2252 2253 /* scan NAND device connected to chip controller */ 2254 nand_chip->options |= info->devsize & NAND_BUSWIDTH_16; 2255 2256 err = nand_scan(nand_chip, 1); 2257 if (err) 2258 goto return_error; 2259 2260 err = mtd_device_register(mtd, NULL, 0); 2261 if (err) 2262 goto cleanup_nand; 2263 2264 platform_set_drvdata(pdev, mtd); 2265 2266 return 0; 2267 2268 cleanup_nand: 2269 nand_cleanup(nand_chip); 2270 2271 return_error: 2272 if (!IS_ERR_OR_NULL(info->dma)) 2273 dma_release_channel(info->dma); 2274 2275 rawnand_sw_bch_cleanup(nand_chip); 2276 2277 return err; 2278 } 2279 2280 static int omap_nand_remove(struct platform_device *pdev) 2281 { 2282 struct mtd_info *mtd = platform_get_drvdata(pdev); 2283 struct nand_chip *nand_chip = mtd_to_nand(mtd); 2284 struct omap_nand_info *info = mtd_to_omap(mtd); 2285 int ret; 2286 2287 rawnand_sw_bch_cleanup(nand_chip); 2288 2289 if (info->dma) 2290 dma_release_channel(info->dma); 2291 ret = mtd_device_unregister(mtd); 2292 WARN_ON(ret); 2293 nand_cleanup(nand_chip); 2294 return ret; 2295 } 2296 2297 static const struct of_device_id omap_nand_ids[] = { 2298 { .compatible = "ti,omap2-nand", }, 2299 {}, 2300 }; 2301 MODULE_DEVICE_TABLE(of, omap_nand_ids); 2302 2303 static struct platform_driver omap_nand_driver = { 2304 .probe = omap_nand_probe, 2305 .remove = omap_nand_remove, 2306 .driver = { 2307 .name = DRIVER_NAME, 2308 .of_match_table = of_match_ptr(omap_nand_ids), 2309 }, 2310 }; 2311 2312 module_platform_driver(omap_nand_driver); 2313 2314 MODULE_ALIAS("platform:" DRIVER_NAME); 2315 MODULE_LICENSE("GPL"); 2316 MODULE_DESCRIPTION("Glue layer for NAND flash on TI OMAP boards"); 2317