1 /* 2 * Copyright (c) 2016, The Linux Foundation. All rights reserved. 3 * 4 * This software is licensed under the terms of the GNU General Public 5 * License version 2, as published by the Free Software Foundation, and 6 * may be copied, distributed, and modified under those terms. 7 * 8 * This program is distributed in the hope that it will be useful, 9 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * GNU General Public License for more details. 12 */ 13 14 #include <linux/clk.h> 15 #include <linux/slab.h> 16 #include <linux/bitops.h> 17 #include <linux/dma-mapping.h> 18 #include <linux/dmaengine.h> 19 #include <linux/module.h> 20 #include <linux/mtd/rawnand.h> 21 #include <linux/mtd/partitions.h> 22 #include <linux/of.h> 23 #include <linux/of_device.h> 24 #include <linux/delay.h> 25 #include <linux/dma/qcom_bam_dma.h> 26 #include <linux/dma-direct.h> /* XXX: drivers shall never use this directly! */ 27 28 /* NANDc reg offsets */ 29 #define NAND_FLASH_CMD 0x00 30 #define NAND_ADDR0 0x04 31 #define NAND_ADDR1 0x08 32 #define NAND_FLASH_CHIP_SELECT 0x0c 33 #define NAND_EXEC_CMD 0x10 34 #define NAND_FLASH_STATUS 0x14 35 #define NAND_BUFFER_STATUS 0x18 36 #define NAND_DEV0_CFG0 0x20 37 #define NAND_DEV0_CFG1 0x24 38 #define NAND_DEV0_ECC_CFG 0x28 39 #define NAND_DEV1_ECC_CFG 0x2c 40 #define NAND_DEV1_CFG0 0x30 41 #define NAND_DEV1_CFG1 0x34 42 #define NAND_READ_ID 0x40 43 #define NAND_READ_STATUS 0x44 44 #define NAND_DEV_CMD0 0xa0 45 #define NAND_DEV_CMD1 0xa4 46 #define NAND_DEV_CMD2 0xa8 47 #define NAND_DEV_CMD_VLD 0xac 48 #define SFLASHC_BURST_CFG 0xe0 49 #define NAND_ERASED_CW_DETECT_CFG 0xe8 50 #define NAND_ERASED_CW_DETECT_STATUS 0xec 51 #define NAND_EBI2_ECC_BUF_CFG 0xf0 52 #define FLASH_BUF_ACC 0x100 53 54 #define NAND_CTRL 0xf00 55 #define NAND_VERSION 0xf08 56 #define NAND_READ_LOCATION_0 0xf20 57 #define NAND_READ_LOCATION_1 0xf24 58 #define NAND_READ_LOCATION_2 0xf28 59 #define NAND_READ_LOCATION_3 0xf2c 60 61 /* dummy register offsets, used by write_reg_dma */ 62 #define NAND_DEV_CMD1_RESTORE 0xdead 63 #define NAND_DEV_CMD_VLD_RESTORE 0xbeef 64 65 /* NAND_FLASH_CMD bits */ 66 #define PAGE_ACC BIT(4) 67 #define LAST_PAGE BIT(5) 68 69 /* NAND_FLASH_CHIP_SELECT bits */ 70 #define NAND_DEV_SEL 0 71 #define DM_EN BIT(2) 72 73 /* NAND_FLASH_STATUS bits */ 74 #define FS_OP_ERR BIT(4) 75 #define FS_READY_BSY_N BIT(5) 76 #define FS_MPU_ERR BIT(8) 77 #define FS_DEVICE_STS_ERR BIT(16) 78 #define FS_DEVICE_WP BIT(23) 79 80 /* NAND_BUFFER_STATUS bits */ 81 #define BS_UNCORRECTABLE_BIT BIT(8) 82 #define BS_CORRECTABLE_ERR_MSK 0x1f 83 84 /* NAND_DEVn_CFG0 bits */ 85 #define DISABLE_STATUS_AFTER_WRITE 4 86 #define CW_PER_PAGE 6 87 #define UD_SIZE_BYTES 9 88 #define ECC_PARITY_SIZE_BYTES_RS 19 89 #define SPARE_SIZE_BYTES 23 90 #define NUM_ADDR_CYCLES 27 91 #define STATUS_BFR_READ 30 92 #define SET_RD_MODE_AFTER_STATUS 31 93 94 /* NAND_DEVn_CFG0 bits */ 95 #define DEV0_CFG1_ECC_DISABLE 0 96 #define WIDE_FLASH 1 97 #define NAND_RECOVERY_CYCLES 2 98 #define CS_ACTIVE_BSY 5 99 #define BAD_BLOCK_BYTE_NUM 6 100 #define BAD_BLOCK_IN_SPARE_AREA 16 101 #define WR_RD_BSY_GAP 17 102 #define ENABLE_BCH_ECC 27 103 104 /* NAND_DEV0_ECC_CFG bits */ 105 #define ECC_CFG_ECC_DISABLE 0 106 #define ECC_SW_RESET 1 107 #define ECC_MODE 4 108 #define ECC_PARITY_SIZE_BYTES_BCH 8 109 #define ECC_NUM_DATA_BYTES 16 110 #define ECC_FORCE_CLK_OPEN 30 111 112 /* NAND_DEV_CMD1 bits */ 113 #define READ_ADDR 0 114 115 /* NAND_DEV_CMD_VLD bits */ 116 #define READ_START_VLD BIT(0) 117 #define READ_STOP_VLD BIT(1) 118 #define WRITE_START_VLD BIT(2) 119 #define ERASE_START_VLD BIT(3) 120 #define SEQ_READ_START_VLD BIT(4) 121 122 /* NAND_EBI2_ECC_BUF_CFG bits */ 123 #define NUM_STEPS 0 124 125 /* NAND_ERASED_CW_DETECT_CFG bits */ 126 #define ERASED_CW_ECC_MASK 1 127 #define AUTO_DETECT_RES 0 128 #define MASK_ECC (1 << ERASED_CW_ECC_MASK) 129 #define RESET_ERASED_DET (1 << AUTO_DETECT_RES) 130 #define ACTIVE_ERASED_DET (0 << AUTO_DETECT_RES) 131 #define CLR_ERASED_PAGE_DET (RESET_ERASED_DET | MASK_ECC) 132 #define SET_ERASED_PAGE_DET (ACTIVE_ERASED_DET | MASK_ECC) 133 134 /* NAND_ERASED_CW_DETECT_STATUS bits */ 135 #define PAGE_ALL_ERASED BIT(7) 136 #define CODEWORD_ALL_ERASED BIT(6) 137 #define PAGE_ERASED BIT(5) 138 #define CODEWORD_ERASED BIT(4) 139 #define ERASED_PAGE (PAGE_ALL_ERASED | PAGE_ERASED) 140 #define ERASED_CW (CODEWORD_ALL_ERASED | CODEWORD_ERASED) 141 142 /* NAND_READ_LOCATION_n bits */ 143 #define READ_LOCATION_OFFSET 0 144 #define READ_LOCATION_SIZE 16 145 #define READ_LOCATION_LAST 31 146 147 /* Version Mask */ 148 #define NAND_VERSION_MAJOR_MASK 0xf0000000 149 #define NAND_VERSION_MAJOR_SHIFT 28 150 #define NAND_VERSION_MINOR_MASK 0x0fff0000 151 #define NAND_VERSION_MINOR_SHIFT 16 152 153 /* NAND OP_CMDs */ 154 #define PAGE_READ 0x2 155 #define PAGE_READ_WITH_ECC 0x3 156 #define PAGE_READ_WITH_ECC_SPARE 0x4 157 #define PROGRAM_PAGE 0x6 158 #define PAGE_PROGRAM_WITH_ECC 0x7 159 #define PROGRAM_PAGE_SPARE 0x9 160 #define BLOCK_ERASE 0xa 161 #define FETCH_ID 0xb 162 #define RESET_DEVICE 0xd 163 164 /* Default Value for NAND_DEV_CMD_VLD */ 165 #define NAND_DEV_CMD_VLD_VAL (READ_START_VLD | WRITE_START_VLD | \ 166 ERASE_START_VLD | SEQ_READ_START_VLD) 167 168 /* NAND_CTRL bits */ 169 #define BAM_MODE_EN BIT(0) 170 171 /* 172 * the NAND controller performs reads/writes with ECC in 516 byte chunks. 173 * the driver calls the chunks 'step' or 'codeword' interchangeably 174 */ 175 #define NANDC_STEP_SIZE 512 176 177 /* 178 * the largest page size we support is 8K, this will have 16 steps/codewords 179 * of 512 bytes each 180 */ 181 #define MAX_NUM_STEPS (SZ_8K / NANDC_STEP_SIZE) 182 183 /* we read at most 3 registers per codeword scan */ 184 #define MAX_REG_RD (3 * MAX_NUM_STEPS) 185 186 /* ECC modes supported by the controller */ 187 #define ECC_NONE BIT(0) 188 #define ECC_RS_4BIT BIT(1) 189 #define ECC_BCH_4BIT BIT(2) 190 #define ECC_BCH_8BIT BIT(3) 191 192 #define nandc_set_read_loc(nandc, reg, offset, size, is_last) \ 193 nandc_set_reg(nandc, NAND_READ_LOCATION_##reg, \ 194 ((offset) << READ_LOCATION_OFFSET) | \ 195 ((size) << READ_LOCATION_SIZE) | \ 196 ((is_last) << READ_LOCATION_LAST)) 197 198 /* 199 * Returns the actual register address for all NAND_DEV_ registers 200 * (i.e. NAND_DEV_CMD0, NAND_DEV_CMD1, NAND_DEV_CMD2 and NAND_DEV_CMD_VLD) 201 */ 202 #define dev_cmd_reg_addr(nandc, reg) ((nandc)->props->dev_cmd_reg_start + (reg)) 203 204 /* Returns the NAND register physical address */ 205 #define nandc_reg_phys(chip, offset) ((chip)->base_phys + (offset)) 206 207 /* Returns the dma address for reg read buffer */ 208 #define reg_buf_dma_addr(chip, vaddr) \ 209 ((chip)->reg_read_dma + \ 210 ((uint8_t *)(vaddr) - (uint8_t *)(chip)->reg_read_buf)) 211 212 #define QPIC_PER_CW_CMD_ELEMENTS 32 213 #define QPIC_PER_CW_CMD_SGL 32 214 #define QPIC_PER_CW_DATA_SGL 8 215 216 /* 217 * Flags used in DMA descriptor preparation helper functions 218 * (i.e. read_reg_dma/write_reg_dma/read_data_dma/write_data_dma) 219 */ 220 /* Don't set the EOT in current tx BAM sgl */ 221 #define NAND_BAM_NO_EOT BIT(0) 222 /* Set the NWD flag in current BAM sgl */ 223 #define NAND_BAM_NWD BIT(1) 224 /* Finish writing in the current BAM sgl and start writing in another BAM sgl */ 225 #define NAND_BAM_NEXT_SGL BIT(2) 226 /* 227 * Erased codeword status is being used two times in single transfer so this 228 * flag will determine the current value of erased codeword status register 229 */ 230 #define NAND_ERASED_CW_SET BIT(4) 231 232 /* 233 * This data type corresponds to the BAM transaction which will be used for all 234 * NAND transfers. 235 * @bam_ce - the array of BAM command elements 236 * @cmd_sgl - sgl for NAND BAM command pipe 237 * @data_sgl - sgl for NAND BAM consumer/producer pipe 238 * @bam_ce_pos - the index in bam_ce which is available for next sgl 239 * @bam_ce_start - the index in bam_ce which marks the start position ce 240 * for current sgl. It will be used for size calculation 241 * for current sgl 242 * @cmd_sgl_pos - current index in command sgl. 243 * @cmd_sgl_start - start index in command sgl. 244 * @tx_sgl_pos - current index in data sgl for tx. 245 * @tx_sgl_start - start index in data sgl for tx. 246 * @rx_sgl_pos - current index in data sgl for rx. 247 * @rx_sgl_start - start index in data sgl for rx. 248 */ 249 struct bam_transaction { 250 struct bam_cmd_element *bam_ce; 251 struct scatterlist *cmd_sgl; 252 struct scatterlist *data_sgl; 253 u32 bam_ce_pos; 254 u32 bam_ce_start; 255 u32 cmd_sgl_pos; 256 u32 cmd_sgl_start; 257 u32 tx_sgl_pos; 258 u32 tx_sgl_start; 259 u32 rx_sgl_pos; 260 u32 rx_sgl_start; 261 }; 262 263 /* 264 * This data type corresponds to the nand dma descriptor 265 * @list - list for desc_info 266 * @dir - DMA transfer direction 267 * @adm_sgl - sgl which will be used for single sgl dma descriptor. Only used by 268 * ADM 269 * @bam_sgl - sgl which will be used for dma descriptor. Only used by BAM 270 * @sgl_cnt - number of SGL in bam_sgl. Only used by BAM 271 * @dma_desc - low level DMA engine descriptor 272 */ 273 struct desc_info { 274 struct list_head node; 275 276 enum dma_data_direction dir; 277 union { 278 struct scatterlist adm_sgl; 279 struct { 280 struct scatterlist *bam_sgl; 281 int sgl_cnt; 282 }; 283 }; 284 struct dma_async_tx_descriptor *dma_desc; 285 }; 286 287 /* 288 * holds the current register values that we want to write. acts as a contiguous 289 * chunk of memory which we use to write the controller registers through DMA. 290 */ 291 struct nandc_regs { 292 __le32 cmd; 293 __le32 addr0; 294 __le32 addr1; 295 __le32 chip_sel; 296 __le32 exec; 297 298 __le32 cfg0; 299 __le32 cfg1; 300 __le32 ecc_bch_cfg; 301 302 __le32 clrflashstatus; 303 __le32 clrreadstatus; 304 305 __le32 cmd1; 306 __le32 vld; 307 308 __le32 orig_cmd1; 309 __le32 orig_vld; 310 311 __le32 ecc_buf_cfg; 312 __le32 read_location0; 313 __le32 read_location1; 314 __le32 read_location2; 315 __le32 read_location3; 316 317 __le32 erased_cw_detect_cfg_clr; 318 __le32 erased_cw_detect_cfg_set; 319 }; 320 321 /* 322 * NAND controller data struct 323 * 324 * @controller: base controller structure 325 * @host_list: list containing all the chips attached to the 326 * controller 327 * @dev: parent device 328 * @base: MMIO base 329 * @base_phys: physical base address of controller registers 330 * @base_dma: dma base address of controller registers 331 * @core_clk: controller clock 332 * @aon_clk: another controller clock 333 * 334 * @chan: dma channel 335 * @cmd_crci: ADM DMA CRCI for command flow control 336 * @data_crci: ADM DMA CRCI for data flow control 337 * @desc_list: DMA descriptor list (list of desc_infos) 338 * 339 * @data_buffer: our local DMA buffer for page read/writes, 340 * used when we can't use the buffer provided 341 * by upper layers directly 342 * @buf_size/count/start: markers for chip->read_buf/write_buf functions 343 * @reg_read_buf: local buffer for reading back registers via DMA 344 * @reg_read_dma: contains dma address for register read buffer 345 * @reg_read_pos: marker for data read in reg_read_buf 346 * 347 * @regs: a contiguous chunk of memory for DMA register 348 * writes. contains the register values to be 349 * written to controller 350 * @cmd1/vld: some fixed controller register values 351 * @props: properties of current NAND controller, 352 * initialized via DT match data 353 * @max_cwperpage: maximum QPIC codewords required. calculated 354 * from all connected NAND devices pagesize 355 */ 356 struct qcom_nand_controller { 357 struct nand_hw_control controller; 358 struct list_head host_list; 359 360 struct device *dev; 361 362 void __iomem *base; 363 phys_addr_t base_phys; 364 dma_addr_t base_dma; 365 366 struct clk *core_clk; 367 struct clk *aon_clk; 368 369 union { 370 /* will be used only by QPIC for BAM DMA */ 371 struct { 372 struct dma_chan *tx_chan; 373 struct dma_chan *rx_chan; 374 struct dma_chan *cmd_chan; 375 }; 376 377 /* will be used only by EBI2 for ADM DMA */ 378 struct { 379 struct dma_chan *chan; 380 unsigned int cmd_crci; 381 unsigned int data_crci; 382 }; 383 }; 384 385 struct list_head desc_list; 386 struct bam_transaction *bam_txn; 387 388 u8 *data_buffer; 389 int buf_size; 390 int buf_count; 391 int buf_start; 392 unsigned int max_cwperpage; 393 394 __le32 *reg_read_buf; 395 dma_addr_t reg_read_dma; 396 int reg_read_pos; 397 398 struct nandc_regs *regs; 399 400 u32 cmd1, vld; 401 const struct qcom_nandc_props *props; 402 }; 403 404 /* 405 * NAND chip structure 406 * 407 * @chip: base NAND chip structure 408 * @node: list node to add itself to host_list in 409 * qcom_nand_controller 410 * 411 * @cs: chip select value for this chip 412 * @cw_size: the number of bytes in a single step/codeword 413 * of a page, consisting of all data, ecc, spare 414 * and reserved bytes 415 * @cw_data: the number of bytes within a codeword protected 416 * by ECC 417 * @use_ecc: request the controller to use ECC for the 418 * upcoming read/write 419 * @bch_enabled: flag to tell whether BCH ECC mode is used 420 * @ecc_bytes_hw: ECC bytes used by controller hardware for this 421 * chip 422 * @status: value to be returned if NAND_CMD_STATUS command 423 * is executed 424 * @last_command: keeps track of last command on this chip. used 425 * for reading correct status 426 * 427 * @cfg0, cfg1, cfg0_raw..: NANDc register configurations needed for 428 * ecc/non-ecc mode for the current nand flash 429 * device 430 */ 431 struct qcom_nand_host { 432 struct nand_chip chip; 433 struct list_head node; 434 435 int cs; 436 int cw_size; 437 int cw_data; 438 bool use_ecc; 439 bool bch_enabled; 440 int ecc_bytes_hw; 441 int spare_bytes; 442 int bbm_size; 443 u8 status; 444 int last_command; 445 446 u32 cfg0, cfg1; 447 u32 cfg0_raw, cfg1_raw; 448 u32 ecc_buf_cfg; 449 u32 ecc_bch_cfg; 450 u32 clrflashstatus; 451 u32 clrreadstatus; 452 }; 453 454 /* 455 * This data type corresponds to the NAND controller properties which varies 456 * among different NAND controllers. 457 * @ecc_modes - ecc mode for NAND 458 * @is_bam - whether NAND controller is using BAM 459 * @dev_cmd_reg_start - NAND_DEV_CMD_* registers starting offset 460 */ 461 struct qcom_nandc_props { 462 u32 ecc_modes; 463 bool is_bam; 464 u32 dev_cmd_reg_start; 465 }; 466 467 /* Frees the BAM transaction memory */ 468 static void free_bam_transaction(struct qcom_nand_controller *nandc) 469 { 470 struct bam_transaction *bam_txn = nandc->bam_txn; 471 472 devm_kfree(nandc->dev, bam_txn); 473 } 474 475 /* Allocates and Initializes the BAM transaction */ 476 static struct bam_transaction * 477 alloc_bam_transaction(struct qcom_nand_controller *nandc) 478 { 479 struct bam_transaction *bam_txn; 480 size_t bam_txn_size; 481 unsigned int num_cw = nandc->max_cwperpage; 482 void *bam_txn_buf; 483 484 bam_txn_size = 485 sizeof(*bam_txn) + num_cw * 486 ((sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS) + 487 (sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL) + 488 (sizeof(*bam_txn->data_sgl) * QPIC_PER_CW_DATA_SGL)); 489 490 bam_txn_buf = devm_kzalloc(nandc->dev, bam_txn_size, GFP_KERNEL); 491 if (!bam_txn_buf) 492 return NULL; 493 494 bam_txn = bam_txn_buf; 495 bam_txn_buf += sizeof(*bam_txn); 496 497 bam_txn->bam_ce = bam_txn_buf; 498 bam_txn_buf += 499 sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS * num_cw; 500 501 bam_txn->cmd_sgl = bam_txn_buf; 502 bam_txn_buf += 503 sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL * num_cw; 504 505 bam_txn->data_sgl = bam_txn_buf; 506 507 return bam_txn; 508 } 509 510 /* Clears the BAM transaction indexes */ 511 static void clear_bam_transaction(struct qcom_nand_controller *nandc) 512 { 513 struct bam_transaction *bam_txn = nandc->bam_txn; 514 515 if (!nandc->props->is_bam) 516 return; 517 518 bam_txn->bam_ce_pos = 0; 519 bam_txn->bam_ce_start = 0; 520 bam_txn->cmd_sgl_pos = 0; 521 bam_txn->cmd_sgl_start = 0; 522 bam_txn->tx_sgl_pos = 0; 523 bam_txn->tx_sgl_start = 0; 524 bam_txn->rx_sgl_pos = 0; 525 bam_txn->rx_sgl_start = 0; 526 527 sg_init_table(bam_txn->cmd_sgl, nandc->max_cwperpage * 528 QPIC_PER_CW_CMD_SGL); 529 sg_init_table(bam_txn->data_sgl, nandc->max_cwperpage * 530 QPIC_PER_CW_DATA_SGL); 531 } 532 533 static inline struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip) 534 { 535 return container_of(chip, struct qcom_nand_host, chip); 536 } 537 538 static inline struct qcom_nand_controller * 539 get_qcom_nand_controller(struct nand_chip *chip) 540 { 541 return container_of(chip->controller, struct qcom_nand_controller, 542 controller); 543 } 544 545 static inline u32 nandc_read(struct qcom_nand_controller *nandc, int offset) 546 { 547 return ioread32(nandc->base + offset); 548 } 549 550 static inline void nandc_write(struct qcom_nand_controller *nandc, int offset, 551 u32 val) 552 { 553 iowrite32(val, nandc->base + offset); 554 } 555 556 static inline void nandc_read_buffer_sync(struct qcom_nand_controller *nandc, 557 bool is_cpu) 558 { 559 if (!nandc->props->is_bam) 560 return; 561 562 if (is_cpu) 563 dma_sync_single_for_cpu(nandc->dev, nandc->reg_read_dma, 564 MAX_REG_RD * 565 sizeof(*nandc->reg_read_buf), 566 DMA_FROM_DEVICE); 567 else 568 dma_sync_single_for_device(nandc->dev, nandc->reg_read_dma, 569 MAX_REG_RD * 570 sizeof(*nandc->reg_read_buf), 571 DMA_FROM_DEVICE); 572 } 573 574 static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset) 575 { 576 switch (offset) { 577 case NAND_FLASH_CMD: 578 return ®s->cmd; 579 case NAND_ADDR0: 580 return ®s->addr0; 581 case NAND_ADDR1: 582 return ®s->addr1; 583 case NAND_FLASH_CHIP_SELECT: 584 return ®s->chip_sel; 585 case NAND_EXEC_CMD: 586 return ®s->exec; 587 case NAND_FLASH_STATUS: 588 return ®s->clrflashstatus; 589 case NAND_DEV0_CFG0: 590 return ®s->cfg0; 591 case NAND_DEV0_CFG1: 592 return ®s->cfg1; 593 case NAND_DEV0_ECC_CFG: 594 return ®s->ecc_bch_cfg; 595 case NAND_READ_STATUS: 596 return ®s->clrreadstatus; 597 case NAND_DEV_CMD1: 598 return ®s->cmd1; 599 case NAND_DEV_CMD1_RESTORE: 600 return ®s->orig_cmd1; 601 case NAND_DEV_CMD_VLD: 602 return ®s->vld; 603 case NAND_DEV_CMD_VLD_RESTORE: 604 return ®s->orig_vld; 605 case NAND_EBI2_ECC_BUF_CFG: 606 return ®s->ecc_buf_cfg; 607 case NAND_READ_LOCATION_0: 608 return ®s->read_location0; 609 case NAND_READ_LOCATION_1: 610 return ®s->read_location1; 611 case NAND_READ_LOCATION_2: 612 return ®s->read_location2; 613 case NAND_READ_LOCATION_3: 614 return ®s->read_location3; 615 default: 616 return NULL; 617 } 618 } 619 620 static void nandc_set_reg(struct qcom_nand_controller *nandc, int offset, 621 u32 val) 622 { 623 struct nandc_regs *regs = nandc->regs; 624 __le32 *reg; 625 626 reg = offset_to_nandc_reg(regs, offset); 627 628 if (reg) 629 *reg = cpu_to_le32(val); 630 } 631 632 /* helper to configure address register values */ 633 static void set_address(struct qcom_nand_host *host, u16 column, int page) 634 { 635 struct nand_chip *chip = &host->chip; 636 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 637 638 if (chip->options & NAND_BUSWIDTH_16) 639 column >>= 1; 640 641 nandc_set_reg(nandc, NAND_ADDR0, page << 16 | column); 642 nandc_set_reg(nandc, NAND_ADDR1, page >> 16 & 0xff); 643 } 644 645 /* 646 * update_rw_regs: set up read/write register values, these will be 647 * written to the NAND controller registers via DMA 648 * 649 * @num_cw: number of steps for the read/write operation 650 * @read: read or write operation 651 */ 652 static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read) 653 { 654 struct nand_chip *chip = &host->chip; 655 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 656 u32 cmd, cfg0, cfg1, ecc_bch_cfg; 657 658 if (read) { 659 if (host->use_ecc) 660 cmd = PAGE_READ_WITH_ECC | PAGE_ACC | LAST_PAGE; 661 else 662 cmd = PAGE_READ | PAGE_ACC | LAST_PAGE; 663 } else { 664 cmd = PROGRAM_PAGE | PAGE_ACC | LAST_PAGE; 665 } 666 667 if (host->use_ecc) { 668 cfg0 = (host->cfg0 & ~(7U << CW_PER_PAGE)) | 669 (num_cw - 1) << CW_PER_PAGE; 670 671 cfg1 = host->cfg1; 672 ecc_bch_cfg = host->ecc_bch_cfg; 673 } else { 674 cfg0 = (host->cfg0_raw & ~(7U << CW_PER_PAGE)) | 675 (num_cw - 1) << CW_PER_PAGE; 676 677 cfg1 = host->cfg1_raw; 678 ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE; 679 } 680 681 nandc_set_reg(nandc, NAND_FLASH_CMD, cmd); 682 nandc_set_reg(nandc, NAND_DEV0_CFG0, cfg0); 683 nandc_set_reg(nandc, NAND_DEV0_CFG1, cfg1); 684 nandc_set_reg(nandc, NAND_DEV0_ECC_CFG, ecc_bch_cfg); 685 nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, host->ecc_buf_cfg); 686 nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus); 687 nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus); 688 nandc_set_reg(nandc, NAND_EXEC_CMD, 1); 689 690 if (read) 691 nandc_set_read_loc(nandc, 0, 0, host->use_ecc ? 692 host->cw_data : host->cw_size, 1); 693 } 694 695 /* 696 * Maps the scatter gather list for DMA transfer and forms the DMA descriptor 697 * for BAM. This descriptor will be added in the NAND DMA descriptor queue 698 * which will be submitted to DMA engine. 699 */ 700 static int prepare_bam_async_desc(struct qcom_nand_controller *nandc, 701 struct dma_chan *chan, 702 unsigned long flags) 703 { 704 struct desc_info *desc; 705 struct scatterlist *sgl; 706 unsigned int sgl_cnt; 707 int ret; 708 struct bam_transaction *bam_txn = nandc->bam_txn; 709 enum dma_transfer_direction dir_eng; 710 struct dma_async_tx_descriptor *dma_desc; 711 712 desc = kzalloc(sizeof(*desc), GFP_KERNEL); 713 if (!desc) 714 return -ENOMEM; 715 716 if (chan == nandc->cmd_chan) { 717 sgl = &bam_txn->cmd_sgl[bam_txn->cmd_sgl_start]; 718 sgl_cnt = bam_txn->cmd_sgl_pos - bam_txn->cmd_sgl_start; 719 bam_txn->cmd_sgl_start = bam_txn->cmd_sgl_pos; 720 dir_eng = DMA_MEM_TO_DEV; 721 desc->dir = DMA_TO_DEVICE; 722 } else if (chan == nandc->tx_chan) { 723 sgl = &bam_txn->data_sgl[bam_txn->tx_sgl_start]; 724 sgl_cnt = bam_txn->tx_sgl_pos - bam_txn->tx_sgl_start; 725 bam_txn->tx_sgl_start = bam_txn->tx_sgl_pos; 726 dir_eng = DMA_MEM_TO_DEV; 727 desc->dir = DMA_TO_DEVICE; 728 } else { 729 sgl = &bam_txn->data_sgl[bam_txn->rx_sgl_start]; 730 sgl_cnt = bam_txn->rx_sgl_pos - bam_txn->rx_sgl_start; 731 bam_txn->rx_sgl_start = bam_txn->rx_sgl_pos; 732 dir_eng = DMA_DEV_TO_MEM; 733 desc->dir = DMA_FROM_DEVICE; 734 } 735 736 sg_mark_end(sgl + sgl_cnt - 1); 737 ret = dma_map_sg(nandc->dev, sgl, sgl_cnt, desc->dir); 738 if (ret == 0) { 739 dev_err(nandc->dev, "failure in mapping desc\n"); 740 kfree(desc); 741 return -ENOMEM; 742 } 743 744 desc->sgl_cnt = sgl_cnt; 745 desc->bam_sgl = sgl; 746 747 dma_desc = dmaengine_prep_slave_sg(chan, sgl, sgl_cnt, dir_eng, 748 flags); 749 750 if (!dma_desc) { 751 dev_err(nandc->dev, "failure in prep desc\n"); 752 dma_unmap_sg(nandc->dev, sgl, sgl_cnt, desc->dir); 753 kfree(desc); 754 return -EINVAL; 755 } 756 757 desc->dma_desc = dma_desc; 758 759 list_add_tail(&desc->node, &nandc->desc_list); 760 761 return 0; 762 } 763 764 /* 765 * Prepares the command descriptor for BAM DMA which will be used for NAND 766 * register reads and writes. The command descriptor requires the command 767 * to be formed in command element type so this function uses the command 768 * element from bam transaction ce array and fills the same with required 769 * data. A single SGL can contain multiple command elements so 770 * NAND_BAM_NEXT_SGL will be used for starting the separate SGL 771 * after the current command element. 772 */ 773 static int prep_bam_dma_desc_cmd(struct qcom_nand_controller *nandc, bool read, 774 int reg_off, const void *vaddr, 775 int size, unsigned int flags) 776 { 777 int bam_ce_size; 778 int i, ret; 779 struct bam_cmd_element *bam_ce_buffer; 780 struct bam_transaction *bam_txn = nandc->bam_txn; 781 782 bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_pos]; 783 784 /* fill the command desc */ 785 for (i = 0; i < size; i++) { 786 if (read) 787 bam_prep_ce(&bam_ce_buffer[i], 788 nandc_reg_phys(nandc, reg_off + 4 * i), 789 BAM_READ_COMMAND, 790 reg_buf_dma_addr(nandc, 791 (__le32 *)vaddr + i)); 792 else 793 bam_prep_ce_le32(&bam_ce_buffer[i], 794 nandc_reg_phys(nandc, reg_off + 4 * i), 795 BAM_WRITE_COMMAND, 796 *((__le32 *)vaddr + i)); 797 } 798 799 bam_txn->bam_ce_pos += size; 800 801 /* use the separate sgl after this command */ 802 if (flags & NAND_BAM_NEXT_SGL) { 803 bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_start]; 804 bam_ce_size = (bam_txn->bam_ce_pos - 805 bam_txn->bam_ce_start) * 806 sizeof(struct bam_cmd_element); 807 sg_set_buf(&bam_txn->cmd_sgl[bam_txn->cmd_sgl_pos], 808 bam_ce_buffer, bam_ce_size); 809 bam_txn->cmd_sgl_pos++; 810 bam_txn->bam_ce_start = bam_txn->bam_ce_pos; 811 812 if (flags & NAND_BAM_NWD) { 813 ret = prepare_bam_async_desc(nandc, nandc->cmd_chan, 814 DMA_PREP_FENCE | 815 DMA_PREP_CMD); 816 if (ret) 817 return ret; 818 } 819 } 820 821 return 0; 822 } 823 824 /* 825 * Prepares the data descriptor for BAM DMA which will be used for NAND 826 * data reads and writes. 827 */ 828 static int prep_bam_dma_desc_data(struct qcom_nand_controller *nandc, bool read, 829 const void *vaddr, 830 int size, unsigned int flags) 831 { 832 int ret; 833 struct bam_transaction *bam_txn = nandc->bam_txn; 834 835 if (read) { 836 sg_set_buf(&bam_txn->data_sgl[bam_txn->rx_sgl_pos], 837 vaddr, size); 838 bam_txn->rx_sgl_pos++; 839 } else { 840 sg_set_buf(&bam_txn->data_sgl[bam_txn->tx_sgl_pos], 841 vaddr, size); 842 bam_txn->tx_sgl_pos++; 843 844 /* 845 * BAM will only set EOT for DMA_PREP_INTERRUPT so if this flag 846 * is not set, form the DMA descriptor 847 */ 848 if (!(flags & NAND_BAM_NO_EOT)) { 849 ret = prepare_bam_async_desc(nandc, nandc->tx_chan, 850 DMA_PREP_INTERRUPT); 851 if (ret) 852 return ret; 853 } 854 } 855 856 return 0; 857 } 858 859 static int prep_adm_dma_desc(struct qcom_nand_controller *nandc, bool read, 860 int reg_off, const void *vaddr, int size, 861 bool flow_control) 862 { 863 struct desc_info *desc; 864 struct dma_async_tx_descriptor *dma_desc; 865 struct scatterlist *sgl; 866 struct dma_slave_config slave_conf; 867 enum dma_transfer_direction dir_eng; 868 int ret; 869 870 desc = kzalloc(sizeof(*desc), GFP_KERNEL); 871 if (!desc) 872 return -ENOMEM; 873 874 sgl = &desc->adm_sgl; 875 876 sg_init_one(sgl, vaddr, size); 877 878 if (read) { 879 dir_eng = DMA_DEV_TO_MEM; 880 desc->dir = DMA_FROM_DEVICE; 881 } else { 882 dir_eng = DMA_MEM_TO_DEV; 883 desc->dir = DMA_TO_DEVICE; 884 } 885 886 ret = dma_map_sg(nandc->dev, sgl, 1, desc->dir); 887 if (ret == 0) { 888 ret = -ENOMEM; 889 goto err; 890 } 891 892 memset(&slave_conf, 0x00, sizeof(slave_conf)); 893 894 slave_conf.device_fc = flow_control; 895 if (read) { 896 slave_conf.src_maxburst = 16; 897 slave_conf.src_addr = nandc->base_dma + reg_off; 898 slave_conf.slave_id = nandc->data_crci; 899 } else { 900 slave_conf.dst_maxburst = 16; 901 slave_conf.dst_addr = nandc->base_dma + reg_off; 902 slave_conf.slave_id = nandc->cmd_crci; 903 } 904 905 ret = dmaengine_slave_config(nandc->chan, &slave_conf); 906 if (ret) { 907 dev_err(nandc->dev, "failed to configure dma channel\n"); 908 goto err; 909 } 910 911 dma_desc = dmaengine_prep_slave_sg(nandc->chan, sgl, 1, dir_eng, 0); 912 if (!dma_desc) { 913 dev_err(nandc->dev, "failed to prepare desc\n"); 914 ret = -EINVAL; 915 goto err; 916 } 917 918 desc->dma_desc = dma_desc; 919 920 list_add_tail(&desc->node, &nandc->desc_list); 921 922 return 0; 923 err: 924 kfree(desc); 925 926 return ret; 927 } 928 929 /* 930 * read_reg_dma: prepares a descriptor to read a given number of 931 * contiguous registers to the reg_read_buf pointer 932 * 933 * @first: offset of the first register in the contiguous block 934 * @num_regs: number of registers to read 935 * @flags: flags to control DMA descriptor preparation 936 */ 937 static int read_reg_dma(struct qcom_nand_controller *nandc, int first, 938 int num_regs, unsigned int flags) 939 { 940 bool flow_control = false; 941 void *vaddr; 942 943 vaddr = nandc->reg_read_buf + nandc->reg_read_pos; 944 nandc->reg_read_pos += num_regs; 945 946 if (first == NAND_DEV_CMD_VLD || first == NAND_DEV_CMD1) 947 first = dev_cmd_reg_addr(nandc, first); 948 949 if (nandc->props->is_bam) 950 return prep_bam_dma_desc_cmd(nandc, true, first, vaddr, 951 num_regs, flags); 952 953 if (first == NAND_READ_ID || first == NAND_FLASH_STATUS) 954 flow_control = true; 955 956 return prep_adm_dma_desc(nandc, true, first, vaddr, 957 num_regs * sizeof(u32), flow_control); 958 } 959 960 /* 961 * write_reg_dma: prepares a descriptor to write a given number of 962 * contiguous registers 963 * 964 * @first: offset of the first register in the contiguous block 965 * @num_regs: number of registers to write 966 * @flags: flags to control DMA descriptor preparation 967 */ 968 static int write_reg_dma(struct qcom_nand_controller *nandc, int first, 969 int num_regs, unsigned int flags) 970 { 971 bool flow_control = false; 972 struct nandc_regs *regs = nandc->regs; 973 void *vaddr; 974 975 vaddr = offset_to_nandc_reg(regs, first); 976 977 if (first == NAND_ERASED_CW_DETECT_CFG) { 978 if (flags & NAND_ERASED_CW_SET) 979 vaddr = ®s->erased_cw_detect_cfg_set; 980 else 981 vaddr = ®s->erased_cw_detect_cfg_clr; 982 } 983 984 if (first == NAND_EXEC_CMD) 985 flags |= NAND_BAM_NWD; 986 987 if (first == NAND_DEV_CMD1_RESTORE || first == NAND_DEV_CMD1) 988 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD1); 989 990 if (first == NAND_DEV_CMD_VLD_RESTORE || first == NAND_DEV_CMD_VLD) 991 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD); 992 993 if (nandc->props->is_bam) 994 return prep_bam_dma_desc_cmd(nandc, false, first, vaddr, 995 num_regs, flags); 996 997 if (first == NAND_FLASH_CMD) 998 flow_control = true; 999 1000 return prep_adm_dma_desc(nandc, false, first, vaddr, 1001 num_regs * sizeof(u32), flow_control); 1002 } 1003 1004 /* 1005 * read_data_dma: prepares a DMA descriptor to transfer data from the 1006 * controller's internal buffer to the buffer 'vaddr' 1007 * 1008 * @reg_off: offset within the controller's data buffer 1009 * @vaddr: virtual address of the buffer we want to write to 1010 * @size: DMA transaction size in bytes 1011 * @flags: flags to control DMA descriptor preparation 1012 */ 1013 static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off, 1014 const u8 *vaddr, int size, unsigned int flags) 1015 { 1016 if (nandc->props->is_bam) 1017 return prep_bam_dma_desc_data(nandc, true, vaddr, size, flags); 1018 1019 return prep_adm_dma_desc(nandc, true, reg_off, vaddr, size, false); 1020 } 1021 1022 /* 1023 * write_data_dma: prepares a DMA descriptor to transfer data from 1024 * 'vaddr' to the controller's internal buffer 1025 * 1026 * @reg_off: offset within the controller's data buffer 1027 * @vaddr: virtual address of the buffer we want to read from 1028 * @size: DMA transaction size in bytes 1029 * @flags: flags to control DMA descriptor preparation 1030 */ 1031 static int write_data_dma(struct qcom_nand_controller *nandc, int reg_off, 1032 const u8 *vaddr, int size, unsigned int flags) 1033 { 1034 if (nandc->props->is_bam) 1035 return prep_bam_dma_desc_data(nandc, false, vaddr, size, flags); 1036 1037 return prep_adm_dma_desc(nandc, false, reg_off, vaddr, size, false); 1038 } 1039 1040 /* 1041 * Helper to prepare DMA descriptors for configuring registers 1042 * before reading a NAND page. 1043 */ 1044 static void config_nand_page_read(struct qcom_nand_controller *nandc) 1045 { 1046 write_reg_dma(nandc, NAND_ADDR0, 2, 0); 1047 write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0); 1048 write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1, 0); 1049 write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1, 0); 1050 write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1, 1051 NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL); 1052 } 1053 1054 /* 1055 * Helper to prepare DMA descriptors for configuring registers 1056 * before reading each codeword in NAND page. 1057 */ 1058 static void config_nand_cw_read(struct qcom_nand_controller *nandc) 1059 { 1060 if (nandc->props->is_bam) 1061 write_reg_dma(nandc, NAND_READ_LOCATION_0, 4, 1062 NAND_BAM_NEXT_SGL); 1063 1064 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL); 1065 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1066 1067 read_reg_dma(nandc, NAND_FLASH_STATUS, 2, 0); 1068 read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1, 1069 NAND_BAM_NEXT_SGL); 1070 } 1071 1072 /* 1073 * Helper to prepare dma descriptors to configure registers needed for reading a 1074 * single codeword in page 1075 */ 1076 static void config_nand_single_cw_page_read(struct qcom_nand_controller *nandc) 1077 { 1078 config_nand_page_read(nandc); 1079 config_nand_cw_read(nandc); 1080 } 1081 1082 /* 1083 * Helper to prepare DMA descriptors used to configure registers needed for 1084 * before writing a NAND page. 1085 */ 1086 static void config_nand_page_write(struct qcom_nand_controller *nandc) 1087 { 1088 write_reg_dma(nandc, NAND_ADDR0, 2, 0); 1089 write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0); 1090 write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1, 1091 NAND_BAM_NEXT_SGL); 1092 } 1093 1094 /* 1095 * Helper to prepare DMA descriptors for configuring registers 1096 * before writing each codeword in NAND page. 1097 */ 1098 static void config_nand_cw_write(struct qcom_nand_controller *nandc) 1099 { 1100 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL); 1101 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1102 1103 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL); 1104 1105 write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0); 1106 write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL); 1107 } 1108 1109 /* 1110 * the following functions are used within chip->cmdfunc() to perform different 1111 * NAND_CMD_* commands 1112 */ 1113 1114 /* sets up descriptors for NAND_CMD_PARAM */ 1115 static int nandc_param(struct qcom_nand_host *host) 1116 { 1117 struct nand_chip *chip = &host->chip; 1118 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1119 1120 /* 1121 * NAND_CMD_PARAM is called before we know much about the FLASH chip 1122 * in use. we configure the controller to perform a raw read of 512 1123 * bytes to read onfi params 1124 */ 1125 nandc_set_reg(nandc, NAND_FLASH_CMD, PAGE_READ | PAGE_ACC | LAST_PAGE); 1126 nandc_set_reg(nandc, NAND_ADDR0, 0); 1127 nandc_set_reg(nandc, NAND_ADDR1, 0); 1128 nandc_set_reg(nandc, NAND_DEV0_CFG0, 0 << CW_PER_PAGE 1129 | 512 << UD_SIZE_BYTES 1130 | 5 << NUM_ADDR_CYCLES 1131 | 0 << SPARE_SIZE_BYTES); 1132 nandc_set_reg(nandc, NAND_DEV0_CFG1, 7 << NAND_RECOVERY_CYCLES 1133 | 0 << CS_ACTIVE_BSY 1134 | 17 << BAD_BLOCK_BYTE_NUM 1135 | 1 << BAD_BLOCK_IN_SPARE_AREA 1136 | 2 << WR_RD_BSY_GAP 1137 | 0 << WIDE_FLASH 1138 | 1 << DEV0_CFG1_ECC_DISABLE); 1139 nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, 1 << ECC_CFG_ECC_DISABLE); 1140 1141 /* configure CMD1 and VLD for ONFI param probing */ 1142 nandc_set_reg(nandc, NAND_DEV_CMD_VLD, 1143 (nandc->vld & ~READ_START_VLD)); 1144 nandc_set_reg(nandc, NAND_DEV_CMD1, 1145 (nandc->cmd1 & ~(0xFF << READ_ADDR)) 1146 | NAND_CMD_PARAM << READ_ADDR); 1147 1148 nandc_set_reg(nandc, NAND_EXEC_CMD, 1); 1149 1150 nandc_set_reg(nandc, NAND_DEV_CMD1_RESTORE, nandc->cmd1); 1151 nandc_set_reg(nandc, NAND_DEV_CMD_VLD_RESTORE, nandc->vld); 1152 nandc_set_read_loc(nandc, 0, 0, 512, 1); 1153 1154 write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1, 0); 1155 write_reg_dma(nandc, NAND_DEV_CMD1, 1, NAND_BAM_NEXT_SGL); 1156 1157 nandc->buf_count = 512; 1158 memset(nandc->data_buffer, 0xff, nandc->buf_count); 1159 1160 config_nand_single_cw_page_read(nandc); 1161 1162 read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, 1163 nandc->buf_count, 0); 1164 1165 /* restore CMD1 and VLD regs */ 1166 write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1, 0); 1167 write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1, NAND_BAM_NEXT_SGL); 1168 1169 return 0; 1170 } 1171 1172 /* sets up descriptors for NAND_CMD_ERASE1 */ 1173 static int erase_block(struct qcom_nand_host *host, int page_addr) 1174 { 1175 struct nand_chip *chip = &host->chip; 1176 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1177 1178 nandc_set_reg(nandc, NAND_FLASH_CMD, 1179 BLOCK_ERASE | PAGE_ACC | LAST_PAGE); 1180 nandc_set_reg(nandc, NAND_ADDR0, page_addr); 1181 nandc_set_reg(nandc, NAND_ADDR1, 0); 1182 nandc_set_reg(nandc, NAND_DEV0_CFG0, 1183 host->cfg0_raw & ~(7 << CW_PER_PAGE)); 1184 nandc_set_reg(nandc, NAND_DEV0_CFG1, host->cfg1_raw); 1185 nandc_set_reg(nandc, NAND_EXEC_CMD, 1); 1186 nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus); 1187 nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus); 1188 1189 write_reg_dma(nandc, NAND_FLASH_CMD, 3, NAND_BAM_NEXT_SGL); 1190 write_reg_dma(nandc, NAND_DEV0_CFG0, 2, NAND_BAM_NEXT_SGL); 1191 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1192 1193 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL); 1194 1195 write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0); 1196 write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL); 1197 1198 return 0; 1199 } 1200 1201 /* sets up descriptors for NAND_CMD_READID */ 1202 static int read_id(struct qcom_nand_host *host, int column) 1203 { 1204 struct nand_chip *chip = &host->chip; 1205 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1206 1207 if (column == -1) 1208 return 0; 1209 1210 nandc_set_reg(nandc, NAND_FLASH_CMD, FETCH_ID); 1211 nandc_set_reg(nandc, NAND_ADDR0, column); 1212 nandc_set_reg(nandc, NAND_ADDR1, 0); 1213 nandc_set_reg(nandc, NAND_FLASH_CHIP_SELECT, 1214 nandc->props->is_bam ? 0 : DM_EN); 1215 nandc_set_reg(nandc, NAND_EXEC_CMD, 1); 1216 1217 write_reg_dma(nandc, NAND_FLASH_CMD, 4, NAND_BAM_NEXT_SGL); 1218 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1219 1220 read_reg_dma(nandc, NAND_READ_ID, 1, NAND_BAM_NEXT_SGL); 1221 1222 return 0; 1223 } 1224 1225 /* sets up descriptors for NAND_CMD_RESET */ 1226 static int reset(struct qcom_nand_host *host) 1227 { 1228 struct nand_chip *chip = &host->chip; 1229 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1230 1231 nandc_set_reg(nandc, NAND_FLASH_CMD, RESET_DEVICE); 1232 nandc_set_reg(nandc, NAND_EXEC_CMD, 1); 1233 1234 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL); 1235 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1236 1237 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL); 1238 1239 return 0; 1240 } 1241 1242 /* helpers to submit/free our list of dma descriptors */ 1243 static int submit_descs(struct qcom_nand_controller *nandc) 1244 { 1245 struct desc_info *desc; 1246 dma_cookie_t cookie = 0; 1247 struct bam_transaction *bam_txn = nandc->bam_txn; 1248 int r; 1249 1250 if (nandc->props->is_bam) { 1251 if (bam_txn->rx_sgl_pos > bam_txn->rx_sgl_start) { 1252 r = prepare_bam_async_desc(nandc, nandc->rx_chan, 0); 1253 if (r) 1254 return r; 1255 } 1256 1257 if (bam_txn->tx_sgl_pos > bam_txn->tx_sgl_start) { 1258 r = prepare_bam_async_desc(nandc, nandc->tx_chan, 1259 DMA_PREP_INTERRUPT); 1260 if (r) 1261 return r; 1262 } 1263 1264 if (bam_txn->cmd_sgl_pos > bam_txn->cmd_sgl_start) { 1265 r = prepare_bam_async_desc(nandc, nandc->cmd_chan, 1266 DMA_PREP_CMD); 1267 if (r) 1268 return r; 1269 } 1270 } 1271 1272 list_for_each_entry(desc, &nandc->desc_list, node) 1273 cookie = dmaengine_submit(desc->dma_desc); 1274 1275 if (nandc->props->is_bam) { 1276 dma_async_issue_pending(nandc->tx_chan); 1277 dma_async_issue_pending(nandc->rx_chan); 1278 1279 if (dma_sync_wait(nandc->cmd_chan, cookie) != DMA_COMPLETE) 1280 return -ETIMEDOUT; 1281 } else { 1282 if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE) 1283 return -ETIMEDOUT; 1284 } 1285 1286 return 0; 1287 } 1288 1289 static void free_descs(struct qcom_nand_controller *nandc) 1290 { 1291 struct desc_info *desc, *n; 1292 1293 list_for_each_entry_safe(desc, n, &nandc->desc_list, node) { 1294 list_del(&desc->node); 1295 1296 if (nandc->props->is_bam) 1297 dma_unmap_sg(nandc->dev, desc->bam_sgl, 1298 desc->sgl_cnt, desc->dir); 1299 else 1300 dma_unmap_sg(nandc->dev, &desc->adm_sgl, 1, 1301 desc->dir); 1302 1303 kfree(desc); 1304 } 1305 } 1306 1307 /* reset the register read buffer for next NAND operation */ 1308 static void clear_read_regs(struct qcom_nand_controller *nandc) 1309 { 1310 nandc->reg_read_pos = 0; 1311 nandc_read_buffer_sync(nandc, false); 1312 } 1313 1314 static void pre_command(struct qcom_nand_host *host, int command) 1315 { 1316 struct nand_chip *chip = &host->chip; 1317 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1318 1319 nandc->buf_count = 0; 1320 nandc->buf_start = 0; 1321 host->use_ecc = false; 1322 host->last_command = command; 1323 1324 clear_read_regs(nandc); 1325 1326 if (command == NAND_CMD_RESET || command == NAND_CMD_READID || 1327 command == NAND_CMD_PARAM || command == NAND_CMD_ERASE1) 1328 clear_bam_transaction(nandc); 1329 } 1330 1331 /* 1332 * this is called after NAND_CMD_PAGEPROG and NAND_CMD_ERASE1 to set our 1333 * privately maintained status byte, this status byte can be read after 1334 * NAND_CMD_STATUS is called 1335 */ 1336 static void parse_erase_write_errors(struct qcom_nand_host *host, int command) 1337 { 1338 struct nand_chip *chip = &host->chip; 1339 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1340 struct nand_ecc_ctrl *ecc = &chip->ecc; 1341 int num_cw; 1342 int i; 1343 1344 num_cw = command == NAND_CMD_PAGEPROG ? ecc->steps : 1; 1345 nandc_read_buffer_sync(nandc, true); 1346 1347 for (i = 0; i < num_cw; i++) { 1348 u32 flash_status = le32_to_cpu(nandc->reg_read_buf[i]); 1349 1350 if (flash_status & FS_MPU_ERR) 1351 host->status &= ~NAND_STATUS_WP; 1352 1353 if (flash_status & FS_OP_ERR || (i == (num_cw - 1) && 1354 (flash_status & 1355 FS_DEVICE_STS_ERR))) 1356 host->status |= NAND_STATUS_FAIL; 1357 } 1358 } 1359 1360 static void post_command(struct qcom_nand_host *host, int command) 1361 { 1362 struct nand_chip *chip = &host->chip; 1363 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1364 1365 switch (command) { 1366 case NAND_CMD_READID: 1367 nandc_read_buffer_sync(nandc, true); 1368 memcpy(nandc->data_buffer, nandc->reg_read_buf, 1369 nandc->buf_count); 1370 break; 1371 case NAND_CMD_PAGEPROG: 1372 case NAND_CMD_ERASE1: 1373 parse_erase_write_errors(host, command); 1374 break; 1375 default: 1376 break; 1377 } 1378 } 1379 1380 /* 1381 * Implements chip->cmdfunc. It's only used for a limited set of commands. 1382 * The rest of the commands wouldn't be called by upper layers. For example, 1383 * NAND_CMD_READOOB would never be called because we have our own versions 1384 * of read_oob ops for nand_ecc_ctrl. 1385 */ 1386 static void qcom_nandc_command(struct mtd_info *mtd, unsigned int command, 1387 int column, int page_addr) 1388 { 1389 struct nand_chip *chip = mtd_to_nand(mtd); 1390 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1391 struct nand_ecc_ctrl *ecc = &chip->ecc; 1392 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1393 bool wait = false; 1394 int ret = 0; 1395 1396 pre_command(host, command); 1397 1398 switch (command) { 1399 case NAND_CMD_RESET: 1400 ret = reset(host); 1401 wait = true; 1402 break; 1403 1404 case NAND_CMD_READID: 1405 nandc->buf_count = 4; 1406 ret = read_id(host, column); 1407 wait = true; 1408 break; 1409 1410 case NAND_CMD_PARAM: 1411 ret = nandc_param(host); 1412 wait = true; 1413 break; 1414 1415 case NAND_CMD_ERASE1: 1416 ret = erase_block(host, page_addr); 1417 wait = true; 1418 break; 1419 1420 case NAND_CMD_READ0: 1421 /* we read the entire page for now */ 1422 WARN_ON(column != 0); 1423 1424 host->use_ecc = true; 1425 set_address(host, 0, page_addr); 1426 update_rw_regs(host, ecc->steps, true); 1427 break; 1428 1429 case NAND_CMD_SEQIN: 1430 WARN_ON(column != 0); 1431 set_address(host, 0, page_addr); 1432 break; 1433 1434 case NAND_CMD_PAGEPROG: 1435 case NAND_CMD_STATUS: 1436 case NAND_CMD_NONE: 1437 default: 1438 break; 1439 } 1440 1441 if (ret) { 1442 dev_err(nandc->dev, "failure executing command %d\n", 1443 command); 1444 free_descs(nandc); 1445 return; 1446 } 1447 1448 if (wait) { 1449 ret = submit_descs(nandc); 1450 if (ret) 1451 dev_err(nandc->dev, 1452 "failure submitting descs for command %d\n", 1453 command); 1454 } 1455 1456 free_descs(nandc); 1457 1458 post_command(host, command); 1459 } 1460 1461 /* 1462 * when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read 1463 * an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS. 1464 * 1465 * when using RS ECC, the HW reports the same erros when reading an erased CW, 1466 * but it notifies that it is an erased CW by placing special characters at 1467 * certain offsets in the buffer. 1468 * 1469 * verify if the page is erased or not, and fix up the page for RS ECC by 1470 * replacing the special characters with 0xff. 1471 */ 1472 static bool erased_chunk_check_and_fixup(u8 *data_buf, int data_len) 1473 { 1474 u8 empty1, empty2; 1475 1476 /* 1477 * an erased page flags an error in NAND_FLASH_STATUS, check if the page 1478 * is erased by looking for 0x54s at offsets 3 and 175 from the 1479 * beginning of each codeword 1480 */ 1481 1482 empty1 = data_buf[3]; 1483 empty2 = data_buf[175]; 1484 1485 /* 1486 * if the erased codework markers, if they exist override them with 1487 * 0xffs 1488 */ 1489 if ((empty1 == 0x54 && empty2 == 0xff) || 1490 (empty1 == 0xff && empty2 == 0x54)) { 1491 data_buf[3] = 0xff; 1492 data_buf[175] = 0xff; 1493 } 1494 1495 /* 1496 * check if the entire chunk contains 0xffs or not. if it doesn't, then 1497 * restore the original values at the special offsets 1498 */ 1499 if (memchr_inv(data_buf, 0xff, data_len)) { 1500 data_buf[3] = empty1; 1501 data_buf[175] = empty2; 1502 1503 return false; 1504 } 1505 1506 return true; 1507 } 1508 1509 struct read_stats { 1510 __le32 flash; 1511 __le32 buffer; 1512 __le32 erased_cw; 1513 }; 1514 1515 /* 1516 * reads back status registers set by the controller to notify page read 1517 * errors. this is equivalent to what 'ecc->correct()' would do. 1518 */ 1519 static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf, 1520 u8 *oob_buf) 1521 { 1522 struct nand_chip *chip = &host->chip; 1523 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1524 struct mtd_info *mtd = nand_to_mtd(chip); 1525 struct nand_ecc_ctrl *ecc = &chip->ecc; 1526 unsigned int max_bitflips = 0; 1527 struct read_stats *buf; 1528 int i; 1529 1530 buf = (struct read_stats *)nandc->reg_read_buf; 1531 nandc_read_buffer_sync(nandc, true); 1532 1533 for (i = 0; i < ecc->steps; i++, buf++) { 1534 u32 flash, buffer, erased_cw; 1535 int data_len, oob_len; 1536 1537 if (i == (ecc->steps - 1)) { 1538 data_len = ecc->size - ((ecc->steps - 1) << 2); 1539 oob_len = ecc->steps << 2; 1540 } else { 1541 data_len = host->cw_data; 1542 oob_len = 0; 1543 } 1544 1545 flash = le32_to_cpu(buf->flash); 1546 buffer = le32_to_cpu(buf->buffer); 1547 erased_cw = le32_to_cpu(buf->erased_cw); 1548 1549 if (flash & (FS_OP_ERR | FS_MPU_ERR)) { 1550 bool erased; 1551 1552 /* ignore erased codeword errors */ 1553 if (host->bch_enabled) { 1554 erased = (erased_cw & ERASED_CW) == ERASED_CW ? 1555 true : false; 1556 } else { 1557 erased = erased_chunk_check_and_fixup(data_buf, 1558 data_len); 1559 } 1560 1561 if (erased) { 1562 data_buf += data_len; 1563 if (oob_buf) 1564 oob_buf += oob_len + ecc->bytes; 1565 continue; 1566 } 1567 1568 if (buffer & BS_UNCORRECTABLE_BIT) { 1569 int ret, ecclen, extraooblen; 1570 void *eccbuf; 1571 1572 eccbuf = oob_buf ? oob_buf + oob_len : NULL; 1573 ecclen = oob_buf ? host->ecc_bytes_hw : 0; 1574 extraooblen = oob_buf ? oob_len : 0; 1575 1576 /* 1577 * make sure it isn't an erased page reported 1578 * as not-erased by HW because of a few bitflips 1579 */ 1580 ret = nand_check_erased_ecc_chunk(data_buf, 1581 data_len, eccbuf, ecclen, oob_buf, 1582 extraooblen, ecc->strength); 1583 if (ret < 0) { 1584 mtd->ecc_stats.failed++; 1585 } else { 1586 mtd->ecc_stats.corrected += ret; 1587 max_bitflips = 1588 max_t(unsigned int, max_bitflips, ret); 1589 } 1590 } 1591 } else { 1592 unsigned int stat; 1593 1594 stat = buffer & BS_CORRECTABLE_ERR_MSK; 1595 mtd->ecc_stats.corrected += stat; 1596 max_bitflips = max(max_bitflips, stat); 1597 } 1598 1599 data_buf += data_len; 1600 if (oob_buf) 1601 oob_buf += oob_len + ecc->bytes; 1602 } 1603 1604 return max_bitflips; 1605 } 1606 1607 /* 1608 * helper to perform the actual page read operation, used by ecc->read_page(), 1609 * ecc->read_oob() 1610 */ 1611 static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf, 1612 u8 *oob_buf) 1613 { 1614 struct nand_chip *chip = &host->chip; 1615 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1616 struct nand_ecc_ctrl *ecc = &chip->ecc; 1617 int i, ret; 1618 1619 config_nand_page_read(nandc); 1620 1621 /* queue cmd descs for each codeword */ 1622 for (i = 0; i < ecc->steps; i++) { 1623 int data_size, oob_size; 1624 1625 if (i == (ecc->steps - 1)) { 1626 data_size = ecc->size - ((ecc->steps - 1) << 2); 1627 oob_size = (ecc->steps << 2) + host->ecc_bytes_hw + 1628 host->spare_bytes; 1629 } else { 1630 data_size = host->cw_data; 1631 oob_size = host->ecc_bytes_hw + host->spare_bytes; 1632 } 1633 1634 if (nandc->props->is_bam) { 1635 if (data_buf && oob_buf) { 1636 nandc_set_read_loc(nandc, 0, 0, data_size, 0); 1637 nandc_set_read_loc(nandc, 1, data_size, 1638 oob_size, 1); 1639 } else if (data_buf) { 1640 nandc_set_read_loc(nandc, 0, 0, data_size, 1); 1641 } else { 1642 nandc_set_read_loc(nandc, 0, data_size, 1643 oob_size, 1); 1644 } 1645 } 1646 1647 config_nand_cw_read(nandc); 1648 1649 if (data_buf) 1650 read_data_dma(nandc, FLASH_BUF_ACC, data_buf, 1651 data_size, 0); 1652 1653 /* 1654 * when ecc is enabled, the controller doesn't read the real 1655 * or dummy bad block markers in each chunk. To maintain a 1656 * consistent layout across RAW and ECC reads, we just 1657 * leave the real/dummy BBM offsets empty (i.e, filled with 1658 * 0xffs) 1659 */ 1660 if (oob_buf) { 1661 int j; 1662 1663 for (j = 0; j < host->bbm_size; j++) 1664 *oob_buf++ = 0xff; 1665 1666 read_data_dma(nandc, FLASH_BUF_ACC + data_size, 1667 oob_buf, oob_size, 0); 1668 } 1669 1670 if (data_buf) 1671 data_buf += data_size; 1672 if (oob_buf) 1673 oob_buf += oob_size; 1674 } 1675 1676 ret = submit_descs(nandc); 1677 if (ret) 1678 dev_err(nandc->dev, "failure to read page/oob\n"); 1679 1680 free_descs(nandc); 1681 1682 return ret; 1683 } 1684 1685 /* 1686 * a helper that copies the last step/codeword of a page (containing free oob) 1687 * into our local buffer 1688 */ 1689 static int copy_last_cw(struct qcom_nand_host *host, int page) 1690 { 1691 struct nand_chip *chip = &host->chip; 1692 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1693 struct nand_ecc_ctrl *ecc = &chip->ecc; 1694 int size; 1695 int ret; 1696 1697 clear_read_regs(nandc); 1698 1699 size = host->use_ecc ? host->cw_data : host->cw_size; 1700 1701 /* prepare a clean read buffer */ 1702 memset(nandc->data_buffer, 0xff, size); 1703 1704 set_address(host, host->cw_size * (ecc->steps - 1), page); 1705 update_rw_regs(host, 1, true); 1706 1707 config_nand_single_cw_page_read(nandc); 1708 1709 read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size, 0); 1710 1711 ret = submit_descs(nandc); 1712 if (ret) 1713 dev_err(nandc->dev, "failed to copy last codeword\n"); 1714 1715 free_descs(nandc); 1716 1717 return ret; 1718 } 1719 1720 /* implements ecc->read_page() */ 1721 static int qcom_nandc_read_page(struct mtd_info *mtd, struct nand_chip *chip, 1722 uint8_t *buf, int oob_required, int page) 1723 { 1724 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1725 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1726 u8 *data_buf, *oob_buf = NULL; 1727 int ret; 1728 1729 nand_read_page_op(chip, page, 0, NULL, 0); 1730 data_buf = buf; 1731 oob_buf = oob_required ? chip->oob_poi : NULL; 1732 1733 clear_bam_transaction(nandc); 1734 ret = read_page_ecc(host, data_buf, oob_buf); 1735 if (ret) { 1736 dev_err(nandc->dev, "failure to read page\n"); 1737 return ret; 1738 } 1739 1740 return parse_read_errors(host, data_buf, oob_buf); 1741 } 1742 1743 /* implements ecc->read_page_raw() */ 1744 static int qcom_nandc_read_page_raw(struct mtd_info *mtd, 1745 struct nand_chip *chip, uint8_t *buf, 1746 int oob_required, int page) 1747 { 1748 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1749 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1750 u8 *data_buf, *oob_buf; 1751 struct nand_ecc_ctrl *ecc = &chip->ecc; 1752 int i, ret; 1753 int read_loc; 1754 1755 nand_read_page_op(chip, page, 0, NULL, 0); 1756 data_buf = buf; 1757 oob_buf = chip->oob_poi; 1758 1759 host->use_ecc = false; 1760 1761 clear_bam_transaction(nandc); 1762 update_rw_regs(host, ecc->steps, true); 1763 config_nand_page_read(nandc); 1764 1765 for (i = 0; i < ecc->steps; i++) { 1766 int data_size1, data_size2, oob_size1, oob_size2; 1767 int reg_off = FLASH_BUF_ACC; 1768 1769 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1); 1770 oob_size1 = host->bbm_size; 1771 1772 if (i == (ecc->steps - 1)) { 1773 data_size2 = ecc->size - data_size1 - 1774 ((ecc->steps - 1) << 2); 1775 oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw + 1776 host->spare_bytes; 1777 } else { 1778 data_size2 = host->cw_data - data_size1; 1779 oob_size2 = host->ecc_bytes_hw + host->spare_bytes; 1780 } 1781 1782 if (nandc->props->is_bam) { 1783 read_loc = 0; 1784 nandc_set_read_loc(nandc, 0, read_loc, data_size1, 0); 1785 read_loc += data_size1; 1786 1787 nandc_set_read_loc(nandc, 1, read_loc, oob_size1, 0); 1788 read_loc += oob_size1; 1789 1790 nandc_set_read_loc(nandc, 2, read_loc, data_size2, 0); 1791 read_loc += data_size2; 1792 1793 nandc_set_read_loc(nandc, 3, read_loc, oob_size2, 1); 1794 } 1795 1796 config_nand_cw_read(nandc); 1797 1798 read_data_dma(nandc, reg_off, data_buf, data_size1, 0); 1799 reg_off += data_size1; 1800 data_buf += data_size1; 1801 1802 read_data_dma(nandc, reg_off, oob_buf, oob_size1, 0); 1803 reg_off += oob_size1; 1804 oob_buf += oob_size1; 1805 1806 read_data_dma(nandc, reg_off, data_buf, data_size2, 0); 1807 reg_off += data_size2; 1808 data_buf += data_size2; 1809 1810 read_data_dma(nandc, reg_off, oob_buf, oob_size2, 0); 1811 oob_buf += oob_size2; 1812 } 1813 1814 ret = submit_descs(nandc); 1815 if (ret) 1816 dev_err(nandc->dev, "failure to read raw page\n"); 1817 1818 free_descs(nandc); 1819 1820 return 0; 1821 } 1822 1823 /* implements ecc->read_oob() */ 1824 static int qcom_nandc_read_oob(struct mtd_info *mtd, struct nand_chip *chip, 1825 int page) 1826 { 1827 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1828 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1829 struct nand_ecc_ctrl *ecc = &chip->ecc; 1830 int ret; 1831 1832 clear_read_regs(nandc); 1833 clear_bam_transaction(nandc); 1834 1835 host->use_ecc = true; 1836 set_address(host, 0, page); 1837 update_rw_regs(host, ecc->steps, true); 1838 1839 ret = read_page_ecc(host, NULL, chip->oob_poi); 1840 if (ret) 1841 dev_err(nandc->dev, "failure to read oob\n"); 1842 1843 return ret; 1844 } 1845 1846 /* implements ecc->write_page() */ 1847 static int qcom_nandc_write_page(struct mtd_info *mtd, struct nand_chip *chip, 1848 const uint8_t *buf, int oob_required, int page) 1849 { 1850 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1851 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1852 struct nand_ecc_ctrl *ecc = &chip->ecc; 1853 u8 *data_buf, *oob_buf; 1854 int i, ret; 1855 1856 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1857 1858 clear_read_regs(nandc); 1859 clear_bam_transaction(nandc); 1860 1861 data_buf = (u8 *)buf; 1862 oob_buf = chip->oob_poi; 1863 1864 host->use_ecc = true; 1865 update_rw_regs(host, ecc->steps, false); 1866 config_nand_page_write(nandc); 1867 1868 for (i = 0; i < ecc->steps; i++) { 1869 int data_size, oob_size; 1870 1871 if (i == (ecc->steps - 1)) { 1872 data_size = ecc->size - ((ecc->steps - 1) << 2); 1873 oob_size = (ecc->steps << 2) + host->ecc_bytes_hw + 1874 host->spare_bytes; 1875 } else { 1876 data_size = host->cw_data; 1877 oob_size = ecc->bytes; 1878 } 1879 1880 1881 write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size, 1882 i == (ecc->steps - 1) ? NAND_BAM_NO_EOT : 0); 1883 1884 /* 1885 * when ECC is enabled, we don't really need to write anything 1886 * to oob for the first n - 1 codewords since these oob regions 1887 * just contain ECC bytes that's written by the controller 1888 * itself. For the last codeword, we skip the bbm positions and 1889 * write to the free oob area. 1890 */ 1891 if (i == (ecc->steps - 1)) { 1892 oob_buf += host->bbm_size; 1893 1894 write_data_dma(nandc, FLASH_BUF_ACC + data_size, 1895 oob_buf, oob_size, 0); 1896 } 1897 1898 config_nand_cw_write(nandc); 1899 1900 data_buf += data_size; 1901 oob_buf += oob_size; 1902 } 1903 1904 ret = submit_descs(nandc); 1905 if (ret) 1906 dev_err(nandc->dev, "failure to write page\n"); 1907 1908 free_descs(nandc); 1909 1910 if (!ret) 1911 ret = nand_prog_page_end_op(chip); 1912 1913 return ret; 1914 } 1915 1916 /* implements ecc->write_page_raw() */ 1917 static int qcom_nandc_write_page_raw(struct mtd_info *mtd, 1918 struct nand_chip *chip, const uint8_t *buf, 1919 int oob_required, int page) 1920 { 1921 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1922 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1923 struct nand_ecc_ctrl *ecc = &chip->ecc; 1924 u8 *data_buf, *oob_buf; 1925 int i, ret; 1926 1927 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1928 clear_read_regs(nandc); 1929 clear_bam_transaction(nandc); 1930 1931 data_buf = (u8 *)buf; 1932 oob_buf = chip->oob_poi; 1933 1934 host->use_ecc = false; 1935 update_rw_regs(host, ecc->steps, false); 1936 config_nand_page_write(nandc); 1937 1938 for (i = 0; i < ecc->steps; i++) { 1939 int data_size1, data_size2, oob_size1, oob_size2; 1940 int reg_off = FLASH_BUF_ACC; 1941 1942 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1); 1943 oob_size1 = host->bbm_size; 1944 1945 if (i == (ecc->steps - 1)) { 1946 data_size2 = ecc->size - data_size1 - 1947 ((ecc->steps - 1) << 2); 1948 oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw + 1949 host->spare_bytes; 1950 } else { 1951 data_size2 = host->cw_data - data_size1; 1952 oob_size2 = host->ecc_bytes_hw + host->spare_bytes; 1953 } 1954 1955 write_data_dma(nandc, reg_off, data_buf, data_size1, 1956 NAND_BAM_NO_EOT); 1957 reg_off += data_size1; 1958 data_buf += data_size1; 1959 1960 write_data_dma(nandc, reg_off, oob_buf, oob_size1, 1961 NAND_BAM_NO_EOT); 1962 reg_off += oob_size1; 1963 oob_buf += oob_size1; 1964 1965 write_data_dma(nandc, reg_off, data_buf, data_size2, 1966 NAND_BAM_NO_EOT); 1967 reg_off += data_size2; 1968 data_buf += data_size2; 1969 1970 write_data_dma(nandc, reg_off, oob_buf, oob_size2, 0); 1971 oob_buf += oob_size2; 1972 1973 config_nand_cw_write(nandc); 1974 } 1975 1976 ret = submit_descs(nandc); 1977 if (ret) 1978 dev_err(nandc->dev, "failure to write raw page\n"); 1979 1980 free_descs(nandc); 1981 1982 if (!ret) 1983 ret = nand_prog_page_end_op(chip); 1984 1985 return ret; 1986 } 1987 1988 /* 1989 * implements ecc->write_oob() 1990 * 1991 * the NAND controller cannot write only data or only oob within a codeword, 1992 * since ecc is calculated for the combined codeword. we first copy the 1993 * entire contents for the last codeword(data + oob), replace the old oob 1994 * with the new one in chip->oob_poi, and then write the entire codeword. 1995 * this read-copy-write operation results in a slight performance loss. 1996 */ 1997 static int qcom_nandc_write_oob(struct mtd_info *mtd, struct nand_chip *chip, 1998 int page) 1999 { 2000 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2001 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2002 struct nand_ecc_ctrl *ecc = &chip->ecc; 2003 u8 *oob = chip->oob_poi; 2004 int data_size, oob_size; 2005 int ret; 2006 2007 host->use_ecc = true; 2008 2009 clear_bam_transaction(nandc); 2010 ret = copy_last_cw(host, page); 2011 if (ret) 2012 return ret; 2013 2014 clear_read_regs(nandc); 2015 clear_bam_transaction(nandc); 2016 2017 /* calculate the data and oob size for the last codeword/step */ 2018 data_size = ecc->size - ((ecc->steps - 1) << 2); 2019 oob_size = mtd->oobavail; 2020 2021 /* override new oob content to last codeword */ 2022 mtd_ooblayout_get_databytes(mtd, nandc->data_buffer + data_size, oob, 2023 0, mtd->oobavail); 2024 2025 set_address(host, host->cw_size * (ecc->steps - 1), page); 2026 update_rw_regs(host, 1, false); 2027 2028 config_nand_page_write(nandc); 2029 write_data_dma(nandc, FLASH_BUF_ACC, 2030 nandc->data_buffer, data_size + oob_size, 0); 2031 config_nand_cw_write(nandc); 2032 2033 ret = submit_descs(nandc); 2034 2035 free_descs(nandc); 2036 2037 if (ret) { 2038 dev_err(nandc->dev, "failure to write oob\n"); 2039 return -EIO; 2040 } 2041 2042 return nand_prog_page_end_op(chip); 2043 } 2044 2045 static int qcom_nandc_block_bad(struct mtd_info *mtd, loff_t ofs) 2046 { 2047 struct nand_chip *chip = mtd_to_nand(mtd); 2048 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2049 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2050 struct nand_ecc_ctrl *ecc = &chip->ecc; 2051 int page, ret, bbpos, bad = 0; 2052 u32 flash_status; 2053 2054 page = (int)(ofs >> chip->page_shift) & chip->pagemask; 2055 2056 /* 2057 * configure registers for a raw sub page read, the address is set to 2058 * the beginning of the last codeword, we don't care about reading ecc 2059 * portion of oob. we just want the first few bytes from this codeword 2060 * that contains the BBM 2061 */ 2062 host->use_ecc = false; 2063 2064 clear_bam_transaction(nandc); 2065 ret = copy_last_cw(host, page); 2066 if (ret) 2067 goto err; 2068 2069 flash_status = le32_to_cpu(nandc->reg_read_buf[0]); 2070 2071 if (flash_status & (FS_OP_ERR | FS_MPU_ERR)) { 2072 dev_warn(nandc->dev, "error when trying to read BBM\n"); 2073 goto err; 2074 } 2075 2076 bbpos = mtd->writesize - host->cw_size * (ecc->steps - 1); 2077 2078 bad = nandc->data_buffer[bbpos] != 0xff; 2079 2080 if (chip->options & NAND_BUSWIDTH_16) 2081 bad = bad || (nandc->data_buffer[bbpos + 1] != 0xff); 2082 err: 2083 return bad; 2084 } 2085 2086 static int qcom_nandc_block_markbad(struct mtd_info *mtd, loff_t ofs) 2087 { 2088 struct nand_chip *chip = mtd_to_nand(mtd); 2089 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2090 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2091 struct nand_ecc_ctrl *ecc = &chip->ecc; 2092 int page, ret; 2093 2094 clear_read_regs(nandc); 2095 clear_bam_transaction(nandc); 2096 2097 /* 2098 * to mark the BBM as bad, we flash the entire last codeword with 0s. 2099 * we don't care about the rest of the content in the codeword since 2100 * we aren't going to use this block again 2101 */ 2102 memset(nandc->data_buffer, 0x00, host->cw_size); 2103 2104 page = (int)(ofs >> chip->page_shift) & chip->pagemask; 2105 2106 /* prepare write */ 2107 host->use_ecc = false; 2108 set_address(host, host->cw_size * (ecc->steps - 1), page); 2109 update_rw_regs(host, 1, false); 2110 2111 config_nand_page_write(nandc); 2112 write_data_dma(nandc, FLASH_BUF_ACC, 2113 nandc->data_buffer, host->cw_size, 0); 2114 config_nand_cw_write(nandc); 2115 2116 ret = submit_descs(nandc); 2117 2118 free_descs(nandc); 2119 2120 if (ret) { 2121 dev_err(nandc->dev, "failure to update BBM\n"); 2122 return -EIO; 2123 } 2124 2125 return nand_prog_page_end_op(chip); 2126 } 2127 2128 /* 2129 * the three functions below implement chip->read_byte(), chip->read_buf() 2130 * and chip->write_buf() respectively. these aren't used for 2131 * reading/writing page data, they are used for smaller data like reading 2132 * id, status etc 2133 */ 2134 static uint8_t qcom_nandc_read_byte(struct mtd_info *mtd) 2135 { 2136 struct nand_chip *chip = mtd_to_nand(mtd); 2137 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2138 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2139 u8 *buf = nandc->data_buffer; 2140 u8 ret = 0x0; 2141 2142 if (host->last_command == NAND_CMD_STATUS) { 2143 ret = host->status; 2144 2145 host->status = NAND_STATUS_READY | NAND_STATUS_WP; 2146 2147 return ret; 2148 } 2149 2150 if (nandc->buf_start < nandc->buf_count) 2151 ret = buf[nandc->buf_start++]; 2152 2153 return ret; 2154 } 2155 2156 static void qcom_nandc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) 2157 { 2158 struct nand_chip *chip = mtd_to_nand(mtd); 2159 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2160 int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start); 2161 2162 memcpy(buf, nandc->data_buffer + nandc->buf_start, real_len); 2163 nandc->buf_start += real_len; 2164 } 2165 2166 static void qcom_nandc_write_buf(struct mtd_info *mtd, const uint8_t *buf, 2167 int len) 2168 { 2169 struct nand_chip *chip = mtd_to_nand(mtd); 2170 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2171 int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start); 2172 2173 memcpy(nandc->data_buffer + nandc->buf_start, buf, real_len); 2174 2175 nandc->buf_start += real_len; 2176 } 2177 2178 /* we support only one external chip for now */ 2179 static void qcom_nandc_select_chip(struct mtd_info *mtd, int chipnr) 2180 { 2181 struct nand_chip *chip = mtd_to_nand(mtd); 2182 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2183 2184 if (chipnr <= 0) 2185 return; 2186 2187 dev_warn(nandc->dev, "invalid chip select\n"); 2188 } 2189 2190 /* 2191 * NAND controller page layout info 2192 * 2193 * Layout with ECC enabled: 2194 * 2195 * |----------------------| |---------------------------------| 2196 * | xx.......yy| | *********xx.......yy| 2197 * | DATA xx..ECC..yy| | DATA **SPARE**xx..ECC..yy| 2198 * | (516) xx.......yy| | (516-n*4) **(n*4)**xx.......yy| 2199 * | xx.......yy| | *********xx.......yy| 2200 * |----------------------| |---------------------------------| 2201 * codeword 1,2..n-1 codeword n 2202 * <---(528/532 Bytes)--> <-------(528/532 Bytes)---------> 2203 * 2204 * n = Number of codewords in the page 2205 * . = ECC bytes 2206 * * = Spare/free bytes 2207 * x = Unused byte(s) 2208 * y = Reserved byte(s) 2209 * 2210 * 2K page: n = 4, spare = 16 bytes 2211 * 4K page: n = 8, spare = 32 bytes 2212 * 8K page: n = 16, spare = 64 bytes 2213 * 2214 * the qcom nand controller operates at a sub page/codeword level. each 2215 * codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively. 2216 * the number of ECC bytes vary based on the ECC strength and the bus width. 2217 * 2218 * the first n - 1 codewords contains 516 bytes of user data, the remaining 2219 * 12/16 bytes consist of ECC and reserved data. The nth codeword contains 2220 * both user data and spare(oobavail) bytes that sum up to 516 bytes. 2221 * 2222 * When we access a page with ECC enabled, the reserved bytes(s) are not 2223 * accessible at all. When reading, we fill up these unreadable positions 2224 * with 0xffs. When writing, the controller skips writing the inaccessible 2225 * bytes. 2226 * 2227 * Layout with ECC disabled: 2228 * 2229 * |------------------------------| |---------------------------------------| 2230 * | yy xx.......| | bb *********xx.......| 2231 * | DATA1 yy DATA2 xx..ECC..| | DATA1 bb DATA2 **SPARE**xx..ECC..| 2232 * | (size1) yy (size2) xx.......| | (size1) bb (size2) **(n*4)**xx.......| 2233 * | yy xx.......| | bb *********xx.......| 2234 * |------------------------------| |---------------------------------------| 2235 * codeword 1,2..n-1 codeword n 2236 * <-------(528/532 Bytes)------> <-----------(528/532 Bytes)-----------> 2237 * 2238 * n = Number of codewords in the page 2239 * . = ECC bytes 2240 * * = Spare/free bytes 2241 * x = Unused byte(s) 2242 * y = Dummy Bad Bock byte(s) 2243 * b = Real Bad Block byte(s) 2244 * size1/size2 = function of codeword size and 'n' 2245 * 2246 * when the ECC block is disabled, one reserved byte (or two for 16 bit bus 2247 * width) is now accessible. For the first n - 1 codewords, these are dummy Bad 2248 * Block Markers. In the last codeword, this position contains the real BBM 2249 * 2250 * In order to have a consistent layout between RAW and ECC modes, we assume 2251 * the following OOB layout arrangement: 2252 * 2253 * |-----------| |--------------------| 2254 * |yyxx.......| |bb*********xx.......| 2255 * |yyxx..ECC..| |bb*FREEOOB*xx..ECC..| 2256 * |yyxx.......| |bb*********xx.......| 2257 * |yyxx.......| |bb*********xx.......| 2258 * |-----------| |--------------------| 2259 * first n - 1 nth OOB region 2260 * OOB regions 2261 * 2262 * n = Number of codewords in the page 2263 * . = ECC bytes 2264 * * = FREE OOB bytes 2265 * y = Dummy bad block byte(s) (inaccessible when ECC enabled) 2266 * x = Unused byte(s) 2267 * b = Real bad block byte(s) (inaccessible when ECC enabled) 2268 * 2269 * This layout is read as is when ECC is disabled. When ECC is enabled, the 2270 * inaccessible Bad Block byte(s) are ignored when we write to a page/oob, 2271 * and assumed as 0xffs when we read a page/oob. The ECC, unused and 2272 * dummy/real bad block bytes are grouped as ecc bytes (i.e, ecc->bytes is 2273 * the sum of the three). 2274 */ 2275 static int qcom_nand_ooblayout_ecc(struct mtd_info *mtd, int section, 2276 struct mtd_oob_region *oobregion) 2277 { 2278 struct nand_chip *chip = mtd_to_nand(mtd); 2279 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2280 struct nand_ecc_ctrl *ecc = &chip->ecc; 2281 2282 if (section > 1) 2283 return -ERANGE; 2284 2285 if (!section) { 2286 oobregion->length = (ecc->bytes * (ecc->steps - 1)) + 2287 host->bbm_size; 2288 oobregion->offset = 0; 2289 } else { 2290 oobregion->length = host->ecc_bytes_hw + host->spare_bytes; 2291 oobregion->offset = mtd->oobsize - oobregion->length; 2292 } 2293 2294 return 0; 2295 } 2296 2297 static int qcom_nand_ooblayout_free(struct mtd_info *mtd, int section, 2298 struct mtd_oob_region *oobregion) 2299 { 2300 struct nand_chip *chip = mtd_to_nand(mtd); 2301 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2302 struct nand_ecc_ctrl *ecc = &chip->ecc; 2303 2304 if (section) 2305 return -ERANGE; 2306 2307 oobregion->length = ecc->steps * 4; 2308 oobregion->offset = ((ecc->steps - 1) * ecc->bytes) + host->bbm_size; 2309 2310 return 0; 2311 } 2312 2313 static const struct mtd_ooblayout_ops qcom_nand_ooblayout_ops = { 2314 .ecc = qcom_nand_ooblayout_ecc, 2315 .free = qcom_nand_ooblayout_free, 2316 }; 2317 2318 static int qcom_nand_host_setup(struct qcom_nand_host *host) 2319 { 2320 struct nand_chip *chip = &host->chip; 2321 struct mtd_info *mtd = nand_to_mtd(chip); 2322 struct nand_ecc_ctrl *ecc = &chip->ecc; 2323 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2324 int cwperpage, bad_block_byte; 2325 bool wide_bus; 2326 int ecc_mode = 1; 2327 2328 /* 2329 * the controller requires each step consists of 512 bytes of data. 2330 * bail out if DT has populated a wrong step size. 2331 */ 2332 if (ecc->size != NANDC_STEP_SIZE) { 2333 dev_err(nandc->dev, "invalid ecc size\n"); 2334 return -EINVAL; 2335 } 2336 2337 wide_bus = chip->options & NAND_BUSWIDTH_16 ? true : false; 2338 2339 if (ecc->strength >= 8) { 2340 /* 8 bit ECC defaults to BCH ECC on all platforms */ 2341 host->bch_enabled = true; 2342 ecc_mode = 1; 2343 2344 if (wide_bus) { 2345 host->ecc_bytes_hw = 14; 2346 host->spare_bytes = 0; 2347 host->bbm_size = 2; 2348 } else { 2349 host->ecc_bytes_hw = 13; 2350 host->spare_bytes = 2; 2351 host->bbm_size = 1; 2352 } 2353 } else { 2354 /* 2355 * if the controller supports BCH for 4 bit ECC, the controller 2356 * uses lesser bytes for ECC. If RS is used, the ECC bytes is 2357 * always 10 bytes 2358 */ 2359 if (nandc->props->ecc_modes & ECC_BCH_4BIT) { 2360 /* BCH */ 2361 host->bch_enabled = true; 2362 ecc_mode = 0; 2363 2364 if (wide_bus) { 2365 host->ecc_bytes_hw = 8; 2366 host->spare_bytes = 2; 2367 host->bbm_size = 2; 2368 } else { 2369 host->ecc_bytes_hw = 7; 2370 host->spare_bytes = 4; 2371 host->bbm_size = 1; 2372 } 2373 } else { 2374 /* RS */ 2375 host->ecc_bytes_hw = 10; 2376 2377 if (wide_bus) { 2378 host->spare_bytes = 0; 2379 host->bbm_size = 2; 2380 } else { 2381 host->spare_bytes = 1; 2382 host->bbm_size = 1; 2383 } 2384 } 2385 } 2386 2387 /* 2388 * we consider ecc->bytes as the sum of all the non-data content in a 2389 * step. It gives us a clean representation of the oob area (even if 2390 * all the bytes aren't used for ECC).It is always 16 bytes for 8 bit 2391 * ECC and 12 bytes for 4 bit ECC 2392 */ 2393 ecc->bytes = host->ecc_bytes_hw + host->spare_bytes + host->bbm_size; 2394 2395 ecc->read_page = qcom_nandc_read_page; 2396 ecc->read_page_raw = qcom_nandc_read_page_raw; 2397 ecc->read_oob = qcom_nandc_read_oob; 2398 ecc->write_page = qcom_nandc_write_page; 2399 ecc->write_page_raw = qcom_nandc_write_page_raw; 2400 ecc->write_oob = qcom_nandc_write_oob; 2401 2402 ecc->mode = NAND_ECC_HW; 2403 2404 mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops); 2405 2406 cwperpage = mtd->writesize / ecc->size; 2407 nandc->max_cwperpage = max_t(unsigned int, nandc->max_cwperpage, 2408 cwperpage); 2409 2410 /* 2411 * DATA_UD_BYTES varies based on whether the read/write command protects 2412 * spare data with ECC too. We protect spare data by default, so we set 2413 * it to main + spare data, which are 512 and 4 bytes respectively. 2414 */ 2415 host->cw_data = 516; 2416 2417 /* 2418 * total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes 2419 * for 8 bit ECC 2420 */ 2421 host->cw_size = host->cw_data + ecc->bytes; 2422 2423 if (ecc->bytes * (mtd->writesize / ecc->size) > mtd->oobsize) { 2424 dev_err(nandc->dev, "ecc data doesn't fit in OOB area\n"); 2425 return -EINVAL; 2426 } 2427 2428 bad_block_byte = mtd->writesize - host->cw_size * (cwperpage - 1) + 1; 2429 2430 host->cfg0 = (cwperpage - 1) << CW_PER_PAGE 2431 | host->cw_data << UD_SIZE_BYTES 2432 | 0 << DISABLE_STATUS_AFTER_WRITE 2433 | 5 << NUM_ADDR_CYCLES 2434 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_RS 2435 | 0 << STATUS_BFR_READ 2436 | 1 << SET_RD_MODE_AFTER_STATUS 2437 | host->spare_bytes << SPARE_SIZE_BYTES; 2438 2439 host->cfg1 = 7 << NAND_RECOVERY_CYCLES 2440 | 0 << CS_ACTIVE_BSY 2441 | bad_block_byte << BAD_BLOCK_BYTE_NUM 2442 | 0 << BAD_BLOCK_IN_SPARE_AREA 2443 | 2 << WR_RD_BSY_GAP 2444 | wide_bus << WIDE_FLASH 2445 | host->bch_enabled << ENABLE_BCH_ECC; 2446 2447 host->cfg0_raw = (cwperpage - 1) << CW_PER_PAGE 2448 | host->cw_size << UD_SIZE_BYTES 2449 | 5 << NUM_ADDR_CYCLES 2450 | 0 << SPARE_SIZE_BYTES; 2451 2452 host->cfg1_raw = 7 << NAND_RECOVERY_CYCLES 2453 | 0 << CS_ACTIVE_BSY 2454 | 17 << BAD_BLOCK_BYTE_NUM 2455 | 1 << BAD_BLOCK_IN_SPARE_AREA 2456 | 2 << WR_RD_BSY_GAP 2457 | wide_bus << WIDE_FLASH 2458 | 1 << DEV0_CFG1_ECC_DISABLE; 2459 2460 host->ecc_bch_cfg = !host->bch_enabled << ECC_CFG_ECC_DISABLE 2461 | 0 << ECC_SW_RESET 2462 | host->cw_data << ECC_NUM_DATA_BYTES 2463 | 1 << ECC_FORCE_CLK_OPEN 2464 | ecc_mode << ECC_MODE 2465 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_BCH; 2466 2467 host->ecc_buf_cfg = 0x203 << NUM_STEPS; 2468 2469 host->clrflashstatus = FS_READY_BSY_N; 2470 host->clrreadstatus = 0xc0; 2471 nandc->regs->erased_cw_detect_cfg_clr = 2472 cpu_to_le32(CLR_ERASED_PAGE_DET); 2473 nandc->regs->erased_cw_detect_cfg_set = 2474 cpu_to_le32(SET_ERASED_PAGE_DET); 2475 2476 dev_dbg(nandc->dev, 2477 "cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n", 2478 host->cfg0, host->cfg1, host->ecc_buf_cfg, host->ecc_bch_cfg, 2479 host->cw_size, host->cw_data, ecc->strength, ecc->bytes, 2480 cwperpage); 2481 2482 return 0; 2483 } 2484 2485 static int qcom_nandc_alloc(struct qcom_nand_controller *nandc) 2486 { 2487 int ret; 2488 2489 ret = dma_set_coherent_mask(nandc->dev, DMA_BIT_MASK(32)); 2490 if (ret) { 2491 dev_err(nandc->dev, "failed to set DMA mask\n"); 2492 return ret; 2493 } 2494 2495 /* 2496 * we use the internal buffer for reading ONFI params, reading small 2497 * data like ID and status, and preforming read-copy-write operations 2498 * when writing to a codeword partially. 532 is the maximum possible 2499 * size of a codeword for our nand controller 2500 */ 2501 nandc->buf_size = 532; 2502 2503 nandc->data_buffer = devm_kzalloc(nandc->dev, nandc->buf_size, 2504 GFP_KERNEL); 2505 if (!nandc->data_buffer) 2506 return -ENOMEM; 2507 2508 nandc->regs = devm_kzalloc(nandc->dev, sizeof(*nandc->regs), 2509 GFP_KERNEL); 2510 if (!nandc->regs) 2511 return -ENOMEM; 2512 2513 nandc->reg_read_buf = devm_kcalloc(nandc->dev, 2514 MAX_REG_RD, sizeof(*nandc->reg_read_buf), 2515 GFP_KERNEL); 2516 if (!nandc->reg_read_buf) 2517 return -ENOMEM; 2518 2519 if (nandc->props->is_bam) { 2520 nandc->reg_read_dma = 2521 dma_map_single(nandc->dev, nandc->reg_read_buf, 2522 MAX_REG_RD * 2523 sizeof(*nandc->reg_read_buf), 2524 DMA_FROM_DEVICE); 2525 if (dma_mapping_error(nandc->dev, nandc->reg_read_dma)) { 2526 dev_err(nandc->dev, "failed to DMA MAP reg buffer\n"); 2527 return -EIO; 2528 } 2529 2530 nandc->tx_chan = dma_request_slave_channel(nandc->dev, "tx"); 2531 if (!nandc->tx_chan) { 2532 dev_err(nandc->dev, "failed to request tx channel\n"); 2533 return -ENODEV; 2534 } 2535 2536 nandc->rx_chan = dma_request_slave_channel(nandc->dev, "rx"); 2537 if (!nandc->rx_chan) { 2538 dev_err(nandc->dev, "failed to request rx channel\n"); 2539 return -ENODEV; 2540 } 2541 2542 nandc->cmd_chan = dma_request_slave_channel(nandc->dev, "cmd"); 2543 if (!nandc->cmd_chan) { 2544 dev_err(nandc->dev, "failed to request cmd channel\n"); 2545 return -ENODEV; 2546 } 2547 2548 /* 2549 * Initially allocate BAM transaction to read ONFI param page. 2550 * After detecting all the devices, this BAM transaction will 2551 * be freed and the next BAM tranasction will be allocated with 2552 * maximum codeword size 2553 */ 2554 nandc->max_cwperpage = 1; 2555 nandc->bam_txn = alloc_bam_transaction(nandc); 2556 if (!nandc->bam_txn) { 2557 dev_err(nandc->dev, 2558 "failed to allocate bam transaction\n"); 2559 return -ENOMEM; 2560 } 2561 } else { 2562 nandc->chan = dma_request_slave_channel(nandc->dev, "rxtx"); 2563 if (!nandc->chan) { 2564 dev_err(nandc->dev, 2565 "failed to request slave channel\n"); 2566 return -ENODEV; 2567 } 2568 } 2569 2570 INIT_LIST_HEAD(&nandc->desc_list); 2571 INIT_LIST_HEAD(&nandc->host_list); 2572 2573 nand_hw_control_init(&nandc->controller); 2574 2575 return 0; 2576 } 2577 2578 static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc) 2579 { 2580 if (nandc->props->is_bam) { 2581 if (!dma_mapping_error(nandc->dev, nandc->reg_read_dma)) 2582 dma_unmap_single(nandc->dev, nandc->reg_read_dma, 2583 MAX_REG_RD * 2584 sizeof(*nandc->reg_read_buf), 2585 DMA_FROM_DEVICE); 2586 2587 if (nandc->tx_chan) 2588 dma_release_channel(nandc->tx_chan); 2589 2590 if (nandc->rx_chan) 2591 dma_release_channel(nandc->rx_chan); 2592 2593 if (nandc->cmd_chan) 2594 dma_release_channel(nandc->cmd_chan); 2595 } else { 2596 if (nandc->chan) 2597 dma_release_channel(nandc->chan); 2598 } 2599 } 2600 2601 /* one time setup of a few nand controller registers */ 2602 static int qcom_nandc_setup(struct qcom_nand_controller *nandc) 2603 { 2604 u32 nand_ctrl; 2605 2606 /* kill onenand */ 2607 nandc_write(nandc, SFLASHC_BURST_CFG, 0); 2608 nandc_write(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD), 2609 NAND_DEV_CMD_VLD_VAL); 2610 2611 /* enable ADM or BAM DMA */ 2612 if (nandc->props->is_bam) { 2613 nand_ctrl = nandc_read(nandc, NAND_CTRL); 2614 nandc_write(nandc, NAND_CTRL, nand_ctrl | BAM_MODE_EN); 2615 } else { 2616 nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN); 2617 } 2618 2619 /* save the original values of these registers */ 2620 nandc->cmd1 = nandc_read(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD1)); 2621 nandc->vld = NAND_DEV_CMD_VLD_VAL; 2622 2623 return 0; 2624 } 2625 2626 static int qcom_nand_host_init(struct qcom_nand_controller *nandc, 2627 struct qcom_nand_host *host, 2628 struct device_node *dn) 2629 { 2630 struct nand_chip *chip = &host->chip; 2631 struct mtd_info *mtd = nand_to_mtd(chip); 2632 struct device *dev = nandc->dev; 2633 int ret; 2634 2635 ret = of_property_read_u32(dn, "reg", &host->cs); 2636 if (ret) { 2637 dev_err(dev, "can't get chip-select\n"); 2638 return -ENXIO; 2639 } 2640 2641 nand_set_flash_node(chip, dn); 2642 mtd->name = devm_kasprintf(dev, GFP_KERNEL, "qcom_nand.%d", host->cs); 2643 if (!mtd->name) 2644 return -ENOMEM; 2645 2646 mtd->owner = THIS_MODULE; 2647 mtd->dev.parent = dev; 2648 2649 chip->cmdfunc = qcom_nandc_command; 2650 chip->select_chip = qcom_nandc_select_chip; 2651 chip->read_byte = qcom_nandc_read_byte; 2652 chip->read_buf = qcom_nandc_read_buf; 2653 chip->write_buf = qcom_nandc_write_buf; 2654 chip->set_features = nand_get_set_features_notsupp; 2655 chip->get_features = nand_get_set_features_notsupp; 2656 2657 /* 2658 * the bad block marker is readable only when we read the last codeword 2659 * of a page with ECC disabled. currently, the nand_base and nand_bbt 2660 * helpers don't allow us to read BB from a nand chip with ECC 2661 * disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad 2662 * and block_markbad helpers until we permanently switch to using 2663 * MTD_OPS_RAW for all drivers (with the help of badblockbits) 2664 */ 2665 chip->block_bad = qcom_nandc_block_bad; 2666 chip->block_markbad = qcom_nandc_block_markbad; 2667 2668 chip->controller = &nandc->controller; 2669 chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USE_BOUNCE_BUFFER | 2670 NAND_SKIP_BBTSCAN; 2671 2672 /* set up initial status value */ 2673 host->status = NAND_STATUS_READY | NAND_STATUS_WP; 2674 2675 ret = nand_scan_ident(mtd, 1, NULL); 2676 if (ret) 2677 return ret; 2678 2679 ret = qcom_nand_host_setup(host); 2680 2681 return ret; 2682 } 2683 2684 static int qcom_nand_mtd_register(struct qcom_nand_controller *nandc, 2685 struct qcom_nand_host *host, 2686 struct device_node *dn) 2687 { 2688 struct nand_chip *chip = &host->chip; 2689 struct mtd_info *mtd = nand_to_mtd(chip); 2690 int ret; 2691 2692 ret = nand_scan_tail(mtd); 2693 if (ret) 2694 return ret; 2695 2696 ret = mtd_device_register(mtd, NULL, 0); 2697 if (ret) 2698 nand_cleanup(mtd_to_nand(mtd)); 2699 2700 return ret; 2701 } 2702 2703 static int qcom_probe_nand_devices(struct qcom_nand_controller *nandc) 2704 { 2705 struct device *dev = nandc->dev; 2706 struct device_node *dn = dev->of_node, *child; 2707 struct qcom_nand_host *host, *tmp; 2708 int ret; 2709 2710 for_each_available_child_of_node(dn, child) { 2711 host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL); 2712 if (!host) { 2713 of_node_put(child); 2714 return -ENOMEM; 2715 } 2716 2717 ret = qcom_nand_host_init(nandc, host, child); 2718 if (ret) { 2719 devm_kfree(dev, host); 2720 continue; 2721 } 2722 2723 list_add_tail(&host->node, &nandc->host_list); 2724 } 2725 2726 if (list_empty(&nandc->host_list)) 2727 return -ENODEV; 2728 2729 if (nandc->props->is_bam) { 2730 free_bam_transaction(nandc); 2731 nandc->bam_txn = alloc_bam_transaction(nandc); 2732 if (!nandc->bam_txn) { 2733 dev_err(nandc->dev, 2734 "failed to allocate bam transaction\n"); 2735 return -ENOMEM; 2736 } 2737 } 2738 2739 list_for_each_entry_safe(host, tmp, &nandc->host_list, node) { 2740 ret = qcom_nand_mtd_register(nandc, host, child); 2741 if (ret) { 2742 list_del(&host->node); 2743 devm_kfree(dev, host); 2744 } 2745 } 2746 2747 if (list_empty(&nandc->host_list)) 2748 return -ENODEV; 2749 2750 return 0; 2751 } 2752 2753 /* parse custom DT properties here */ 2754 static int qcom_nandc_parse_dt(struct platform_device *pdev) 2755 { 2756 struct qcom_nand_controller *nandc = platform_get_drvdata(pdev); 2757 struct device_node *np = nandc->dev->of_node; 2758 int ret; 2759 2760 if (!nandc->props->is_bam) { 2761 ret = of_property_read_u32(np, "qcom,cmd-crci", 2762 &nandc->cmd_crci); 2763 if (ret) { 2764 dev_err(nandc->dev, "command CRCI unspecified\n"); 2765 return ret; 2766 } 2767 2768 ret = of_property_read_u32(np, "qcom,data-crci", 2769 &nandc->data_crci); 2770 if (ret) { 2771 dev_err(nandc->dev, "data CRCI unspecified\n"); 2772 return ret; 2773 } 2774 } 2775 2776 return 0; 2777 } 2778 2779 static int qcom_nandc_probe(struct platform_device *pdev) 2780 { 2781 struct qcom_nand_controller *nandc; 2782 const void *dev_data; 2783 struct device *dev = &pdev->dev; 2784 struct resource *res; 2785 int ret; 2786 2787 nandc = devm_kzalloc(&pdev->dev, sizeof(*nandc), GFP_KERNEL); 2788 if (!nandc) 2789 return -ENOMEM; 2790 2791 platform_set_drvdata(pdev, nandc); 2792 nandc->dev = dev; 2793 2794 dev_data = of_device_get_match_data(dev); 2795 if (!dev_data) { 2796 dev_err(&pdev->dev, "failed to get device data\n"); 2797 return -ENODEV; 2798 } 2799 2800 nandc->props = dev_data; 2801 2802 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2803 nandc->base = devm_ioremap_resource(dev, res); 2804 if (IS_ERR(nandc->base)) 2805 return PTR_ERR(nandc->base); 2806 2807 nandc->base_phys = res->start; 2808 nandc->base_dma = phys_to_dma(dev, (phys_addr_t)res->start); 2809 2810 nandc->core_clk = devm_clk_get(dev, "core"); 2811 if (IS_ERR(nandc->core_clk)) 2812 return PTR_ERR(nandc->core_clk); 2813 2814 nandc->aon_clk = devm_clk_get(dev, "aon"); 2815 if (IS_ERR(nandc->aon_clk)) 2816 return PTR_ERR(nandc->aon_clk); 2817 2818 ret = qcom_nandc_parse_dt(pdev); 2819 if (ret) 2820 return ret; 2821 2822 ret = qcom_nandc_alloc(nandc); 2823 if (ret) 2824 goto err_core_clk; 2825 2826 ret = clk_prepare_enable(nandc->core_clk); 2827 if (ret) 2828 goto err_core_clk; 2829 2830 ret = clk_prepare_enable(nandc->aon_clk); 2831 if (ret) 2832 goto err_aon_clk; 2833 2834 ret = qcom_nandc_setup(nandc); 2835 if (ret) 2836 goto err_setup; 2837 2838 ret = qcom_probe_nand_devices(nandc); 2839 if (ret) 2840 goto err_setup; 2841 2842 return 0; 2843 2844 err_setup: 2845 clk_disable_unprepare(nandc->aon_clk); 2846 err_aon_clk: 2847 clk_disable_unprepare(nandc->core_clk); 2848 err_core_clk: 2849 qcom_nandc_unalloc(nandc); 2850 2851 return ret; 2852 } 2853 2854 static int qcom_nandc_remove(struct platform_device *pdev) 2855 { 2856 struct qcom_nand_controller *nandc = platform_get_drvdata(pdev); 2857 struct qcom_nand_host *host; 2858 2859 list_for_each_entry(host, &nandc->host_list, node) 2860 nand_release(nand_to_mtd(&host->chip)); 2861 2862 qcom_nandc_unalloc(nandc); 2863 2864 clk_disable_unprepare(nandc->aon_clk); 2865 clk_disable_unprepare(nandc->core_clk); 2866 2867 return 0; 2868 } 2869 2870 static const struct qcom_nandc_props ipq806x_nandc_props = { 2871 .ecc_modes = (ECC_RS_4BIT | ECC_BCH_8BIT), 2872 .is_bam = false, 2873 .dev_cmd_reg_start = 0x0, 2874 }; 2875 2876 static const struct qcom_nandc_props ipq4019_nandc_props = { 2877 .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT), 2878 .is_bam = true, 2879 .dev_cmd_reg_start = 0x0, 2880 }; 2881 2882 static const struct qcom_nandc_props ipq8074_nandc_props = { 2883 .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT), 2884 .is_bam = true, 2885 .dev_cmd_reg_start = 0x7000, 2886 }; 2887 2888 /* 2889 * data will hold a struct pointer containing more differences once we support 2890 * more controller variants 2891 */ 2892 static const struct of_device_id qcom_nandc_of_match[] = { 2893 { 2894 .compatible = "qcom,ipq806x-nand", 2895 .data = &ipq806x_nandc_props, 2896 }, 2897 { 2898 .compatible = "qcom,ipq4019-nand", 2899 .data = &ipq4019_nandc_props, 2900 }, 2901 { 2902 .compatible = "qcom,ipq8074-nand", 2903 .data = &ipq8074_nandc_props, 2904 }, 2905 {} 2906 }; 2907 MODULE_DEVICE_TABLE(of, qcom_nandc_of_match); 2908 2909 static struct platform_driver qcom_nandc_driver = { 2910 .driver = { 2911 .name = "qcom-nandc", 2912 .of_match_table = qcom_nandc_of_match, 2913 }, 2914 .probe = qcom_nandc_probe, 2915 .remove = qcom_nandc_remove, 2916 }; 2917 module_platform_driver(qcom_nandc_driver); 2918 2919 MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>"); 2920 MODULE_DESCRIPTION("Qualcomm NAND Controller driver"); 2921 MODULE_LICENSE("GPL v2"); 2922