1 /* 2 * Copyright (c) 2016, The Linux Foundation. All rights reserved. 3 * 4 * This software is licensed under the terms of the GNU General Public 5 * License version 2, as published by the Free Software Foundation, and 6 * may be copied, distributed, and modified under those terms. 7 * 8 * This program is distributed in the hope that it will be useful, 9 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * GNU General Public License for more details. 12 */ 13 14 #include <linux/clk.h> 15 #include <linux/slab.h> 16 #include <linux/bitops.h> 17 #include <linux/dma-mapping.h> 18 #include <linux/dmaengine.h> 19 #include <linux/module.h> 20 #include <linux/mtd/rawnand.h> 21 #include <linux/mtd/partitions.h> 22 #include <linux/of.h> 23 #include <linux/of_device.h> 24 #include <linux/delay.h> 25 #include <linux/dma/qcom_bam_dma.h> 26 27 /* NANDc reg offsets */ 28 #define NAND_FLASH_CMD 0x00 29 #define NAND_ADDR0 0x04 30 #define NAND_ADDR1 0x08 31 #define NAND_FLASH_CHIP_SELECT 0x0c 32 #define NAND_EXEC_CMD 0x10 33 #define NAND_FLASH_STATUS 0x14 34 #define NAND_BUFFER_STATUS 0x18 35 #define NAND_DEV0_CFG0 0x20 36 #define NAND_DEV0_CFG1 0x24 37 #define NAND_DEV0_ECC_CFG 0x28 38 #define NAND_DEV1_ECC_CFG 0x2c 39 #define NAND_DEV1_CFG0 0x30 40 #define NAND_DEV1_CFG1 0x34 41 #define NAND_READ_ID 0x40 42 #define NAND_READ_STATUS 0x44 43 #define NAND_DEV_CMD0 0xa0 44 #define NAND_DEV_CMD1 0xa4 45 #define NAND_DEV_CMD2 0xa8 46 #define NAND_DEV_CMD_VLD 0xac 47 #define SFLASHC_BURST_CFG 0xe0 48 #define NAND_ERASED_CW_DETECT_CFG 0xe8 49 #define NAND_ERASED_CW_DETECT_STATUS 0xec 50 #define NAND_EBI2_ECC_BUF_CFG 0xf0 51 #define FLASH_BUF_ACC 0x100 52 53 #define NAND_CTRL 0xf00 54 #define NAND_VERSION 0xf08 55 #define NAND_READ_LOCATION_0 0xf20 56 #define NAND_READ_LOCATION_1 0xf24 57 #define NAND_READ_LOCATION_2 0xf28 58 #define NAND_READ_LOCATION_3 0xf2c 59 60 /* dummy register offsets, used by write_reg_dma */ 61 #define NAND_DEV_CMD1_RESTORE 0xdead 62 #define NAND_DEV_CMD_VLD_RESTORE 0xbeef 63 64 /* NAND_FLASH_CMD bits */ 65 #define PAGE_ACC BIT(4) 66 #define LAST_PAGE BIT(5) 67 68 /* NAND_FLASH_CHIP_SELECT bits */ 69 #define NAND_DEV_SEL 0 70 #define DM_EN BIT(2) 71 72 /* NAND_FLASH_STATUS bits */ 73 #define FS_OP_ERR BIT(4) 74 #define FS_READY_BSY_N BIT(5) 75 #define FS_MPU_ERR BIT(8) 76 #define FS_DEVICE_STS_ERR BIT(16) 77 #define FS_DEVICE_WP BIT(23) 78 79 /* NAND_BUFFER_STATUS bits */ 80 #define BS_UNCORRECTABLE_BIT BIT(8) 81 #define BS_CORRECTABLE_ERR_MSK 0x1f 82 83 /* NAND_DEVn_CFG0 bits */ 84 #define DISABLE_STATUS_AFTER_WRITE 4 85 #define CW_PER_PAGE 6 86 #define UD_SIZE_BYTES 9 87 #define ECC_PARITY_SIZE_BYTES_RS 19 88 #define SPARE_SIZE_BYTES 23 89 #define NUM_ADDR_CYCLES 27 90 #define STATUS_BFR_READ 30 91 #define SET_RD_MODE_AFTER_STATUS 31 92 93 /* NAND_DEVn_CFG0 bits */ 94 #define DEV0_CFG1_ECC_DISABLE 0 95 #define WIDE_FLASH 1 96 #define NAND_RECOVERY_CYCLES 2 97 #define CS_ACTIVE_BSY 5 98 #define BAD_BLOCK_BYTE_NUM 6 99 #define BAD_BLOCK_IN_SPARE_AREA 16 100 #define WR_RD_BSY_GAP 17 101 #define ENABLE_BCH_ECC 27 102 103 /* NAND_DEV0_ECC_CFG bits */ 104 #define ECC_CFG_ECC_DISABLE 0 105 #define ECC_SW_RESET 1 106 #define ECC_MODE 4 107 #define ECC_PARITY_SIZE_BYTES_BCH 8 108 #define ECC_NUM_DATA_BYTES 16 109 #define ECC_FORCE_CLK_OPEN 30 110 111 /* NAND_DEV_CMD1 bits */ 112 #define READ_ADDR 0 113 114 /* NAND_DEV_CMD_VLD bits */ 115 #define READ_START_VLD BIT(0) 116 #define READ_STOP_VLD BIT(1) 117 #define WRITE_START_VLD BIT(2) 118 #define ERASE_START_VLD BIT(3) 119 #define SEQ_READ_START_VLD BIT(4) 120 121 /* NAND_EBI2_ECC_BUF_CFG bits */ 122 #define NUM_STEPS 0 123 124 /* NAND_ERASED_CW_DETECT_CFG bits */ 125 #define ERASED_CW_ECC_MASK 1 126 #define AUTO_DETECT_RES 0 127 #define MASK_ECC (1 << ERASED_CW_ECC_MASK) 128 #define RESET_ERASED_DET (1 << AUTO_DETECT_RES) 129 #define ACTIVE_ERASED_DET (0 << AUTO_DETECT_RES) 130 #define CLR_ERASED_PAGE_DET (RESET_ERASED_DET | MASK_ECC) 131 #define SET_ERASED_PAGE_DET (ACTIVE_ERASED_DET | MASK_ECC) 132 133 /* NAND_ERASED_CW_DETECT_STATUS bits */ 134 #define PAGE_ALL_ERASED BIT(7) 135 #define CODEWORD_ALL_ERASED BIT(6) 136 #define PAGE_ERASED BIT(5) 137 #define CODEWORD_ERASED BIT(4) 138 #define ERASED_PAGE (PAGE_ALL_ERASED | PAGE_ERASED) 139 #define ERASED_CW (CODEWORD_ALL_ERASED | CODEWORD_ERASED) 140 141 /* NAND_READ_LOCATION_n bits */ 142 #define READ_LOCATION_OFFSET 0 143 #define READ_LOCATION_SIZE 16 144 #define READ_LOCATION_LAST 31 145 146 /* Version Mask */ 147 #define NAND_VERSION_MAJOR_MASK 0xf0000000 148 #define NAND_VERSION_MAJOR_SHIFT 28 149 #define NAND_VERSION_MINOR_MASK 0x0fff0000 150 #define NAND_VERSION_MINOR_SHIFT 16 151 152 /* NAND OP_CMDs */ 153 #define PAGE_READ 0x2 154 #define PAGE_READ_WITH_ECC 0x3 155 #define PAGE_READ_WITH_ECC_SPARE 0x4 156 #define PROGRAM_PAGE 0x6 157 #define PAGE_PROGRAM_WITH_ECC 0x7 158 #define PROGRAM_PAGE_SPARE 0x9 159 #define BLOCK_ERASE 0xa 160 #define FETCH_ID 0xb 161 #define RESET_DEVICE 0xd 162 163 /* Default Value for NAND_DEV_CMD_VLD */ 164 #define NAND_DEV_CMD_VLD_VAL (READ_START_VLD | WRITE_START_VLD | \ 165 ERASE_START_VLD | SEQ_READ_START_VLD) 166 167 /* NAND_CTRL bits */ 168 #define BAM_MODE_EN BIT(0) 169 170 /* 171 * the NAND controller performs reads/writes with ECC in 516 byte chunks. 172 * the driver calls the chunks 'step' or 'codeword' interchangeably 173 */ 174 #define NANDC_STEP_SIZE 512 175 176 /* 177 * the largest page size we support is 8K, this will have 16 steps/codewords 178 * of 512 bytes each 179 */ 180 #define MAX_NUM_STEPS (SZ_8K / NANDC_STEP_SIZE) 181 182 /* we read at most 3 registers per codeword scan */ 183 #define MAX_REG_RD (3 * MAX_NUM_STEPS) 184 185 /* ECC modes supported by the controller */ 186 #define ECC_NONE BIT(0) 187 #define ECC_RS_4BIT BIT(1) 188 #define ECC_BCH_4BIT BIT(2) 189 #define ECC_BCH_8BIT BIT(3) 190 191 #define nandc_set_read_loc(nandc, reg, offset, size, is_last) \ 192 nandc_set_reg(nandc, NAND_READ_LOCATION_##reg, \ 193 ((offset) << READ_LOCATION_OFFSET) | \ 194 ((size) << READ_LOCATION_SIZE) | \ 195 ((is_last) << READ_LOCATION_LAST)) 196 197 /* 198 * Returns the actual register address for all NAND_DEV_ registers 199 * (i.e. NAND_DEV_CMD0, NAND_DEV_CMD1, NAND_DEV_CMD2 and NAND_DEV_CMD_VLD) 200 */ 201 #define dev_cmd_reg_addr(nandc, reg) ((nandc)->props->dev_cmd_reg_start + (reg)) 202 203 /* Returns the NAND register physical address */ 204 #define nandc_reg_phys(chip, offset) ((chip)->base_phys + (offset)) 205 206 /* Returns the dma address for reg read buffer */ 207 #define reg_buf_dma_addr(chip, vaddr) \ 208 ((chip)->reg_read_dma + \ 209 ((uint8_t *)(vaddr) - (uint8_t *)(chip)->reg_read_buf)) 210 211 #define QPIC_PER_CW_CMD_ELEMENTS 32 212 #define QPIC_PER_CW_CMD_SGL 32 213 #define QPIC_PER_CW_DATA_SGL 8 214 215 #define QPIC_NAND_COMPLETION_TIMEOUT msecs_to_jiffies(2000) 216 217 /* 218 * Flags used in DMA descriptor preparation helper functions 219 * (i.e. read_reg_dma/write_reg_dma/read_data_dma/write_data_dma) 220 */ 221 /* Don't set the EOT in current tx BAM sgl */ 222 #define NAND_BAM_NO_EOT BIT(0) 223 /* Set the NWD flag in current BAM sgl */ 224 #define NAND_BAM_NWD BIT(1) 225 /* Finish writing in the current BAM sgl and start writing in another BAM sgl */ 226 #define NAND_BAM_NEXT_SGL BIT(2) 227 /* 228 * Erased codeword status is being used two times in single transfer so this 229 * flag will determine the current value of erased codeword status register 230 */ 231 #define NAND_ERASED_CW_SET BIT(4) 232 233 /* 234 * This data type corresponds to the BAM transaction which will be used for all 235 * NAND transfers. 236 * @bam_ce - the array of BAM command elements 237 * @cmd_sgl - sgl for NAND BAM command pipe 238 * @data_sgl - sgl for NAND BAM consumer/producer pipe 239 * @bam_ce_pos - the index in bam_ce which is available for next sgl 240 * @bam_ce_start - the index in bam_ce which marks the start position ce 241 * for current sgl. It will be used for size calculation 242 * for current sgl 243 * @cmd_sgl_pos - current index in command sgl. 244 * @cmd_sgl_start - start index in command sgl. 245 * @tx_sgl_pos - current index in data sgl for tx. 246 * @tx_sgl_start - start index in data sgl for tx. 247 * @rx_sgl_pos - current index in data sgl for rx. 248 * @rx_sgl_start - start index in data sgl for rx. 249 * @wait_second_completion - wait for second DMA desc completion before making 250 * the NAND transfer completion. 251 * @txn_done - completion for NAND transfer. 252 * @last_data_desc - last DMA desc in data channel (tx/rx). 253 * @last_cmd_desc - last DMA desc in command channel. 254 */ 255 struct bam_transaction { 256 struct bam_cmd_element *bam_ce; 257 struct scatterlist *cmd_sgl; 258 struct scatterlist *data_sgl; 259 u32 bam_ce_pos; 260 u32 bam_ce_start; 261 u32 cmd_sgl_pos; 262 u32 cmd_sgl_start; 263 u32 tx_sgl_pos; 264 u32 tx_sgl_start; 265 u32 rx_sgl_pos; 266 u32 rx_sgl_start; 267 bool wait_second_completion; 268 struct completion txn_done; 269 struct dma_async_tx_descriptor *last_data_desc; 270 struct dma_async_tx_descriptor *last_cmd_desc; 271 }; 272 273 /* 274 * This data type corresponds to the nand dma descriptor 275 * @list - list for desc_info 276 * @dir - DMA transfer direction 277 * @adm_sgl - sgl which will be used for single sgl dma descriptor. Only used by 278 * ADM 279 * @bam_sgl - sgl which will be used for dma descriptor. Only used by BAM 280 * @sgl_cnt - number of SGL in bam_sgl. Only used by BAM 281 * @dma_desc - low level DMA engine descriptor 282 */ 283 struct desc_info { 284 struct list_head node; 285 286 enum dma_data_direction dir; 287 union { 288 struct scatterlist adm_sgl; 289 struct { 290 struct scatterlist *bam_sgl; 291 int sgl_cnt; 292 }; 293 }; 294 struct dma_async_tx_descriptor *dma_desc; 295 }; 296 297 /* 298 * holds the current register values that we want to write. acts as a contiguous 299 * chunk of memory which we use to write the controller registers through DMA. 300 */ 301 struct nandc_regs { 302 __le32 cmd; 303 __le32 addr0; 304 __le32 addr1; 305 __le32 chip_sel; 306 __le32 exec; 307 308 __le32 cfg0; 309 __le32 cfg1; 310 __le32 ecc_bch_cfg; 311 312 __le32 clrflashstatus; 313 __le32 clrreadstatus; 314 315 __le32 cmd1; 316 __le32 vld; 317 318 __le32 orig_cmd1; 319 __le32 orig_vld; 320 321 __le32 ecc_buf_cfg; 322 __le32 read_location0; 323 __le32 read_location1; 324 __le32 read_location2; 325 __le32 read_location3; 326 327 __le32 erased_cw_detect_cfg_clr; 328 __le32 erased_cw_detect_cfg_set; 329 }; 330 331 /* 332 * NAND controller data struct 333 * 334 * @controller: base controller structure 335 * @host_list: list containing all the chips attached to the 336 * controller 337 * @dev: parent device 338 * @base: MMIO base 339 * @base_phys: physical base address of controller registers 340 * @base_dma: dma base address of controller registers 341 * @core_clk: controller clock 342 * @aon_clk: another controller clock 343 * 344 * @chan: dma channel 345 * @cmd_crci: ADM DMA CRCI for command flow control 346 * @data_crci: ADM DMA CRCI for data flow control 347 * @desc_list: DMA descriptor list (list of desc_infos) 348 * 349 * @data_buffer: our local DMA buffer for page read/writes, 350 * used when we can't use the buffer provided 351 * by upper layers directly 352 * @buf_size/count/start: markers for chip->legacy.read_buf/write_buf 353 * functions 354 * @reg_read_buf: local buffer for reading back registers via DMA 355 * @reg_read_dma: contains dma address for register read buffer 356 * @reg_read_pos: marker for data read in reg_read_buf 357 * 358 * @regs: a contiguous chunk of memory for DMA register 359 * writes. contains the register values to be 360 * written to controller 361 * @cmd1/vld: some fixed controller register values 362 * @props: properties of current NAND controller, 363 * initialized via DT match data 364 * @max_cwperpage: maximum QPIC codewords required. calculated 365 * from all connected NAND devices pagesize 366 */ 367 struct qcom_nand_controller { 368 struct nand_controller controller; 369 struct list_head host_list; 370 371 struct device *dev; 372 373 void __iomem *base; 374 phys_addr_t base_phys; 375 dma_addr_t base_dma; 376 377 struct clk *core_clk; 378 struct clk *aon_clk; 379 380 union { 381 /* will be used only by QPIC for BAM DMA */ 382 struct { 383 struct dma_chan *tx_chan; 384 struct dma_chan *rx_chan; 385 struct dma_chan *cmd_chan; 386 }; 387 388 /* will be used only by EBI2 for ADM DMA */ 389 struct { 390 struct dma_chan *chan; 391 unsigned int cmd_crci; 392 unsigned int data_crci; 393 }; 394 }; 395 396 struct list_head desc_list; 397 struct bam_transaction *bam_txn; 398 399 u8 *data_buffer; 400 int buf_size; 401 int buf_count; 402 int buf_start; 403 unsigned int max_cwperpage; 404 405 __le32 *reg_read_buf; 406 dma_addr_t reg_read_dma; 407 int reg_read_pos; 408 409 struct nandc_regs *regs; 410 411 u32 cmd1, vld; 412 const struct qcom_nandc_props *props; 413 }; 414 415 /* 416 * NAND chip structure 417 * 418 * @chip: base NAND chip structure 419 * @node: list node to add itself to host_list in 420 * qcom_nand_controller 421 * 422 * @cs: chip select value for this chip 423 * @cw_size: the number of bytes in a single step/codeword 424 * of a page, consisting of all data, ecc, spare 425 * and reserved bytes 426 * @cw_data: the number of bytes within a codeword protected 427 * by ECC 428 * @use_ecc: request the controller to use ECC for the 429 * upcoming read/write 430 * @bch_enabled: flag to tell whether BCH ECC mode is used 431 * @ecc_bytes_hw: ECC bytes used by controller hardware for this 432 * chip 433 * @status: value to be returned if NAND_CMD_STATUS command 434 * is executed 435 * @last_command: keeps track of last command on this chip. used 436 * for reading correct status 437 * 438 * @cfg0, cfg1, cfg0_raw..: NANDc register configurations needed for 439 * ecc/non-ecc mode for the current nand flash 440 * device 441 */ 442 struct qcom_nand_host { 443 struct nand_chip chip; 444 struct list_head node; 445 446 int cs; 447 int cw_size; 448 int cw_data; 449 bool use_ecc; 450 bool bch_enabled; 451 int ecc_bytes_hw; 452 int spare_bytes; 453 int bbm_size; 454 u8 status; 455 int last_command; 456 457 u32 cfg0, cfg1; 458 u32 cfg0_raw, cfg1_raw; 459 u32 ecc_buf_cfg; 460 u32 ecc_bch_cfg; 461 u32 clrflashstatus; 462 u32 clrreadstatus; 463 }; 464 465 /* 466 * This data type corresponds to the NAND controller properties which varies 467 * among different NAND controllers. 468 * @ecc_modes - ecc mode for NAND 469 * @is_bam - whether NAND controller is using BAM 470 * @dev_cmd_reg_start - NAND_DEV_CMD_* registers starting offset 471 */ 472 struct qcom_nandc_props { 473 u32 ecc_modes; 474 bool is_bam; 475 u32 dev_cmd_reg_start; 476 }; 477 478 /* Frees the BAM transaction memory */ 479 static void free_bam_transaction(struct qcom_nand_controller *nandc) 480 { 481 struct bam_transaction *bam_txn = nandc->bam_txn; 482 483 devm_kfree(nandc->dev, bam_txn); 484 } 485 486 /* Allocates and Initializes the BAM transaction */ 487 static struct bam_transaction * 488 alloc_bam_transaction(struct qcom_nand_controller *nandc) 489 { 490 struct bam_transaction *bam_txn; 491 size_t bam_txn_size; 492 unsigned int num_cw = nandc->max_cwperpage; 493 void *bam_txn_buf; 494 495 bam_txn_size = 496 sizeof(*bam_txn) + num_cw * 497 ((sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS) + 498 (sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL) + 499 (sizeof(*bam_txn->data_sgl) * QPIC_PER_CW_DATA_SGL)); 500 501 bam_txn_buf = devm_kzalloc(nandc->dev, bam_txn_size, GFP_KERNEL); 502 if (!bam_txn_buf) 503 return NULL; 504 505 bam_txn = bam_txn_buf; 506 bam_txn_buf += sizeof(*bam_txn); 507 508 bam_txn->bam_ce = bam_txn_buf; 509 bam_txn_buf += 510 sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS * num_cw; 511 512 bam_txn->cmd_sgl = bam_txn_buf; 513 bam_txn_buf += 514 sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL * num_cw; 515 516 bam_txn->data_sgl = bam_txn_buf; 517 518 init_completion(&bam_txn->txn_done); 519 520 return bam_txn; 521 } 522 523 /* Clears the BAM transaction indexes */ 524 static void clear_bam_transaction(struct qcom_nand_controller *nandc) 525 { 526 struct bam_transaction *bam_txn = nandc->bam_txn; 527 528 if (!nandc->props->is_bam) 529 return; 530 531 bam_txn->bam_ce_pos = 0; 532 bam_txn->bam_ce_start = 0; 533 bam_txn->cmd_sgl_pos = 0; 534 bam_txn->cmd_sgl_start = 0; 535 bam_txn->tx_sgl_pos = 0; 536 bam_txn->tx_sgl_start = 0; 537 bam_txn->rx_sgl_pos = 0; 538 bam_txn->rx_sgl_start = 0; 539 bam_txn->last_data_desc = NULL; 540 bam_txn->wait_second_completion = false; 541 542 sg_init_table(bam_txn->cmd_sgl, nandc->max_cwperpage * 543 QPIC_PER_CW_CMD_SGL); 544 sg_init_table(bam_txn->data_sgl, nandc->max_cwperpage * 545 QPIC_PER_CW_DATA_SGL); 546 547 reinit_completion(&bam_txn->txn_done); 548 } 549 550 /* Callback for DMA descriptor completion */ 551 static void qpic_bam_dma_done(void *data) 552 { 553 struct bam_transaction *bam_txn = data; 554 555 /* 556 * In case of data transfer with NAND, 2 callbacks will be generated. 557 * One for command channel and another one for data channel. 558 * If current transaction has data descriptors 559 * (i.e. wait_second_completion is true), then set this to false 560 * and wait for second DMA descriptor completion. 561 */ 562 if (bam_txn->wait_second_completion) 563 bam_txn->wait_second_completion = false; 564 else 565 complete(&bam_txn->txn_done); 566 } 567 568 static inline struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip) 569 { 570 return container_of(chip, struct qcom_nand_host, chip); 571 } 572 573 static inline struct qcom_nand_controller * 574 get_qcom_nand_controller(struct nand_chip *chip) 575 { 576 return container_of(chip->controller, struct qcom_nand_controller, 577 controller); 578 } 579 580 static inline u32 nandc_read(struct qcom_nand_controller *nandc, int offset) 581 { 582 return ioread32(nandc->base + offset); 583 } 584 585 static inline void nandc_write(struct qcom_nand_controller *nandc, int offset, 586 u32 val) 587 { 588 iowrite32(val, nandc->base + offset); 589 } 590 591 static inline void nandc_read_buffer_sync(struct qcom_nand_controller *nandc, 592 bool is_cpu) 593 { 594 if (!nandc->props->is_bam) 595 return; 596 597 if (is_cpu) 598 dma_sync_single_for_cpu(nandc->dev, nandc->reg_read_dma, 599 MAX_REG_RD * 600 sizeof(*nandc->reg_read_buf), 601 DMA_FROM_DEVICE); 602 else 603 dma_sync_single_for_device(nandc->dev, nandc->reg_read_dma, 604 MAX_REG_RD * 605 sizeof(*nandc->reg_read_buf), 606 DMA_FROM_DEVICE); 607 } 608 609 static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset) 610 { 611 switch (offset) { 612 case NAND_FLASH_CMD: 613 return ®s->cmd; 614 case NAND_ADDR0: 615 return ®s->addr0; 616 case NAND_ADDR1: 617 return ®s->addr1; 618 case NAND_FLASH_CHIP_SELECT: 619 return ®s->chip_sel; 620 case NAND_EXEC_CMD: 621 return ®s->exec; 622 case NAND_FLASH_STATUS: 623 return ®s->clrflashstatus; 624 case NAND_DEV0_CFG0: 625 return ®s->cfg0; 626 case NAND_DEV0_CFG1: 627 return ®s->cfg1; 628 case NAND_DEV0_ECC_CFG: 629 return ®s->ecc_bch_cfg; 630 case NAND_READ_STATUS: 631 return ®s->clrreadstatus; 632 case NAND_DEV_CMD1: 633 return ®s->cmd1; 634 case NAND_DEV_CMD1_RESTORE: 635 return ®s->orig_cmd1; 636 case NAND_DEV_CMD_VLD: 637 return ®s->vld; 638 case NAND_DEV_CMD_VLD_RESTORE: 639 return ®s->orig_vld; 640 case NAND_EBI2_ECC_BUF_CFG: 641 return ®s->ecc_buf_cfg; 642 case NAND_READ_LOCATION_0: 643 return ®s->read_location0; 644 case NAND_READ_LOCATION_1: 645 return ®s->read_location1; 646 case NAND_READ_LOCATION_2: 647 return ®s->read_location2; 648 case NAND_READ_LOCATION_3: 649 return ®s->read_location3; 650 default: 651 return NULL; 652 } 653 } 654 655 static void nandc_set_reg(struct qcom_nand_controller *nandc, int offset, 656 u32 val) 657 { 658 struct nandc_regs *regs = nandc->regs; 659 __le32 *reg; 660 661 reg = offset_to_nandc_reg(regs, offset); 662 663 if (reg) 664 *reg = cpu_to_le32(val); 665 } 666 667 /* helper to configure address register values */ 668 static void set_address(struct qcom_nand_host *host, u16 column, int page) 669 { 670 struct nand_chip *chip = &host->chip; 671 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 672 673 if (chip->options & NAND_BUSWIDTH_16) 674 column >>= 1; 675 676 nandc_set_reg(nandc, NAND_ADDR0, page << 16 | column); 677 nandc_set_reg(nandc, NAND_ADDR1, page >> 16 & 0xff); 678 } 679 680 /* 681 * update_rw_regs: set up read/write register values, these will be 682 * written to the NAND controller registers via DMA 683 * 684 * @num_cw: number of steps for the read/write operation 685 * @read: read or write operation 686 */ 687 static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read) 688 { 689 struct nand_chip *chip = &host->chip; 690 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 691 u32 cmd, cfg0, cfg1, ecc_bch_cfg; 692 693 if (read) { 694 if (host->use_ecc) 695 cmd = PAGE_READ_WITH_ECC | PAGE_ACC | LAST_PAGE; 696 else 697 cmd = PAGE_READ | PAGE_ACC | LAST_PAGE; 698 } else { 699 cmd = PROGRAM_PAGE | PAGE_ACC | LAST_PAGE; 700 } 701 702 if (host->use_ecc) { 703 cfg0 = (host->cfg0 & ~(7U << CW_PER_PAGE)) | 704 (num_cw - 1) << CW_PER_PAGE; 705 706 cfg1 = host->cfg1; 707 ecc_bch_cfg = host->ecc_bch_cfg; 708 } else { 709 cfg0 = (host->cfg0_raw & ~(7U << CW_PER_PAGE)) | 710 (num_cw - 1) << CW_PER_PAGE; 711 712 cfg1 = host->cfg1_raw; 713 ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE; 714 } 715 716 nandc_set_reg(nandc, NAND_FLASH_CMD, cmd); 717 nandc_set_reg(nandc, NAND_DEV0_CFG0, cfg0); 718 nandc_set_reg(nandc, NAND_DEV0_CFG1, cfg1); 719 nandc_set_reg(nandc, NAND_DEV0_ECC_CFG, ecc_bch_cfg); 720 nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, host->ecc_buf_cfg); 721 nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus); 722 nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus); 723 nandc_set_reg(nandc, NAND_EXEC_CMD, 1); 724 725 if (read) 726 nandc_set_read_loc(nandc, 0, 0, host->use_ecc ? 727 host->cw_data : host->cw_size, 1); 728 } 729 730 /* 731 * Maps the scatter gather list for DMA transfer and forms the DMA descriptor 732 * for BAM. This descriptor will be added in the NAND DMA descriptor queue 733 * which will be submitted to DMA engine. 734 */ 735 static int prepare_bam_async_desc(struct qcom_nand_controller *nandc, 736 struct dma_chan *chan, 737 unsigned long flags) 738 { 739 struct desc_info *desc; 740 struct scatterlist *sgl; 741 unsigned int sgl_cnt; 742 int ret; 743 struct bam_transaction *bam_txn = nandc->bam_txn; 744 enum dma_transfer_direction dir_eng; 745 struct dma_async_tx_descriptor *dma_desc; 746 747 desc = kzalloc(sizeof(*desc), GFP_KERNEL); 748 if (!desc) 749 return -ENOMEM; 750 751 if (chan == nandc->cmd_chan) { 752 sgl = &bam_txn->cmd_sgl[bam_txn->cmd_sgl_start]; 753 sgl_cnt = bam_txn->cmd_sgl_pos - bam_txn->cmd_sgl_start; 754 bam_txn->cmd_sgl_start = bam_txn->cmd_sgl_pos; 755 dir_eng = DMA_MEM_TO_DEV; 756 desc->dir = DMA_TO_DEVICE; 757 } else if (chan == nandc->tx_chan) { 758 sgl = &bam_txn->data_sgl[bam_txn->tx_sgl_start]; 759 sgl_cnt = bam_txn->tx_sgl_pos - bam_txn->tx_sgl_start; 760 bam_txn->tx_sgl_start = bam_txn->tx_sgl_pos; 761 dir_eng = DMA_MEM_TO_DEV; 762 desc->dir = DMA_TO_DEVICE; 763 } else { 764 sgl = &bam_txn->data_sgl[bam_txn->rx_sgl_start]; 765 sgl_cnt = bam_txn->rx_sgl_pos - bam_txn->rx_sgl_start; 766 bam_txn->rx_sgl_start = bam_txn->rx_sgl_pos; 767 dir_eng = DMA_DEV_TO_MEM; 768 desc->dir = DMA_FROM_DEVICE; 769 } 770 771 sg_mark_end(sgl + sgl_cnt - 1); 772 ret = dma_map_sg(nandc->dev, sgl, sgl_cnt, desc->dir); 773 if (ret == 0) { 774 dev_err(nandc->dev, "failure in mapping desc\n"); 775 kfree(desc); 776 return -ENOMEM; 777 } 778 779 desc->sgl_cnt = sgl_cnt; 780 desc->bam_sgl = sgl; 781 782 dma_desc = dmaengine_prep_slave_sg(chan, sgl, sgl_cnt, dir_eng, 783 flags); 784 785 if (!dma_desc) { 786 dev_err(nandc->dev, "failure in prep desc\n"); 787 dma_unmap_sg(nandc->dev, sgl, sgl_cnt, desc->dir); 788 kfree(desc); 789 return -EINVAL; 790 } 791 792 desc->dma_desc = dma_desc; 793 794 /* update last data/command descriptor */ 795 if (chan == nandc->cmd_chan) 796 bam_txn->last_cmd_desc = dma_desc; 797 else 798 bam_txn->last_data_desc = dma_desc; 799 800 list_add_tail(&desc->node, &nandc->desc_list); 801 802 return 0; 803 } 804 805 /* 806 * Prepares the command descriptor for BAM DMA which will be used for NAND 807 * register reads and writes. The command descriptor requires the command 808 * to be formed in command element type so this function uses the command 809 * element from bam transaction ce array and fills the same with required 810 * data. A single SGL can contain multiple command elements so 811 * NAND_BAM_NEXT_SGL will be used for starting the separate SGL 812 * after the current command element. 813 */ 814 static int prep_bam_dma_desc_cmd(struct qcom_nand_controller *nandc, bool read, 815 int reg_off, const void *vaddr, 816 int size, unsigned int flags) 817 { 818 int bam_ce_size; 819 int i, ret; 820 struct bam_cmd_element *bam_ce_buffer; 821 struct bam_transaction *bam_txn = nandc->bam_txn; 822 823 bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_pos]; 824 825 /* fill the command desc */ 826 for (i = 0; i < size; i++) { 827 if (read) 828 bam_prep_ce(&bam_ce_buffer[i], 829 nandc_reg_phys(nandc, reg_off + 4 * i), 830 BAM_READ_COMMAND, 831 reg_buf_dma_addr(nandc, 832 (__le32 *)vaddr + i)); 833 else 834 bam_prep_ce_le32(&bam_ce_buffer[i], 835 nandc_reg_phys(nandc, reg_off + 4 * i), 836 BAM_WRITE_COMMAND, 837 *((__le32 *)vaddr + i)); 838 } 839 840 bam_txn->bam_ce_pos += size; 841 842 /* use the separate sgl after this command */ 843 if (flags & NAND_BAM_NEXT_SGL) { 844 bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_start]; 845 bam_ce_size = (bam_txn->bam_ce_pos - 846 bam_txn->bam_ce_start) * 847 sizeof(struct bam_cmd_element); 848 sg_set_buf(&bam_txn->cmd_sgl[bam_txn->cmd_sgl_pos], 849 bam_ce_buffer, bam_ce_size); 850 bam_txn->cmd_sgl_pos++; 851 bam_txn->bam_ce_start = bam_txn->bam_ce_pos; 852 853 if (flags & NAND_BAM_NWD) { 854 ret = prepare_bam_async_desc(nandc, nandc->cmd_chan, 855 DMA_PREP_FENCE | 856 DMA_PREP_CMD); 857 if (ret) 858 return ret; 859 } 860 } 861 862 return 0; 863 } 864 865 /* 866 * Prepares the data descriptor for BAM DMA which will be used for NAND 867 * data reads and writes. 868 */ 869 static int prep_bam_dma_desc_data(struct qcom_nand_controller *nandc, bool read, 870 const void *vaddr, 871 int size, unsigned int flags) 872 { 873 int ret; 874 struct bam_transaction *bam_txn = nandc->bam_txn; 875 876 if (read) { 877 sg_set_buf(&bam_txn->data_sgl[bam_txn->rx_sgl_pos], 878 vaddr, size); 879 bam_txn->rx_sgl_pos++; 880 } else { 881 sg_set_buf(&bam_txn->data_sgl[bam_txn->tx_sgl_pos], 882 vaddr, size); 883 bam_txn->tx_sgl_pos++; 884 885 /* 886 * BAM will only set EOT for DMA_PREP_INTERRUPT so if this flag 887 * is not set, form the DMA descriptor 888 */ 889 if (!(flags & NAND_BAM_NO_EOT)) { 890 ret = prepare_bam_async_desc(nandc, nandc->tx_chan, 891 DMA_PREP_INTERRUPT); 892 if (ret) 893 return ret; 894 } 895 } 896 897 return 0; 898 } 899 900 static int prep_adm_dma_desc(struct qcom_nand_controller *nandc, bool read, 901 int reg_off, const void *vaddr, int size, 902 bool flow_control) 903 { 904 struct desc_info *desc; 905 struct dma_async_tx_descriptor *dma_desc; 906 struct scatterlist *sgl; 907 struct dma_slave_config slave_conf; 908 enum dma_transfer_direction dir_eng; 909 int ret; 910 911 desc = kzalloc(sizeof(*desc), GFP_KERNEL); 912 if (!desc) 913 return -ENOMEM; 914 915 sgl = &desc->adm_sgl; 916 917 sg_init_one(sgl, vaddr, size); 918 919 if (read) { 920 dir_eng = DMA_DEV_TO_MEM; 921 desc->dir = DMA_FROM_DEVICE; 922 } else { 923 dir_eng = DMA_MEM_TO_DEV; 924 desc->dir = DMA_TO_DEVICE; 925 } 926 927 ret = dma_map_sg(nandc->dev, sgl, 1, desc->dir); 928 if (ret == 0) { 929 ret = -ENOMEM; 930 goto err; 931 } 932 933 memset(&slave_conf, 0x00, sizeof(slave_conf)); 934 935 slave_conf.device_fc = flow_control; 936 if (read) { 937 slave_conf.src_maxburst = 16; 938 slave_conf.src_addr = nandc->base_dma + reg_off; 939 slave_conf.slave_id = nandc->data_crci; 940 } else { 941 slave_conf.dst_maxburst = 16; 942 slave_conf.dst_addr = nandc->base_dma + reg_off; 943 slave_conf.slave_id = nandc->cmd_crci; 944 } 945 946 ret = dmaengine_slave_config(nandc->chan, &slave_conf); 947 if (ret) { 948 dev_err(nandc->dev, "failed to configure dma channel\n"); 949 goto err; 950 } 951 952 dma_desc = dmaengine_prep_slave_sg(nandc->chan, sgl, 1, dir_eng, 0); 953 if (!dma_desc) { 954 dev_err(nandc->dev, "failed to prepare desc\n"); 955 ret = -EINVAL; 956 goto err; 957 } 958 959 desc->dma_desc = dma_desc; 960 961 list_add_tail(&desc->node, &nandc->desc_list); 962 963 return 0; 964 err: 965 kfree(desc); 966 967 return ret; 968 } 969 970 /* 971 * read_reg_dma: prepares a descriptor to read a given number of 972 * contiguous registers to the reg_read_buf pointer 973 * 974 * @first: offset of the first register in the contiguous block 975 * @num_regs: number of registers to read 976 * @flags: flags to control DMA descriptor preparation 977 */ 978 static int read_reg_dma(struct qcom_nand_controller *nandc, int first, 979 int num_regs, unsigned int flags) 980 { 981 bool flow_control = false; 982 void *vaddr; 983 984 vaddr = nandc->reg_read_buf + nandc->reg_read_pos; 985 nandc->reg_read_pos += num_regs; 986 987 if (first == NAND_DEV_CMD_VLD || first == NAND_DEV_CMD1) 988 first = dev_cmd_reg_addr(nandc, first); 989 990 if (nandc->props->is_bam) 991 return prep_bam_dma_desc_cmd(nandc, true, first, vaddr, 992 num_regs, flags); 993 994 if (first == NAND_READ_ID || first == NAND_FLASH_STATUS) 995 flow_control = true; 996 997 return prep_adm_dma_desc(nandc, true, first, vaddr, 998 num_regs * sizeof(u32), flow_control); 999 } 1000 1001 /* 1002 * write_reg_dma: prepares a descriptor to write a given number of 1003 * contiguous registers 1004 * 1005 * @first: offset of the first register in the contiguous block 1006 * @num_regs: number of registers to write 1007 * @flags: flags to control DMA descriptor preparation 1008 */ 1009 static int write_reg_dma(struct qcom_nand_controller *nandc, int first, 1010 int num_regs, unsigned int flags) 1011 { 1012 bool flow_control = false; 1013 struct nandc_regs *regs = nandc->regs; 1014 void *vaddr; 1015 1016 vaddr = offset_to_nandc_reg(regs, first); 1017 1018 if (first == NAND_ERASED_CW_DETECT_CFG) { 1019 if (flags & NAND_ERASED_CW_SET) 1020 vaddr = ®s->erased_cw_detect_cfg_set; 1021 else 1022 vaddr = ®s->erased_cw_detect_cfg_clr; 1023 } 1024 1025 if (first == NAND_EXEC_CMD) 1026 flags |= NAND_BAM_NWD; 1027 1028 if (first == NAND_DEV_CMD1_RESTORE || first == NAND_DEV_CMD1) 1029 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD1); 1030 1031 if (first == NAND_DEV_CMD_VLD_RESTORE || first == NAND_DEV_CMD_VLD) 1032 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD); 1033 1034 if (nandc->props->is_bam) 1035 return prep_bam_dma_desc_cmd(nandc, false, first, vaddr, 1036 num_regs, flags); 1037 1038 if (first == NAND_FLASH_CMD) 1039 flow_control = true; 1040 1041 return prep_adm_dma_desc(nandc, false, first, vaddr, 1042 num_regs * sizeof(u32), flow_control); 1043 } 1044 1045 /* 1046 * read_data_dma: prepares a DMA descriptor to transfer data from the 1047 * controller's internal buffer to the buffer 'vaddr' 1048 * 1049 * @reg_off: offset within the controller's data buffer 1050 * @vaddr: virtual address of the buffer we want to write to 1051 * @size: DMA transaction size in bytes 1052 * @flags: flags to control DMA descriptor preparation 1053 */ 1054 static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off, 1055 const u8 *vaddr, int size, unsigned int flags) 1056 { 1057 if (nandc->props->is_bam) 1058 return prep_bam_dma_desc_data(nandc, true, vaddr, size, flags); 1059 1060 return prep_adm_dma_desc(nandc, true, reg_off, vaddr, size, false); 1061 } 1062 1063 /* 1064 * write_data_dma: prepares a DMA descriptor to transfer data from 1065 * 'vaddr' to the controller's internal buffer 1066 * 1067 * @reg_off: offset within the controller's data buffer 1068 * @vaddr: virtual address of the buffer we want to read from 1069 * @size: DMA transaction size in bytes 1070 * @flags: flags to control DMA descriptor preparation 1071 */ 1072 static int write_data_dma(struct qcom_nand_controller *nandc, int reg_off, 1073 const u8 *vaddr, int size, unsigned int flags) 1074 { 1075 if (nandc->props->is_bam) 1076 return prep_bam_dma_desc_data(nandc, false, vaddr, size, flags); 1077 1078 return prep_adm_dma_desc(nandc, false, reg_off, vaddr, size, false); 1079 } 1080 1081 /* 1082 * Helper to prepare DMA descriptors for configuring registers 1083 * before reading a NAND page. 1084 */ 1085 static void config_nand_page_read(struct qcom_nand_controller *nandc) 1086 { 1087 write_reg_dma(nandc, NAND_ADDR0, 2, 0); 1088 write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0); 1089 write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1, 0); 1090 write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1, 0); 1091 write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1, 1092 NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL); 1093 } 1094 1095 /* 1096 * Helper to prepare DMA descriptors for configuring registers 1097 * before reading each codeword in NAND page. 1098 */ 1099 static void 1100 config_nand_cw_read(struct qcom_nand_controller *nandc, bool use_ecc) 1101 { 1102 if (nandc->props->is_bam) 1103 write_reg_dma(nandc, NAND_READ_LOCATION_0, 4, 1104 NAND_BAM_NEXT_SGL); 1105 1106 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL); 1107 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1108 1109 if (use_ecc) { 1110 read_reg_dma(nandc, NAND_FLASH_STATUS, 2, 0); 1111 read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1, 1112 NAND_BAM_NEXT_SGL); 1113 } else { 1114 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL); 1115 } 1116 } 1117 1118 /* 1119 * Helper to prepare dma descriptors to configure registers needed for reading a 1120 * single codeword in page 1121 */ 1122 static void 1123 config_nand_single_cw_page_read(struct qcom_nand_controller *nandc, 1124 bool use_ecc) 1125 { 1126 config_nand_page_read(nandc); 1127 config_nand_cw_read(nandc, use_ecc); 1128 } 1129 1130 /* 1131 * Helper to prepare DMA descriptors used to configure registers needed for 1132 * before writing a NAND page. 1133 */ 1134 static void config_nand_page_write(struct qcom_nand_controller *nandc) 1135 { 1136 write_reg_dma(nandc, NAND_ADDR0, 2, 0); 1137 write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0); 1138 write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1, 1139 NAND_BAM_NEXT_SGL); 1140 } 1141 1142 /* 1143 * Helper to prepare DMA descriptors for configuring registers 1144 * before writing each codeword in NAND page. 1145 */ 1146 static void config_nand_cw_write(struct qcom_nand_controller *nandc) 1147 { 1148 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL); 1149 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1150 1151 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL); 1152 1153 write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0); 1154 write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL); 1155 } 1156 1157 /* 1158 * the following functions are used within chip->legacy.cmdfunc() to 1159 * perform different NAND_CMD_* commands 1160 */ 1161 1162 /* sets up descriptors for NAND_CMD_PARAM */ 1163 static int nandc_param(struct qcom_nand_host *host) 1164 { 1165 struct nand_chip *chip = &host->chip; 1166 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1167 1168 /* 1169 * NAND_CMD_PARAM is called before we know much about the FLASH chip 1170 * in use. we configure the controller to perform a raw read of 512 1171 * bytes to read onfi params 1172 */ 1173 nandc_set_reg(nandc, NAND_FLASH_CMD, PAGE_READ | PAGE_ACC | LAST_PAGE); 1174 nandc_set_reg(nandc, NAND_ADDR0, 0); 1175 nandc_set_reg(nandc, NAND_ADDR1, 0); 1176 nandc_set_reg(nandc, NAND_DEV0_CFG0, 0 << CW_PER_PAGE 1177 | 512 << UD_SIZE_BYTES 1178 | 5 << NUM_ADDR_CYCLES 1179 | 0 << SPARE_SIZE_BYTES); 1180 nandc_set_reg(nandc, NAND_DEV0_CFG1, 7 << NAND_RECOVERY_CYCLES 1181 | 0 << CS_ACTIVE_BSY 1182 | 17 << BAD_BLOCK_BYTE_NUM 1183 | 1 << BAD_BLOCK_IN_SPARE_AREA 1184 | 2 << WR_RD_BSY_GAP 1185 | 0 << WIDE_FLASH 1186 | 1 << DEV0_CFG1_ECC_DISABLE); 1187 nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, 1 << ECC_CFG_ECC_DISABLE); 1188 1189 /* configure CMD1 and VLD for ONFI param probing */ 1190 nandc_set_reg(nandc, NAND_DEV_CMD_VLD, 1191 (nandc->vld & ~READ_START_VLD)); 1192 nandc_set_reg(nandc, NAND_DEV_CMD1, 1193 (nandc->cmd1 & ~(0xFF << READ_ADDR)) 1194 | NAND_CMD_PARAM << READ_ADDR); 1195 1196 nandc_set_reg(nandc, NAND_EXEC_CMD, 1); 1197 1198 nandc_set_reg(nandc, NAND_DEV_CMD1_RESTORE, nandc->cmd1); 1199 nandc_set_reg(nandc, NAND_DEV_CMD_VLD_RESTORE, nandc->vld); 1200 nandc_set_read_loc(nandc, 0, 0, 512, 1); 1201 1202 write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1, 0); 1203 write_reg_dma(nandc, NAND_DEV_CMD1, 1, NAND_BAM_NEXT_SGL); 1204 1205 nandc->buf_count = 512; 1206 memset(nandc->data_buffer, 0xff, nandc->buf_count); 1207 1208 config_nand_single_cw_page_read(nandc, false); 1209 1210 read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, 1211 nandc->buf_count, 0); 1212 1213 /* restore CMD1 and VLD regs */ 1214 write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1, 0); 1215 write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1, NAND_BAM_NEXT_SGL); 1216 1217 return 0; 1218 } 1219 1220 /* sets up descriptors for NAND_CMD_ERASE1 */ 1221 static int erase_block(struct qcom_nand_host *host, int page_addr) 1222 { 1223 struct nand_chip *chip = &host->chip; 1224 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1225 1226 nandc_set_reg(nandc, NAND_FLASH_CMD, 1227 BLOCK_ERASE | PAGE_ACC | LAST_PAGE); 1228 nandc_set_reg(nandc, NAND_ADDR0, page_addr); 1229 nandc_set_reg(nandc, NAND_ADDR1, 0); 1230 nandc_set_reg(nandc, NAND_DEV0_CFG0, 1231 host->cfg0_raw & ~(7 << CW_PER_PAGE)); 1232 nandc_set_reg(nandc, NAND_DEV0_CFG1, host->cfg1_raw); 1233 nandc_set_reg(nandc, NAND_EXEC_CMD, 1); 1234 nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus); 1235 nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus); 1236 1237 write_reg_dma(nandc, NAND_FLASH_CMD, 3, NAND_BAM_NEXT_SGL); 1238 write_reg_dma(nandc, NAND_DEV0_CFG0, 2, NAND_BAM_NEXT_SGL); 1239 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1240 1241 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL); 1242 1243 write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0); 1244 write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL); 1245 1246 return 0; 1247 } 1248 1249 /* sets up descriptors for NAND_CMD_READID */ 1250 static int read_id(struct qcom_nand_host *host, int column) 1251 { 1252 struct nand_chip *chip = &host->chip; 1253 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1254 1255 if (column == -1) 1256 return 0; 1257 1258 nandc_set_reg(nandc, NAND_FLASH_CMD, FETCH_ID); 1259 nandc_set_reg(nandc, NAND_ADDR0, column); 1260 nandc_set_reg(nandc, NAND_ADDR1, 0); 1261 nandc_set_reg(nandc, NAND_FLASH_CHIP_SELECT, 1262 nandc->props->is_bam ? 0 : DM_EN); 1263 nandc_set_reg(nandc, NAND_EXEC_CMD, 1); 1264 1265 write_reg_dma(nandc, NAND_FLASH_CMD, 4, NAND_BAM_NEXT_SGL); 1266 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1267 1268 read_reg_dma(nandc, NAND_READ_ID, 1, NAND_BAM_NEXT_SGL); 1269 1270 return 0; 1271 } 1272 1273 /* sets up descriptors for NAND_CMD_RESET */ 1274 static int reset(struct qcom_nand_host *host) 1275 { 1276 struct nand_chip *chip = &host->chip; 1277 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1278 1279 nandc_set_reg(nandc, NAND_FLASH_CMD, RESET_DEVICE); 1280 nandc_set_reg(nandc, NAND_EXEC_CMD, 1); 1281 1282 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL); 1283 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1284 1285 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL); 1286 1287 return 0; 1288 } 1289 1290 /* helpers to submit/free our list of dma descriptors */ 1291 static int submit_descs(struct qcom_nand_controller *nandc) 1292 { 1293 struct desc_info *desc; 1294 dma_cookie_t cookie = 0; 1295 struct bam_transaction *bam_txn = nandc->bam_txn; 1296 int r; 1297 1298 if (nandc->props->is_bam) { 1299 if (bam_txn->rx_sgl_pos > bam_txn->rx_sgl_start) { 1300 r = prepare_bam_async_desc(nandc, nandc->rx_chan, 0); 1301 if (r) 1302 return r; 1303 } 1304 1305 if (bam_txn->tx_sgl_pos > bam_txn->tx_sgl_start) { 1306 r = prepare_bam_async_desc(nandc, nandc->tx_chan, 1307 DMA_PREP_INTERRUPT); 1308 if (r) 1309 return r; 1310 } 1311 1312 if (bam_txn->cmd_sgl_pos > bam_txn->cmd_sgl_start) { 1313 r = prepare_bam_async_desc(nandc, nandc->cmd_chan, 1314 DMA_PREP_CMD); 1315 if (r) 1316 return r; 1317 } 1318 } 1319 1320 list_for_each_entry(desc, &nandc->desc_list, node) 1321 cookie = dmaengine_submit(desc->dma_desc); 1322 1323 if (nandc->props->is_bam) { 1324 bam_txn->last_cmd_desc->callback = qpic_bam_dma_done; 1325 bam_txn->last_cmd_desc->callback_param = bam_txn; 1326 if (bam_txn->last_data_desc) { 1327 bam_txn->last_data_desc->callback = qpic_bam_dma_done; 1328 bam_txn->last_data_desc->callback_param = bam_txn; 1329 bam_txn->wait_second_completion = true; 1330 } 1331 1332 dma_async_issue_pending(nandc->tx_chan); 1333 dma_async_issue_pending(nandc->rx_chan); 1334 dma_async_issue_pending(nandc->cmd_chan); 1335 1336 if (!wait_for_completion_timeout(&bam_txn->txn_done, 1337 QPIC_NAND_COMPLETION_TIMEOUT)) 1338 return -ETIMEDOUT; 1339 } else { 1340 if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE) 1341 return -ETIMEDOUT; 1342 } 1343 1344 return 0; 1345 } 1346 1347 static void free_descs(struct qcom_nand_controller *nandc) 1348 { 1349 struct desc_info *desc, *n; 1350 1351 list_for_each_entry_safe(desc, n, &nandc->desc_list, node) { 1352 list_del(&desc->node); 1353 1354 if (nandc->props->is_bam) 1355 dma_unmap_sg(nandc->dev, desc->bam_sgl, 1356 desc->sgl_cnt, desc->dir); 1357 else 1358 dma_unmap_sg(nandc->dev, &desc->adm_sgl, 1, 1359 desc->dir); 1360 1361 kfree(desc); 1362 } 1363 } 1364 1365 /* reset the register read buffer for next NAND operation */ 1366 static void clear_read_regs(struct qcom_nand_controller *nandc) 1367 { 1368 nandc->reg_read_pos = 0; 1369 nandc_read_buffer_sync(nandc, false); 1370 } 1371 1372 static void pre_command(struct qcom_nand_host *host, int command) 1373 { 1374 struct nand_chip *chip = &host->chip; 1375 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1376 1377 nandc->buf_count = 0; 1378 nandc->buf_start = 0; 1379 host->use_ecc = false; 1380 host->last_command = command; 1381 1382 clear_read_regs(nandc); 1383 1384 if (command == NAND_CMD_RESET || command == NAND_CMD_READID || 1385 command == NAND_CMD_PARAM || command == NAND_CMD_ERASE1) 1386 clear_bam_transaction(nandc); 1387 } 1388 1389 /* 1390 * this is called after NAND_CMD_PAGEPROG and NAND_CMD_ERASE1 to set our 1391 * privately maintained status byte, this status byte can be read after 1392 * NAND_CMD_STATUS is called 1393 */ 1394 static void parse_erase_write_errors(struct qcom_nand_host *host, int command) 1395 { 1396 struct nand_chip *chip = &host->chip; 1397 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1398 struct nand_ecc_ctrl *ecc = &chip->ecc; 1399 int num_cw; 1400 int i; 1401 1402 num_cw = command == NAND_CMD_PAGEPROG ? ecc->steps : 1; 1403 nandc_read_buffer_sync(nandc, true); 1404 1405 for (i = 0; i < num_cw; i++) { 1406 u32 flash_status = le32_to_cpu(nandc->reg_read_buf[i]); 1407 1408 if (flash_status & FS_MPU_ERR) 1409 host->status &= ~NAND_STATUS_WP; 1410 1411 if (flash_status & FS_OP_ERR || (i == (num_cw - 1) && 1412 (flash_status & 1413 FS_DEVICE_STS_ERR))) 1414 host->status |= NAND_STATUS_FAIL; 1415 } 1416 } 1417 1418 static void post_command(struct qcom_nand_host *host, int command) 1419 { 1420 struct nand_chip *chip = &host->chip; 1421 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1422 1423 switch (command) { 1424 case NAND_CMD_READID: 1425 nandc_read_buffer_sync(nandc, true); 1426 memcpy(nandc->data_buffer, nandc->reg_read_buf, 1427 nandc->buf_count); 1428 break; 1429 case NAND_CMD_PAGEPROG: 1430 case NAND_CMD_ERASE1: 1431 parse_erase_write_errors(host, command); 1432 break; 1433 default: 1434 break; 1435 } 1436 } 1437 1438 /* 1439 * Implements chip->legacy.cmdfunc. It's only used for a limited set of 1440 * commands. The rest of the commands wouldn't be called by upper layers. 1441 * For example, NAND_CMD_READOOB would never be called because we have our own 1442 * versions of read_oob ops for nand_ecc_ctrl. 1443 */ 1444 static void qcom_nandc_command(struct nand_chip *chip, unsigned int command, 1445 int column, int page_addr) 1446 { 1447 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1448 struct nand_ecc_ctrl *ecc = &chip->ecc; 1449 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1450 bool wait = false; 1451 int ret = 0; 1452 1453 pre_command(host, command); 1454 1455 switch (command) { 1456 case NAND_CMD_RESET: 1457 ret = reset(host); 1458 wait = true; 1459 break; 1460 1461 case NAND_CMD_READID: 1462 nandc->buf_count = 4; 1463 ret = read_id(host, column); 1464 wait = true; 1465 break; 1466 1467 case NAND_CMD_PARAM: 1468 ret = nandc_param(host); 1469 wait = true; 1470 break; 1471 1472 case NAND_CMD_ERASE1: 1473 ret = erase_block(host, page_addr); 1474 wait = true; 1475 break; 1476 1477 case NAND_CMD_READ0: 1478 /* we read the entire page for now */ 1479 WARN_ON(column != 0); 1480 1481 host->use_ecc = true; 1482 set_address(host, 0, page_addr); 1483 update_rw_regs(host, ecc->steps, true); 1484 break; 1485 1486 case NAND_CMD_SEQIN: 1487 WARN_ON(column != 0); 1488 set_address(host, 0, page_addr); 1489 break; 1490 1491 case NAND_CMD_PAGEPROG: 1492 case NAND_CMD_STATUS: 1493 case NAND_CMD_NONE: 1494 default: 1495 break; 1496 } 1497 1498 if (ret) { 1499 dev_err(nandc->dev, "failure executing command %d\n", 1500 command); 1501 free_descs(nandc); 1502 return; 1503 } 1504 1505 if (wait) { 1506 ret = submit_descs(nandc); 1507 if (ret) 1508 dev_err(nandc->dev, 1509 "failure submitting descs for command %d\n", 1510 command); 1511 } 1512 1513 free_descs(nandc); 1514 1515 post_command(host, command); 1516 } 1517 1518 /* 1519 * when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read 1520 * an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS. 1521 * 1522 * when using RS ECC, the HW reports the same erros when reading an erased CW, 1523 * but it notifies that it is an erased CW by placing special characters at 1524 * certain offsets in the buffer. 1525 * 1526 * verify if the page is erased or not, and fix up the page for RS ECC by 1527 * replacing the special characters with 0xff. 1528 */ 1529 static bool erased_chunk_check_and_fixup(u8 *data_buf, int data_len) 1530 { 1531 u8 empty1, empty2; 1532 1533 /* 1534 * an erased page flags an error in NAND_FLASH_STATUS, check if the page 1535 * is erased by looking for 0x54s at offsets 3 and 175 from the 1536 * beginning of each codeword 1537 */ 1538 1539 empty1 = data_buf[3]; 1540 empty2 = data_buf[175]; 1541 1542 /* 1543 * if the erased codework markers, if they exist override them with 1544 * 0xffs 1545 */ 1546 if ((empty1 == 0x54 && empty2 == 0xff) || 1547 (empty1 == 0xff && empty2 == 0x54)) { 1548 data_buf[3] = 0xff; 1549 data_buf[175] = 0xff; 1550 } 1551 1552 /* 1553 * check if the entire chunk contains 0xffs or not. if it doesn't, then 1554 * restore the original values at the special offsets 1555 */ 1556 if (memchr_inv(data_buf, 0xff, data_len)) { 1557 data_buf[3] = empty1; 1558 data_buf[175] = empty2; 1559 1560 return false; 1561 } 1562 1563 return true; 1564 } 1565 1566 struct read_stats { 1567 __le32 flash; 1568 __le32 buffer; 1569 __le32 erased_cw; 1570 }; 1571 1572 /* reads back FLASH_STATUS register set by the controller */ 1573 static int check_flash_errors(struct qcom_nand_host *host, int cw_cnt) 1574 { 1575 struct nand_chip *chip = &host->chip; 1576 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1577 int i; 1578 1579 for (i = 0; i < cw_cnt; i++) { 1580 u32 flash = le32_to_cpu(nandc->reg_read_buf[i]); 1581 1582 if (flash & (FS_OP_ERR | FS_MPU_ERR)) 1583 return -EIO; 1584 } 1585 1586 return 0; 1587 } 1588 1589 /* performs raw read for one codeword */ 1590 static int 1591 qcom_nandc_read_cw_raw(struct mtd_info *mtd, struct nand_chip *chip, 1592 u8 *data_buf, u8 *oob_buf, int page, int cw) 1593 { 1594 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1595 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1596 struct nand_ecc_ctrl *ecc = &chip->ecc; 1597 int data_size1, data_size2, oob_size1, oob_size2; 1598 int ret, reg_off = FLASH_BUF_ACC, read_loc = 0; 1599 1600 nand_read_page_op(chip, page, 0, NULL, 0); 1601 host->use_ecc = false; 1602 1603 clear_bam_transaction(nandc); 1604 set_address(host, host->cw_size * cw, page); 1605 update_rw_regs(host, 1, true); 1606 config_nand_page_read(nandc); 1607 1608 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1); 1609 oob_size1 = host->bbm_size; 1610 1611 if (cw == (ecc->steps - 1)) { 1612 data_size2 = ecc->size - data_size1 - 1613 ((ecc->steps - 1) * 4); 1614 oob_size2 = (ecc->steps * 4) + host->ecc_bytes_hw + 1615 host->spare_bytes; 1616 } else { 1617 data_size2 = host->cw_data - data_size1; 1618 oob_size2 = host->ecc_bytes_hw + host->spare_bytes; 1619 } 1620 1621 if (nandc->props->is_bam) { 1622 nandc_set_read_loc(nandc, 0, read_loc, data_size1, 0); 1623 read_loc += data_size1; 1624 1625 nandc_set_read_loc(nandc, 1, read_loc, oob_size1, 0); 1626 read_loc += oob_size1; 1627 1628 nandc_set_read_loc(nandc, 2, read_loc, data_size2, 0); 1629 read_loc += data_size2; 1630 1631 nandc_set_read_loc(nandc, 3, read_loc, oob_size2, 1); 1632 } 1633 1634 config_nand_cw_read(nandc, false); 1635 1636 read_data_dma(nandc, reg_off, data_buf, data_size1, 0); 1637 reg_off += data_size1; 1638 1639 read_data_dma(nandc, reg_off, oob_buf, oob_size1, 0); 1640 reg_off += oob_size1; 1641 1642 read_data_dma(nandc, reg_off, data_buf + data_size1, data_size2, 0); 1643 reg_off += data_size2; 1644 1645 read_data_dma(nandc, reg_off, oob_buf + oob_size1, oob_size2, 0); 1646 1647 ret = submit_descs(nandc); 1648 free_descs(nandc); 1649 if (ret) { 1650 dev_err(nandc->dev, "failure to read raw cw %d\n", cw); 1651 return ret; 1652 } 1653 1654 return check_flash_errors(host, 1); 1655 } 1656 1657 /* 1658 * Bitflips can happen in erased codewords also so this function counts the 1659 * number of 0 in each CW for which ECC engine returns the uncorrectable 1660 * error. The page will be assumed as erased if this count is less than or 1661 * equal to the ecc->strength for each CW. 1662 * 1663 * 1. Both DATA and OOB need to be checked for number of 0. The 1664 * top-level API can be called with only data buf or OOB buf so use 1665 * chip->data_buf if data buf is null and chip->oob_poi if oob buf 1666 * is null for copying the raw bytes. 1667 * 2. Perform raw read for all the CW which has uncorrectable errors. 1668 * 3. For each CW, check the number of 0 in cw_data and usable OOB bytes. 1669 * The BBM and spare bytes bit flip won’t affect the ECC so don’t check 1670 * the number of bitflips in this area. 1671 */ 1672 static int 1673 check_for_erased_page(struct qcom_nand_host *host, u8 *data_buf, 1674 u8 *oob_buf, unsigned long uncorrectable_cws, 1675 int page, unsigned int max_bitflips) 1676 { 1677 struct nand_chip *chip = &host->chip; 1678 struct mtd_info *mtd = nand_to_mtd(chip); 1679 struct nand_ecc_ctrl *ecc = &chip->ecc; 1680 u8 *cw_data_buf, *cw_oob_buf; 1681 int cw, data_size, oob_size, ret = 0; 1682 1683 if (!data_buf) { 1684 data_buf = chip->data_buf; 1685 chip->pagebuf = -1; 1686 } 1687 1688 if (!oob_buf) { 1689 oob_buf = chip->oob_poi; 1690 chip->pagebuf = -1; 1691 } 1692 1693 for_each_set_bit(cw, &uncorrectable_cws, ecc->steps) { 1694 if (cw == (ecc->steps - 1)) { 1695 data_size = ecc->size - ((ecc->steps - 1) * 4); 1696 oob_size = (ecc->steps * 4) + host->ecc_bytes_hw; 1697 } else { 1698 data_size = host->cw_data; 1699 oob_size = host->ecc_bytes_hw; 1700 } 1701 1702 /* determine starting buffer address for current CW */ 1703 cw_data_buf = data_buf + (cw * host->cw_data); 1704 cw_oob_buf = oob_buf + (cw * ecc->bytes); 1705 1706 ret = qcom_nandc_read_cw_raw(mtd, chip, cw_data_buf, 1707 cw_oob_buf, page, cw); 1708 if (ret) 1709 return ret; 1710 1711 /* 1712 * make sure it isn't an erased page reported 1713 * as not-erased by HW because of a few bitflips 1714 */ 1715 ret = nand_check_erased_ecc_chunk(cw_data_buf, data_size, 1716 cw_oob_buf + host->bbm_size, 1717 oob_size, NULL, 1718 0, ecc->strength); 1719 if (ret < 0) { 1720 mtd->ecc_stats.failed++; 1721 } else { 1722 mtd->ecc_stats.corrected += ret; 1723 max_bitflips = max_t(unsigned int, max_bitflips, ret); 1724 } 1725 } 1726 1727 return max_bitflips; 1728 } 1729 1730 /* 1731 * reads back status registers set by the controller to notify page read 1732 * errors. this is equivalent to what 'ecc->correct()' would do. 1733 */ 1734 static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf, 1735 u8 *oob_buf, int page) 1736 { 1737 struct nand_chip *chip = &host->chip; 1738 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1739 struct mtd_info *mtd = nand_to_mtd(chip); 1740 struct nand_ecc_ctrl *ecc = &chip->ecc; 1741 unsigned int max_bitflips = 0, uncorrectable_cws = 0; 1742 struct read_stats *buf; 1743 bool flash_op_err = false, erased; 1744 int i; 1745 u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf; 1746 1747 buf = (struct read_stats *)nandc->reg_read_buf; 1748 nandc_read_buffer_sync(nandc, true); 1749 1750 for (i = 0; i < ecc->steps; i++, buf++) { 1751 u32 flash, buffer, erased_cw; 1752 int data_len, oob_len; 1753 1754 if (i == (ecc->steps - 1)) { 1755 data_len = ecc->size - ((ecc->steps - 1) << 2); 1756 oob_len = ecc->steps << 2; 1757 } else { 1758 data_len = host->cw_data; 1759 oob_len = 0; 1760 } 1761 1762 flash = le32_to_cpu(buf->flash); 1763 buffer = le32_to_cpu(buf->buffer); 1764 erased_cw = le32_to_cpu(buf->erased_cw); 1765 1766 /* 1767 * Check ECC failure for each codeword. ECC failure can 1768 * happen in either of the following conditions 1769 * 1. If number of bitflips are greater than ECC engine 1770 * capability. 1771 * 2. If this codeword contains all 0xff for which erased 1772 * codeword detection check will be done. 1773 */ 1774 if ((flash & FS_OP_ERR) && (buffer & BS_UNCORRECTABLE_BIT)) { 1775 /* 1776 * For BCH ECC, ignore erased codeword errors, if 1777 * ERASED_CW bits are set. 1778 */ 1779 if (host->bch_enabled) { 1780 erased = (erased_cw & ERASED_CW) == ERASED_CW ? 1781 true : false; 1782 /* 1783 * For RS ECC, HW reports the erased CW by placing 1784 * special characters at certain offsets in the buffer. 1785 * These special characters will be valid only if 1786 * complete page is read i.e. data_buf is not NULL. 1787 */ 1788 } else if (data_buf) { 1789 erased = erased_chunk_check_and_fixup(data_buf, 1790 data_len); 1791 } else { 1792 erased = false; 1793 } 1794 1795 if (!erased) 1796 uncorrectable_cws |= BIT(i); 1797 /* 1798 * Check if MPU or any other operational error (timeout, 1799 * device failure, etc.) happened for this codeword and 1800 * make flash_op_err true. If flash_op_err is set, then 1801 * EIO will be returned for page read. 1802 */ 1803 } else if (flash & (FS_OP_ERR | FS_MPU_ERR)) { 1804 flash_op_err = true; 1805 /* 1806 * No ECC or operational errors happened. Check the number of 1807 * bits corrected and update the ecc_stats.corrected. 1808 */ 1809 } else { 1810 unsigned int stat; 1811 1812 stat = buffer & BS_CORRECTABLE_ERR_MSK; 1813 mtd->ecc_stats.corrected += stat; 1814 max_bitflips = max(max_bitflips, stat); 1815 } 1816 1817 if (data_buf) 1818 data_buf += data_len; 1819 if (oob_buf) 1820 oob_buf += oob_len + ecc->bytes; 1821 } 1822 1823 if (flash_op_err) 1824 return -EIO; 1825 1826 if (!uncorrectable_cws) 1827 return max_bitflips; 1828 1829 return check_for_erased_page(host, data_buf_start, oob_buf_start, 1830 uncorrectable_cws, page, 1831 max_bitflips); 1832 } 1833 1834 /* 1835 * helper to perform the actual page read operation, used by ecc->read_page(), 1836 * ecc->read_oob() 1837 */ 1838 static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf, 1839 u8 *oob_buf, int page) 1840 { 1841 struct nand_chip *chip = &host->chip; 1842 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1843 struct nand_ecc_ctrl *ecc = &chip->ecc; 1844 u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf; 1845 int i, ret; 1846 1847 config_nand_page_read(nandc); 1848 1849 /* queue cmd descs for each codeword */ 1850 for (i = 0; i < ecc->steps; i++) { 1851 int data_size, oob_size; 1852 1853 if (i == (ecc->steps - 1)) { 1854 data_size = ecc->size - ((ecc->steps - 1) << 2); 1855 oob_size = (ecc->steps << 2) + host->ecc_bytes_hw + 1856 host->spare_bytes; 1857 } else { 1858 data_size = host->cw_data; 1859 oob_size = host->ecc_bytes_hw + host->spare_bytes; 1860 } 1861 1862 if (nandc->props->is_bam) { 1863 if (data_buf && oob_buf) { 1864 nandc_set_read_loc(nandc, 0, 0, data_size, 0); 1865 nandc_set_read_loc(nandc, 1, data_size, 1866 oob_size, 1); 1867 } else if (data_buf) { 1868 nandc_set_read_loc(nandc, 0, 0, data_size, 1); 1869 } else { 1870 nandc_set_read_loc(nandc, 0, data_size, 1871 oob_size, 1); 1872 } 1873 } 1874 1875 config_nand_cw_read(nandc, true); 1876 1877 if (data_buf) 1878 read_data_dma(nandc, FLASH_BUF_ACC, data_buf, 1879 data_size, 0); 1880 1881 /* 1882 * when ecc is enabled, the controller doesn't read the real 1883 * or dummy bad block markers in each chunk. To maintain a 1884 * consistent layout across RAW and ECC reads, we just 1885 * leave the real/dummy BBM offsets empty (i.e, filled with 1886 * 0xffs) 1887 */ 1888 if (oob_buf) { 1889 int j; 1890 1891 for (j = 0; j < host->bbm_size; j++) 1892 *oob_buf++ = 0xff; 1893 1894 read_data_dma(nandc, FLASH_BUF_ACC + data_size, 1895 oob_buf, oob_size, 0); 1896 } 1897 1898 if (data_buf) 1899 data_buf += data_size; 1900 if (oob_buf) 1901 oob_buf += oob_size; 1902 } 1903 1904 ret = submit_descs(nandc); 1905 free_descs(nandc); 1906 1907 if (ret) { 1908 dev_err(nandc->dev, "failure to read page/oob\n"); 1909 return ret; 1910 } 1911 1912 return parse_read_errors(host, data_buf_start, oob_buf_start, page); 1913 } 1914 1915 /* 1916 * a helper that copies the last step/codeword of a page (containing free oob) 1917 * into our local buffer 1918 */ 1919 static int copy_last_cw(struct qcom_nand_host *host, int page) 1920 { 1921 struct nand_chip *chip = &host->chip; 1922 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1923 struct nand_ecc_ctrl *ecc = &chip->ecc; 1924 int size; 1925 int ret; 1926 1927 clear_read_regs(nandc); 1928 1929 size = host->use_ecc ? host->cw_data : host->cw_size; 1930 1931 /* prepare a clean read buffer */ 1932 memset(nandc->data_buffer, 0xff, size); 1933 1934 set_address(host, host->cw_size * (ecc->steps - 1), page); 1935 update_rw_regs(host, 1, true); 1936 1937 config_nand_single_cw_page_read(nandc, host->use_ecc); 1938 1939 read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size, 0); 1940 1941 ret = submit_descs(nandc); 1942 if (ret) 1943 dev_err(nandc->dev, "failed to copy last codeword\n"); 1944 1945 free_descs(nandc); 1946 1947 return ret; 1948 } 1949 1950 /* implements ecc->read_page() */ 1951 static int qcom_nandc_read_page(struct nand_chip *chip, uint8_t *buf, 1952 int oob_required, int page) 1953 { 1954 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1955 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1956 u8 *data_buf, *oob_buf = NULL; 1957 1958 nand_read_page_op(chip, page, 0, NULL, 0); 1959 data_buf = buf; 1960 oob_buf = oob_required ? chip->oob_poi : NULL; 1961 1962 clear_bam_transaction(nandc); 1963 1964 return read_page_ecc(host, data_buf, oob_buf, page); 1965 } 1966 1967 /* implements ecc->read_page_raw() */ 1968 static int qcom_nandc_read_page_raw(struct nand_chip *chip, uint8_t *buf, 1969 int oob_required, int page) 1970 { 1971 struct mtd_info *mtd = nand_to_mtd(chip); 1972 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1973 struct nand_ecc_ctrl *ecc = &chip->ecc; 1974 int cw, ret; 1975 u8 *data_buf = buf, *oob_buf = chip->oob_poi; 1976 1977 for (cw = 0; cw < ecc->steps; cw++) { 1978 ret = qcom_nandc_read_cw_raw(mtd, chip, data_buf, oob_buf, 1979 page, cw); 1980 if (ret) 1981 return ret; 1982 1983 data_buf += host->cw_data; 1984 oob_buf += ecc->bytes; 1985 } 1986 1987 return 0; 1988 } 1989 1990 /* implements ecc->read_oob() */ 1991 static int qcom_nandc_read_oob(struct nand_chip *chip, int page) 1992 { 1993 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1994 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1995 struct nand_ecc_ctrl *ecc = &chip->ecc; 1996 1997 clear_read_regs(nandc); 1998 clear_bam_transaction(nandc); 1999 2000 host->use_ecc = true; 2001 set_address(host, 0, page); 2002 update_rw_regs(host, ecc->steps, true); 2003 2004 return read_page_ecc(host, NULL, chip->oob_poi, page); 2005 } 2006 2007 /* implements ecc->write_page() */ 2008 static int qcom_nandc_write_page(struct nand_chip *chip, const uint8_t *buf, 2009 int oob_required, int page) 2010 { 2011 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2012 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2013 struct nand_ecc_ctrl *ecc = &chip->ecc; 2014 u8 *data_buf, *oob_buf; 2015 int i, ret; 2016 2017 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 2018 2019 clear_read_regs(nandc); 2020 clear_bam_transaction(nandc); 2021 2022 data_buf = (u8 *)buf; 2023 oob_buf = chip->oob_poi; 2024 2025 host->use_ecc = true; 2026 update_rw_regs(host, ecc->steps, false); 2027 config_nand_page_write(nandc); 2028 2029 for (i = 0; i < ecc->steps; i++) { 2030 int data_size, oob_size; 2031 2032 if (i == (ecc->steps - 1)) { 2033 data_size = ecc->size - ((ecc->steps - 1) << 2); 2034 oob_size = (ecc->steps << 2) + host->ecc_bytes_hw + 2035 host->spare_bytes; 2036 } else { 2037 data_size = host->cw_data; 2038 oob_size = ecc->bytes; 2039 } 2040 2041 2042 write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size, 2043 i == (ecc->steps - 1) ? NAND_BAM_NO_EOT : 0); 2044 2045 /* 2046 * when ECC is enabled, we don't really need to write anything 2047 * to oob for the first n - 1 codewords since these oob regions 2048 * just contain ECC bytes that's written by the controller 2049 * itself. For the last codeword, we skip the bbm positions and 2050 * write to the free oob area. 2051 */ 2052 if (i == (ecc->steps - 1)) { 2053 oob_buf += host->bbm_size; 2054 2055 write_data_dma(nandc, FLASH_BUF_ACC + data_size, 2056 oob_buf, oob_size, 0); 2057 } 2058 2059 config_nand_cw_write(nandc); 2060 2061 data_buf += data_size; 2062 oob_buf += oob_size; 2063 } 2064 2065 ret = submit_descs(nandc); 2066 if (ret) 2067 dev_err(nandc->dev, "failure to write page\n"); 2068 2069 free_descs(nandc); 2070 2071 if (!ret) 2072 ret = nand_prog_page_end_op(chip); 2073 2074 return ret; 2075 } 2076 2077 /* implements ecc->write_page_raw() */ 2078 static int qcom_nandc_write_page_raw(struct nand_chip *chip, 2079 const uint8_t *buf, int oob_required, 2080 int page) 2081 { 2082 struct mtd_info *mtd = nand_to_mtd(chip); 2083 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2084 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2085 struct nand_ecc_ctrl *ecc = &chip->ecc; 2086 u8 *data_buf, *oob_buf; 2087 int i, ret; 2088 2089 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 2090 clear_read_regs(nandc); 2091 clear_bam_transaction(nandc); 2092 2093 data_buf = (u8 *)buf; 2094 oob_buf = chip->oob_poi; 2095 2096 host->use_ecc = false; 2097 update_rw_regs(host, ecc->steps, false); 2098 config_nand_page_write(nandc); 2099 2100 for (i = 0; i < ecc->steps; i++) { 2101 int data_size1, data_size2, oob_size1, oob_size2; 2102 int reg_off = FLASH_BUF_ACC; 2103 2104 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1); 2105 oob_size1 = host->bbm_size; 2106 2107 if (i == (ecc->steps - 1)) { 2108 data_size2 = ecc->size - data_size1 - 2109 ((ecc->steps - 1) << 2); 2110 oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw + 2111 host->spare_bytes; 2112 } else { 2113 data_size2 = host->cw_data - data_size1; 2114 oob_size2 = host->ecc_bytes_hw + host->spare_bytes; 2115 } 2116 2117 write_data_dma(nandc, reg_off, data_buf, data_size1, 2118 NAND_BAM_NO_EOT); 2119 reg_off += data_size1; 2120 data_buf += data_size1; 2121 2122 write_data_dma(nandc, reg_off, oob_buf, oob_size1, 2123 NAND_BAM_NO_EOT); 2124 reg_off += oob_size1; 2125 oob_buf += oob_size1; 2126 2127 write_data_dma(nandc, reg_off, data_buf, data_size2, 2128 NAND_BAM_NO_EOT); 2129 reg_off += data_size2; 2130 data_buf += data_size2; 2131 2132 write_data_dma(nandc, reg_off, oob_buf, oob_size2, 0); 2133 oob_buf += oob_size2; 2134 2135 config_nand_cw_write(nandc); 2136 } 2137 2138 ret = submit_descs(nandc); 2139 if (ret) 2140 dev_err(nandc->dev, "failure to write raw page\n"); 2141 2142 free_descs(nandc); 2143 2144 if (!ret) 2145 ret = nand_prog_page_end_op(chip); 2146 2147 return ret; 2148 } 2149 2150 /* 2151 * implements ecc->write_oob() 2152 * 2153 * the NAND controller cannot write only data or only OOB within a codeword 2154 * since ECC is calculated for the combined codeword. So update the OOB from 2155 * chip->oob_poi, and pad the data area with OxFF before writing. 2156 */ 2157 static int qcom_nandc_write_oob(struct nand_chip *chip, int page) 2158 { 2159 struct mtd_info *mtd = nand_to_mtd(chip); 2160 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2161 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2162 struct nand_ecc_ctrl *ecc = &chip->ecc; 2163 u8 *oob = chip->oob_poi; 2164 int data_size, oob_size; 2165 int ret; 2166 2167 host->use_ecc = true; 2168 clear_bam_transaction(nandc); 2169 2170 /* calculate the data and oob size for the last codeword/step */ 2171 data_size = ecc->size - ((ecc->steps - 1) << 2); 2172 oob_size = mtd->oobavail; 2173 2174 memset(nandc->data_buffer, 0xff, host->cw_data); 2175 /* override new oob content to last codeword */ 2176 mtd_ooblayout_get_databytes(mtd, nandc->data_buffer + data_size, oob, 2177 0, mtd->oobavail); 2178 2179 set_address(host, host->cw_size * (ecc->steps - 1), page); 2180 update_rw_regs(host, 1, false); 2181 2182 config_nand_page_write(nandc); 2183 write_data_dma(nandc, FLASH_BUF_ACC, 2184 nandc->data_buffer, data_size + oob_size, 0); 2185 config_nand_cw_write(nandc); 2186 2187 ret = submit_descs(nandc); 2188 2189 free_descs(nandc); 2190 2191 if (ret) { 2192 dev_err(nandc->dev, "failure to write oob\n"); 2193 return -EIO; 2194 } 2195 2196 return nand_prog_page_end_op(chip); 2197 } 2198 2199 static int qcom_nandc_block_bad(struct nand_chip *chip, loff_t ofs) 2200 { 2201 struct mtd_info *mtd = nand_to_mtd(chip); 2202 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2203 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2204 struct nand_ecc_ctrl *ecc = &chip->ecc; 2205 int page, ret, bbpos, bad = 0; 2206 2207 page = (int)(ofs >> chip->page_shift) & chip->pagemask; 2208 2209 /* 2210 * configure registers for a raw sub page read, the address is set to 2211 * the beginning of the last codeword, we don't care about reading ecc 2212 * portion of oob. we just want the first few bytes from this codeword 2213 * that contains the BBM 2214 */ 2215 host->use_ecc = false; 2216 2217 clear_bam_transaction(nandc); 2218 ret = copy_last_cw(host, page); 2219 if (ret) 2220 goto err; 2221 2222 if (check_flash_errors(host, 1)) { 2223 dev_warn(nandc->dev, "error when trying to read BBM\n"); 2224 goto err; 2225 } 2226 2227 bbpos = mtd->writesize - host->cw_size * (ecc->steps - 1); 2228 2229 bad = nandc->data_buffer[bbpos] != 0xff; 2230 2231 if (chip->options & NAND_BUSWIDTH_16) 2232 bad = bad || (nandc->data_buffer[bbpos + 1] != 0xff); 2233 err: 2234 return bad; 2235 } 2236 2237 static int qcom_nandc_block_markbad(struct nand_chip *chip, loff_t ofs) 2238 { 2239 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2240 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2241 struct nand_ecc_ctrl *ecc = &chip->ecc; 2242 int page, ret; 2243 2244 clear_read_regs(nandc); 2245 clear_bam_transaction(nandc); 2246 2247 /* 2248 * to mark the BBM as bad, we flash the entire last codeword with 0s. 2249 * we don't care about the rest of the content in the codeword since 2250 * we aren't going to use this block again 2251 */ 2252 memset(nandc->data_buffer, 0x00, host->cw_size); 2253 2254 page = (int)(ofs >> chip->page_shift) & chip->pagemask; 2255 2256 /* prepare write */ 2257 host->use_ecc = false; 2258 set_address(host, host->cw_size * (ecc->steps - 1), page); 2259 update_rw_regs(host, 1, false); 2260 2261 config_nand_page_write(nandc); 2262 write_data_dma(nandc, FLASH_BUF_ACC, 2263 nandc->data_buffer, host->cw_size, 0); 2264 config_nand_cw_write(nandc); 2265 2266 ret = submit_descs(nandc); 2267 2268 free_descs(nandc); 2269 2270 if (ret) { 2271 dev_err(nandc->dev, "failure to update BBM\n"); 2272 return -EIO; 2273 } 2274 2275 return nand_prog_page_end_op(chip); 2276 } 2277 2278 /* 2279 * the three functions below implement chip->legacy.read_byte(), 2280 * chip->legacy.read_buf() and chip->legacy.write_buf() respectively. these 2281 * aren't used for reading/writing page data, they are used for smaller data 2282 * like reading id, status etc 2283 */ 2284 static uint8_t qcom_nandc_read_byte(struct nand_chip *chip) 2285 { 2286 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2287 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2288 u8 *buf = nandc->data_buffer; 2289 u8 ret = 0x0; 2290 2291 if (host->last_command == NAND_CMD_STATUS) { 2292 ret = host->status; 2293 2294 host->status = NAND_STATUS_READY | NAND_STATUS_WP; 2295 2296 return ret; 2297 } 2298 2299 if (nandc->buf_start < nandc->buf_count) 2300 ret = buf[nandc->buf_start++]; 2301 2302 return ret; 2303 } 2304 2305 static void qcom_nandc_read_buf(struct nand_chip *chip, uint8_t *buf, int len) 2306 { 2307 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2308 int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start); 2309 2310 memcpy(buf, nandc->data_buffer + nandc->buf_start, real_len); 2311 nandc->buf_start += real_len; 2312 } 2313 2314 static void qcom_nandc_write_buf(struct nand_chip *chip, const uint8_t *buf, 2315 int len) 2316 { 2317 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2318 int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start); 2319 2320 memcpy(nandc->data_buffer + nandc->buf_start, buf, real_len); 2321 2322 nandc->buf_start += real_len; 2323 } 2324 2325 /* we support only one external chip for now */ 2326 static void qcom_nandc_select_chip(struct nand_chip *chip, int chipnr) 2327 { 2328 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2329 2330 if (chipnr <= 0) 2331 return; 2332 2333 dev_warn(nandc->dev, "invalid chip select\n"); 2334 } 2335 2336 /* 2337 * NAND controller page layout info 2338 * 2339 * Layout with ECC enabled: 2340 * 2341 * |----------------------| |---------------------------------| 2342 * | xx.......yy| | *********xx.......yy| 2343 * | DATA xx..ECC..yy| | DATA **SPARE**xx..ECC..yy| 2344 * | (516) xx.......yy| | (516-n*4) **(n*4)**xx.......yy| 2345 * | xx.......yy| | *********xx.......yy| 2346 * |----------------------| |---------------------------------| 2347 * codeword 1,2..n-1 codeword n 2348 * <---(528/532 Bytes)--> <-------(528/532 Bytes)---------> 2349 * 2350 * n = Number of codewords in the page 2351 * . = ECC bytes 2352 * * = Spare/free bytes 2353 * x = Unused byte(s) 2354 * y = Reserved byte(s) 2355 * 2356 * 2K page: n = 4, spare = 16 bytes 2357 * 4K page: n = 8, spare = 32 bytes 2358 * 8K page: n = 16, spare = 64 bytes 2359 * 2360 * the qcom nand controller operates at a sub page/codeword level. each 2361 * codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively. 2362 * the number of ECC bytes vary based on the ECC strength and the bus width. 2363 * 2364 * the first n - 1 codewords contains 516 bytes of user data, the remaining 2365 * 12/16 bytes consist of ECC and reserved data. The nth codeword contains 2366 * both user data and spare(oobavail) bytes that sum up to 516 bytes. 2367 * 2368 * When we access a page with ECC enabled, the reserved bytes(s) are not 2369 * accessible at all. When reading, we fill up these unreadable positions 2370 * with 0xffs. When writing, the controller skips writing the inaccessible 2371 * bytes. 2372 * 2373 * Layout with ECC disabled: 2374 * 2375 * |------------------------------| |---------------------------------------| 2376 * | yy xx.......| | bb *********xx.......| 2377 * | DATA1 yy DATA2 xx..ECC..| | DATA1 bb DATA2 **SPARE**xx..ECC..| 2378 * | (size1) yy (size2) xx.......| | (size1) bb (size2) **(n*4)**xx.......| 2379 * | yy xx.......| | bb *********xx.......| 2380 * |------------------------------| |---------------------------------------| 2381 * codeword 1,2..n-1 codeword n 2382 * <-------(528/532 Bytes)------> <-----------(528/532 Bytes)-----------> 2383 * 2384 * n = Number of codewords in the page 2385 * . = ECC bytes 2386 * * = Spare/free bytes 2387 * x = Unused byte(s) 2388 * y = Dummy Bad Bock byte(s) 2389 * b = Real Bad Block byte(s) 2390 * size1/size2 = function of codeword size and 'n' 2391 * 2392 * when the ECC block is disabled, one reserved byte (or two for 16 bit bus 2393 * width) is now accessible. For the first n - 1 codewords, these are dummy Bad 2394 * Block Markers. In the last codeword, this position contains the real BBM 2395 * 2396 * In order to have a consistent layout between RAW and ECC modes, we assume 2397 * the following OOB layout arrangement: 2398 * 2399 * |-----------| |--------------------| 2400 * |yyxx.......| |bb*********xx.......| 2401 * |yyxx..ECC..| |bb*FREEOOB*xx..ECC..| 2402 * |yyxx.......| |bb*********xx.......| 2403 * |yyxx.......| |bb*********xx.......| 2404 * |-----------| |--------------------| 2405 * first n - 1 nth OOB region 2406 * OOB regions 2407 * 2408 * n = Number of codewords in the page 2409 * . = ECC bytes 2410 * * = FREE OOB bytes 2411 * y = Dummy bad block byte(s) (inaccessible when ECC enabled) 2412 * x = Unused byte(s) 2413 * b = Real bad block byte(s) (inaccessible when ECC enabled) 2414 * 2415 * This layout is read as is when ECC is disabled. When ECC is enabled, the 2416 * inaccessible Bad Block byte(s) are ignored when we write to a page/oob, 2417 * and assumed as 0xffs when we read a page/oob. The ECC, unused and 2418 * dummy/real bad block bytes are grouped as ecc bytes (i.e, ecc->bytes is 2419 * the sum of the three). 2420 */ 2421 static int qcom_nand_ooblayout_ecc(struct mtd_info *mtd, int section, 2422 struct mtd_oob_region *oobregion) 2423 { 2424 struct nand_chip *chip = mtd_to_nand(mtd); 2425 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2426 struct nand_ecc_ctrl *ecc = &chip->ecc; 2427 2428 if (section > 1) 2429 return -ERANGE; 2430 2431 if (!section) { 2432 oobregion->length = (ecc->bytes * (ecc->steps - 1)) + 2433 host->bbm_size; 2434 oobregion->offset = 0; 2435 } else { 2436 oobregion->length = host->ecc_bytes_hw + host->spare_bytes; 2437 oobregion->offset = mtd->oobsize - oobregion->length; 2438 } 2439 2440 return 0; 2441 } 2442 2443 static int qcom_nand_ooblayout_free(struct mtd_info *mtd, int section, 2444 struct mtd_oob_region *oobregion) 2445 { 2446 struct nand_chip *chip = mtd_to_nand(mtd); 2447 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2448 struct nand_ecc_ctrl *ecc = &chip->ecc; 2449 2450 if (section) 2451 return -ERANGE; 2452 2453 oobregion->length = ecc->steps * 4; 2454 oobregion->offset = ((ecc->steps - 1) * ecc->bytes) + host->bbm_size; 2455 2456 return 0; 2457 } 2458 2459 static const struct mtd_ooblayout_ops qcom_nand_ooblayout_ops = { 2460 .ecc = qcom_nand_ooblayout_ecc, 2461 .free = qcom_nand_ooblayout_free, 2462 }; 2463 2464 static int 2465 qcom_nandc_calc_ecc_bytes(int step_size, int strength) 2466 { 2467 return strength == 4 ? 12 : 16; 2468 } 2469 NAND_ECC_CAPS_SINGLE(qcom_nandc_ecc_caps, qcom_nandc_calc_ecc_bytes, 2470 NANDC_STEP_SIZE, 4, 8); 2471 2472 static int qcom_nand_attach_chip(struct nand_chip *chip) 2473 { 2474 struct mtd_info *mtd = nand_to_mtd(chip); 2475 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2476 struct nand_ecc_ctrl *ecc = &chip->ecc; 2477 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2478 int cwperpage, bad_block_byte, ret; 2479 bool wide_bus; 2480 int ecc_mode = 1; 2481 2482 /* controller only supports 512 bytes data steps */ 2483 ecc->size = NANDC_STEP_SIZE; 2484 wide_bus = chip->options & NAND_BUSWIDTH_16 ? true : false; 2485 cwperpage = mtd->writesize / NANDC_STEP_SIZE; 2486 2487 /* 2488 * Each CW has 4 available OOB bytes which will be protected with ECC 2489 * so remaining bytes can be used for ECC. 2490 */ 2491 ret = nand_ecc_choose_conf(chip, &qcom_nandc_ecc_caps, 2492 mtd->oobsize - (cwperpage * 4)); 2493 if (ret) { 2494 dev_err(nandc->dev, "No valid ECC settings possible\n"); 2495 return ret; 2496 } 2497 2498 if (ecc->strength >= 8) { 2499 /* 8 bit ECC defaults to BCH ECC on all platforms */ 2500 host->bch_enabled = true; 2501 ecc_mode = 1; 2502 2503 if (wide_bus) { 2504 host->ecc_bytes_hw = 14; 2505 host->spare_bytes = 0; 2506 host->bbm_size = 2; 2507 } else { 2508 host->ecc_bytes_hw = 13; 2509 host->spare_bytes = 2; 2510 host->bbm_size = 1; 2511 } 2512 } else { 2513 /* 2514 * if the controller supports BCH for 4 bit ECC, the controller 2515 * uses lesser bytes for ECC. If RS is used, the ECC bytes is 2516 * always 10 bytes 2517 */ 2518 if (nandc->props->ecc_modes & ECC_BCH_4BIT) { 2519 /* BCH */ 2520 host->bch_enabled = true; 2521 ecc_mode = 0; 2522 2523 if (wide_bus) { 2524 host->ecc_bytes_hw = 8; 2525 host->spare_bytes = 2; 2526 host->bbm_size = 2; 2527 } else { 2528 host->ecc_bytes_hw = 7; 2529 host->spare_bytes = 4; 2530 host->bbm_size = 1; 2531 } 2532 } else { 2533 /* RS */ 2534 host->ecc_bytes_hw = 10; 2535 2536 if (wide_bus) { 2537 host->spare_bytes = 0; 2538 host->bbm_size = 2; 2539 } else { 2540 host->spare_bytes = 1; 2541 host->bbm_size = 1; 2542 } 2543 } 2544 } 2545 2546 /* 2547 * we consider ecc->bytes as the sum of all the non-data content in a 2548 * step. It gives us a clean representation of the oob area (even if 2549 * all the bytes aren't used for ECC).It is always 16 bytes for 8 bit 2550 * ECC and 12 bytes for 4 bit ECC 2551 */ 2552 ecc->bytes = host->ecc_bytes_hw + host->spare_bytes + host->bbm_size; 2553 2554 ecc->read_page = qcom_nandc_read_page; 2555 ecc->read_page_raw = qcom_nandc_read_page_raw; 2556 ecc->read_oob = qcom_nandc_read_oob; 2557 ecc->write_page = qcom_nandc_write_page; 2558 ecc->write_page_raw = qcom_nandc_write_page_raw; 2559 ecc->write_oob = qcom_nandc_write_oob; 2560 2561 ecc->mode = NAND_ECC_HW; 2562 2563 mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops); 2564 2565 nandc->max_cwperpage = max_t(unsigned int, nandc->max_cwperpage, 2566 cwperpage); 2567 2568 /* 2569 * DATA_UD_BYTES varies based on whether the read/write command protects 2570 * spare data with ECC too. We protect spare data by default, so we set 2571 * it to main + spare data, which are 512 and 4 bytes respectively. 2572 */ 2573 host->cw_data = 516; 2574 2575 /* 2576 * total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes 2577 * for 8 bit ECC 2578 */ 2579 host->cw_size = host->cw_data + ecc->bytes; 2580 bad_block_byte = mtd->writesize - host->cw_size * (cwperpage - 1) + 1; 2581 2582 host->cfg0 = (cwperpage - 1) << CW_PER_PAGE 2583 | host->cw_data << UD_SIZE_BYTES 2584 | 0 << DISABLE_STATUS_AFTER_WRITE 2585 | 5 << NUM_ADDR_CYCLES 2586 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_RS 2587 | 0 << STATUS_BFR_READ 2588 | 1 << SET_RD_MODE_AFTER_STATUS 2589 | host->spare_bytes << SPARE_SIZE_BYTES; 2590 2591 host->cfg1 = 7 << NAND_RECOVERY_CYCLES 2592 | 0 << CS_ACTIVE_BSY 2593 | bad_block_byte << BAD_BLOCK_BYTE_NUM 2594 | 0 << BAD_BLOCK_IN_SPARE_AREA 2595 | 2 << WR_RD_BSY_GAP 2596 | wide_bus << WIDE_FLASH 2597 | host->bch_enabled << ENABLE_BCH_ECC; 2598 2599 host->cfg0_raw = (cwperpage - 1) << CW_PER_PAGE 2600 | host->cw_size << UD_SIZE_BYTES 2601 | 5 << NUM_ADDR_CYCLES 2602 | 0 << SPARE_SIZE_BYTES; 2603 2604 host->cfg1_raw = 7 << NAND_RECOVERY_CYCLES 2605 | 0 << CS_ACTIVE_BSY 2606 | 17 << BAD_BLOCK_BYTE_NUM 2607 | 1 << BAD_BLOCK_IN_SPARE_AREA 2608 | 2 << WR_RD_BSY_GAP 2609 | wide_bus << WIDE_FLASH 2610 | 1 << DEV0_CFG1_ECC_DISABLE; 2611 2612 host->ecc_bch_cfg = !host->bch_enabled << ECC_CFG_ECC_DISABLE 2613 | 0 << ECC_SW_RESET 2614 | host->cw_data << ECC_NUM_DATA_BYTES 2615 | 1 << ECC_FORCE_CLK_OPEN 2616 | ecc_mode << ECC_MODE 2617 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_BCH; 2618 2619 host->ecc_buf_cfg = 0x203 << NUM_STEPS; 2620 2621 host->clrflashstatus = FS_READY_BSY_N; 2622 host->clrreadstatus = 0xc0; 2623 nandc->regs->erased_cw_detect_cfg_clr = 2624 cpu_to_le32(CLR_ERASED_PAGE_DET); 2625 nandc->regs->erased_cw_detect_cfg_set = 2626 cpu_to_le32(SET_ERASED_PAGE_DET); 2627 2628 dev_dbg(nandc->dev, 2629 "cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n", 2630 host->cfg0, host->cfg1, host->ecc_buf_cfg, host->ecc_bch_cfg, 2631 host->cw_size, host->cw_data, ecc->strength, ecc->bytes, 2632 cwperpage); 2633 2634 return 0; 2635 } 2636 2637 static const struct nand_controller_ops qcom_nandc_ops = { 2638 .attach_chip = qcom_nand_attach_chip, 2639 }; 2640 2641 static int qcom_nandc_alloc(struct qcom_nand_controller *nandc) 2642 { 2643 int ret; 2644 2645 ret = dma_set_coherent_mask(nandc->dev, DMA_BIT_MASK(32)); 2646 if (ret) { 2647 dev_err(nandc->dev, "failed to set DMA mask\n"); 2648 return ret; 2649 } 2650 2651 /* 2652 * we use the internal buffer for reading ONFI params, reading small 2653 * data like ID and status, and preforming read-copy-write operations 2654 * when writing to a codeword partially. 532 is the maximum possible 2655 * size of a codeword for our nand controller 2656 */ 2657 nandc->buf_size = 532; 2658 2659 nandc->data_buffer = devm_kzalloc(nandc->dev, nandc->buf_size, 2660 GFP_KERNEL); 2661 if (!nandc->data_buffer) 2662 return -ENOMEM; 2663 2664 nandc->regs = devm_kzalloc(nandc->dev, sizeof(*nandc->regs), 2665 GFP_KERNEL); 2666 if (!nandc->regs) 2667 return -ENOMEM; 2668 2669 nandc->reg_read_buf = devm_kcalloc(nandc->dev, 2670 MAX_REG_RD, sizeof(*nandc->reg_read_buf), 2671 GFP_KERNEL); 2672 if (!nandc->reg_read_buf) 2673 return -ENOMEM; 2674 2675 if (nandc->props->is_bam) { 2676 nandc->reg_read_dma = 2677 dma_map_single(nandc->dev, nandc->reg_read_buf, 2678 MAX_REG_RD * 2679 sizeof(*nandc->reg_read_buf), 2680 DMA_FROM_DEVICE); 2681 if (dma_mapping_error(nandc->dev, nandc->reg_read_dma)) { 2682 dev_err(nandc->dev, "failed to DMA MAP reg buffer\n"); 2683 return -EIO; 2684 } 2685 2686 nandc->tx_chan = dma_request_slave_channel(nandc->dev, "tx"); 2687 if (!nandc->tx_chan) { 2688 dev_err(nandc->dev, "failed to request tx channel\n"); 2689 return -ENODEV; 2690 } 2691 2692 nandc->rx_chan = dma_request_slave_channel(nandc->dev, "rx"); 2693 if (!nandc->rx_chan) { 2694 dev_err(nandc->dev, "failed to request rx channel\n"); 2695 return -ENODEV; 2696 } 2697 2698 nandc->cmd_chan = dma_request_slave_channel(nandc->dev, "cmd"); 2699 if (!nandc->cmd_chan) { 2700 dev_err(nandc->dev, "failed to request cmd channel\n"); 2701 return -ENODEV; 2702 } 2703 2704 /* 2705 * Initially allocate BAM transaction to read ONFI param page. 2706 * After detecting all the devices, this BAM transaction will 2707 * be freed and the next BAM tranasction will be allocated with 2708 * maximum codeword size 2709 */ 2710 nandc->max_cwperpage = 1; 2711 nandc->bam_txn = alloc_bam_transaction(nandc); 2712 if (!nandc->bam_txn) { 2713 dev_err(nandc->dev, 2714 "failed to allocate bam transaction\n"); 2715 return -ENOMEM; 2716 } 2717 } else { 2718 nandc->chan = dma_request_slave_channel(nandc->dev, "rxtx"); 2719 if (!nandc->chan) { 2720 dev_err(nandc->dev, 2721 "failed to request slave channel\n"); 2722 return -ENODEV; 2723 } 2724 } 2725 2726 INIT_LIST_HEAD(&nandc->desc_list); 2727 INIT_LIST_HEAD(&nandc->host_list); 2728 2729 nand_controller_init(&nandc->controller); 2730 nandc->controller.ops = &qcom_nandc_ops; 2731 2732 return 0; 2733 } 2734 2735 static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc) 2736 { 2737 if (nandc->props->is_bam) { 2738 if (!dma_mapping_error(nandc->dev, nandc->reg_read_dma)) 2739 dma_unmap_single(nandc->dev, nandc->reg_read_dma, 2740 MAX_REG_RD * 2741 sizeof(*nandc->reg_read_buf), 2742 DMA_FROM_DEVICE); 2743 2744 if (nandc->tx_chan) 2745 dma_release_channel(nandc->tx_chan); 2746 2747 if (nandc->rx_chan) 2748 dma_release_channel(nandc->rx_chan); 2749 2750 if (nandc->cmd_chan) 2751 dma_release_channel(nandc->cmd_chan); 2752 } else { 2753 if (nandc->chan) 2754 dma_release_channel(nandc->chan); 2755 } 2756 } 2757 2758 /* one time setup of a few nand controller registers */ 2759 static int qcom_nandc_setup(struct qcom_nand_controller *nandc) 2760 { 2761 u32 nand_ctrl; 2762 2763 /* kill onenand */ 2764 nandc_write(nandc, SFLASHC_BURST_CFG, 0); 2765 nandc_write(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD), 2766 NAND_DEV_CMD_VLD_VAL); 2767 2768 /* enable ADM or BAM DMA */ 2769 if (nandc->props->is_bam) { 2770 nand_ctrl = nandc_read(nandc, NAND_CTRL); 2771 nandc_write(nandc, NAND_CTRL, nand_ctrl | BAM_MODE_EN); 2772 } else { 2773 nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN); 2774 } 2775 2776 /* save the original values of these registers */ 2777 nandc->cmd1 = nandc_read(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD1)); 2778 nandc->vld = NAND_DEV_CMD_VLD_VAL; 2779 2780 return 0; 2781 } 2782 2783 static int qcom_nand_host_init_and_register(struct qcom_nand_controller *nandc, 2784 struct qcom_nand_host *host, 2785 struct device_node *dn) 2786 { 2787 struct nand_chip *chip = &host->chip; 2788 struct mtd_info *mtd = nand_to_mtd(chip); 2789 struct device *dev = nandc->dev; 2790 int ret; 2791 2792 ret = of_property_read_u32(dn, "reg", &host->cs); 2793 if (ret) { 2794 dev_err(dev, "can't get chip-select\n"); 2795 return -ENXIO; 2796 } 2797 2798 nand_set_flash_node(chip, dn); 2799 mtd->name = devm_kasprintf(dev, GFP_KERNEL, "qcom_nand.%d", host->cs); 2800 if (!mtd->name) 2801 return -ENOMEM; 2802 2803 mtd->owner = THIS_MODULE; 2804 mtd->dev.parent = dev; 2805 2806 chip->legacy.cmdfunc = qcom_nandc_command; 2807 chip->select_chip = qcom_nandc_select_chip; 2808 chip->legacy.read_byte = qcom_nandc_read_byte; 2809 chip->legacy.read_buf = qcom_nandc_read_buf; 2810 chip->legacy.write_buf = qcom_nandc_write_buf; 2811 chip->legacy.set_features = nand_get_set_features_notsupp; 2812 chip->legacy.get_features = nand_get_set_features_notsupp; 2813 2814 /* 2815 * the bad block marker is readable only when we read the last codeword 2816 * of a page with ECC disabled. currently, the nand_base and nand_bbt 2817 * helpers don't allow us to read BB from a nand chip with ECC 2818 * disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad 2819 * and block_markbad helpers until we permanently switch to using 2820 * MTD_OPS_RAW for all drivers (with the help of badblockbits) 2821 */ 2822 chip->legacy.block_bad = qcom_nandc_block_bad; 2823 chip->legacy.block_markbad = qcom_nandc_block_markbad; 2824 2825 chip->controller = &nandc->controller; 2826 chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USE_BOUNCE_BUFFER | 2827 NAND_SKIP_BBTSCAN; 2828 2829 /* set up initial status value */ 2830 host->status = NAND_STATUS_READY | NAND_STATUS_WP; 2831 2832 ret = nand_scan(chip, 1); 2833 if (ret) 2834 return ret; 2835 2836 ret = mtd_device_register(mtd, NULL, 0); 2837 if (ret) 2838 nand_cleanup(chip); 2839 2840 return ret; 2841 } 2842 2843 static int qcom_probe_nand_devices(struct qcom_nand_controller *nandc) 2844 { 2845 struct device *dev = nandc->dev; 2846 struct device_node *dn = dev->of_node, *child; 2847 struct qcom_nand_host *host; 2848 int ret; 2849 2850 if (nandc->props->is_bam) { 2851 free_bam_transaction(nandc); 2852 nandc->bam_txn = alloc_bam_transaction(nandc); 2853 if (!nandc->bam_txn) { 2854 dev_err(nandc->dev, 2855 "failed to allocate bam transaction\n"); 2856 return -ENOMEM; 2857 } 2858 } 2859 2860 for_each_available_child_of_node(dn, child) { 2861 host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL); 2862 if (!host) { 2863 of_node_put(child); 2864 return -ENOMEM; 2865 } 2866 2867 ret = qcom_nand_host_init_and_register(nandc, host, child); 2868 if (ret) { 2869 devm_kfree(dev, host); 2870 continue; 2871 } 2872 2873 list_add_tail(&host->node, &nandc->host_list); 2874 } 2875 2876 if (list_empty(&nandc->host_list)) 2877 return -ENODEV; 2878 2879 return 0; 2880 } 2881 2882 /* parse custom DT properties here */ 2883 static int qcom_nandc_parse_dt(struct platform_device *pdev) 2884 { 2885 struct qcom_nand_controller *nandc = platform_get_drvdata(pdev); 2886 struct device_node *np = nandc->dev->of_node; 2887 int ret; 2888 2889 if (!nandc->props->is_bam) { 2890 ret = of_property_read_u32(np, "qcom,cmd-crci", 2891 &nandc->cmd_crci); 2892 if (ret) { 2893 dev_err(nandc->dev, "command CRCI unspecified\n"); 2894 return ret; 2895 } 2896 2897 ret = of_property_read_u32(np, "qcom,data-crci", 2898 &nandc->data_crci); 2899 if (ret) { 2900 dev_err(nandc->dev, "data CRCI unspecified\n"); 2901 return ret; 2902 } 2903 } 2904 2905 return 0; 2906 } 2907 2908 static int qcom_nandc_probe(struct platform_device *pdev) 2909 { 2910 struct qcom_nand_controller *nandc; 2911 const void *dev_data; 2912 struct device *dev = &pdev->dev; 2913 struct resource *res; 2914 int ret; 2915 2916 nandc = devm_kzalloc(&pdev->dev, sizeof(*nandc), GFP_KERNEL); 2917 if (!nandc) 2918 return -ENOMEM; 2919 2920 platform_set_drvdata(pdev, nandc); 2921 nandc->dev = dev; 2922 2923 dev_data = of_device_get_match_data(dev); 2924 if (!dev_data) { 2925 dev_err(&pdev->dev, "failed to get device data\n"); 2926 return -ENODEV; 2927 } 2928 2929 nandc->props = dev_data; 2930 2931 nandc->core_clk = devm_clk_get(dev, "core"); 2932 if (IS_ERR(nandc->core_clk)) 2933 return PTR_ERR(nandc->core_clk); 2934 2935 nandc->aon_clk = devm_clk_get(dev, "aon"); 2936 if (IS_ERR(nandc->aon_clk)) 2937 return PTR_ERR(nandc->aon_clk); 2938 2939 ret = qcom_nandc_parse_dt(pdev); 2940 if (ret) 2941 return ret; 2942 2943 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2944 nandc->base = devm_ioremap_resource(dev, res); 2945 if (IS_ERR(nandc->base)) 2946 return PTR_ERR(nandc->base); 2947 2948 nandc->base_phys = res->start; 2949 nandc->base_dma = dma_map_resource(dev, res->start, 2950 resource_size(res), 2951 DMA_BIDIRECTIONAL, 0); 2952 if (!nandc->base_dma) 2953 return -ENXIO; 2954 2955 ret = qcom_nandc_alloc(nandc); 2956 if (ret) 2957 goto err_nandc_alloc; 2958 2959 ret = clk_prepare_enable(nandc->core_clk); 2960 if (ret) 2961 goto err_core_clk; 2962 2963 ret = clk_prepare_enable(nandc->aon_clk); 2964 if (ret) 2965 goto err_aon_clk; 2966 2967 ret = qcom_nandc_setup(nandc); 2968 if (ret) 2969 goto err_setup; 2970 2971 ret = qcom_probe_nand_devices(nandc); 2972 if (ret) 2973 goto err_setup; 2974 2975 return 0; 2976 2977 err_setup: 2978 clk_disable_unprepare(nandc->aon_clk); 2979 err_aon_clk: 2980 clk_disable_unprepare(nandc->core_clk); 2981 err_core_clk: 2982 qcom_nandc_unalloc(nandc); 2983 err_nandc_alloc: 2984 dma_unmap_resource(dev, res->start, resource_size(res), 2985 DMA_BIDIRECTIONAL, 0); 2986 2987 return ret; 2988 } 2989 2990 static int qcom_nandc_remove(struct platform_device *pdev) 2991 { 2992 struct qcom_nand_controller *nandc = platform_get_drvdata(pdev); 2993 struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2994 struct qcom_nand_host *host; 2995 2996 list_for_each_entry(host, &nandc->host_list, node) 2997 nand_release(&host->chip); 2998 2999 3000 qcom_nandc_unalloc(nandc); 3001 3002 clk_disable_unprepare(nandc->aon_clk); 3003 clk_disable_unprepare(nandc->core_clk); 3004 3005 dma_unmap_resource(&pdev->dev, nandc->base_dma, resource_size(res), 3006 DMA_BIDIRECTIONAL, 0); 3007 3008 return 0; 3009 } 3010 3011 static const struct qcom_nandc_props ipq806x_nandc_props = { 3012 .ecc_modes = (ECC_RS_4BIT | ECC_BCH_8BIT), 3013 .is_bam = false, 3014 .dev_cmd_reg_start = 0x0, 3015 }; 3016 3017 static const struct qcom_nandc_props ipq4019_nandc_props = { 3018 .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT), 3019 .is_bam = true, 3020 .dev_cmd_reg_start = 0x0, 3021 }; 3022 3023 static const struct qcom_nandc_props ipq8074_nandc_props = { 3024 .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT), 3025 .is_bam = true, 3026 .dev_cmd_reg_start = 0x7000, 3027 }; 3028 3029 /* 3030 * data will hold a struct pointer containing more differences once we support 3031 * more controller variants 3032 */ 3033 static const struct of_device_id qcom_nandc_of_match[] = { 3034 { 3035 .compatible = "qcom,ipq806x-nand", 3036 .data = &ipq806x_nandc_props, 3037 }, 3038 { 3039 .compatible = "qcom,ipq4019-nand", 3040 .data = &ipq4019_nandc_props, 3041 }, 3042 { 3043 .compatible = "qcom,ipq8074-nand", 3044 .data = &ipq8074_nandc_props, 3045 }, 3046 {} 3047 }; 3048 MODULE_DEVICE_TABLE(of, qcom_nandc_of_match); 3049 3050 static struct platform_driver qcom_nandc_driver = { 3051 .driver = { 3052 .name = "qcom-nandc", 3053 .of_match_table = qcom_nandc_of_match, 3054 }, 3055 .probe = qcom_nandc_probe, 3056 .remove = qcom_nandc_remove, 3057 }; 3058 module_platform_driver(qcom_nandc_driver); 3059 3060 MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>"); 3061 MODULE_DESCRIPTION("Qualcomm NAND Controller driver"); 3062 MODULE_LICENSE("GPL v2"); 3063