1 /* 2 * Copyright (c) 2016, The Linux Foundation. All rights reserved. 3 * 4 * This software is licensed under the terms of the GNU General Public 5 * License version 2, as published by the Free Software Foundation, and 6 * may be copied, distributed, and modified under those terms. 7 * 8 * This program is distributed in the hope that it will be useful, 9 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * GNU General Public License for more details. 12 */ 13 14 #include <linux/clk.h> 15 #include <linux/slab.h> 16 #include <linux/bitops.h> 17 #include <linux/dma-mapping.h> 18 #include <linux/dmaengine.h> 19 #include <linux/module.h> 20 #include <linux/mtd/rawnand.h> 21 #include <linux/mtd/partitions.h> 22 #include <linux/of.h> 23 #include <linux/of_device.h> 24 #include <linux/delay.h> 25 #include <linux/dma/qcom_bam_dma.h> 26 27 /* NANDc reg offsets */ 28 #define NAND_FLASH_CMD 0x00 29 #define NAND_ADDR0 0x04 30 #define NAND_ADDR1 0x08 31 #define NAND_FLASH_CHIP_SELECT 0x0c 32 #define NAND_EXEC_CMD 0x10 33 #define NAND_FLASH_STATUS 0x14 34 #define NAND_BUFFER_STATUS 0x18 35 #define NAND_DEV0_CFG0 0x20 36 #define NAND_DEV0_CFG1 0x24 37 #define NAND_DEV0_ECC_CFG 0x28 38 #define NAND_DEV1_ECC_CFG 0x2c 39 #define NAND_DEV1_CFG0 0x30 40 #define NAND_DEV1_CFG1 0x34 41 #define NAND_READ_ID 0x40 42 #define NAND_READ_STATUS 0x44 43 #define NAND_DEV_CMD0 0xa0 44 #define NAND_DEV_CMD1 0xa4 45 #define NAND_DEV_CMD2 0xa8 46 #define NAND_DEV_CMD_VLD 0xac 47 #define SFLASHC_BURST_CFG 0xe0 48 #define NAND_ERASED_CW_DETECT_CFG 0xe8 49 #define NAND_ERASED_CW_DETECT_STATUS 0xec 50 #define NAND_EBI2_ECC_BUF_CFG 0xf0 51 #define FLASH_BUF_ACC 0x100 52 53 #define NAND_CTRL 0xf00 54 #define NAND_VERSION 0xf08 55 #define NAND_READ_LOCATION_0 0xf20 56 #define NAND_READ_LOCATION_1 0xf24 57 #define NAND_READ_LOCATION_2 0xf28 58 #define NAND_READ_LOCATION_3 0xf2c 59 60 /* dummy register offsets, used by write_reg_dma */ 61 #define NAND_DEV_CMD1_RESTORE 0xdead 62 #define NAND_DEV_CMD_VLD_RESTORE 0xbeef 63 64 /* NAND_FLASH_CMD bits */ 65 #define PAGE_ACC BIT(4) 66 #define LAST_PAGE BIT(5) 67 68 /* NAND_FLASH_CHIP_SELECT bits */ 69 #define NAND_DEV_SEL 0 70 #define DM_EN BIT(2) 71 72 /* NAND_FLASH_STATUS bits */ 73 #define FS_OP_ERR BIT(4) 74 #define FS_READY_BSY_N BIT(5) 75 #define FS_MPU_ERR BIT(8) 76 #define FS_DEVICE_STS_ERR BIT(16) 77 #define FS_DEVICE_WP BIT(23) 78 79 /* NAND_BUFFER_STATUS bits */ 80 #define BS_UNCORRECTABLE_BIT BIT(8) 81 #define BS_CORRECTABLE_ERR_MSK 0x1f 82 83 /* NAND_DEVn_CFG0 bits */ 84 #define DISABLE_STATUS_AFTER_WRITE 4 85 #define CW_PER_PAGE 6 86 #define UD_SIZE_BYTES 9 87 #define ECC_PARITY_SIZE_BYTES_RS 19 88 #define SPARE_SIZE_BYTES 23 89 #define NUM_ADDR_CYCLES 27 90 #define STATUS_BFR_READ 30 91 #define SET_RD_MODE_AFTER_STATUS 31 92 93 /* NAND_DEVn_CFG0 bits */ 94 #define DEV0_CFG1_ECC_DISABLE 0 95 #define WIDE_FLASH 1 96 #define NAND_RECOVERY_CYCLES 2 97 #define CS_ACTIVE_BSY 5 98 #define BAD_BLOCK_BYTE_NUM 6 99 #define BAD_BLOCK_IN_SPARE_AREA 16 100 #define WR_RD_BSY_GAP 17 101 #define ENABLE_BCH_ECC 27 102 103 /* NAND_DEV0_ECC_CFG bits */ 104 #define ECC_CFG_ECC_DISABLE 0 105 #define ECC_SW_RESET 1 106 #define ECC_MODE 4 107 #define ECC_PARITY_SIZE_BYTES_BCH 8 108 #define ECC_NUM_DATA_BYTES 16 109 #define ECC_FORCE_CLK_OPEN 30 110 111 /* NAND_DEV_CMD1 bits */ 112 #define READ_ADDR 0 113 114 /* NAND_DEV_CMD_VLD bits */ 115 #define READ_START_VLD BIT(0) 116 #define READ_STOP_VLD BIT(1) 117 #define WRITE_START_VLD BIT(2) 118 #define ERASE_START_VLD BIT(3) 119 #define SEQ_READ_START_VLD BIT(4) 120 121 /* NAND_EBI2_ECC_BUF_CFG bits */ 122 #define NUM_STEPS 0 123 124 /* NAND_ERASED_CW_DETECT_CFG bits */ 125 #define ERASED_CW_ECC_MASK 1 126 #define AUTO_DETECT_RES 0 127 #define MASK_ECC (1 << ERASED_CW_ECC_MASK) 128 #define RESET_ERASED_DET (1 << AUTO_DETECT_RES) 129 #define ACTIVE_ERASED_DET (0 << AUTO_DETECT_RES) 130 #define CLR_ERASED_PAGE_DET (RESET_ERASED_DET | MASK_ECC) 131 #define SET_ERASED_PAGE_DET (ACTIVE_ERASED_DET | MASK_ECC) 132 133 /* NAND_ERASED_CW_DETECT_STATUS bits */ 134 #define PAGE_ALL_ERASED BIT(7) 135 #define CODEWORD_ALL_ERASED BIT(6) 136 #define PAGE_ERASED BIT(5) 137 #define CODEWORD_ERASED BIT(4) 138 #define ERASED_PAGE (PAGE_ALL_ERASED | PAGE_ERASED) 139 #define ERASED_CW (CODEWORD_ALL_ERASED | CODEWORD_ERASED) 140 141 /* NAND_READ_LOCATION_n bits */ 142 #define READ_LOCATION_OFFSET 0 143 #define READ_LOCATION_SIZE 16 144 #define READ_LOCATION_LAST 31 145 146 /* Version Mask */ 147 #define NAND_VERSION_MAJOR_MASK 0xf0000000 148 #define NAND_VERSION_MAJOR_SHIFT 28 149 #define NAND_VERSION_MINOR_MASK 0x0fff0000 150 #define NAND_VERSION_MINOR_SHIFT 16 151 152 /* NAND OP_CMDs */ 153 #define OP_PAGE_READ 0x2 154 #define OP_PAGE_READ_WITH_ECC 0x3 155 #define OP_PAGE_READ_WITH_ECC_SPARE 0x4 156 #define OP_PROGRAM_PAGE 0x6 157 #define OP_PAGE_PROGRAM_WITH_ECC 0x7 158 #define OP_PROGRAM_PAGE_SPARE 0x9 159 #define OP_BLOCK_ERASE 0xa 160 #define OP_FETCH_ID 0xb 161 #define OP_RESET_DEVICE 0xd 162 163 /* Default Value for NAND_DEV_CMD_VLD */ 164 #define NAND_DEV_CMD_VLD_VAL (READ_START_VLD | WRITE_START_VLD | \ 165 ERASE_START_VLD | SEQ_READ_START_VLD) 166 167 /* NAND_CTRL bits */ 168 #define BAM_MODE_EN BIT(0) 169 170 /* 171 * the NAND controller performs reads/writes with ECC in 516 byte chunks. 172 * the driver calls the chunks 'step' or 'codeword' interchangeably 173 */ 174 #define NANDC_STEP_SIZE 512 175 176 /* 177 * the largest page size we support is 8K, this will have 16 steps/codewords 178 * of 512 bytes each 179 */ 180 #define MAX_NUM_STEPS (SZ_8K / NANDC_STEP_SIZE) 181 182 /* we read at most 3 registers per codeword scan */ 183 #define MAX_REG_RD (3 * MAX_NUM_STEPS) 184 185 /* ECC modes supported by the controller */ 186 #define ECC_NONE BIT(0) 187 #define ECC_RS_4BIT BIT(1) 188 #define ECC_BCH_4BIT BIT(2) 189 #define ECC_BCH_8BIT BIT(3) 190 191 #define nandc_set_read_loc(nandc, reg, offset, size, is_last) \ 192 nandc_set_reg(nandc, NAND_READ_LOCATION_##reg, \ 193 ((offset) << READ_LOCATION_OFFSET) | \ 194 ((size) << READ_LOCATION_SIZE) | \ 195 ((is_last) << READ_LOCATION_LAST)) 196 197 /* 198 * Returns the actual register address for all NAND_DEV_ registers 199 * (i.e. NAND_DEV_CMD0, NAND_DEV_CMD1, NAND_DEV_CMD2 and NAND_DEV_CMD_VLD) 200 */ 201 #define dev_cmd_reg_addr(nandc, reg) ((nandc)->props->dev_cmd_reg_start + (reg)) 202 203 /* Returns the NAND register physical address */ 204 #define nandc_reg_phys(chip, offset) ((chip)->base_phys + (offset)) 205 206 /* Returns the dma address for reg read buffer */ 207 #define reg_buf_dma_addr(chip, vaddr) \ 208 ((chip)->reg_read_dma + \ 209 ((uint8_t *)(vaddr) - (uint8_t *)(chip)->reg_read_buf)) 210 211 #define QPIC_PER_CW_CMD_ELEMENTS 32 212 #define QPIC_PER_CW_CMD_SGL 32 213 #define QPIC_PER_CW_DATA_SGL 8 214 215 #define QPIC_NAND_COMPLETION_TIMEOUT msecs_to_jiffies(2000) 216 217 /* 218 * Flags used in DMA descriptor preparation helper functions 219 * (i.e. read_reg_dma/write_reg_dma/read_data_dma/write_data_dma) 220 */ 221 /* Don't set the EOT in current tx BAM sgl */ 222 #define NAND_BAM_NO_EOT BIT(0) 223 /* Set the NWD flag in current BAM sgl */ 224 #define NAND_BAM_NWD BIT(1) 225 /* Finish writing in the current BAM sgl and start writing in another BAM sgl */ 226 #define NAND_BAM_NEXT_SGL BIT(2) 227 /* 228 * Erased codeword status is being used two times in single transfer so this 229 * flag will determine the current value of erased codeword status register 230 */ 231 #define NAND_ERASED_CW_SET BIT(4) 232 233 /* 234 * This data type corresponds to the BAM transaction which will be used for all 235 * NAND transfers. 236 * @bam_ce - the array of BAM command elements 237 * @cmd_sgl - sgl for NAND BAM command pipe 238 * @data_sgl - sgl for NAND BAM consumer/producer pipe 239 * @bam_ce_pos - the index in bam_ce which is available for next sgl 240 * @bam_ce_start - the index in bam_ce which marks the start position ce 241 * for current sgl. It will be used for size calculation 242 * for current sgl 243 * @cmd_sgl_pos - current index in command sgl. 244 * @cmd_sgl_start - start index in command sgl. 245 * @tx_sgl_pos - current index in data sgl for tx. 246 * @tx_sgl_start - start index in data sgl for tx. 247 * @rx_sgl_pos - current index in data sgl for rx. 248 * @rx_sgl_start - start index in data sgl for rx. 249 * @wait_second_completion - wait for second DMA desc completion before making 250 * the NAND transfer completion. 251 * @txn_done - completion for NAND transfer. 252 * @last_data_desc - last DMA desc in data channel (tx/rx). 253 * @last_cmd_desc - last DMA desc in command channel. 254 */ 255 struct bam_transaction { 256 struct bam_cmd_element *bam_ce; 257 struct scatterlist *cmd_sgl; 258 struct scatterlist *data_sgl; 259 u32 bam_ce_pos; 260 u32 bam_ce_start; 261 u32 cmd_sgl_pos; 262 u32 cmd_sgl_start; 263 u32 tx_sgl_pos; 264 u32 tx_sgl_start; 265 u32 rx_sgl_pos; 266 u32 rx_sgl_start; 267 bool wait_second_completion; 268 struct completion txn_done; 269 struct dma_async_tx_descriptor *last_data_desc; 270 struct dma_async_tx_descriptor *last_cmd_desc; 271 }; 272 273 /* 274 * This data type corresponds to the nand dma descriptor 275 * @list - list for desc_info 276 * @dir - DMA transfer direction 277 * @adm_sgl - sgl which will be used for single sgl dma descriptor. Only used by 278 * ADM 279 * @bam_sgl - sgl which will be used for dma descriptor. Only used by BAM 280 * @sgl_cnt - number of SGL in bam_sgl. Only used by BAM 281 * @dma_desc - low level DMA engine descriptor 282 */ 283 struct desc_info { 284 struct list_head node; 285 286 enum dma_data_direction dir; 287 union { 288 struct scatterlist adm_sgl; 289 struct { 290 struct scatterlist *bam_sgl; 291 int sgl_cnt; 292 }; 293 }; 294 struct dma_async_tx_descriptor *dma_desc; 295 }; 296 297 /* 298 * holds the current register values that we want to write. acts as a contiguous 299 * chunk of memory which we use to write the controller registers through DMA. 300 */ 301 struct nandc_regs { 302 __le32 cmd; 303 __le32 addr0; 304 __le32 addr1; 305 __le32 chip_sel; 306 __le32 exec; 307 308 __le32 cfg0; 309 __le32 cfg1; 310 __le32 ecc_bch_cfg; 311 312 __le32 clrflashstatus; 313 __le32 clrreadstatus; 314 315 __le32 cmd1; 316 __le32 vld; 317 318 __le32 orig_cmd1; 319 __le32 orig_vld; 320 321 __le32 ecc_buf_cfg; 322 __le32 read_location0; 323 __le32 read_location1; 324 __le32 read_location2; 325 __le32 read_location3; 326 327 __le32 erased_cw_detect_cfg_clr; 328 __le32 erased_cw_detect_cfg_set; 329 }; 330 331 /* 332 * NAND controller data struct 333 * 334 * @controller: base controller structure 335 * @host_list: list containing all the chips attached to the 336 * controller 337 * @dev: parent device 338 * @base: MMIO base 339 * @base_phys: physical base address of controller registers 340 * @base_dma: dma base address of controller registers 341 * @core_clk: controller clock 342 * @aon_clk: another controller clock 343 * 344 * @chan: dma channel 345 * @cmd_crci: ADM DMA CRCI for command flow control 346 * @data_crci: ADM DMA CRCI for data flow control 347 * @desc_list: DMA descriptor list (list of desc_infos) 348 * 349 * @data_buffer: our local DMA buffer for page read/writes, 350 * used when we can't use the buffer provided 351 * by upper layers directly 352 * @buf_size/count/start: markers for chip->legacy.read_buf/write_buf 353 * functions 354 * @reg_read_buf: local buffer for reading back registers via DMA 355 * @reg_read_dma: contains dma address for register read buffer 356 * @reg_read_pos: marker for data read in reg_read_buf 357 * 358 * @regs: a contiguous chunk of memory for DMA register 359 * writes. contains the register values to be 360 * written to controller 361 * @cmd1/vld: some fixed controller register values 362 * @props: properties of current NAND controller, 363 * initialized via DT match data 364 * @max_cwperpage: maximum QPIC codewords required. calculated 365 * from all connected NAND devices pagesize 366 */ 367 struct qcom_nand_controller { 368 struct nand_controller controller; 369 struct list_head host_list; 370 371 struct device *dev; 372 373 void __iomem *base; 374 phys_addr_t base_phys; 375 dma_addr_t base_dma; 376 377 struct clk *core_clk; 378 struct clk *aon_clk; 379 380 union { 381 /* will be used only by QPIC for BAM DMA */ 382 struct { 383 struct dma_chan *tx_chan; 384 struct dma_chan *rx_chan; 385 struct dma_chan *cmd_chan; 386 }; 387 388 /* will be used only by EBI2 for ADM DMA */ 389 struct { 390 struct dma_chan *chan; 391 unsigned int cmd_crci; 392 unsigned int data_crci; 393 }; 394 }; 395 396 struct list_head desc_list; 397 struct bam_transaction *bam_txn; 398 399 u8 *data_buffer; 400 int buf_size; 401 int buf_count; 402 int buf_start; 403 unsigned int max_cwperpage; 404 405 __le32 *reg_read_buf; 406 dma_addr_t reg_read_dma; 407 int reg_read_pos; 408 409 struct nandc_regs *regs; 410 411 u32 cmd1, vld; 412 const struct qcom_nandc_props *props; 413 }; 414 415 /* 416 * NAND chip structure 417 * 418 * @chip: base NAND chip structure 419 * @node: list node to add itself to host_list in 420 * qcom_nand_controller 421 * 422 * @cs: chip select value for this chip 423 * @cw_size: the number of bytes in a single step/codeword 424 * of a page, consisting of all data, ecc, spare 425 * and reserved bytes 426 * @cw_data: the number of bytes within a codeword protected 427 * by ECC 428 * @use_ecc: request the controller to use ECC for the 429 * upcoming read/write 430 * @bch_enabled: flag to tell whether BCH ECC mode is used 431 * @ecc_bytes_hw: ECC bytes used by controller hardware for this 432 * chip 433 * @status: value to be returned if NAND_CMD_STATUS command 434 * is executed 435 * @last_command: keeps track of last command on this chip. used 436 * for reading correct status 437 * 438 * @cfg0, cfg1, cfg0_raw..: NANDc register configurations needed for 439 * ecc/non-ecc mode for the current nand flash 440 * device 441 */ 442 struct qcom_nand_host { 443 struct nand_chip chip; 444 struct list_head node; 445 446 int cs; 447 int cw_size; 448 int cw_data; 449 bool use_ecc; 450 bool bch_enabled; 451 int ecc_bytes_hw; 452 int spare_bytes; 453 int bbm_size; 454 u8 status; 455 int last_command; 456 457 u32 cfg0, cfg1; 458 u32 cfg0_raw, cfg1_raw; 459 u32 ecc_buf_cfg; 460 u32 ecc_bch_cfg; 461 u32 clrflashstatus; 462 u32 clrreadstatus; 463 }; 464 465 /* 466 * This data type corresponds to the NAND controller properties which varies 467 * among different NAND controllers. 468 * @ecc_modes - ecc mode for NAND 469 * @is_bam - whether NAND controller is using BAM 470 * @dev_cmd_reg_start - NAND_DEV_CMD_* registers starting offset 471 */ 472 struct qcom_nandc_props { 473 u32 ecc_modes; 474 bool is_bam; 475 u32 dev_cmd_reg_start; 476 }; 477 478 /* Frees the BAM transaction memory */ 479 static void free_bam_transaction(struct qcom_nand_controller *nandc) 480 { 481 struct bam_transaction *bam_txn = nandc->bam_txn; 482 483 devm_kfree(nandc->dev, bam_txn); 484 } 485 486 /* Allocates and Initializes the BAM transaction */ 487 static struct bam_transaction * 488 alloc_bam_transaction(struct qcom_nand_controller *nandc) 489 { 490 struct bam_transaction *bam_txn; 491 size_t bam_txn_size; 492 unsigned int num_cw = nandc->max_cwperpage; 493 void *bam_txn_buf; 494 495 bam_txn_size = 496 sizeof(*bam_txn) + num_cw * 497 ((sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS) + 498 (sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL) + 499 (sizeof(*bam_txn->data_sgl) * QPIC_PER_CW_DATA_SGL)); 500 501 bam_txn_buf = devm_kzalloc(nandc->dev, bam_txn_size, GFP_KERNEL); 502 if (!bam_txn_buf) 503 return NULL; 504 505 bam_txn = bam_txn_buf; 506 bam_txn_buf += sizeof(*bam_txn); 507 508 bam_txn->bam_ce = bam_txn_buf; 509 bam_txn_buf += 510 sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS * num_cw; 511 512 bam_txn->cmd_sgl = bam_txn_buf; 513 bam_txn_buf += 514 sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL * num_cw; 515 516 bam_txn->data_sgl = bam_txn_buf; 517 518 init_completion(&bam_txn->txn_done); 519 520 return bam_txn; 521 } 522 523 /* Clears the BAM transaction indexes */ 524 static void clear_bam_transaction(struct qcom_nand_controller *nandc) 525 { 526 struct bam_transaction *bam_txn = nandc->bam_txn; 527 528 if (!nandc->props->is_bam) 529 return; 530 531 bam_txn->bam_ce_pos = 0; 532 bam_txn->bam_ce_start = 0; 533 bam_txn->cmd_sgl_pos = 0; 534 bam_txn->cmd_sgl_start = 0; 535 bam_txn->tx_sgl_pos = 0; 536 bam_txn->tx_sgl_start = 0; 537 bam_txn->rx_sgl_pos = 0; 538 bam_txn->rx_sgl_start = 0; 539 bam_txn->last_data_desc = NULL; 540 bam_txn->wait_second_completion = false; 541 542 sg_init_table(bam_txn->cmd_sgl, nandc->max_cwperpage * 543 QPIC_PER_CW_CMD_SGL); 544 sg_init_table(bam_txn->data_sgl, nandc->max_cwperpage * 545 QPIC_PER_CW_DATA_SGL); 546 547 reinit_completion(&bam_txn->txn_done); 548 } 549 550 /* Callback for DMA descriptor completion */ 551 static void qpic_bam_dma_done(void *data) 552 { 553 struct bam_transaction *bam_txn = data; 554 555 /* 556 * In case of data transfer with NAND, 2 callbacks will be generated. 557 * One for command channel and another one for data channel. 558 * If current transaction has data descriptors 559 * (i.e. wait_second_completion is true), then set this to false 560 * and wait for second DMA descriptor completion. 561 */ 562 if (bam_txn->wait_second_completion) 563 bam_txn->wait_second_completion = false; 564 else 565 complete(&bam_txn->txn_done); 566 } 567 568 static inline struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip) 569 { 570 return container_of(chip, struct qcom_nand_host, chip); 571 } 572 573 static inline struct qcom_nand_controller * 574 get_qcom_nand_controller(struct nand_chip *chip) 575 { 576 return container_of(chip->controller, struct qcom_nand_controller, 577 controller); 578 } 579 580 static inline u32 nandc_read(struct qcom_nand_controller *nandc, int offset) 581 { 582 return ioread32(nandc->base + offset); 583 } 584 585 static inline void nandc_write(struct qcom_nand_controller *nandc, int offset, 586 u32 val) 587 { 588 iowrite32(val, nandc->base + offset); 589 } 590 591 static inline void nandc_read_buffer_sync(struct qcom_nand_controller *nandc, 592 bool is_cpu) 593 { 594 if (!nandc->props->is_bam) 595 return; 596 597 if (is_cpu) 598 dma_sync_single_for_cpu(nandc->dev, nandc->reg_read_dma, 599 MAX_REG_RD * 600 sizeof(*nandc->reg_read_buf), 601 DMA_FROM_DEVICE); 602 else 603 dma_sync_single_for_device(nandc->dev, nandc->reg_read_dma, 604 MAX_REG_RD * 605 sizeof(*nandc->reg_read_buf), 606 DMA_FROM_DEVICE); 607 } 608 609 static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset) 610 { 611 switch (offset) { 612 case NAND_FLASH_CMD: 613 return ®s->cmd; 614 case NAND_ADDR0: 615 return ®s->addr0; 616 case NAND_ADDR1: 617 return ®s->addr1; 618 case NAND_FLASH_CHIP_SELECT: 619 return ®s->chip_sel; 620 case NAND_EXEC_CMD: 621 return ®s->exec; 622 case NAND_FLASH_STATUS: 623 return ®s->clrflashstatus; 624 case NAND_DEV0_CFG0: 625 return ®s->cfg0; 626 case NAND_DEV0_CFG1: 627 return ®s->cfg1; 628 case NAND_DEV0_ECC_CFG: 629 return ®s->ecc_bch_cfg; 630 case NAND_READ_STATUS: 631 return ®s->clrreadstatus; 632 case NAND_DEV_CMD1: 633 return ®s->cmd1; 634 case NAND_DEV_CMD1_RESTORE: 635 return ®s->orig_cmd1; 636 case NAND_DEV_CMD_VLD: 637 return ®s->vld; 638 case NAND_DEV_CMD_VLD_RESTORE: 639 return ®s->orig_vld; 640 case NAND_EBI2_ECC_BUF_CFG: 641 return ®s->ecc_buf_cfg; 642 case NAND_READ_LOCATION_0: 643 return ®s->read_location0; 644 case NAND_READ_LOCATION_1: 645 return ®s->read_location1; 646 case NAND_READ_LOCATION_2: 647 return ®s->read_location2; 648 case NAND_READ_LOCATION_3: 649 return ®s->read_location3; 650 default: 651 return NULL; 652 } 653 } 654 655 static void nandc_set_reg(struct qcom_nand_controller *nandc, int offset, 656 u32 val) 657 { 658 struct nandc_regs *regs = nandc->regs; 659 __le32 *reg; 660 661 reg = offset_to_nandc_reg(regs, offset); 662 663 if (reg) 664 *reg = cpu_to_le32(val); 665 } 666 667 /* helper to configure address register values */ 668 static void set_address(struct qcom_nand_host *host, u16 column, int page) 669 { 670 struct nand_chip *chip = &host->chip; 671 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 672 673 if (chip->options & NAND_BUSWIDTH_16) 674 column >>= 1; 675 676 nandc_set_reg(nandc, NAND_ADDR0, page << 16 | column); 677 nandc_set_reg(nandc, NAND_ADDR1, page >> 16 & 0xff); 678 } 679 680 /* 681 * update_rw_regs: set up read/write register values, these will be 682 * written to the NAND controller registers via DMA 683 * 684 * @num_cw: number of steps for the read/write operation 685 * @read: read or write operation 686 */ 687 static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read) 688 { 689 struct nand_chip *chip = &host->chip; 690 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 691 u32 cmd, cfg0, cfg1, ecc_bch_cfg; 692 693 if (read) { 694 if (host->use_ecc) 695 cmd = OP_PAGE_READ_WITH_ECC | PAGE_ACC | LAST_PAGE; 696 else 697 cmd = OP_PAGE_READ | PAGE_ACC | LAST_PAGE; 698 } else { 699 cmd = OP_PROGRAM_PAGE | PAGE_ACC | LAST_PAGE; 700 } 701 702 if (host->use_ecc) { 703 cfg0 = (host->cfg0 & ~(7U << CW_PER_PAGE)) | 704 (num_cw - 1) << CW_PER_PAGE; 705 706 cfg1 = host->cfg1; 707 ecc_bch_cfg = host->ecc_bch_cfg; 708 } else { 709 cfg0 = (host->cfg0_raw & ~(7U << CW_PER_PAGE)) | 710 (num_cw - 1) << CW_PER_PAGE; 711 712 cfg1 = host->cfg1_raw; 713 ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE; 714 } 715 716 nandc_set_reg(nandc, NAND_FLASH_CMD, cmd); 717 nandc_set_reg(nandc, NAND_DEV0_CFG0, cfg0); 718 nandc_set_reg(nandc, NAND_DEV0_CFG1, cfg1); 719 nandc_set_reg(nandc, NAND_DEV0_ECC_CFG, ecc_bch_cfg); 720 nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, host->ecc_buf_cfg); 721 nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus); 722 nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus); 723 nandc_set_reg(nandc, NAND_EXEC_CMD, 1); 724 725 if (read) 726 nandc_set_read_loc(nandc, 0, 0, host->use_ecc ? 727 host->cw_data : host->cw_size, 1); 728 } 729 730 /* 731 * Maps the scatter gather list for DMA transfer and forms the DMA descriptor 732 * for BAM. This descriptor will be added in the NAND DMA descriptor queue 733 * which will be submitted to DMA engine. 734 */ 735 static int prepare_bam_async_desc(struct qcom_nand_controller *nandc, 736 struct dma_chan *chan, 737 unsigned long flags) 738 { 739 struct desc_info *desc; 740 struct scatterlist *sgl; 741 unsigned int sgl_cnt; 742 int ret; 743 struct bam_transaction *bam_txn = nandc->bam_txn; 744 enum dma_transfer_direction dir_eng; 745 struct dma_async_tx_descriptor *dma_desc; 746 747 desc = kzalloc(sizeof(*desc), GFP_KERNEL); 748 if (!desc) 749 return -ENOMEM; 750 751 if (chan == nandc->cmd_chan) { 752 sgl = &bam_txn->cmd_sgl[bam_txn->cmd_sgl_start]; 753 sgl_cnt = bam_txn->cmd_sgl_pos - bam_txn->cmd_sgl_start; 754 bam_txn->cmd_sgl_start = bam_txn->cmd_sgl_pos; 755 dir_eng = DMA_MEM_TO_DEV; 756 desc->dir = DMA_TO_DEVICE; 757 } else if (chan == nandc->tx_chan) { 758 sgl = &bam_txn->data_sgl[bam_txn->tx_sgl_start]; 759 sgl_cnt = bam_txn->tx_sgl_pos - bam_txn->tx_sgl_start; 760 bam_txn->tx_sgl_start = bam_txn->tx_sgl_pos; 761 dir_eng = DMA_MEM_TO_DEV; 762 desc->dir = DMA_TO_DEVICE; 763 } else { 764 sgl = &bam_txn->data_sgl[bam_txn->rx_sgl_start]; 765 sgl_cnt = bam_txn->rx_sgl_pos - bam_txn->rx_sgl_start; 766 bam_txn->rx_sgl_start = bam_txn->rx_sgl_pos; 767 dir_eng = DMA_DEV_TO_MEM; 768 desc->dir = DMA_FROM_DEVICE; 769 } 770 771 sg_mark_end(sgl + sgl_cnt - 1); 772 ret = dma_map_sg(nandc->dev, sgl, sgl_cnt, desc->dir); 773 if (ret == 0) { 774 dev_err(nandc->dev, "failure in mapping desc\n"); 775 kfree(desc); 776 return -ENOMEM; 777 } 778 779 desc->sgl_cnt = sgl_cnt; 780 desc->bam_sgl = sgl; 781 782 dma_desc = dmaengine_prep_slave_sg(chan, sgl, sgl_cnt, dir_eng, 783 flags); 784 785 if (!dma_desc) { 786 dev_err(nandc->dev, "failure in prep desc\n"); 787 dma_unmap_sg(nandc->dev, sgl, sgl_cnt, desc->dir); 788 kfree(desc); 789 return -EINVAL; 790 } 791 792 desc->dma_desc = dma_desc; 793 794 /* update last data/command descriptor */ 795 if (chan == nandc->cmd_chan) 796 bam_txn->last_cmd_desc = dma_desc; 797 else 798 bam_txn->last_data_desc = dma_desc; 799 800 list_add_tail(&desc->node, &nandc->desc_list); 801 802 return 0; 803 } 804 805 /* 806 * Prepares the command descriptor for BAM DMA which will be used for NAND 807 * register reads and writes. The command descriptor requires the command 808 * to be formed in command element type so this function uses the command 809 * element from bam transaction ce array and fills the same with required 810 * data. A single SGL can contain multiple command elements so 811 * NAND_BAM_NEXT_SGL will be used for starting the separate SGL 812 * after the current command element. 813 */ 814 static int prep_bam_dma_desc_cmd(struct qcom_nand_controller *nandc, bool read, 815 int reg_off, const void *vaddr, 816 int size, unsigned int flags) 817 { 818 int bam_ce_size; 819 int i, ret; 820 struct bam_cmd_element *bam_ce_buffer; 821 struct bam_transaction *bam_txn = nandc->bam_txn; 822 823 bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_pos]; 824 825 /* fill the command desc */ 826 for (i = 0; i < size; i++) { 827 if (read) 828 bam_prep_ce(&bam_ce_buffer[i], 829 nandc_reg_phys(nandc, reg_off + 4 * i), 830 BAM_READ_COMMAND, 831 reg_buf_dma_addr(nandc, 832 (__le32 *)vaddr + i)); 833 else 834 bam_prep_ce_le32(&bam_ce_buffer[i], 835 nandc_reg_phys(nandc, reg_off + 4 * i), 836 BAM_WRITE_COMMAND, 837 *((__le32 *)vaddr + i)); 838 } 839 840 bam_txn->bam_ce_pos += size; 841 842 /* use the separate sgl after this command */ 843 if (flags & NAND_BAM_NEXT_SGL) { 844 bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_start]; 845 bam_ce_size = (bam_txn->bam_ce_pos - 846 bam_txn->bam_ce_start) * 847 sizeof(struct bam_cmd_element); 848 sg_set_buf(&bam_txn->cmd_sgl[bam_txn->cmd_sgl_pos], 849 bam_ce_buffer, bam_ce_size); 850 bam_txn->cmd_sgl_pos++; 851 bam_txn->bam_ce_start = bam_txn->bam_ce_pos; 852 853 if (flags & NAND_BAM_NWD) { 854 ret = prepare_bam_async_desc(nandc, nandc->cmd_chan, 855 DMA_PREP_FENCE | 856 DMA_PREP_CMD); 857 if (ret) 858 return ret; 859 } 860 } 861 862 return 0; 863 } 864 865 /* 866 * Prepares the data descriptor for BAM DMA which will be used for NAND 867 * data reads and writes. 868 */ 869 static int prep_bam_dma_desc_data(struct qcom_nand_controller *nandc, bool read, 870 const void *vaddr, 871 int size, unsigned int flags) 872 { 873 int ret; 874 struct bam_transaction *bam_txn = nandc->bam_txn; 875 876 if (read) { 877 sg_set_buf(&bam_txn->data_sgl[bam_txn->rx_sgl_pos], 878 vaddr, size); 879 bam_txn->rx_sgl_pos++; 880 } else { 881 sg_set_buf(&bam_txn->data_sgl[bam_txn->tx_sgl_pos], 882 vaddr, size); 883 bam_txn->tx_sgl_pos++; 884 885 /* 886 * BAM will only set EOT for DMA_PREP_INTERRUPT so if this flag 887 * is not set, form the DMA descriptor 888 */ 889 if (!(flags & NAND_BAM_NO_EOT)) { 890 ret = prepare_bam_async_desc(nandc, nandc->tx_chan, 891 DMA_PREP_INTERRUPT); 892 if (ret) 893 return ret; 894 } 895 } 896 897 return 0; 898 } 899 900 static int prep_adm_dma_desc(struct qcom_nand_controller *nandc, bool read, 901 int reg_off, const void *vaddr, int size, 902 bool flow_control) 903 { 904 struct desc_info *desc; 905 struct dma_async_tx_descriptor *dma_desc; 906 struct scatterlist *sgl; 907 struct dma_slave_config slave_conf; 908 enum dma_transfer_direction dir_eng; 909 int ret; 910 911 desc = kzalloc(sizeof(*desc), GFP_KERNEL); 912 if (!desc) 913 return -ENOMEM; 914 915 sgl = &desc->adm_sgl; 916 917 sg_init_one(sgl, vaddr, size); 918 919 if (read) { 920 dir_eng = DMA_DEV_TO_MEM; 921 desc->dir = DMA_FROM_DEVICE; 922 } else { 923 dir_eng = DMA_MEM_TO_DEV; 924 desc->dir = DMA_TO_DEVICE; 925 } 926 927 ret = dma_map_sg(nandc->dev, sgl, 1, desc->dir); 928 if (ret == 0) { 929 ret = -ENOMEM; 930 goto err; 931 } 932 933 memset(&slave_conf, 0x00, sizeof(slave_conf)); 934 935 slave_conf.device_fc = flow_control; 936 if (read) { 937 slave_conf.src_maxburst = 16; 938 slave_conf.src_addr = nandc->base_dma + reg_off; 939 slave_conf.slave_id = nandc->data_crci; 940 } else { 941 slave_conf.dst_maxburst = 16; 942 slave_conf.dst_addr = nandc->base_dma + reg_off; 943 slave_conf.slave_id = nandc->cmd_crci; 944 } 945 946 ret = dmaengine_slave_config(nandc->chan, &slave_conf); 947 if (ret) { 948 dev_err(nandc->dev, "failed to configure dma channel\n"); 949 goto err; 950 } 951 952 dma_desc = dmaengine_prep_slave_sg(nandc->chan, sgl, 1, dir_eng, 0); 953 if (!dma_desc) { 954 dev_err(nandc->dev, "failed to prepare desc\n"); 955 ret = -EINVAL; 956 goto err; 957 } 958 959 desc->dma_desc = dma_desc; 960 961 list_add_tail(&desc->node, &nandc->desc_list); 962 963 return 0; 964 err: 965 kfree(desc); 966 967 return ret; 968 } 969 970 /* 971 * read_reg_dma: prepares a descriptor to read a given number of 972 * contiguous registers to the reg_read_buf pointer 973 * 974 * @first: offset of the first register in the contiguous block 975 * @num_regs: number of registers to read 976 * @flags: flags to control DMA descriptor preparation 977 */ 978 static int read_reg_dma(struct qcom_nand_controller *nandc, int first, 979 int num_regs, unsigned int flags) 980 { 981 bool flow_control = false; 982 void *vaddr; 983 984 vaddr = nandc->reg_read_buf + nandc->reg_read_pos; 985 nandc->reg_read_pos += num_regs; 986 987 if (first == NAND_DEV_CMD_VLD || first == NAND_DEV_CMD1) 988 first = dev_cmd_reg_addr(nandc, first); 989 990 if (nandc->props->is_bam) 991 return prep_bam_dma_desc_cmd(nandc, true, first, vaddr, 992 num_regs, flags); 993 994 if (first == NAND_READ_ID || first == NAND_FLASH_STATUS) 995 flow_control = true; 996 997 return prep_adm_dma_desc(nandc, true, first, vaddr, 998 num_regs * sizeof(u32), flow_control); 999 } 1000 1001 /* 1002 * write_reg_dma: prepares a descriptor to write a given number of 1003 * contiguous registers 1004 * 1005 * @first: offset of the first register in the contiguous block 1006 * @num_regs: number of registers to write 1007 * @flags: flags to control DMA descriptor preparation 1008 */ 1009 static int write_reg_dma(struct qcom_nand_controller *nandc, int first, 1010 int num_regs, unsigned int flags) 1011 { 1012 bool flow_control = false; 1013 struct nandc_regs *regs = nandc->regs; 1014 void *vaddr; 1015 1016 vaddr = offset_to_nandc_reg(regs, first); 1017 1018 if (first == NAND_ERASED_CW_DETECT_CFG) { 1019 if (flags & NAND_ERASED_CW_SET) 1020 vaddr = ®s->erased_cw_detect_cfg_set; 1021 else 1022 vaddr = ®s->erased_cw_detect_cfg_clr; 1023 } 1024 1025 if (first == NAND_EXEC_CMD) 1026 flags |= NAND_BAM_NWD; 1027 1028 if (first == NAND_DEV_CMD1_RESTORE || first == NAND_DEV_CMD1) 1029 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD1); 1030 1031 if (first == NAND_DEV_CMD_VLD_RESTORE || first == NAND_DEV_CMD_VLD) 1032 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD); 1033 1034 if (nandc->props->is_bam) 1035 return prep_bam_dma_desc_cmd(nandc, false, first, vaddr, 1036 num_regs, flags); 1037 1038 if (first == NAND_FLASH_CMD) 1039 flow_control = true; 1040 1041 return prep_adm_dma_desc(nandc, false, first, vaddr, 1042 num_regs * sizeof(u32), flow_control); 1043 } 1044 1045 /* 1046 * read_data_dma: prepares a DMA descriptor to transfer data from the 1047 * controller's internal buffer to the buffer 'vaddr' 1048 * 1049 * @reg_off: offset within the controller's data buffer 1050 * @vaddr: virtual address of the buffer we want to write to 1051 * @size: DMA transaction size in bytes 1052 * @flags: flags to control DMA descriptor preparation 1053 */ 1054 static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off, 1055 const u8 *vaddr, int size, unsigned int flags) 1056 { 1057 if (nandc->props->is_bam) 1058 return prep_bam_dma_desc_data(nandc, true, vaddr, size, flags); 1059 1060 return prep_adm_dma_desc(nandc, true, reg_off, vaddr, size, false); 1061 } 1062 1063 /* 1064 * write_data_dma: prepares a DMA descriptor to transfer data from 1065 * 'vaddr' to the controller's internal buffer 1066 * 1067 * @reg_off: offset within the controller's data buffer 1068 * @vaddr: virtual address of the buffer we want to read from 1069 * @size: DMA transaction size in bytes 1070 * @flags: flags to control DMA descriptor preparation 1071 */ 1072 static int write_data_dma(struct qcom_nand_controller *nandc, int reg_off, 1073 const u8 *vaddr, int size, unsigned int flags) 1074 { 1075 if (nandc->props->is_bam) 1076 return prep_bam_dma_desc_data(nandc, false, vaddr, size, flags); 1077 1078 return prep_adm_dma_desc(nandc, false, reg_off, vaddr, size, false); 1079 } 1080 1081 /* 1082 * Helper to prepare DMA descriptors for configuring registers 1083 * before reading a NAND page. 1084 */ 1085 static void config_nand_page_read(struct qcom_nand_controller *nandc) 1086 { 1087 write_reg_dma(nandc, NAND_ADDR0, 2, 0); 1088 write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0); 1089 write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1, 0); 1090 write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1, 0); 1091 write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1, 1092 NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL); 1093 } 1094 1095 /* 1096 * Helper to prepare DMA descriptors for configuring registers 1097 * before reading each codeword in NAND page. 1098 */ 1099 static void 1100 config_nand_cw_read(struct qcom_nand_controller *nandc, bool use_ecc) 1101 { 1102 if (nandc->props->is_bam) 1103 write_reg_dma(nandc, NAND_READ_LOCATION_0, 4, 1104 NAND_BAM_NEXT_SGL); 1105 1106 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL); 1107 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1108 1109 if (use_ecc) { 1110 read_reg_dma(nandc, NAND_FLASH_STATUS, 2, 0); 1111 read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1, 1112 NAND_BAM_NEXT_SGL); 1113 } else { 1114 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL); 1115 } 1116 } 1117 1118 /* 1119 * Helper to prepare dma descriptors to configure registers needed for reading a 1120 * single codeword in page 1121 */ 1122 static void 1123 config_nand_single_cw_page_read(struct qcom_nand_controller *nandc, 1124 bool use_ecc) 1125 { 1126 config_nand_page_read(nandc); 1127 config_nand_cw_read(nandc, use_ecc); 1128 } 1129 1130 /* 1131 * Helper to prepare DMA descriptors used to configure registers needed for 1132 * before writing a NAND page. 1133 */ 1134 static void config_nand_page_write(struct qcom_nand_controller *nandc) 1135 { 1136 write_reg_dma(nandc, NAND_ADDR0, 2, 0); 1137 write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0); 1138 write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1, 1139 NAND_BAM_NEXT_SGL); 1140 } 1141 1142 /* 1143 * Helper to prepare DMA descriptors for configuring registers 1144 * before writing each codeword in NAND page. 1145 */ 1146 static void config_nand_cw_write(struct qcom_nand_controller *nandc) 1147 { 1148 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL); 1149 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1150 1151 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL); 1152 1153 write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0); 1154 write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL); 1155 } 1156 1157 /* 1158 * the following functions are used within chip->legacy.cmdfunc() to 1159 * perform different NAND_CMD_* commands 1160 */ 1161 1162 /* sets up descriptors for NAND_CMD_PARAM */ 1163 static int nandc_param(struct qcom_nand_host *host) 1164 { 1165 struct nand_chip *chip = &host->chip; 1166 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1167 1168 /* 1169 * NAND_CMD_PARAM is called before we know much about the FLASH chip 1170 * in use. we configure the controller to perform a raw read of 512 1171 * bytes to read onfi params 1172 */ 1173 nandc_set_reg(nandc, NAND_FLASH_CMD, OP_PAGE_READ | PAGE_ACC | LAST_PAGE); 1174 nandc_set_reg(nandc, NAND_ADDR0, 0); 1175 nandc_set_reg(nandc, NAND_ADDR1, 0); 1176 nandc_set_reg(nandc, NAND_DEV0_CFG0, 0 << CW_PER_PAGE 1177 | 512 << UD_SIZE_BYTES 1178 | 5 << NUM_ADDR_CYCLES 1179 | 0 << SPARE_SIZE_BYTES); 1180 nandc_set_reg(nandc, NAND_DEV0_CFG1, 7 << NAND_RECOVERY_CYCLES 1181 | 0 << CS_ACTIVE_BSY 1182 | 17 << BAD_BLOCK_BYTE_NUM 1183 | 1 << BAD_BLOCK_IN_SPARE_AREA 1184 | 2 << WR_RD_BSY_GAP 1185 | 0 << WIDE_FLASH 1186 | 1 << DEV0_CFG1_ECC_DISABLE); 1187 nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, 1 << ECC_CFG_ECC_DISABLE); 1188 1189 /* configure CMD1 and VLD for ONFI param probing */ 1190 nandc_set_reg(nandc, NAND_DEV_CMD_VLD, 1191 (nandc->vld & ~READ_START_VLD)); 1192 nandc_set_reg(nandc, NAND_DEV_CMD1, 1193 (nandc->cmd1 & ~(0xFF << READ_ADDR)) 1194 | NAND_CMD_PARAM << READ_ADDR); 1195 1196 nandc_set_reg(nandc, NAND_EXEC_CMD, 1); 1197 1198 nandc_set_reg(nandc, NAND_DEV_CMD1_RESTORE, nandc->cmd1); 1199 nandc_set_reg(nandc, NAND_DEV_CMD_VLD_RESTORE, nandc->vld); 1200 nandc_set_read_loc(nandc, 0, 0, 512, 1); 1201 1202 write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1, 0); 1203 write_reg_dma(nandc, NAND_DEV_CMD1, 1, NAND_BAM_NEXT_SGL); 1204 1205 nandc->buf_count = 512; 1206 memset(nandc->data_buffer, 0xff, nandc->buf_count); 1207 1208 config_nand_single_cw_page_read(nandc, false); 1209 1210 read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, 1211 nandc->buf_count, 0); 1212 1213 /* restore CMD1 and VLD regs */ 1214 write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1, 0); 1215 write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1, NAND_BAM_NEXT_SGL); 1216 1217 return 0; 1218 } 1219 1220 /* sets up descriptors for NAND_CMD_ERASE1 */ 1221 static int erase_block(struct qcom_nand_host *host, int page_addr) 1222 { 1223 struct nand_chip *chip = &host->chip; 1224 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1225 1226 nandc_set_reg(nandc, NAND_FLASH_CMD, 1227 OP_BLOCK_ERASE | PAGE_ACC | LAST_PAGE); 1228 nandc_set_reg(nandc, NAND_ADDR0, page_addr); 1229 nandc_set_reg(nandc, NAND_ADDR1, 0); 1230 nandc_set_reg(nandc, NAND_DEV0_CFG0, 1231 host->cfg0_raw & ~(7 << CW_PER_PAGE)); 1232 nandc_set_reg(nandc, NAND_DEV0_CFG1, host->cfg1_raw); 1233 nandc_set_reg(nandc, NAND_EXEC_CMD, 1); 1234 nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus); 1235 nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus); 1236 1237 write_reg_dma(nandc, NAND_FLASH_CMD, 3, NAND_BAM_NEXT_SGL); 1238 write_reg_dma(nandc, NAND_DEV0_CFG0, 2, NAND_BAM_NEXT_SGL); 1239 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1240 1241 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL); 1242 1243 write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0); 1244 write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL); 1245 1246 return 0; 1247 } 1248 1249 /* sets up descriptors for NAND_CMD_READID */ 1250 static int read_id(struct qcom_nand_host *host, int column) 1251 { 1252 struct nand_chip *chip = &host->chip; 1253 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1254 1255 if (column == -1) 1256 return 0; 1257 1258 nandc_set_reg(nandc, NAND_FLASH_CMD, OP_FETCH_ID); 1259 nandc_set_reg(nandc, NAND_ADDR0, column); 1260 nandc_set_reg(nandc, NAND_ADDR1, 0); 1261 nandc_set_reg(nandc, NAND_FLASH_CHIP_SELECT, 1262 nandc->props->is_bam ? 0 : DM_EN); 1263 nandc_set_reg(nandc, NAND_EXEC_CMD, 1); 1264 1265 write_reg_dma(nandc, NAND_FLASH_CMD, 4, NAND_BAM_NEXT_SGL); 1266 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1267 1268 read_reg_dma(nandc, NAND_READ_ID, 1, NAND_BAM_NEXT_SGL); 1269 1270 return 0; 1271 } 1272 1273 /* sets up descriptors for NAND_CMD_RESET */ 1274 static int reset(struct qcom_nand_host *host) 1275 { 1276 struct nand_chip *chip = &host->chip; 1277 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1278 1279 nandc_set_reg(nandc, NAND_FLASH_CMD, OP_RESET_DEVICE); 1280 nandc_set_reg(nandc, NAND_EXEC_CMD, 1); 1281 1282 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL); 1283 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1284 1285 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL); 1286 1287 return 0; 1288 } 1289 1290 /* helpers to submit/free our list of dma descriptors */ 1291 static int submit_descs(struct qcom_nand_controller *nandc) 1292 { 1293 struct desc_info *desc; 1294 dma_cookie_t cookie = 0; 1295 struct bam_transaction *bam_txn = nandc->bam_txn; 1296 int r; 1297 1298 if (nandc->props->is_bam) { 1299 if (bam_txn->rx_sgl_pos > bam_txn->rx_sgl_start) { 1300 r = prepare_bam_async_desc(nandc, nandc->rx_chan, 0); 1301 if (r) 1302 return r; 1303 } 1304 1305 if (bam_txn->tx_sgl_pos > bam_txn->tx_sgl_start) { 1306 r = prepare_bam_async_desc(nandc, nandc->tx_chan, 1307 DMA_PREP_INTERRUPT); 1308 if (r) 1309 return r; 1310 } 1311 1312 if (bam_txn->cmd_sgl_pos > bam_txn->cmd_sgl_start) { 1313 r = prepare_bam_async_desc(nandc, nandc->cmd_chan, 1314 DMA_PREP_CMD); 1315 if (r) 1316 return r; 1317 } 1318 } 1319 1320 list_for_each_entry(desc, &nandc->desc_list, node) 1321 cookie = dmaengine_submit(desc->dma_desc); 1322 1323 if (nandc->props->is_bam) { 1324 bam_txn->last_cmd_desc->callback = qpic_bam_dma_done; 1325 bam_txn->last_cmd_desc->callback_param = bam_txn; 1326 if (bam_txn->last_data_desc) { 1327 bam_txn->last_data_desc->callback = qpic_bam_dma_done; 1328 bam_txn->last_data_desc->callback_param = bam_txn; 1329 bam_txn->wait_second_completion = true; 1330 } 1331 1332 dma_async_issue_pending(nandc->tx_chan); 1333 dma_async_issue_pending(nandc->rx_chan); 1334 dma_async_issue_pending(nandc->cmd_chan); 1335 1336 if (!wait_for_completion_timeout(&bam_txn->txn_done, 1337 QPIC_NAND_COMPLETION_TIMEOUT)) 1338 return -ETIMEDOUT; 1339 } else { 1340 if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE) 1341 return -ETIMEDOUT; 1342 } 1343 1344 return 0; 1345 } 1346 1347 static void free_descs(struct qcom_nand_controller *nandc) 1348 { 1349 struct desc_info *desc, *n; 1350 1351 list_for_each_entry_safe(desc, n, &nandc->desc_list, node) { 1352 list_del(&desc->node); 1353 1354 if (nandc->props->is_bam) 1355 dma_unmap_sg(nandc->dev, desc->bam_sgl, 1356 desc->sgl_cnt, desc->dir); 1357 else 1358 dma_unmap_sg(nandc->dev, &desc->adm_sgl, 1, 1359 desc->dir); 1360 1361 kfree(desc); 1362 } 1363 } 1364 1365 /* reset the register read buffer for next NAND operation */ 1366 static void clear_read_regs(struct qcom_nand_controller *nandc) 1367 { 1368 nandc->reg_read_pos = 0; 1369 nandc_read_buffer_sync(nandc, false); 1370 } 1371 1372 static void pre_command(struct qcom_nand_host *host, int command) 1373 { 1374 struct nand_chip *chip = &host->chip; 1375 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1376 1377 nandc->buf_count = 0; 1378 nandc->buf_start = 0; 1379 host->use_ecc = false; 1380 host->last_command = command; 1381 1382 clear_read_regs(nandc); 1383 1384 if (command == NAND_CMD_RESET || command == NAND_CMD_READID || 1385 command == NAND_CMD_PARAM || command == NAND_CMD_ERASE1) 1386 clear_bam_transaction(nandc); 1387 } 1388 1389 /* 1390 * this is called after NAND_CMD_PAGEPROG and NAND_CMD_ERASE1 to set our 1391 * privately maintained status byte, this status byte can be read after 1392 * NAND_CMD_STATUS is called 1393 */ 1394 static void parse_erase_write_errors(struct qcom_nand_host *host, int command) 1395 { 1396 struct nand_chip *chip = &host->chip; 1397 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1398 struct nand_ecc_ctrl *ecc = &chip->ecc; 1399 int num_cw; 1400 int i; 1401 1402 num_cw = command == NAND_CMD_PAGEPROG ? ecc->steps : 1; 1403 nandc_read_buffer_sync(nandc, true); 1404 1405 for (i = 0; i < num_cw; i++) { 1406 u32 flash_status = le32_to_cpu(nandc->reg_read_buf[i]); 1407 1408 if (flash_status & FS_MPU_ERR) 1409 host->status &= ~NAND_STATUS_WP; 1410 1411 if (flash_status & FS_OP_ERR || (i == (num_cw - 1) && 1412 (flash_status & 1413 FS_DEVICE_STS_ERR))) 1414 host->status |= NAND_STATUS_FAIL; 1415 } 1416 } 1417 1418 static void post_command(struct qcom_nand_host *host, int command) 1419 { 1420 struct nand_chip *chip = &host->chip; 1421 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1422 1423 switch (command) { 1424 case NAND_CMD_READID: 1425 nandc_read_buffer_sync(nandc, true); 1426 memcpy(nandc->data_buffer, nandc->reg_read_buf, 1427 nandc->buf_count); 1428 break; 1429 case NAND_CMD_PAGEPROG: 1430 case NAND_CMD_ERASE1: 1431 parse_erase_write_errors(host, command); 1432 break; 1433 default: 1434 break; 1435 } 1436 } 1437 1438 /* 1439 * Implements chip->legacy.cmdfunc. It's only used for a limited set of 1440 * commands. The rest of the commands wouldn't be called by upper layers. 1441 * For example, NAND_CMD_READOOB would never be called because we have our own 1442 * versions of read_oob ops for nand_ecc_ctrl. 1443 */ 1444 static void qcom_nandc_command(struct nand_chip *chip, unsigned int command, 1445 int column, int page_addr) 1446 { 1447 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1448 struct nand_ecc_ctrl *ecc = &chip->ecc; 1449 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1450 bool wait = false; 1451 int ret = 0; 1452 1453 pre_command(host, command); 1454 1455 switch (command) { 1456 case NAND_CMD_RESET: 1457 ret = reset(host); 1458 wait = true; 1459 break; 1460 1461 case NAND_CMD_READID: 1462 nandc->buf_count = 4; 1463 ret = read_id(host, column); 1464 wait = true; 1465 break; 1466 1467 case NAND_CMD_PARAM: 1468 ret = nandc_param(host); 1469 wait = true; 1470 break; 1471 1472 case NAND_CMD_ERASE1: 1473 ret = erase_block(host, page_addr); 1474 wait = true; 1475 break; 1476 1477 case NAND_CMD_READ0: 1478 /* we read the entire page for now */ 1479 WARN_ON(column != 0); 1480 1481 host->use_ecc = true; 1482 set_address(host, 0, page_addr); 1483 update_rw_regs(host, ecc->steps, true); 1484 break; 1485 1486 case NAND_CMD_SEQIN: 1487 WARN_ON(column != 0); 1488 set_address(host, 0, page_addr); 1489 break; 1490 1491 case NAND_CMD_PAGEPROG: 1492 case NAND_CMD_STATUS: 1493 case NAND_CMD_NONE: 1494 default: 1495 break; 1496 } 1497 1498 if (ret) { 1499 dev_err(nandc->dev, "failure executing command %d\n", 1500 command); 1501 free_descs(nandc); 1502 return; 1503 } 1504 1505 if (wait) { 1506 ret = submit_descs(nandc); 1507 if (ret) 1508 dev_err(nandc->dev, 1509 "failure submitting descs for command %d\n", 1510 command); 1511 } 1512 1513 free_descs(nandc); 1514 1515 post_command(host, command); 1516 } 1517 1518 /* 1519 * when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read 1520 * an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS. 1521 * 1522 * when using RS ECC, the HW reports the same erros when reading an erased CW, 1523 * but it notifies that it is an erased CW by placing special characters at 1524 * certain offsets in the buffer. 1525 * 1526 * verify if the page is erased or not, and fix up the page for RS ECC by 1527 * replacing the special characters with 0xff. 1528 */ 1529 static bool erased_chunk_check_and_fixup(u8 *data_buf, int data_len) 1530 { 1531 u8 empty1, empty2; 1532 1533 /* 1534 * an erased page flags an error in NAND_FLASH_STATUS, check if the page 1535 * is erased by looking for 0x54s at offsets 3 and 175 from the 1536 * beginning of each codeword 1537 */ 1538 1539 empty1 = data_buf[3]; 1540 empty2 = data_buf[175]; 1541 1542 /* 1543 * if the erased codework markers, if they exist override them with 1544 * 0xffs 1545 */ 1546 if ((empty1 == 0x54 && empty2 == 0xff) || 1547 (empty1 == 0xff && empty2 == 0x54)) { 1548 data_buf[3] = 0xff; 1549 data_buf[175] = 0xff; 1550 } 1551 1552 /* 1553 * check if the entire chunk contains 0xffs or not. if it doesn't, then 1554 * restore the original values at the special offsets 1555 */ 1556 if (memchr_inv(data_buf, 0xff, data_len)) { 1557 data_buf[3] = empty1; 1558 data_buf[175] = empty2; 1559 1560 return false; 1561 } 1562 1563 return true; 1564 } 1565 1566 struct read_stats { 1567 __le32 flash; 1568 __le32 buffer; 1569 __le32 erased_cw; 1570 }; 1571 1572 /* reads back FLASH_STATUS register set by the controller */ 1573 static int check_flash_errors(struct qcom_nand_host *host, int cw_cnt) 1574 { 1575 struct nand_chip *chip = &host->chip; 1576 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1577 int i; 1578 1579 for (i = 0; i < cw_cnt; i++) { 1580 u32 flash = le32_to_cpu(nandc->reg_read_buf[i]); 1581 1582 if (flash & (FS_OP_ERR | FS_MPU_ERR)) 1583 return -EIO; 1584 } 1585 1586 return 0; 1587 } 1588 1589 /* performs raw read for one codeword */ 1590 static int 1591 qcom_nandc_read_cw_raw(struct mtd_info *mtd, struct nand_chip *chip, 1592 u8 *data_buf, u8 *oob_buf, int page, int cw) 1593 { 1594 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1595 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1596 struct nand_ecc_ctrl *ecc = &chip->ecc; 1597 int data_size1, data_size2, oob_size1, oob_size2; 1598 int ret, reg_off = FLASH_BUF_ACC, read_loc = 0; 1599 1600 nand_read_page_op(chip, page, 0, NULL, 0); 1601 host->use_ecc = false; 1602 1603 clear_bam_transaction(nandc); 1604 set_address(host, host->cw_size * cw, page); 1605 update_rw_regs(host, 1, true); 1606 config_nand_page_read(nandc); 1607 1608 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1); 1609 oob_size1 = host->bbm_size; 1610 1611 if (cw == (ecc->steps - 1)) { 1612 data_size2 = ecc->size - data_size1 - 1613 ((ecc->steps - 1) * 4); 1614 oob_size2 = (ecc->steps * 4) + host->ecc_bytes_hw + 1615 host->spare_bytes; 1616 } else { 1617 data_size2 = host->cw_data - data_size1; 1618 oob_size2 = host->ecc_bytes_hw + host->spare_bytes; 1619 } 1620 1621 if (nandc->props->is_bam) { 1622 nandc_set_read_loc(nandc, 0, read_loc, data_size1, 0); 1623 read_loc += data_size1; 1624 1625 nandc_set_read_loc(nandc, 1, read_loc, oob_size1, 0); 1626 read_loc += oob_size1; 1627 1628 nandc_set_read_loc(nandc, 2, read_loc, data_size2, 0); 1629 read_loc += data_size2; 1630 1631 nandc_set_read_loc(nandc, 3, read_loc, oob_size2, 1); 1632 } 1633 1634 config_nand_cw_read(nandc, false); 1635 1636 read_data_dma(nandc, reg_off, data_buf, data_size1, 0); 1637 reg_off += data_size1; 1638 1639 read_data_dma(nandc, reg_off, oob_buf, oob_size1, 0); 1640 reg_off += oob_size1; 1641 1642 read_data_dma(nandc, reg_off, data_buf + data_size1, data_size2, 0); 1643 reg_off += data_size2; 1644 1645 read_data_dma(nandc, reg_off, oob_buf + oob_size1, oob_size2, 0); 1646 1647 ret = submit_descs(nandc); 1648 free_descs(nandc); 1649 if (ret) { 1650 dev_err(nandc->dev, "failure to read raw cw %d\n", cw); 1651 return ret; 1652 } 1653 1654 return check_flash_errors(host, 1); 1655 } 1656 1657 /* 1658 * Bitflips can happen in erased codewords also so this function counts the 1659 * number of 0 in each CW for which ECC engine returns the uncorrectable 1660 * error. The page will be assumed as erased if this count is less than or 1661 * equal to the ecc->strength for each CW. 1662 * 1663 * 1. Both DATA and OOB need to be checked for number of 0. The 1664 * top-level API can be called with only data buf or OOB buf so use 1665 * chip->data_buf if data buf is null and chip->oob_poi if oob buf 1666 * is null for copying the raw bytes. 1667 * 2. Perform raw read for all the CW which has uncorrectable errors. 1668 * 3. For each CW, check the number of 0 in cw_data and usable OOB bytes. 1669 * The BBM and spare bytes bit flip won’t affect the ECC so don’t check 1670 * the number of bitflips in this area. 1671 */ 1672 static int 1673 check_for_erased_page(struct qcom_nand_host *host, u8 *data_buf, 1674 u8 *oob_buf, unsigned long uncorrectable_cws, 1675 int page, unsigned int max_bitflips) 1676 { 1677 struct nand_chip *chip = &host->chip; 1678 struct mtd_info *mtd = nand_to_mtd(chip); 1679 struct nand_ecc_ctrl *ecc = &chip->ecc; 1680 u8 *cw_data_buf, *cw_oob_buf; 1681 int cw, data_size, oob_size, ret = 0; 1682 1683 if (!data_buf) 1684 data_buf = nand_get_data_buf(chip); 1685 1686 if (!oob_buf) { 1687 nand_get_data_buf(chip); 1688 oob_buf = chip->oob_poi; 1689 } 1690 1691 for_each_set_bit(cw, &uncorrectable_cws, ecc->steps) { 1692 if (cw == (ecc->steps - 1)) { 1693 data_size = ecc->size - ((ecc->steps - 1) * 4); 1694 oob_size = (ecc->steps * 4) + host->ecc_bytes_hw; 1695 } else { 1696 data_size = host->cw_data; 1697 oob_size = host->ecc_bytes_hw; 1698 } 1699 1700 /* determine starting buffer address for current CW */ 1701 cw_data_buf = data_buf + (cw * host->cw_data); 1702 cw_oob_buf = oob_buf + (cw * ecc->bytes); 1703 1704 ret = qcom_nandc_read_cw_raw(mtd, chip, cw_data_buf, 1705 cw_oob_buf, page, cw); 1706 if (ret) 1707 return ret; 1708 1709 /* 1710 * make sure it isn't an erased page reported 1711 * as not-erased by HW because of a few bitflips 1712 */ 1713 ret = nand_check_erased_ecc_chunk(cw_data_buf, data_size, 1714 cw_oob_buf + host->bbm_size, 1715 oob_size, NULL, 1716 0, ecc->strength); 1717 if (ret < 0) { 1718 mtd->ecc_stats.failed++; 1719 } else { 1720 mtd->ecc_stats.corrected += ret; 1721 max_bitflips = max_t(unsigned int, max_bitflips, ret); 1722 } 1723 } 1724 1725 return max_bitflips; 1726 } 1727 1728 /* 1729 * reads back status registers set by the controller to notify page read 1730 * errors. this is equivalent to what 'ecc->correct()' would do. 1731 */ 1732 static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf, 1733 u8 *oob_buf, int page) 1734 { 1735 struct nand_chip *chip = &host->chip; 1736 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1737 struct mtd_info *mtd = nand_to_mtd(chip); 1738 struct nand_ecc_ctrl *ecc = &chip->ecc; 1739 unsigned int max_bitflips = 0, uncorrectable_cws = 0; 1740 struct read_stats *buf; 1741 bool flash_op_err = false, erased; 1742 int i; 1743 u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf; 1744 1745 buf = (struct read_stats *)nandc->reg_read_buf; 1746 nandc_read_buffer_sync(nandc, true); 1747 1748 for (i = 0; i < ecc->steps; i++, buf++) { 1749 u32 flash, buffer, erased_cw; 1750 int data_len, oob_len; 1751 1752 if (i == (ecc->steps - 1)) { 1753 data_len = ecc->size - ((ecc->steps - 1) << 2); 1754 oob_len = ecc->steps << 2; 1755 } else { 1756 data_len = host->cw_data; 1757 oob_len = 0; 1758 } 1759 1760 flash = le32_to_cpu(buf->flash); 1761 buffer = le32_to_cpu(buf->buffer); 1762 erased_cw = le32_to_cpu(buf->erased_cw); 1763 1764 /* 1765 * Check ECC failure for each codeword. ECC failure can 1766 * happen in either of the following conditions 1767 * 1. If number of bitflips are greater than ECC engine 1768 * capability. 1769 * 2. If this codeword contains all 0xff for which erased 1770 * codeword detection check will be done. 1771 */ 1772 if ((flash & FS_OP_ERR) && (buffer & BS_UNCORRECTABLE_BIT)) { 1773 /* 1774 * For BCH ECC, ignore erased codeword errors, if 1775 * ERASED_CW bits are set. 1776 */ 1777 if (host->bch_enabled) { 1778 erased = (erased_cw & ERASED_CW) == ERASED_CW ? 1779 true : false; 1780 /* 1781 * For RS ECC, HW reports the erased CW by placing 1782 * special characters at certain offsets in the buffer. 1783 * These special characters will be valid only if 1784 * complete page is read i.e. data_buf is not NULL. 1785 */ 1786 } else if (data_buf) { 1787 erased = erased_chunk_check_and_fixup(data_buf, 1788 data_len); 1789 } else { 1790 erased = false; 1791 } 1792 1793 if (!erased) 1794 uncorrectable_cws |= BIT(i); 1795 /* 1796 * Check if MPU or any other operational error (timeout, 1797 * device failure, etc.) happened for this codeword and 1798 * make flash_op_err true. If flash_op_err is set, then 1799 * EIO will be returned for page read. 1800 */ 1801 } else if (flash & (FS_OP_ERR | FS_MPU_ERR)) { 1802 flash_op_err = true; 1803 /* 1804 * No ECC or operational errors happened. Check the number of 1805 * bits corrected and update the ecc_stats.corrected. 1806 */ 1807 } else { 1808 unsigned int stat; 1809 1810 stat = buffer & BS_CORRECTABLE_ERR_MSK; 1811 mtd->ecc_stats.corrected += stat; 1812 max_bitflips = max(max_bitflips, stat); 1813 } 1814 1815 if (data_buf) 1816 data_buf += data_len; 1817 if (oob_buf) 1818 oob_buf += oob_len + ecc->bytes; 1819 } 1820 1821 if (flash_op_err) 1822 return -EIO; 1823 1824 if (!uncorrectable_cws) 1825 return max_bitflips; 1826 1827 return check_for_erased_page(host, data_buf_start, oob_buf_start, 1828 uncorrectable_cws, page, 1829 max_bitflips); 1830 } 1831 1832 /* 1833 * helper to perform the actual page read operation, used by ecc->read_page(), 1834 * ecc->read_oob() 1835 */ 1836 static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf, 1837 u8 *oob_buf, int page) 1838 { 1839 struct nand_chip *chip = &host->chip; 1840 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1841 struct nand_ecc_ctrl *ecc = &chip->ecc; 1842 u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf; 1843 int i, ret; 1844 1845 config_nand_page_read(nandc); 1846 1847 /* queue cmd descs for each codeword */ 1848 for (i = 0; i < ecc->steps; i++) { 1849 int data_size, oob_size; 1850 1851 if (i == (ecc->steps - 1)) { 1852 data_size = ecc->size - ((ecc->steps - 1) << 2); 1853 oob_size = (ecc->steps << 2) + host->ecc_bytes_hw + 1854 host->spare_bytes; 1855 } else { 1856 data_size = host->cw_data; 1857 oob_size = host->ecc_bytes_hw + host->spare_bytes; 1858 } 1859 1860 if (nandc->props->is_bam) { 1861 if (data_buf && oob_buf) { 1862 nandc_set_read_loc(nandc, 0, 0, data_size, 0); 1863 nandc_set_read_loc(nandc, 1, data_size, 1864 oob_size, 1); 1865 } else if (data_buf) { 1866 nandc_set_read_loc(nandc, 0, 0, data_size, 1); 1867 } else { 1868 nandc_set_read_loc(nandc, 0, data_size, 1869 oob_size, 1); 1870 } 1871 } 1872 1873 config_nand_cw_read(nandc, true); 1874 1875 if (data_buf) 1876 read_data_dma(nandc, FLASH_BUF_ACC, data_buf, 1877 data_size, 0); 1878 1879 /* 1880 * when ecc is enabled, the controller doesn't read the real 1881 * or dummy bad block markers in each chunk. To maintain a 1882 * consistent layout across RAW and ECC reads, we just 1883 * leave the real/dummy BBM offsets empty (i.e, filled with 1884 * 0xffs) 1885 */ 1886 if (oob_buf) { 1887 int j; 1888 1889 for (j = 0; j < host->bbm_size; j++) 1890 *oob_buf++ = 0xff; 1891 1892 read_data_dma(nandc, FLASH_BUF_ACC + data_size, 1893 oob_buf, oob_size, 0); 1894 } 1895 1896 if (data_buf) 1897 data_buf += data_size; 1898 if (oob_buf) 1899 oob_buf += oob_size; 1900 } 1901 1902 ret = submit_descs(nandc); 1903 free_descs(nandc); 1904 1905 if (ret) { 1906 dev_err(nandc->dev, "failure to read page/oob\n"); 1907 return ret; 1908 } 1909 1910 return parse_read_errors(host, data_buf_start, oob_buf_start, page); 1911 } 1912 1913 /* 1914 * a helper that copies the last step/codeword of a page (containing free oob) 1915 * into our local buffer 1916 */ 1917 static int copy_last_cw(struct qcom_nand_host *host, int page) 1918 { 1919 struct nand_chip *chip = &host->chip; 1920 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1921 struct nand_ecc_ctrl *ecc = &chip->ecc; 1922 int size; 1923 int ret; 1924 1925 clear_read_regs(nandc); 1926 1927 size = host->use_ecc ? host->cw_data : host->cw_size; 1928 1929 /* prepare a clean read buffer */ 1930 memset(nandc->data_buffer, 0xff, size); 1931 1932 set_address(host, host->cw_size * (ecc->steps - 1), page); 1933 update_rw_regs(host, 1, true); 1934 1935 config_nand_single_cw_page_read(nandc, host->use_ecc); 1936 1937 read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size, 0); 1938 1939 ret = submit_descs(nandc); 1940 if (ret) 1941 dev_err(nandc->dev, "failed to copy last codeword\n"); 1942 1943 free_descs(nandc); 1944 1945 return ret; 1946 } 1947 1948 /* implements ecc->read_page() */ 1949 static int qcom_nandc_read_page(struct nand_chip *chip, uint8_t *buf, 1950 int oob_required, int page) 1951 { 1952 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1953 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1954 u8 *data_buf, *oob_buf = NULL; 1955 1956 nand_read_page_op(chip, page, 0, NULL, 0); 1957 data_buf = buf; 1958 oob_buf = oob_required ? chip->oob_poi : NULL; 1959 1960 clear_bam_transaction(nandc); 1961 1962 return read_page_ecc(host, data_buf, oob_buf, page); 1963 } 1964 1965 /* implements ecc->read_page_raw() */ 1966 static int qcom_nandc_read_page_raw(struct nand_chip *chip, uint8_t *buf, 1967 int oob_required, int page) 1968 { 1969 struct mtd_info *mtd = nand_to_mtd(chip); 1970 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1971 struct nand_ecc_ctrl *ecc = &chip->ecc; 1972 int cw, ret; 1973 u8 *data_buf = buf, *oob_buf = chip->oob_poi; 1974 1975 for (cw = 0; cw < ecc->steps; cw++) { 1976 ret = qcom_nandc_read_cw_raw(mtd, chip, data_buf, oob_buf, 1977 page, cw); 1978 if (ret) 1979 return ret; 1980 1981 data_buf += host->cw_data; 1982 oob_buf += ecc->bytes; 1983 } 1984 1985 return 0; 1986 } 1987 1988 /* implements ecc->read_oob() */ 1989 static int qcom_nandc_read_oob(struct nand_chip *chip, int page) 1990 { 1991 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1992 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1993 struct nand_ecc_ctrl *ecc = &chip->ecc; 1994 1995 clear_read_regs(nandc); 1996 clear_bam_transaction(nandc); 1997 1998 host->use_ecc = true; 1999 set_address(host, 0, page); 2000 update_rw_regs(host, ecc->steps, true); 2001 2002 return read_page_ecc(host, NULL, chip->oob_poi, page); 2003 } 2004 2005 /* implements ecc->write_page() */ 2006 static int qcom_nandc_write_page(struct nand_chip *chip, const uint8_t *buf, 2007 int oob_required, int page) 2008 { 2009 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2010 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2011 struct nand_ecc_ctrl *ecc = &chip->ecc; 2012 u8 *data_buf, *oob_buf; 2013 int i, ret; 2014 2015 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 2016 2017 clear_read_regs(nandc); 2018 clear_bam_transaction(nandc); 2019 2020 data_buf = (u8 *)buf; 2021 oob_buf = chip->oob_poi; 2022 2023 host->use_ecc = true; 2024 update_rw_regs(host, ecc->steps, false); 2025 config_nand_page_write(nandc); 2026 2027 for (i = 0; i < ecc->steps; i++) { 2028 int data_size, oob_size; 2029 2030 if (i == (ecc->steps - 1)) { 2031 data_size = ecc->size - ((ecc->steps - 1) << 2); 2032 oob_size = (ecc->steps << 2) + host->ecc_bytes_hw + 2033 host->spare_bytes; 2034 } else { 2035 data_size = host->cw_data; 2036 oob_size = ecc->bytes; 2037 } 2038 2039 2040 write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size, 2041 i == (ecc->steps - 1) ? NAND_BAM_NO_EOT : 0); 2042 2043 /* 2044 * when ECC is enabled, we don't really need to write anything 2045 * to oob for the first n - 1 codewords since these oob regions 2046 * just contain ECC bytes that's written by the controller 2047 * itself. For the last codeword, we skip the bbm positions and 2048 * write to the free oob area. 2049 */ 2050 if (i == (ecc->steps - 1)) { 2051 oob_buf += host->bbm_size; 2052 2053 write_data_dma(nandc, FLASH_BUF_ACC + data_size, 2054 oob_buf, oob_size, 0); 2055 } 2056 2057 config_nand_cw_write(nandc); 2058 2059 data_buf += data_size; 2060 oob_buf += oob_size; 2061 } 2062 2063 ret = submit_descs(nandc); 2064 if (ret) 2065 dev_err(nandc->dev, "failure to write page\n"); 2066 2067 free_descs(nandc); 2068 2069 if (!ret) 2070 ret = nand_prog_page_end_op(chip); 2071 2072 return ret; 2073 } 2074 2075 /* implements ecc->write_page_raw() */ 2076 static int qcom_nandc_write_page_raw(struct nand_chip *chip, 2077 const uint8_t *buf, int oob_required, 2078 int page) 2079 { 2080 struct mtd_info *mtd = nand_to_mtd(chip); 2081 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2082 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2083 struct nand_ecc_ctrl *ecc = &chip->ecc; 2084 u8 *data_buf, *oob_buf; 2085 int i, ret; 2086 2087 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 2088 clear_read_regs(nandc); 2089 clear_bam_transaction(nandc); 2090 2091 data_buf = (u8 *)buf; 2092 oob_buf = chip->oob_poi; 2093 2094 host->use_ecc = false; 2095 update_rw_regs(host, ecc->steps, false); 2096 config_nand_page_write(nandc); 2097 2098 for (i = 0; i < ecc->steps; i++) { 2099 int data_size1, data_size2, oob_size1, oob_size2; 2100 int reg_off = FLASH_BUF_ACC; 2101 2102 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1); 2103 oob_size1 = host->bbm_size; 2104 2105 if (i == (ecc->steps - 1)) { 2106 data_size2 = ecc->size - data_size1 - 2107 ((ecc->steps - 1) << 2); 2108 oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw + 2109 host->spare_bytes; 2110 } else { 2111 data_size2 = host->cw_data - data_size1; 2112 oob_size2 = host->ecc_bytes_hw + host->spare_bytes; 2113 } 2114 2115 write_data_dma(nandc, reg_off, data_buf, data_size1, 2116 NAND_BAM_NO_EOT); 2117 reg_off += data_size1; 2118 data_buf += data_size1; 2119 2120 write_data_dma(nandc, reg_off, oob_buf, oob_size1, 2121 NAND_BAM_NO_EOT); 2122 reg_off += oob_size1; 2123 oob_buf += oob_size1; 2124 2125 write_data_dma(nandc, reg_off, data_buf, data_size2, 2126 NAND_BAM_NO_EOT); 2127 reg_off += data_size2; 2128 data_buf += data_size2; 2129 2130 write_data_dma(nandc, reg_off, oob_buf, oob_size2, 0); 2131 oob_buf += oob_size2; 2132 2133 config_nand_cw_write(nandc); 2134 } 2135 2136 ret = submit_descs(nandc); 2137 if (ret) 2138 dev_err(nandc->dev, "failure to write raw page\n"); 2139 2140 free_descs(nandc); 2141 2142 if (!ret) 2143 ret = nand_prog_page_end_op(chip); 2144 2145 return ret; 2146 } 2147 2148 /* 2149 * implements ecc->write_oob() 2150 * 2151 * the NAND controller cannot write only data or only OOB within a codeword 2152 * since ECC is calculated for the combined codeword. So update the OOB from 2153 * chip->oob_poi, and pad the data area with OxFF before writing. 2154 */ 2155 static int qcom_nandc_write_oob(struct nand_chip *chip, int page) 2156 { 2157 struct mtd_info *mtd = nand_to_mtd(chip); 2158 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2159 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2160 struct nand_ecc_ctrl *ecc = &chip->ecc; 2161 u8 *oob = chip->oob_poi; 2162 int data_size, oob_size; 2163 int ret; 2164 2165 host->use_ecc = true; 2166 clear_bam_transaction(nandc); 2167 2168 /* calculate the data and oob size for the last codeword/step */ 2169 data_size = ecc->size - ((ecc->steps - 1) << 2); 2170 oob_size = mtd->oobavail; 2171 2172 memset(nandc->data_buffer, 0xff, host->cw_data); 2173 /* override new oob content to last codeword */ 2174 mtd_ooblayout_get_databytes(mtd, nandc->data_buffer + data_size, oob, 2175 0, mtd->oobavail); 2176 2177 set_address(host, host->cw_size * (ecc->steps - 1), page); 2178 update_rw_regs(host, 1, false); 2179 2180 config_nand_page_write(nandc); 2181 write_data_dma(nandc, FLASH_BUF_ACC, 2182 nandc->data_buffer, data_size + oob_size, 0); 2183 config_nand_cw_write(nandc); 2184 2185 ret = submit_descs(nandc); 2186 2187 free_descs(nandc); 2188 2189 if (ret) { 2190 dev_err(nandc->dev, "failure to write oob\n"); 2191 return -EIO; 2192 } 2193 2194 return nand_prog_page_end_op(chip); 2195 } 2196 2197 static int qcom_nandc_block_bad(struct nand_chip *chip, loff_t ofs) 2198 { 2199 struct mtd_info *mtd = nand_to_mtd(chip); 2200 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2201 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2202 struct nand_ecc_ctrl *ecc = &chip->ecc; 2203 int page, ret, bbpos, bad = 0; 2204 2205 page = (int)(ofs >> chip->page_shift) & chip->pagemask; 2206 2207 /* 2208 * configure registers for a raw sub page read, the address is set to 2209 * the beginning of the last codeword, we don't care about reading ecc 2210 * portion of oob. we just want the first few bytes from this codeword 2211 * that contains the BBM 2212 */ 2213 host->use_ecc = false; 2214 2215 clear_bam_transaction(nandc); 2216 ret = copy_last_cw(host, page); 2217 if (ret) 2218 goto err; 2219 2220 if (check_flash_errors(host, 1)) { 2221 dev_warn(nandc->dev, "error when trying to read BBM\n"); 2222 goto err; 2223 } 2224 2225 bbpos = mtd->writesize - host->cw_size * (ecc->steps - 1); 2226 2227 bad = nandc->data_buffer[bbpos] != 0xff; 2228 2229 if (chip->options & NAND_BUSWIDTH_16) 2230 bad = bad || (nandc->data_buffer[bbpos + 1] != 0xff); 2231 err: 2232 return bad; 2233 } 2234 2235 static int qcom_nandc_block_markbad(struct nand_chip *chip, loff_t ofs) 2236 { 2237 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2238 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2239 struct nand_ecc_ctrl *ecc = &chip->ecc; 2240 int page, ret; 2241 2242 clear_read_regs(nandc); 2243 clear_bam_transaction(nandc); 2244 2245 /* 2246 * to mark the BBM as bad, we flash the entire last codeword with 0s. 2247 * we don't care about the rest of the content in the codeword since 2248 * we aren't going to use this block again 2249 */ 2250 memset(nandc->data_buffer, 0x00, host->cw_size); 2251 2252 page = (int)(ofs >> chip->page_shift) & chip->pagemask; 2253 2254 /* prepare write */ 2255 host->use_ecc = false; 2256 set_address(host, host->cw_size * (ecc->steps - 1), page); 2257 update_rw_regs(host, 1, false); 2258 2259 config_nand_page_write(nandc); 2260 write_data_dma(nandc, FLASH_BUF_ACC, 2261 nandc->data_buffer, host->cw_size, 0); 2262 config_nand_cw_write(nandc); 2263 2264 ret = submit_descs(nandc); 2265 2266 free_descs(nandc); 2267 2268 if (ret) { 2269 dev_err(nandc->dev, "failure to update BBM\n"); 2270 return -EIO; 2271 } 2272 2273 return nand_prog_page_end_op(chip); 2274 } 2275 2276 /* 2277 * the three functions below implement chip->legacy.read_byte(), 2278 * chip->legacy.read_buf() and chip->legacy.write_buf() respectively. these 2279 * aren't used for reading/writing page data, they are used for smaller data 2280 * like reading id, status etc 2281 */ 2282 static uint8_t qcom_nandc_read_byte(struct nand_chip *chip) 2283 { 2284 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2285 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2286 u8 *buf = nandc->data_buffer; 2287 u8 ret = 0x0; 2288 2289 if (host->last_command == NAND_CMD_STATUS) { 2290 ret = host->status; 2291 2292 host->status = NAND_STATUS_READY | NAND_STATUS_WP; 2293 2294 return ret; 2295 } 2296 2297 if (nandc->buf_start < nandc->buf_count) 2298 ret = buf[nandc->buf_start++]; 2299 2300 return ret; 2301 } 2302 2303 static void qcom_nandc_read_buf(struct nand_chip *chip, uint8_t *buf, int len) 2304 { 2305 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2306 int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start); 2307 2308 memcpy(buf, nandc->data_buffer + nandc->buf_start, real_len); 2309 nandc->buf_start += real_len; 2310 } 2311 2312 static void qcom_nandc_write_buf(struct nand_chip *chip, const uint8_t *buf, 2313 int len) 2314 { 2315 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2316 int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start); 2317 2318 memcpy(nandc->data_buffer + nandc->buf_start, buf, real_len); 2319 2320 nandc->buf_start += real_len; 2321 } 2322 2323 /* we support only one external chip for now */ 2324 static void qcom_nandc_select_chip(struct nand_chip *chip, int chipnr) 2325 { 2326 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2327 2328 if (chipnr <= 0) 2329 return; 2330 2331 dev_warn(nandc->dev, "invalid chip select\n"); 2332 } 2333 2334 /* 2335 * NAND controller page layout info 2336 * 2337 * Layout with ECC enabled: 2338 * 2339 * |----------------------| |---------------------------------| 2340 * | xx.......yy| | *********xx.......yy| 2341 * | DATA xx..ECC..yy| | DATA **SPARE**xx..ECC..yy| 2342 * | (516) xx.......yy| | (516-n*4) **(n*4)**xx.......yy| 2343 * | xx.......yy| | *********xx.......yy| 2344 * |----------------------| |---------------------------------| 2345 * codeword 1,2..n-1 codeword n 2346 * <---(528/532 Bytes)--> <-------(528/532 Bytes)---------> 2347 * 2348 * n = Number of codewords in the page 2349 * . = ECC bytes 2350 * * = Spare/free bytes 2351 * x = Unused byte(s) 2352 * y = Reserved byte(s) 2353 * 2354 * 2K page: n = 4, spare = 16 bytes 2355 * 4K page: n = 8, spare = 32 bytes 2356 * 8K page: n = 16, spare = 64 bytes 2357 * 2358 * the qcom nand controller operates at a sub page/codeword level. each 2359 * codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively. 2360 * the number of ECC bytes vary based on the ECC strength and the bus width. 2361 * 2362 * the first n - 1 codewords contains 516 bytes of user data, the remaining 2363 * 12/16 bytes consist of ECC and reserved data. The nth codeword contains 2364 * both user data and spare(oobavail) bytes that sum up to 516 bytes. 2365 * 2366 * When we access a page with ECC enabled, the reserved bytes(s) are not 2367 * accessible at all. When reading, we fill up these unreadable positions 2368 * with 0xffs. When writing, the controller skips writing the inaccessible 2369 * bytes. 2370 * 2371 * Layout with ECC disabled: 2372 * 2373 * |------------------------------| |---------------------------------------| 2374 * | yy xx.......| | bb *********xx.......| 2375 * | DATA1 yy DATA2 xx..ECC..| | DATA1 bb DATA2 **SPARE**xx..ECC..| 2376 * | (size1) yy (size2) xx.......| | (size1) bb (size2) **(n*4)**xx.......| 2377 * | yy xx.......| | bb *********xx.......| 2378 * |------------------------------| |---------------------------------------| 2379 * codeword 1,2..n-1 codeword n 2380 * <-------(528/532 Bytes)------> <-----------(528/532 Bytes)-----------> 2381 * 2382 * n = Number of codewords in the page 2383 * . = ECC bytes 2384 * * = Spare/free bytes 2385 * x = Unused byte(s) 2386 * y = Dummy Bad Bock byte(s) 2387 * b = Real Bad Block byte(s) 2388 * size1/size2 = function of codeword size and 'n' 2389 * 2390 * when the ECC block is disabled, one reserved byte (or two for 16 bit bus 2391 * width) is now accessible. For the first n - 1 codewords, these are dummy Bad 2392 * Block Markers. In the last codeword, this position contains the real BBM 2393 * 2394 * In order to have a consistent layout between RAW and ECC modes, we assume 2395 * the following OOB layout arrangement: 2396 * 2397 * |-----------| |--------------------| 2398 * |yyxx.......| |bb*********xx.......| 2399 * |yyxx..ECC..| |bb*FREEOOB*xx..ECC..| 2400 * |yyxx.......| |bb*********xx.......| 2401 * |yyxx.......| |bb*********xx.......| 2402 * |-----------| |--------------------| 2403 * first n - 1 nth OOB region 2404 * OOB regions 2405 * 2406 * n = Number of codewords in the page 2407 * . = ECC bytes 2408 * * = FREE OOB bytes 2409 * y = Dummy bad block byte(s) (inaccessible when ECC enabled) 2410 * x = Unused byte(s) 2411 * b = Real bad block byte(s) (inaccessible when ECC enabled) 2412 * 2413 * This layout is read as is when ECC is disabled. When ECC is enabled, the 2414 * inaccessible Bad Block byte(s) are ignored when we write to a page/oob, 2415 * and assumed as 0xffs when we read a page/oob. The ECC, unused and 2416 * dummy/real bad block bytes are grouped as ecc bytes (i.e, ecc->bytes is 2417 * the sum of the three). 2418 */ 2419 static int qcom_nand_ooblayout_ecc(struct mtd_info *mtd, int section, 2420 struct mtd_oob_region *oobregion) 2421 { 2422 struct nand_chip *chip = mtd_to_nand(mtd); 2423 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2424 struct nand_ecc_ctrl *ecc = &chip->ecc; 2425 2426 if (section > 1) 2427 return -ERANGE; 2428 2429 if (!section) { 2430 oobregion->length = (ecc->bytes * (ecc->steps - 1)) + 2431 host->bbm_size; 2432 oobregion->offset = 0; 2433 } else { 2434 oobregion->length = host->ecc_bytes_hw + host->spare_bytes; 2435 oobregion->offset = mtd->oobsize - oobregion->length; 2436 } 2437 2438 return 0; 2439 } 2440 2441 static int qcom_nand_ooblayout_free(struct mtd_info *mtd, int section, 2442 struct mtd_oob_region *oobregion) 2443 { 2444 struct nand_chip *chip = mtd_to_nand(mtd); 2445 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2446 struct nand_ecc_ctrl *ecc = &chip->ecc; 2447 2448 if (section) 2449 return -ERANGE; 2450 2451 oobregion->length = ecc->steps * 4; 2452 oobregion->offset = ((ecc->steps - 1) * ecc->bytes) + host->bbm_size; 2453 2454 return 0; 2455 } 2456 2457 static const struct mtd_ooblayout_ops qcom_nand_ooblayout_ops = { 2458 .ecc = qcom_nand_ooblayout_ecc, 2459 .free = qcom_nand_ooblayout_free, 2460 }; 2461 2462 static int 2463 qcom_nandc_calc_ecc_bytes(int step_size, int strength) 2464 { 2465 return strength == 4 ? 12 : 16; 2466 } 2467 NAND_ECC_CAPS_SINGLE(qcom_nandc_ecc_caps, qcom_nandc_calc_ecc_bytes, 2468 NANDC_STEP_SIZE, 4, 8); 2469 2470 static int qcom_nand_attach_chip(struct nand_chip *chip) 2471 { 2472 struct mtd_info *mtd = nand_to_mtd(chip); 2473 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2474 struct nand_ecc_ctrl *ecc = &chip->ecc; 2475 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2476 int cwperpage, bad_block_byte, ret; 2477 bool wide_bus; 2478 int ecc_mode = 1; 2479 2480 /* controller only supports 512 bytes data steps */ 2481 ecc->size = NANDC_STEP_SIZE; 2482 wide_bus = chip->options & NAND_BUSWIDTH_16 ? true : false; 2483 cwperpage = mtd->writesize / NANDC_STEP_SIZE; 2484 2485 /* 2486 * Each CW has 4 available OOB bytes which will be protected with ECC 2487 * so remaining bytes can be used for ECC. 2488 */ 2489 ret = nand_ecc_choose_conf(chip, &qcom_nandc_ecc_caps, 2490 mtd->oobsize - (cwperpage * 4)); 2491 if (ret) { 2492 dev_err(nandc->dev, "No valid ECC settings possible\n"); 2493 return ret; 2494 } 2495 2496 if (ecc->strength >= 8) { 2497 /* 8 bit ECC defaults to BCH ECC on all platforms */ 2498 host->bch_enabled = true; 2499 ecc_mode = 1; 2500 2501 if (wide_bus) { 2502 host->ecc_bytes_hw = 14; 2503 host->spare_bytes = 0; 2504 host->bbm_size = 2; 2505 } else { 2506 host->ecc_bytes_hw = 13; 2507 host->spare_bytes = 2; 2508 host->bbm_size = 1; 2509 } 2510 } else { 2511 /* 2512 * if the controller supports BCH for 4 bit ECC, the controller 2513 * uses lesser bytes for ECC. If RS is used, the ECC bytes is 2514 * always 10 bytes 2515 */ 2516 if (nandc->props->ecc_modes & ECC_BCH_4BIT) { 2517 /* BCH */ 2518 host->bch_enabled = true; 2519 ecc_mode = 0; 2520 2521 if (wide_bus) { 2522 host->ecc_bytes_hw = 8; 2523 host->spare_bytes = 2; 2524 host->bbm_size = 2; 2525 } else { 2526 host->ecc_bytes_hw = 7; 2527 host->spare_bytes = 4; 2528 host->bbm_size = 1; 2529 } 2530 } else { 2531 /* RS */ 2532 host->ecc_bytes_hw = 10; 2533 2534 if (wide_bus) { 2535 host->spare_bytes = 0; 2536 host->bbm_size = 2; 2537 } else { 2538 host->spare_bytes = 1; 2539 host->bbm_size = 1; 2540 } 2541 } 2542 } 2543 2544 /* 2545 * we consider ecc->bytes as the sum of all the non-data content in a 2546 * step. It gives us a clean representation of the oob area (even if 2547 * all the bytes aren't used for ECC).It is always 16 bytes for 8 bit 2548 * ECC and 12 bytes for 4 bit ECC 2549 */ 2550 ecc->bytes = host->ecc_bytes_hw + host->spare_bytes + host->bbm_size; 2551 2552 ecc->read_page = qcom_nandc_read_page; 2553 ecc->read_page_raw = qcom_nandc_read_page_raw; 2554 ecc->read_oob = qcom_nandc_read_oob; 2555 ecc->write_page = qcom_nandc_write_page; 2556 ecc->write_page_raw = qcom_nandc_write_page_raw; 2557 ecc->write_oob = qcom_nandc_write_oob; 2558 2559 ecc->mode = NAND_ECC_HW; 2560 2561 mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops); 2562 2563 nandc->max_cwperpage = max_t(unsigned int, nandc->max_cwperpage, 2564 cwperpage); 2565 2566 /* 2567 * DATA_UD_BYTES varies based on whether the read/write command protects 2568 * spare data with ECC too. We protect spare data by default, so we set 2569 * it to main + spare data, which are 512 and 4 bytes respectively. 2570 */ 2571 host->cw_data = 516; 2572 2573 /* 2574 * total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes 2575 * for 8 bit ECC 2576 */ 2577 host->cw_size = host->cw_data + ecc->bytes; 2578 bad_block_byte = mtd->writesize - host->cw_size * (cwperpage - 1) + 1; 2579 2580 host->cfg0 = (cwperpage - 1) << CW_PER_PAGE 2581 | host->cw_data << UD_SIZE_BYTES 2582 | 0 << DISABLE_STATUS_AFTER_WRITE 2583 | 5 << NUM_ADDR_CYCLES 2584 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_RS 2585 | 0 << STATUS_BFR_READ 2586 | 1 << SET_RD_MODE_AFTER_STATUS 2587 | host->spare_bytes << SPARE_SIZE_BYTES; 2588 2589 host->cfg1 = 7 << NAND_RECOVERY_CYCLES 2590 | 0 << CS_ACTIVE_BSY 2591 | bad_block_byte << BAD_BLOCK_BYTE_NUM 2592 | 0 << BAD_BLOCK_IN_SPARE_AREA 2593 | 2 << WR_RD_BSY_GAP 2594 | wide_bus << WIDE_FLASH 2595 | host->bch_enabled << ENABLE_BCH_ECC; 2596 2597 host->cfg0_raw = (cwperpage - 1) << CW_PER_PAGE 2598 | host->cw_size << UD_SIZE_BYTES 2599 | 5 << NUM_ADDR_CYCLES 2600 | 0 << SPARE_SIZE_BYTES; 2601 2602 host->cfg1_raw = 7 << NAND_RECOVERY_CYCLES 2603 | 0 << CS_ACTIVE_BSY 2604 | 17 << BAD_BLOCK_BYTE_NUM 2605 | 1 << BAD_BLOCK_IN_SPARE_AREA 2606 | 2 << WR_RD_BSY_GAP 2607 | wide_bus << WIDE_FLASH 2608 | 1 << DEV0_CFG1_ECC_DISABLE; 2609 2610 host->ecc_bch_cfg = !host->bch_enabled << ECC_CFG_ECC_DISABLE 2611 | 0 << ECC_SW_RESET 2612 | host->cw_data << ECC_NUM_DATA_BYTES 2613 | 1 << ECC_FORCE_CLK_OPEN 2614 | ecc_mode << ECC_MODE 2615 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_BCH; 2616 2617 host->ecc_buf_cfg = 0x203 << NUM_STEPS; 2618 2619 host->clrflashstatus = FS_READY_BSY_N; 2620 host->clrreadstatus = 0xc0; 2621 nandc->regs->erased_cw_detect_cfg_clr = 2622 cpu_to_le32(CLR_ERASED_PAGE_DET); 2623 nandc->regs->erased_cw_detect_cfg_set = 2624 cpu_to_le32(SET_ERASED_PAGE_DET); 2625 2626 dev_dbg(nandc->dev, 2627 "cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n", 2628 host->cfg0, host->cfg1, host->ecc_buf_cfg, host->ecc_bch_cfg, 2629 host->cw_size, host->cw_data, ecc->strength, ecc->bytes, 2630 cwperpage); 2631 2632 return 0; 2633 } 2634 2635 static const struct nand_controller_ops qcom_nandc_ops = { 2636 .attach_chip = qcom_nand_attach_chip, 2637 }; 2638 2639 static int qcom_nandc_alloc(struct qcom_nand_controller *nandc) 2640 { 2641 int ret; 2642 2643 ret = dma_set_coherent_mask(nandc->dev, DMA_BIT_MASK(32)); 2644 if (ret) { 2645 dev_err(nandc->dev, "failed to set DMA mask\n"); 2646 return ret; 2647 } 2648 2649 /* 2650 * we use the internal buffer for reading ONFI params, reading small 2651 * data like ID and status, and preforming read-copy-write operations 2652 * when writing to a codeword partially. 532 is the maximum possible 2653 * size of a codeword for our nand controller 2654 */ 2655 nandc->buf_size = 532; 2656 2657 nandc->data_buffer = devm_kzalloc(nandc->dev, nandc->buf_size, 2658 GFP_KERNEL); 2659 if (!nandc->data_buffer) 2660 return -ENOMEM; 2661 2662 nandc->regs = devm_kzalloc(nandc->dev, sizeof(*nandc->regs), 2663 GFP_KERNEL); 2664 if (!nandc->regs) 2665 return -ENOMEM; 2666 2667 nandc->reg_read_buf = devm_kcalloc(nandc->dev, 2668 MAX_REG_RD, sizeof(*nandc->reg_read_buf), 2669 GFP_KERNEL); 2670 if (!nandc->reg_read_buf) 2671 return -ENOMEM; 2672 2673 if (nandc->props->is_bam) { 2674 nandc->reg_read_dma = 2675 dma_map_single(nandc->dev, nandc->reg_read_buf, 2676 MAX_REG_RD * 2677 sizeof(*nandc->reg_read_buf), 2678 DMA_FROM_DEVICE); 2679 if (dma_mapping_error(nandc->dev, nandc->reg_read_dma)) { 2680 dev_err(nandc->dev, "failed to DMA MAP reg buffer\n"); 2681 return -EIO; 2682 } 2683 2684 nandc->tx_chan = dma_request_slave_channel(nandc->dev, "tx"); 2685 if (!nandc->tx_chan) { 2686 dev_err(nandc->dev, "failed to request tx channel\n"); 2687 return -ENODEV; 2688 } 2689 2690 nandc->rx_chan = dma_request_slave_channel(nandc->dev, "rx"); 2691 if (!nandc->rx_chan) { 2692 dev_err(nandc->dev, "failed to request rx channel\n"); 2693 return -ENODEV; 2694 } 2695 2696 nandc->cmd_chan = dma_request_slave_channel(nandc->dev, "cmd"); 2697 if (!nandc->cmd_chan) { 2698 dev_err(nandc->dev, "failed to request cmd channel\n"); 2699 return -ENODEV; 2700 } 2701 2702 /* 2703 * Initially allocate BAM transaction to read ONFI param page. 2704 * After detecting all the devices, this BAM transaction will 2705 * be freed and the next BAM tranasction will be allocated with 2706 * maximum codeword size 2707 */ 2708 nandc->max_cwperpage = 1; 2709 nandc->bam_txn = alloc_bam_transaction(nandc); 2710 if (!nandc->bam_txn) { 2711 dev_err(nandc->dev, 2712 "failed to allocate bam transaction\n"); 2713 return -ENOMEM; 2714 } 2715 } else { 2716 nandc->chan = dma_request_slave_channel(nandc->dev, "rxtx"); 2717 if (!nandc->chan) { 2718 dev_err(nandc->dev, 2719 "failed to request slave channel\n"); 2720 return -ENODEV; 2721 } 2722 } 2723 2724 INIT_LIST_HEAD(&nandc->desc_list); 2725 INIT_LIST_HEAD(&nandc->host_list); 2726 2727 nand_controller_init(&nandc->controller); 2728 nandc->controller.ops = &qcom_nandc_ops; 2729 2730 return 0; 2731 } 2732 2733 static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc) 2734 { 2735 if (nandc->props->is_bam) { 2736 if (!dma_mapping_error(nandc->dev, nandc->reg_read_dma)) 2737 dma_unmap_single(nandc->dev, nandc->reg_read_dma, 2738 MAX_REG_RD * 2739 sizeof(*nandc->reg_read_buf), 2740 DMA_FROM_DEVICE); 2741 2742 if (nandc->tx_chan) 2743 dma_release_channel(nandc->tx_chan); 2744 2745 if (nandc->rx_chan) 2746 dma_release_channel(nandc->rx_chan); 2747 2748 if (nandc->cmd_chan) 2749 dma_release_channel(nandc->cmd_chan); 2750 } else { 2751 if (nandc->chan) 2752 dma_release_channel(nandc->chan); 2753 } 2754 } 2755 2756 /* one time setup of a few nand controller registers */ 2757 static int qcom_nandc_setup(struct qcom_nand_controller *nandc) 2758 { 2759 u32 nand_ctrl; 2760 2761 /* kill onenand */ 2762 nandc_write(nandc, SFLASHC_BURST_CFG, 0); 2763 nandc_write(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD), 2764 NAND_DEV_CMD_VLD_VAL); 2765 2766 /* enable ADM or BAM DMA */ 2767 if (nandc->props->is_bam) { 2768 nand_ctrl = nandc_read(nandc, NAND_CTRL); 2769 nandc_write(nandc, NAND_CTRL, nand_ctrl | BAM_MODE_EN); 2770 } else { 2771 nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN); 2772 } 2773 2774 /* save the original values of these registers */ 2775 nandc->cmd1 = nandc_read(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD1)); 2776 nandc->vld = NAND_DEV_CMD_VLD_VAL; 2777 2778 return 0; 2779 } 2780 2781 static int qcom_nand_host_init_and_register(struct qcom_nand_controller *nandc, 2782 struct qcom_nand_host *host, 2783 struct device_node *dn) 2784 { 2785 struct nand_chip *chip = &host->chip; 2786 struct mtd_info *mtd = nand_to_mtd(chip); 2787 struct device *dev = nandc->dev; 2788 int ret; 2789 2790 ret = of_property_read_u32(dn, "reg", &host->cs); 2791 if (ret) { 2792 dev_err(dev, "can't get chip-select\n"); 2793 return -ENXIO; 2794 } 2795 2796 nand_set_flash_node(chip, dn); 2797 mtd->name = devm_kasprintf(dev, GFP_KERNEL, "qcom_nand.%d", host->cs); 2798 if (!mtd->name) 2799 return -ENOMEM; 2800 2801 mtd->owner = THIS_MODULE; 2802 mtd->dev.parent = dev; 2803 2804 chip->legacy.cmdfunc = qcom_nandc_command; 2805 chip->legacy.select_chip = qcom_nandc_select_chip; 2806 chip->legacy.read_byte = qcom_nandc_read_byte; 2807 chip->legacy.read_buf = qcom_nandc_read_buf; 2808 chip->legacy.write_buf = qcom_nandc_write_buf; 2809 chip->legacy.set_features = nand_get_set_features_notsupp; 2810 chip->legacy.get_features = nand_get_set_features_notsupp; 2811 2812 /* 2813 * the bad block marker is readable only when we read the last codeword 2814 * of a page with ECC disabled. currently, the nand_base and nand_bbt 2815 * helpers don't allow us to read BB from a nand chip with ECC 2816 * disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad 2817 * and block_markbad helpers until we permanently switch to using 2818 * MTD_OPS_RAW for all drivers (with the help of badblockbits) 2819 */ 2820 chip->legacy.block_bad = qcom_nandc_block_bad; 2821 chip->legacy.block_markbad = qcom_nandc_block_markbad; 2822 2823 chip->controller = &nandc->controller; 2824 chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USE_BOUNCE_BUFFER | 2825 NAND_SKIP_BBTSCAN; 2826 2827 /* set up initial status value */ 2828 host->status = NAND_STATUS_READY | NAND_STATUS_WP; 2829 2830 ret = nand_scan(chip, 1); 2831 if (ret) 2832 return ret; 2833 2834 if (nandc->props->is_bam) { 2835 free_bam_transaction(nandc); 2836 nandc->bam_txn = alloc_bam_transaction(nandc); 2837 if (!nandc->bam_txn) { 2838 dev_err(nandc->dev, 2839 "failed to allocate bam transaction\n"); 2840 return -ENOMEM; 2841 } 2842 } 2843 2844 ret = mtd_device_register(mtd, NULL, 0); 2845 if (ret) 2846 nand_cleanup(chip); 2847 2848 return ret; 2849 } 2850 2851 static int qcom_probe_nand_devices(struct qcom_nand_controller *nandc) 2852 { 2853 struct device *dev = nandc->dev; 2854 struct device_node *dn = dev->of_node, *child; 2855 struct qcom_nand_host *host; 2856 int ret; 2857 2858 for_each_available_child_of_node(dn, child) { 2859 host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL); 2860 if (!host) { 2861 of_node_put(child); 2862 return -ENOMEM; 2863 } 2864 2865 ret = qcom_nand_host_init_and_register(nandc, host, child); 2866 if (ret) { 2867 devm_kfree(dev, host); 2868 continue; 2869 } 2870 2871 list_add_tail(&host->node, &nandc->host_list); 2872 } 2873 2874 if (list_empty(&nandc->host_list)) 2875 return -ENODEV; 2876 2877 return 0; 2878 } 2879 2880 /* parse custom DT properties here */ 2881 static int qcom_nandc_parse_dt(struct platform_device *pdev) 2882 { 2883 struct qcom_nand_controller *nandc = platform_get_drvdata(pdev); 2884 struct device_node *np = nandc->dev->of_node; 2885 int ret; 2886 2887 if (!nandc->props->is_bam) { 2888 ret = of_property_read_u32(np, "qcom,cmd-crci", 2889 &nandc->cmd_crci); 2890 if (ret) { 2891 dev_err(nandc->dev, "command CRCI unspecified\n"); 2892 return ret; 2893 } 2894 2895 ret = of_property_read_u32(np, "qcom,data-crci", 2896 &nandc->data_crci); 2897 if (ret) { 2898 dev_err(nandc->dev, "data CRCI unspecified\n"); 2899 return ret; 2900 } 2901 } 2902 2903 return 0; 2904 } 2905 2906 static int qcom_nandc_probe(struct platform_device *pdev) 2907 { 2908 struct qcom_nand_controller *nandc; 2909 const void *dev_data; 2910 struct device *dev = &pdev->dev; 2911 struct resource *res; 2912 int ret; 2913 2914 nandc = devm_kzalloc(&pdev->dev, sizeof(*nandc), GFP_KERNEL); 2915 if (!nandc) 2916 return -ENOMEM; 2917 2918 platform_set_drvdata(pdev, nandc); 2919 nandc->dev = dev; 2920 2921 dev_data = of_device_get_match_data(dev); 2922 if (!dev_data) { 2923 dev_err(&pdev->dev, "failed to get device data\n"); 2924 return -ENODEV; 2925 } 2926 2927 nandc->props = dev_data; 2928 2929 nandc->core_clk = devm_clk_get(dev, "core"); 2930 if (IS_ERR(nandc->core_clk)) 2931 return PTR_ERR(nandc->core_clk); 2932 2933 nandc->aon_clk = devm_clk_get(dev, "aon"); 2934 if (IS_ERR(nandc->aon_clk)) 2935 return PTR_ERR(nandc->aon_clk); 2936 2937 ret = qcom_nandc_parse_dt(pdev); 2938 if (ret) 2939 return ret; 2940 2941 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2942 nandc->base = devm_ioremap_resource(dev, res); 2943 if (IS_ERR(nandc->base)) 2944 return PTR_ERR(nandc->base); 2945 2946 nandc->base_phys = res->start; 2947 nandc->base_dma = dma_map_resource(dev, res->start, 2948 resource_size(res), 2949 DMA_BIDIRECTIONAL, 0); 2950 if (!nandc->base_dma) 2951 return -ENXIO; 2952 2953 ret = qcom_nandc_alloc(nandc); 2954 if (ret) 2955 goto err_nandc_alloc; 2956 2957 ret = clk_prepare_enable(nandc->core_clk); 2958 if (ret) 2959 goto err_core_clk; 2960 2961 ret = clk_prepare_enable(nandc->aon_clk); 2962 if (ret) 2963 goto err_aon_clk; 2964 2965 ret = qcom_nandc_setup(nandc); 2966 if (ret) 2967 goto err_setup; 2968 2969 ret = qcom_probe_nand_devices(nandc); 2970 if (ret) 2971 goto err_setup; 2972 2973 return 0; 2974 2975 err_setup: 2976 clk_disable_unprepare(nandc->aon_clk); 2977 err_aon_clk: 2978 clk_disable_unprepare(nandc->core_clk); 2979 err_core_clk: 2980 qcom_nandc_unalloc(nandc); 2981 err_nandc_alloc: 2982 dma_unmap_resource(dev, res->start, resource_size(res), 2983 DMA_BIDIRECTIONAL, 0); 2984 2985 return ret; 2986 } 2987 2988 static int qcom_nandc_remove(struct platform_device *pdev) 2989 { 2990 struct qcom_nand_controller *nandc = platform_get_drvdata(pdev); 2991 struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2992 struct qcom_nand_host *host; 2993 2994 list_for_each_entry(host, &nandc->host_list, node) 2995 nand_release(&host->chip); 2996 2997 2998 qcom_nandc_unalloc(nandc); 2999 3000 clk_disable_unprepare(nandc->aon_clk); 3001 clk_disable_unprepare(nandc->core_clk); 3002 3003 dma_unmap_resource(&pdev->dev, nandc->base_dma, resource_size(res), 3004 DMA_BIDIRECTIONAL, 0); 3005 3006 return 0; 3007 } 3008 3009 static const struct qcom_nandc_props ipq806x_nandc_props = { 3010 .ecc_modes = (ECC_RS_4BIT | ECC_BCH_8BIT), 3011 .is_bam = false, 3012 .dev_cmd_reg_start = 0x0, 3013 }; 3014 3015 static const struct qcom_nandc_props ipq4019_nandc_props = { 3016 .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT), 3017 .is_bam = true, 3018 .dev_cmd_reg_start = 0x0, 3019 }; 3020 3021 static const struct qcom_nandc_props ipq8074_nandc_props = { 3022 .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT), 3023 .is_bam = true, 3024 .dev_cmd_reg_start = 0x7000, 3025 }; 3026 3027 /* 3028 * data will hold a struct pointer containing more differences once we support 3029 * more controller variants 3030 */ 3031 static const struct of_device_id qcom_nandc_of_match[] = { 3032 { 3033 .compatible = "qcom,ipq806x-nand", 3034 .data = &ipq806x_nandc_props, 3035 }, 3036 { 3037 .compatible = "qcom,ipq4019-nand", 3038 .data = &ipq4019_nandc_props, 3039 }, 3040 { 3041 .compatible = "qcom,ipq8074-nand", 3042 .data = &ipq8074_nandc_props, 3043 }, 3044 {} 3045 }; 3046 MODULE_DEVICE_TABLE(of, qcom_nandc_of_match); 3047 3048 static struct platform_driver qcom_nandc_driver = { 3049 .driver = { 3050 .name = "qcom-nandc", 3051 .of_match_table = qcom_nandc_of_match, 3052 }, 3053 .probe = qcom_nandc_probe, 3054 .remove = qcom_nandc_remove, 3055 }; 3056 module_platform_driver(qcom_nandc_driver); 3057 3058 MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>"); 3059 MODULE_DESCRIPTION("Qualcomm NAND Controller driver"); 3060 MODULE_LICENSE("GPL v2"); 3061