1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2016, The Linux Foundation. All rights reserved. 4 */ 5 #include <linux/clk.h> 6 #include <linux/slab.h> 7 #include <linux/bitops.h> 8 #include <linux/dma/qcom_adm.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/dmaengine.h> 11 #include <linux/module.h> 12 #include <linux/mtd/rawnand.h> 13 #include <linux/mtd/partitions.h> 14 #include <linux/of.h> 15 #include <linux/of_device.h> 16 #include <linux/delay.h> 17 #include <linux/dma/qcom_bam_dma.h> 18 19 /* NANDc reg offsets */ 20 #define NAND_FLASH_CMD 0x00 21 #define NAND_ADDR0 0x04 22 #define NAND_ADDR1 0x08 23 #define NAND_FLASH_CHIP_SELECT 0x0c 24 #define NAND_EXEC_CMD 0x10 25 #define NAND_FLASH_STATUS 0x14 26 #define NAND_BUFFER_STATUS 0x18 27 #define NAND_DEV0_CFG0 0x20 28 #define NAND_DEV0_CFG1 0x24 29 #define NAND_DEV0_ECC_CFG 0x28 30 #define NAND_AUTO_STATUS_EN 0x2c 31 #define NAND_DEV1_CFG0 0x30 32 #define NAND_DEV1_CFG1 0x34 33 #define NAND_READ_ID 0x40 34 #define NAND_READ_STATUS 0x44 35 #define NAND_DEV_CMD0 0xa0 36 #define NAND_DEV_CMD1 0xa4 37 #define NAND_DEV_CMD2 0xa8 38 #define NAND_DEV_CMD_VLD 0xac 39 #define SFLASHC_BURST_CFG 0xe0 40 #define NAND_ERASED_CW_DETECT_CFG 0xe8 41 #define NAND_ERASED_CW_DETECT_STATUS 0xec 42 #define NAND_EBI2_ECC_BUF_CFG 0xf0 43 #define FLASH_BUF_ACC 0x100 44 45 #define NAND_CTRL 0xf00 46 #define NAND_VERSION 0xf08 47 #define NAND_READ_LOCATION_0 0xf20 48 #define NAND_READ_LOCATION_1 0xf24 49 #define NAND_READ_LOCATION_2 0xf28 50 #define NAND_READ_LOCATION_3 0xf2c 51 #define NAND_READ_LOCATION_LAST_CW_0 0xf40 52 #define NAND_READ_LOCATION_LAST_CW_1 0xf44 53 #define NAND_READ_LOCATION_LAST_CW_2 0xf48 54 #define NAND_READ_LOCATION_LAST_CW_3 0xf4c 55 56 /* dummy register offsets, used by write_reg_dma */ 57 #define NAND_DEV_CMD1_RESTORE 0xdead 58 #define NAND_DEV_CMD_VLD_RESTORE 0xbeef 59 60 /* NAND_FLASH_CMD bits */ 61 #define PAGE_ACC BIT(4) 62 #define LAST_PAGE BIT(5) 63 64 /* NAND_FLASH_CHIP_SELECT bits */ 65 #define NAND_DEV_SEL 0 66 #define DM_EN BIT(2) 67 68 /* NAND_FLASH_STATUS bits */ 69 #define FS_OP_ERR BIT(4) 70 #define FS_READY_BSY_N BIT(5) 71 #define FS_MPU_ERR BIT(8) 72 #define FS_DEVICE_STS_ERR BIT(16) 73 #define FS_DEVICE_WP BIT(23) 74 75 /* NAND_BUFFER_STATUS bits */ 76 #define BS_UNCORRECTABLE_BIT BIT(8) 77 #define BS_CORRECTABLE_ERR_MSK 0x1f 78 79 /* NAND_DEVn_CFG0 bits */ 80 #define DISABLE_STATUS_AFTER_WRITE 4 81 #define CW_PER_PAGE 6 82 #define UD_SIZE_BYTES 9 83 #define UD_SIZE_BYTES_MASK GENMASK(18, 9) 84 #define ECC_PARITY_SIZE_BYTES_RS 19 85 #define SPARE_SIZE_BYTES 23 86 #define SPARE_SIZE_BYTES_MASK GENMASK(26, 23) 87 #define NUM_ADDR_CYCLES 27 88 #define STATUS_BFR_READ 30 89 #define SET_RD_MODE_AFTER_STATUS 31 90 91 /* NAND_DEVn_CFG0 bits */ 92 #define DEV0_CFG1_ECC_DISABLE 0 93 #define WIDE_FLASH 1 94 #define NAND_RECOVERY_CYCLES 2 95 #define CS_ACTIVE_BSY 5 96 #define BAD_BLOCK_BYTE_NUM 6 97 #define BAD_BLOCK_IN_SPARE_AREA 16 98 #define WR_RD_BSY_GAP 17 99 #define ENABLE_BCH_ECC 27 100 101 /* NAND_DEV0_ECC_CFG bits */ 102 #define ECC_CFG_ECC_DISABLE 0 103 #define ECC_SW_RESET 1 104 #define ECC_MODE 4 105 #define ECC_PARITY_SIZE_BYTES_BCH 8 106 #define ECC_NUM_DATA_BYTES 16 107 #define ECC_NUM_DATA_BYTES_MASK GENMASK(25, 16) 108 #define ECC_FORCE_CLK_OPEN 30 109 110 /* NAND_DEV_CMD1 bits */ 111 #define READ_ADDR 0 112 113 /* NAND_DEV_CMD_VLD bits */ 114 #define READ_START_VLD BIT(0) 115 #define READ_STOP_VLD BIT(1) 116 #define WRITE_START_VLD BIT(2) 117 #define ERASE_START_VLD BIT(3) 118 #define SEQ_READ_START_VLD BIT(4) 119 120 /* NAND_EBI2_ECC_BUF_CFG bits */ 121 #define NUM_STEPS 0 122 123 /* NAND_ERASED_CW_DETECT_CFG bits */ 124 #define ERASED_CW_ECC_MASK 1 125 #define AUTO_DETECT_RES 0 126 #define MASK_ECC (1 << ERASED_CW_ECC_MASK) 127 #define RESET_ERASED_DET (1 << AUTO_DETECT_RES) 128 #define ACTIVE_ERASED_DET (0 << AUTO_DETECT_RES) 129 #define CLR_ERASED_PAGE_DET (RESET_ERASED_DET | MASK_ECC) 130 #define SET_ERASED_PAGE_DET (ACTIVE_ERASED_DET | MASK_ECC) 131 132 /* NAND_ERASED_CW_DETECT_STATUS bits */ 133 #define PAGE_ALL_ERASED BIT(7) 134 #define CODEWORD_ALL_ERASED BIT(6) 135 #define PAGE_ERASED BIT(5) 136 #define CODEWORD_ERASED BIT(4) 137 #define ERASED_PAGE (PAGE_ALL_ERASED | PAGE_ERASED) 138 #define ERASED_CW (CODEWORD_ALL_ERASED | CODEWORD_ERASED) 139 140 /* NAND_READ_LOCATION_n bits */ 141 #define READ_LOCATION_OFFSET 0 142 #define READ_LOCATION_SIZE 16 143 #define READ_LOCATION_LAST 31 144 145 /* Version Mask */ 146 #define NAND_VERSION_MAJOR_MASK 0xf0000000 147 #define NAND_VERSION_MAJOR_SHIFT 28 148 #define NAND_VERSION_MINOR_MASK 0x0fff0000 149 #define NAND_VERSION_MINOR_SHIFT 16 150 151 /* NAND OP_CMDs */ 152 #define OP_PAGE_READ 0x2 153 #define OP_PAGE_READ_WITH_ECC 0x3 154 #define OP_PAGE_READ_WITH_ECC_SPARE 0x4 155 #define OP_PAGE_READ_ONFI_READ 0x5 156 #define OP_PROGRAM_PAGE 0x6 157 #define OP_PAGE_PROGRAM_WITH_ECC 0x7 158 #define OP_PROGRAM_PAGE_SPARE 0x9 159 #define OP_BLOCK_ERASE 0xa 160 #define OP_FETCH_ID 0xb 161 #define OP_RESET_DEVICE 0xd 162 163 /* Default Value for NAND_DEV_CMD_VLD */ 164 #define NAND_DEV_CMD_VLD_VAL (READ_START_VLD | WRITE_START_VLD | \ 165 ERASE_START_VLD | SEQ_READ_START_VLD) 166 167 /* NAND_CTRL bits */ 168 #define BAM_MODE_EN BIT(0) 169 170 /* 171 * the NAND controller performs reads/writes with ECC in 516 byte chunks. 172 * the driver calls the chunks 'step' or 'codeword' interchangeably 173 */ 174 #define NANDC_STEP_SIZE 512 175 176 /* 177 * the largest page size we support is 8K, this will have 16 steps/codewords 178 * of 512 bytes each 179 */ 180 #define MAX_NUM_STEPS (SZ_8K / NANDC_STEP_SIZE) 181 182 /* we read at most 3 registers per codeword scan */ 183 #define MAX_REG_RD (3 * MAX_NUM_STEPS) 184 185 /* ECC modes supported by the controller */ 186 #define ECC_NONE BIT(0) 187 #define ECC_RS_4BIT BIT(1) 188 #define ECC_BCH_4BIT BIT(2) 189 #define ECC_BCH_8BIT BIT(3) 190 191 #define nandc_set_read_loc_first(chip, reg, cw_offset, read_size, is_last_read_loc) \ 192 nandc_set_reg(chip, reg, \ 193 ((cw_offset) << READ_LOCATION_OFFSET) | \ 194 ((read_size) << READ_LOCATION_SIZE) | \ 195 ((is_last_read_loc) << READ_LOCATION_LAST)) 196 197 #define nandc_set_read_loc_last(chip, reg, cw_offset, read_size, is_last_read_loc) \ 198 nandc_set_reg(chip, reg, \ 199 ((cw_offset) << READ_LOCATION_OFFSET) | \ 200 ((read_size) << READ_LOCATION_SIZE) | \ 201 ((is_last_read_loc) << READ_LOCATION_LAST)) 202 /* 203 * Returns the actual register address for all NAND_DEV_ registers 204 * (i.e. NAND_DEV_CMD0, NAND_DEV_CMD1, NAND_DEV_CMD2 and NAND_DEV_CMD_VLD) 205 */ 206 #define dev_cmd_reg_addr(nandc, reg) ((nandc)->props->dev_cmd_reg_start + (reg)) 207 208 /* Returns the NAND register physical address */ 209 #define nandc_reg_phys(chip, offset) ((chip)->base_phys + (offset)) 210 211 /* Returns the dma address for reg read buffer */ 212 #define reg_buf_dma_addr(chip, vaddr) \ 213 ((chip)->reg_read_dma + \ 214 ((uint8_t *)(vaddr) - (uint8_t *)(chip)->reg_read_buf)) 215 216 #define QPIC_PER_CW_CMD_ELEMENTS 32 217 #define QPIC_PER_CW_CMD_SGL 32 218 #define QPIC_PER_CW_DATA_SGL 8 219 220 #define QPIC_NAND_COMPLETION_TIMEOUT msecs_to_jiffies(2000) 221 222 /* 223 * Flags used in DMA descriptor preparation helper functions 224 * (i.e. read_reg_dma/write_reg_dma/read_data_dma/write_data_dma) 225 */ 226 /* Don't set the EOT in current tx BAM sgl */ 227 #define NAND_BAM_NO_EOT BIT(0) 228 /* Set the NWD flag in current BAM sgl */ 229 #define NAND_BAM_NWD BIT(1) 230 /* Finish writing in the current BAM sgl and start writing in another BAM sgl */ 231 #define NAND_BAM_NEXT_SGL BIT(2) 232 /* 233 * Erased codeword status is being used two times in single transfer so this 234 * flag will determine the current value of erased codeword status register 235 */ 236 #define NAND_ERASED_CW_SET BIT(4) 237 238 /* 239 * This data type corresponds to the BAM transaction which will be used for all 240 * NAND transfers. 241 * @bam_ce - the array of BAM command elements 242 * @cmd_sgl - sgl for NAND BAM command pipe 243 * @data_sgl - sgl for NAND BAM consumer/producer pipe 244 * @last_data_desc - last DMA desc in data channel (tx/rx). 245 * @last_cmd_desc - last DMA desc in command channel. 246 * @txn_done - completion for NAND transfer. 247 * @bam_ce_pos - the index in bam_ce which is available for next sgl 248 * @bam_ce_start - the index in bam_ce which marks the start position ce 249 * for current sgl. It will be used for size calculation 250 * for current sgl 251 * @cmd_sgl_pos - current index in command sgl. 252 * @cmd_sgl_start - start index in command sgl. 253 * @tx_sgl_pos - current index in data sgl for tx. 254 * @tx_sgl_start - start index in data sgl for tx. 255 * @rx_sgl_pos - current index in data sgl for rx. 256 * @rx_sgl_start - start index in data sgl for rx. 257 * @wait_second_completion - wait for second DMA desc completion before making 258 * the NAND transfer completion. 259 */ 260 struct bam_transaction { 261 struct bam_cmd_element *bam_ce; 262 struct scatterlist *cmd_sgl; 263 struct scatterlist *data_sgl; 264 struct dma_async_tx_descriptor *last_data_desc; 265 struct dma_async_tx_descriptor *last_cmd_desc; 266 struct completion txn_done; 267 u32 bam_ce_pos; 268 u32 bam_ce_start; 269 u32 cmd_sgl_pos; 270 u32 cmd_sgl_start; 271 u32 tx_sgl_pos; 272 u32 tx_sgl_start; 273 u32 rx_sgl_pos; 274 u32 rx_sgl_start; 275 bool wait_second_completion; 276 }; 277 278 /* 279 * This data type corresponds to the nand dma descriptor 280 * @dma_desc - low level DMA engine descriptor 281 * @list - list for desc_info 282 * 283 * @adm_sgl - sgl which will be used for single sgl dma descriptor. Only used by 284 * ADM 285 * @bam_sgl - sgl which will be used for dma descriptor. Only used by BAM 286 * @sgl_cnt - number of SGL in bam_sgl. Only used by BAM 287 * @dir - DMA transfer direction 288 */ 289 struct desc_info { 290 struct dma_async_tx_descriptor *dma_desc; 291 struct list_head node; 292 293 union { 294 struct scatterlist adm_sgl; 295 struct { 296 struct scatterlist *bam_sgl; 297 int sgl_cnt; 298 }; 299 }; 300 enum dma_data_direction dir; 301 }; 302 303 /* 304 * holds the current register values that we want to write. acts as a contiguous 305 * chunk of memory which we use to write the controller registers through DMA. 306 */ 307 struct nandc_regs { 308 __le32 cmd; 309 __le32 addr0; 310 __le32 addr1; 311 __le32 chip_sel; 312 __le32 exec; 313 314 __le32 cfg0; 315 __le32 cfg1; 316 __le32 ecc_bch_cfg; 317 318 __le32 clrflashstatus; 319 __le32 clrreadstatus; 320 321 __le32 cmd1; 322 __le32 vld; 323 324 __le32 orig_cmd1; 325 __le32 orig_vld; 326 327 __le32 ecc_buf_cfg; 328 __le32 read_location0; 329 __le32 read_location1; 330 __le32 read_location2; 331 __le32 read_location3; 332 __le32 read_location_last0; 333 __le32 read_location_last1; 334 __le32 read_location_last2; 335 __le32 read_location_last3; 336 337 __le32 erased_cw_detect_cfg_clr; 338 __le32 erased_cw_detect_cfg_set; 339 }; 340 341 /* 342 * NAND controller data struct 343 * 344 * @dev: parent device 345 * 346 * @base: MMIO base 347 * 348 * @core_clk: controller clock 349 * @aon_clk: another controller clock 350 * 351 * @regs: a contiguous chunk of memory for DMA register 352 * writes. contains the register values to be 353 * written to controller 354 * 355 * @props: properties of current NAND controller, 356 * initialized via DT match data 357 * 358 * @controller: base controller structure 359 * @host_list: list containing all the chips attached to the 360 * controller 361 * 362 * @chan: dma channel 363 * @cmd_crci: ADM DMA CRCI for command flow control 364 * @data_crci: ADM DMA CRCI for data flow control 365 * 366 * @desc_list: DMA descriptor list (list of desc_infos) 367 * 368 * @data_buffer: our local DMA buffer for page read/writes, 369 * used when we can't use the buffer provided 370 * by upper layers directly 371 * @reg_read_buf: local buffer for reading back registers via DMA 372 * 373 * @base_phys: physical base address of controller registers 374 * @base_dma: dma base address of controller registers 375 * @reg_read_dma: contains dma address for register read buffer 376 * 377 * @buf_size/count/start: markers for chip->legacy.read_buf/write_buf 378 * functions 379 * @max_cwperpage: maximum QPIC codewords required. calculated 380 * from all connected NAND devices pagesize 381 * 382 * @reg_read_pos: marker for data read in reg_read_buf 383 * 384 * @cmd1/vld: some fixed controller register values 385 */ 386 struct qcom_nand_controller { 387 struct device *dev; 388 389 void __iomem *base; 390 391 struct clk *core_clk; 392 struct clk *aon_clk; 393 394 struct nandc_regs *regs; 395 struct bam_transaction *bam_txn; 396 397 const struct qcom_nandc_props *props; 398 399 struct nand_controller controller; 400 struct list_head host_list; 401 402 union { 403 /* will be used only by QPIC for BAM DMA */ 404 struct { 405 struct dma_chan *tx_chan; 406 struct dma_chan *rx_chan; 407 struct dma_chan *cmd_chan; 408 }; 409 410 /* will be used only by EBI2 for ADM DMA */ 411 struct { 412 struct dma_chan *chan; 413 unsigned int cmd_crci; 414 unsigned int data_crci; 415 }; 416 }; 417 418 struct list_head desc_list; 419 420 u8 *data_buffer; 421 __le32 *reg_read_buf; 422 423 phys_addr_t base_phys; 424 dma_addr_t base_dma; 425 dma_addr_t reg_read_dma; 426 427 int buf_size; 428 int buf_count; 429 int buf_start; 430 unsigned int max_cwperpage; 431 432 int reg_read_pos; 433 434 u32 cmd1, vld; 435 }; 436 437 /* 438 * NAND special boot partitions 439 * 440 * @page_offset: offset of the partition where spare data is not protected 441 * by ECC (value in pages) 442 * @page_offset: size of the partition where spare data is not protected 443 * by ECC (value in pages) 444 */ 445 struct qcom_nand_boot_partition { 446 u32 page_offset; 447 u32 page_size; 448 }; 449 450 /* 451 * NAND chip structure 452 * 453 * @boot_partitions: array of boot partitions where offset and size of the 454 * boot partitions are stored 455 * 456 * @chip: base NAND chip structure 457 * @node: list node to add itself to host_list in 458 * qcom_nand_controller 459 * 460 * @nr_boot_partitions: count of the boot partitions where spare data is not 461 * protected by ECC 462 * 463 * @cs: chip select value for this chip 464 * @cw_size: the number of bytes in a single step/codeword 465 * of a page, consisting of all data, ecc, spare 466 * and reserved bytes 467 * @cw_data: the number of bytes within a codeword protected 468 * by ECC 469 * @ecc_bytes_hw: ECC bytes used by controller hardware for this 470 * chip 471 * 472 * @last_command: keeps track of last command on this chip. used 473 * for reading correct status 474 * 475 * @cfg0, cfg1, cfg0_raw..: NANDc register configurations needed for 476 * ecc/non-ecc mode for the current nand flash 477 * device 478 * 479 * @status: value to be returned if NAND_CMD_STATUS command 480 * is executed 481 * @codeword_fixup: keep track of the current layout used by 482 * the driver for read/write operation. 483 * @use_ecc: request the controller to use ECC for the 484 * upcoming read/write 485 * @bch_enabled: flag to tell whether BCH ECC mode is used 486 */ 487 struct qcom_nand_host { 488 struct qcom_nand_boot_partition *boot_partitions; 489 490 struct nand_chip chip; 491 struct list_head node; 492 493 int nr_boot_partitions; 494 495 int cs; 496 int cw_size; 497 int cw_data; 498 int ecc_bytes_hw; 499 int spare_bytes; 500 int bbm_size; 501 502 int last_command; 503 504 u32 cfg0, cfg1; 505 u32 cfg0_raw, cfg1_raw; 506 u32 ecc_buf_cfg; 507 u32 ecc_bch_cfg; 508 u32 clrflashstatus; 509 u32 clrreadstatus; 510 511 u8 status; 512 bool codeword_fixup; 513 bool use_ecc; 514 bool bch_enabled; 515 }; 516 517 /* 518 * This data type corresponds to the NAND controller properties which varies 519 * among different NAND controllers. 520 * @ecc_modes - ecc mode for NAND 521 * @dev_cmd_reg_start - NAND_DEV_CMD_* registers starting offset 522 * @is_bam - whether NAND controller is using BAM 523 * @is_qpic - whether NAND CTRL is part of qpic IP 524 * @qpic_v2 - flag to indicate QPIC IP version 2 525 * @use_codeword_fixup - whether NAND has different layout for boot partitions 526 */ 527 struct qcom_nandc_props { 528 u32 ecc_modes; 529 u32 dev_cmd_reg_start; 530 bool is_bam; 531 bool is_qpic; 532 bool qpic_v2; 533 bool use_codeword_fixup; 534 }; 535 536 /* Frees the BAM transaction memory */ 537 static void free_bam_transaction(struct qcom_nand_controller *nandc) 538 { 539 struct bam_transaction *bam_txn = nandc->bam_txn; 540 541 devm_kfree(nandc->dev, bam_txn); 542 } 543 544 /* Allocates and Initializes the BAM transaction */ 545 static struct bam_transaction * 546 alloc_bam_transaction(struct qcom_nand_controller *nandc) 547 { 548 struct bam_transaction *bam_txn; 549 size_t bam_txn_size; 550 unsigned int num_cw = nandc->max_cwperpage; 551 void *bam_txn_buf; 552 553 bam_txn_size = 554 sizeof(*bam_txn) + num_cw * 555 ((sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS) + 556 (sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL) + 557 (sizeof(*bam_txn->data_sgl) * QPIC_PER_CW_DATA_SGL)); 558 559 bam_txn_buf = devm_kzalloc(nandc->dev, bam_txn_size, GFP_KERNEL); 560 if (!bam_txn_buf) 561 return NULL; 562 563 bam_txn = bam_txn_buf; 564 bam_txn_buf += sizeof(*bam_txn); 565 566 bam_txn->bam_ce = bam_txn_buf; 567 bam_txn_buf += 568 sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS * num_cw; 569 570 bam_txn->cmd_sgl = bam_txn_buf; 571 bam_txn_buf += 572 sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL * num_cw; 573 574 bam_txn->data_sgl = bam_txn_buf; 575 576 init_completion(&bam_txn->txn_done); 577 578 return bam_txn; 579 } 580 581 /* Clears the BAM transaction indexes */ 582 static void clear_bam_transaction(struct qcom_nand_controller *nandc) 583 { 584 struct bam_transaction *bam_txn = nandc->bam_txn; 585 586 if (!nandc->props->is_bam) 587 return; 588 589 bam_txn->bam_ce_pos = 0; 590 bam_txn->bam_ce_start = 0; 591 bam_txn->cmd_sgl_pos = 0; 592 bam_txn->cmd_sgl_start = 0; 593 bam_txn->tx_sgl_pos = 0; 594 bam_txn->tx_sgl_start = 0; 595 bam_txn->rx_sgl_pos = 0; 596 bam_txn->rx_sgl_start = 0; 597 bam_txn->last_data_desc = NULL; 598 bam_txn->wait_second_completion = false; 599 600 sg_init_table(bam_txn->cmd_sgl, nandc->max_cwperpage * 601 QPIC_PER_CW_CMD_SGL); 602 sg_init_table(bam_txn->data_sgl, nandc->max_cwperpage * 603 QPIC_PER_CW_DATA_SGL); 604 605 reinit_completion(&bam_txn->txn_done); 606 } 607 608 /* Callback for DMA descriptor completion */ 609 static void qpic_bam_dma_done(void *data) 610 { 611 struct bam_transaction *bam_txn = data; 612 613 /* 614 * In case of data transfer with NAND, 2 callbacks will be generated. 615 * One for command channel and another one for data channel. 616 * If current transaction has data descriptors 617 * (i.e. wait_second_completion is true), then set this to false 618 * and wait for second DMA descriptor completion. 619 */ 620 if (bam_txn->wait_second_completion) 621 bam_txn->wait_second_completion = false; 622 else 623 complete(&bam_txn->txn_done); 624 } 625 626 static inline struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip) 627 { 628 return container_of(chip, struct qcom_nand_host, chip); 629 } 630 631 static inline struct qcom_nand_controller * 632 get_qcom_nand_controller(struct nand_chip *chip) 633 { 634 return container_of(chip->controller, struct qcom_nand_controller, 635 controller); 636 } 637 638 static inline u32 nandc_read(struct qcom_nand_controller *nandc, int offset) 639 { 640 return ioread32(nandc->base + offset); 641 } 642 643 static inline void nandc_write(struct qcom_nand_controller *nandc, int offset, 644 u32 val) 645 { 646 iowrite32(val, nandc->base + offset); 647 } 648 649 static inline void nandc_read_buffer_sync(struct qcom_nand_controller *nandc, 650 bool is_cpu) 651 { 652 if (!nandc->props->is_bam) 653 return; 654 655 if (is_cpu) 656 dma_sync_single_for_cpu(nandc->dev, nandc->reg_read_dma, 657 MAX_REG_RD * 658 sizeof(*nandc->reg_read_buf), 659 DMA_FROM_DEVICE); 660 else 661 dma_sync_single_for_device(nandc->dev, nandc->reg_read_dma, 662 MAX_REG_RD * 663 sizeof(*nandc->reg_read_buf), 664 DMA_FROM_DEVICE); 665 } 666 667 static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset) 668 { 669 switch (offset) { 670 case NAND_FLASH_CMD: 671 return ®s->cmd; 672 case NAND_ADDR0: 673 return ®s->addr0; 674 case NAND_ADDR1: 675 return ®s->addr1; 676 case NAND_FLASH_CHIP_SELECT: 677 return ®s->chip_sel; 678 case NAND_EXEC_CMD: 679 return ®s->exec; 680 case NAND_FLASH_STATUS: 681 return ®s->clrflashstatus; 682 case NAND_DEV0_CFG0: 683 return ®s->cfg0; 684 case NAND_DEV0_CFG1: 685 return ®s->cfg1; 686 case NAND_DEV0_ECC_CFG: 687 return ®s->ecc_bch_cfg; 688 case NAND_READ_STATUS: 689 return ®s->clrreadstatus; 690 case NAND_DEV_CMD1: 691 return ®s->cmd1; 692 case NAND_DEV_CMD1_RESTORE: 693 return ®s->orig_cmd1; 694 case NAND_DEV_CMD_VLD: 695 return ®s->vld; 696 case NAND_DEV_CMD_VLD_RESTORE: 697 return ®s->orig_vld; 698 case NAND_EBI2_ECC_BUF_CFG: 699 return ®s->ecc_buf_cfg; 700 case NAND_READ_LOCATION_0: 701 return ®s->read_location0; 702 case NAND_READ_LOCATION_1: 703 return ®s->read_location1; 704 case NAND_READ_LOCATION_2: 705 return ®s->read_location2; 706 case NAND_READ_LOCATION_3: 707 return ®s->read_location3; 708 case NAND_READ_LOCATION_LAST_CW_0: 709 return ®s->read_location_last0; 710 case NAND_READ_LOCATION_LAST_CW_1: 711 return ®s->read_location_last1; 712 case NAND_READ_LOCATION_LAST_CW_2: 713 return ®s->read_location_last2; 714 case NAND_READ_LOCATION_LAST_CW_3: 715 return ®s->read_location_last3; 716 default: 717 return NULL; 718 } 719 } 720 721 static void nandc_set_reg(struct nand_chip *chip, int offset, 722 u32 val) 723 { 724 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 725 struct nandc_regs *regs = nandc->regs; 726 __le32 *reg; 727 728 reg = offset_to_nandc_reg(regs, offset); 729 730 if (reg) 731 *reg = cpu_to_le32(val); 732 } 733 734 /* Helper to check the code word, whether it is last cw or not */ 735 static bool qcom_nandc_is_last_cw(struct nand_ecc_ctrl *ecc, int cw) 736 { 737 return cw == (ecc->steps - 1); 738 } 739 740 /* helper to configure location register values */ 741 static void nandc_set_read_loc(struct nand_chip *chip, int cw, int reg, 742 int cw_offset, int read_size, int is_last_read_loc) 743 { 744 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 745 struct nand_ecc_ctrl *ecc = &chip->ecc; 746 int reg_base = NAND_READ_LOCATION_0; 747 748 if (nandc->props->qpic_v2 && qcom_nandc_is_last_cw(ecc, cw)) 749 reg_base = NAND_READ_LOCATION_LAST_CW_0; 750 751 reg_base += reg * 4; 752 753 if (nandc->props->qpic_v2 && qcom_nandc_is_last_cw(ecc, cw)) 754 return nandc_set_read_loc_last(chip, reg_base, cw_offset, 755 read_size, is_last_read_loc); 756 else 757 return nandc_set_read_loc_first(chip, reg_base, cw_offset, 758 read_size, is_last_read_loc); 759 } 760 761 /* helper to configure address register values */ 762 static void set_address(struct qcom_nand_host *host, u16 column, int page) 763 { 764 struct nand_chip *chip = &host->chip; 765 766 if (chip->options & NAND_BUSWIDTH_16) 767 column >>= 1; 768 769 nandc_set_reg(chip, NAND_ADDR0, page << 16 | column); 770 nandc_set_reg(chip, NAND_ADDR1, page >> 16 & 0xff); 771 } 772 773 /* 774 * update_rw_regs: set up read/write register values, these will be 775 * written to the NAND controller registers via DMA 776 * 777 * @num_cw: number of steps for the read/write operation 778 * @read: read or write operation 779 * @cw : which code word 780 */ 781 static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read, int cw) 782 { 783 struct nand_chip *chip = &host->chip; 784 u32 cmd, cfg0, cfg1, ecc_bch_cfg; 785 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 786 787 if (read) { 788 if (host->use_ecc) 789 cmd = OP_PAGE_READ_WITH_ECC | PAGE_ACC | LAST_PAGE; 790 else 791 cmd = OP_PAGE_READ | PAGE_ACC | LAST_PAGE; 792 } else { 793 cmd = OP_PROGRAM_PAGE | PAGE_ACC | LAST_PAGE; 794 } 795 796 if (host->use_ecc) { 797 cfg0 = (host->cfg0 & ~(7U << CW_PER_PAGE)) | 798 (num_cw - 1) << CW_PER_PAGE; 799 800 cfg1 = host->cfg1; 801 ecc_bch_cfg = host->ecc_bch_cfg; 802 } else { 803 cfg0 = (host->cfg0_raw & ~(7U << CW_PER_PAGE)) | 804 (num_cw - 1) << CW_PER_PAGE; 805 806 cfg1 = host->cfg1_raw; 807 ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE; 808 } 809 810 nandc_set_reg(chip, NAND_FLASH_CMD, cmd); 811 nandc_set_reg(chip, NAND_DEV0_CFG0, cfg0); 812 nandc_set_reg(chip, NAND_DEV0_CFG1, cfg1); 813 nandc_set_reg(chip, NAND_DEV0_ECC_CFG, ecc_bch_cfg); 814 if (!nandc->props->qpic_v2) 815 nandc_set_reg(chip, NAND_EBI2_ECC_BUF_CFG, host->ecc_buf_cfg); 816 nandc_set_reg(chip, NAND_FLASH_STATUS, host->clrflashstatus); 817 nandc_set_reg(chip, NAND_READ_STATUS, host->clrreadstatus); 818 nandc_set_reg(chip, NAND_EXEC_CMD, 1); 819 820 if (read) 821 nandc_set_read_loc(chip, cw, 0, 0, host->use_ecc ? 822 host->cw_data : host->cw_size, 1); 823 } 824 825 /* 826 * Maps the scatter gather list for DMA transfer and forms the DMA descriptor 827 * for BAM. This descriptor will be added in the NAND DMA descriptor queue 828 * which will be submitted to DMA engine. 829 */ 830 static int prepare_bam_async_desc(struct qcom_nand_controller *nandc, 831 struct dma_chan *chan, 832 unsigned long flags) 833 { 834 struct desc_info *desc; 835 struct scatterlist *sgl; 836 unsigned int sgl_cnt; 837 int ret; 838 struct bam_transaction *bam_txn = nandc->bam_txn; 839 enum dma_transfer_direction dir_eng; 840 struct dma_async_tx_descriptor *dma_desc; 841 842 desc = kzalloc(sizeof(*desc), GFP_KERNEL); 843 if (!desc) 844 return -ENOMEM; 845 846 if (chan == nandc->cmd_chan) { 847 sgl = &bam_txn->cmd_sgl[bam_txn->cmd_sgl_start]; 848 sgl_cnt = bam_txn->cmd_sgl_pos - bam_txn->cmd_sgl_start; 849 bam_txn->cmd_sgl_start = bam_txn->cmd_sgl_pos; 850 dir_eng = DMA_MEM_TO_DEV; 851 desc->dir = DMA_TO_DEVICE; 852 } else if (chan == nandc->tx_chan) { 853 sgl = &bam_txn->data_sgl[bam_txn->tx_sgl_start]; 854 sgl_cnt = bam_txn->tx_sgl_pos - bam_txn->tx_sgl_start; 855 bam_txn->tx_sgl_start = bam_txn->tx_sgl_pos; 856 dir_eng = DMA_MEM_TO_DEV; 857 desc->dir = DMA_TO_DEVICE; 858 } else { 859 sgl = &bam_txn->data_sgl[bam_txn->rx_sgl_start]; 860 sgl_cnt = bam_txn->rx_sgl_pos - bam_txn->rx_sgl_start; 861 bam_txn->rx_sgl_start = bam_txn->rx_sgl_pos; 862 dir_eng = DMA_DEV_TO_MEM; 863 desc->dir = DMA_FROM_DEVICE; 864 } 865 866 sg_mark_end(sgl + sgl_cnt - 1); 867 ret = dma_map_sg(nandc->dev, sgl, sgl_cnt, desc->dir); 868 if (ret == 0) { 869 dev_err(nandc->dev, "failure in mapping desc\n"); 870 kfree(desc); 871 return -ENOMEM; 872 } 873 874 desc->sgl_cnt = sgl_cnt; 875 desc->bam_sgl = sgl; 876 877 dma_desc = dmaengine_prep_slave_sg(chan, sgl, sgl_cnt, dir_eng, 878 flags); 879 880 if (!dma_desc) { 881 dev_err(nandc->dev, "failure in prep desc\n"); 882 dma_unmap_sg(nandc->dev, sgl, sgl_cnt, desc->dir); 883 kfree(desc); 884 return -EINVAL; 885 } 886 887 desc->dma_desc = dma_desc; 888 889 /* update last data/command descriptor */ 890 if (chan == nandc->cmd_chan) 891 bam_txn->last_cmd_desc = dma_desc; 892 else 893 bam_txn->last_data_desc = dma_desc; 894 895 list_add_tail(&desc->node, &nandc->desc_list); 896 897 return 0; 898 } 899 900 /* 901 * Prepares the command descriptor for BAM DMA which will be used for NAND 902 * register reads and writes. The command descriptor requires the command 903 * to be formed in command element type so this function uses the command 904 * element from bam transaction ce array and fills the same with required 905 * data. A single SGL can contain multiple command elements so 906 * NAND_BAM_NEXT_SGL will be used for starting the separate SGL 907 * after the current command element. 908 */ 909 static int prep_bam_dma_desc_cmd(struct qcom_nand_controller *nandc, bool read, 910 int reg_off, const void *vaddr, 911 int size, unsigned int flags) 912 { 913 int bam_ce_size; 914 int i, ret; 915 struct bam_cmd_element *bam_ce_buffer; 916 struct bam_transaction *bam_txn = nandc->bam_txn; 917 918 bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_pos]; 919 920 /* fill the command desc */ 921 for (i = 0; i < size; i++) { 922 if (read) 923 bam_prep_ce(&bam_ce_buffer[i], 924 nandc_reg_phys(nandc, reg_off + 4 * i), 925 BAM_READ_COMMAND, 926 reg_buf_dma_addr(nandc, 927 (__le32 *)vaddr + i)); 928 else 929 bam_prep_ce_le32(&bam_ce_buffer[i], 930 nandc_reg_phys(nandc, reg_off + 4 * i), 931 BAM_WRITE_COMMAND, 932 *((__le32 *)vaddr + i)); 933 } 934 935 bam_txn->bam_ce_pos += size; 936 937 /* use the separate sgl after this command */ 938 if (flags & NAND_BAM_NEXT_SGL) { 939 bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_start]; 940 bam_ce_size = (bam_txn->bam_ce_pos - 941 bam_txn->bam_ce_start) * 942 sizeof(struct bam_cmd_element); 943 sg_set_buf(&bam_txn->cmd_sgl[bam_txn->cmd_sgl_pos], 944 bam_ce_buffer, bam_ce_size); 945 bam_txn->cmd_sgl_pos++; 946 bam_txn->bam_ce_start = bam_txn->bam_ce_pos; 947 948 if (flags & NAND_BAM_NWD) { 949 ret = prepare_bam_async_desc(nandc, nandc->cmd_chan, 950 DMA_PREP_FENCE | 951 DMA_PREP_CMD); 952 if (ret) 953 return ret; 954 } 955 } 956 957 return 0; 958 } 959 960 /* 961 * Prepares the data descriptor for BAM DMA which will be used for NAND 962 * data reads and writes. 963 */ 964 static int prep_bam_dma_desc_data(struct qcom_nand_controller *nandc, bool read, 965 const void *vaddr, 966 int size, unsigned int flags) 967 { 968 int ret; 969 struct bam_transaction *bam_txn = nandc->bam_txn; 970 971 if (read) { 972 sg_set_buf(&bam_txn->data_sgl[bam_txn->rx_sgl_pos], 973 vaddr, size); 974 bam_txn->rx_sgl_pos++; 975 } else { 976 sg_set_buf(&bam_txn->data_sgl[bam_txn->tx_sgl_pos], 977 vaddr, size); 978 bam_txn->tx_sgl_pos++; 979 980 /* 981 * BAM will only set EOT for DMA_PREP_INTERRUPT so if this flag 982 * is not set, form the DMA descriptor 983 */ 984 if (!(flags & NAND_BAM_NO_EOT)) { 985 ret = prepare_bam_async_desc(nandc, nandc->tx_chan, 986 DMA_PREP_INTERRUPT); 987 if (ret) 988 return ret; 989 } 990 } 991 992 return 0; 993 } 994 995 static int prep_adm_dma_desc(struct qcom_nand_controller *nandc, bool read, 996 int reg_off, const void *vaddr, int size, 997 bool flow_control) 998 { 999 struct desc_info *desc; 1000 struct dma_async_tx_descriptor *dma_desc; 1001 struct scatterlist *sgl; 1002 struct dma_slave_config slave_conf; 1003 struct qcom_adm_peripheral_config periph_conf = {}; 1004 enum dma_transfer_direction dir_eng; 1005 int ret; 1006 1007 desc = kzalloc(sizeof(*desc), GFP_KERNEL); 1008 if (!desc) 1009 return -ENOMEM; 1010 1011 sgl = &desc->adm_sgl; 1012 1013 sg_init_one(sgl, vaddr, size); 1014 1015 if (read) { 1016 dir_eng = DMA_DEV_TO_MEM; 1017 desc->dir = DMA_FROM_DEVICE; 1018 } else { 1019 dir_eng = DMA_MEM_TO_DEV; 1020 desc->dir = DMA_TO_DEVICE; 1021 } 1022 1023 ret = dma_map_sg(nandc->dev, sgl, 1, desc->dir); 1024 if (ret == 0) { 1025 ret = -ENOMEM; 1026 goto err; 1027 } 1028 1029 memset(&slave_conf, 0x00, sizeof(slave_conf)); 1030 1031 slave_conf.device_fc = flow_control; 1032 if (read) { 1033 slave_conf.src_maxburst = 16; 1034 slave_conf.src_addr = nandc->base_dma + reg_off; 1035 if (nandc->data_crci) { 1036 periph_conf.crci = nandc->data_crci; 1037 slave_conf.peripheral_config = &periph_conf; 1038 slave_conf.peripheral_size = sizeof(periph_conf); 1039 } 1040 } else { 1041 slave_conf.dst_maxburst = 16; 1042 slave_conf.dst_addr = nandc->base_dma + reg_off; 1043 if (nandc->cmd_crci) { 1044 periph_conf.crci = nandc->cmd_crci; 1045 slave_conf.peripheral_config = &periph_conf; 1046 slave_conf.peripheral_size = sizeof(periph_conf); 1047 } 1048 } 1049 1050 ret = dmaengine_slave_config(nandc->chan, &slave_conf); 1051 if (ret) { 1052 dev_err(nandc->dev, "failed to configure dma channel\n"); 1053 goto err; 1054 } 1055 1056 dma_desc = dmaengine_prep_slave_sg(nandc->chan, sgl, 1, dir_eng, 0); 1057 if (!dma_desc) { 1058 dev_err(nandc->dev, "failed to prepare desc\n"); 1059 ret = -EINVAL; 1060 goto err; 1061 } 1062 1063 desc->dma_desc = dma_desc; 1064 1065 list_add_tail(&desc->node, &nandc->desc_list); 1066 1067 return 0; 1068 err: 1069 kfree(desc); 1070 1071 return ret; 1072 } 1073 1074 /* 1075 * read_reg_dma: prepares a descriptor to read a given number of 1076 * contiguous registers to the reg_read_buf pointer 1077 * 1078 * @first: offset of the first register in the contiguous block 1079 * @num_regs: number of registers to read 1080 * @flags: flags to control DMA descriptor preparation 1081 */ 1082 static int read_reg_dma(struct qcom_nand_controller *nandc, int first, 1083 int num_regs, unsigned int flags) 1084 { 1085 bool flow_control = false; 1086 void *vaddr; 1087 1088 vaddr = nandc->reg_read_buf + nandc->reg_read_pos; 1089 nandc->reg_read_pos += num_regs; 1090 1091 if (first == NAND_DEV_CMD_VLD || first == NAND_DEV_CMD1) 1092 first = dev_cmd_reg_addr(nandc, first); 1093 1094 if (nandc->props->is_bam) 1095 return prep_bam_dma_desc_cmd(nandc, true, first, vaddr, 1096 num_regs, flags); 1097 1098 if (first == NAND_READ_ID || first == NAND_FLASH_STATUS) 1099 flow_control = true; 1100 1101 return prep_adm_dma_desc(nandc, true, first, vaddr, 1102 num_regs * sizeof(u32), flow_control); 1103 } 1104 1105 /* 1106 * write_reg_dma: prepares a descriptor to write a given number of 1107 * contiguous registers 1108 * 1109 * @first: offset of the first register in the contiguous block 1110 * @num_regs: number of registers to write 1111 * @flags: flags to control DMA descriptor preparation 1112 */ 1113 static int write_reg_dma(struct qcom_nand_controller *nandc, int first, 1114 int num_regs, unsigned int flags) 1115 { 1116 bool flow_control = false; 1117 struct nandc_regs *regs = nandc->regs; 1118 void *vaddr; 1119 1120 vaddr = offset_to_nandc_reg(regs, first); 1121 1122 if (first == NAND_ERASED_CW_DETECT_CFG) { 1123 if (flags & NAND_ERASED_CW_SET) 1124 vaddr = ®s->erased_cw_detect_cfg_set; 1125 else 1126 vaddr = ®s->erased_cw_detect_cfg_clr; 1127 } 1128 1129 if (first == NAND_EXEC_CMD) 1130 flags |= NAND_BAM_NWD; 1131 1132 if (first == NAND_DEV_CMD1_RESTORE || first == NAND_DEV_CMD1) 1133 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD1); 1134 1135 if (first == NAND_DEV_CMD_VLD_RESTORE || first == NAND_DEV_CMD_VLD) 1136 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD); 1137 1138 if (nandc->props->is_bam) 1139 return prep_bam_dma_desc_cmd(nandc, false, first, vaddr, 1140 num_regs, flags); 1141 1142 if (first == NAND_FLASH_CMD) 1143 flow_control = true; 1144 1145 return prep_adm_dma_desc(nandc, false, first, vaddr, 1146 num_regs * sizeof(u32), flow_control); 1147 } 1148 1149 /* 1150 * read_data_dma: prepares a DMA descriptor to transfer data from the 1151 * controller's internal buffer to the buffer 'vaddr' 1152 * 1153 * @reg_off: offset within the controller's data buffer 1154 * @vaddr: virtual address of the buffer we want to write to 1155 * @size: DMA transaction size in bytes 1156 * @flags: flags to control DMA descriptor preparation 1157 */ 1158 static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off, 1159 const u8 *vaddr, int size, unsigned int flags) 1160 { 1161 if (nandc->props->is_bam) 1162 return prep_bam_dma_desc_data(nandc, true, vaddr, size, flags); 1163 1164 return prep_adm_dma_desc(nandc, true, reg_off, vaddr, size, false); 1165 } 1166 1167 /* 1168 * write_data_dma: prepares a DMA descriptor to transfer data from 1169 * 'vaddr' to the controller's internal buffer 1170 * 1171 * @reg_off: offset within the controller's data buffer 1172 * @vaddr: virtual address of the buffer we want to read from 1173 * @size: DMA transaction size in bytes 1174 * @flags: flags to control DMA descriptor preparation 1175 */ 1176 static int write_data_dma(struct qcom_nand_controller *nandc, int reg_off, 1177 const u8 *vaddr, int size, unsigned int flags) 1178 { 1179 if (nandc->props->is_bam) 1180 return prep_bam_dma_desc_data(nandc, false, vaddr, size, flags); 1181 1182 return prep_adm_dma_desc(nandc, false, reg_off, vaddr, size, false); 1183 } 1184 1185 /* 1186 * Helper to prepare DMA descriptors for configuring registers 1187 * before reading a NAND page. 1188 */ 1189 static void config_nand_page_read(struct nand_chip *chip) 1190 { 1191 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1192 1193 write_reg_dma(nandc, NAND_ADDR0, 2, 0); 1194 write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0); 1195 if (!nandc->props->qpic_v2) 1196 write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1, 0); 1197 write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1, 0); 1198 write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1, 1199 NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL); 1200 } 1201 1202 /* 1203 * Helper to prepare DMA descriptors for configuring registers 1204 * before reading each codeword in NAND page. 1205 */ 1206 static void 1207 config_nand_cw_read(struct nand_chip *chip, bool use_ecc, int cw) 1208 { 1209 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1210 struct nand_ecc_ctrl *ecc = &chip->ecc; 1211 1212 int reg = NAND_READ_LOCATION_0; 1213 1214 if (nandc->props->qpic_v2 && qcom_nandc_is_last_cw(ecc, cw)) 1215 reg = NAND_READ_LOCATION_LAST_CW_0; 1216 1217 if (nandc->props->is_bam) 1218 write_reg_dma(nandc, reg, 4, NAND_BAM_NEXT_SGL); 1219 1220 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL); 1221 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1222 1223 if (use_ecc) { 1224 read_reg_dma(nandc, NAND_FLASH_STATUS, 2, 0); 1225 read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1, 1226 NAND_BAM_NEXT_SGL); 1227 } else { 1228 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL); 1229 } 1230 } 1231 1232 /* 1233 * Helper to prepare dma descriptors to configure registers needed for reading a 1234 * single codeword in page 1235 */ 1236 static void 1237 config_nand_single_cw_page_read(struct nand_chip *chip, 1238 bool use_ecc, int cw) 1239 { 1240 config_nand_page_read(chip); 1241 config_nand_cw_read(chip, use_ecc, cw); 1242 } 1243 1244 /* 1245 * Helper to prepare DMA descriptors used to configure registers needed for 1246 * before writing a NAND page. 1247 */ 1248 static void config_nand_page_write(struct nand_chip *chip) 1249 { 1250 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1251 1252 write_reg_dma(nandc, NAND_ADDR0, 2, 0); 1253 write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0); 1254 if (!nandc->props->qpic_v2) 1255 write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1, 1256 NAND_BAM_NEXT_SGL); 1257 } 1258 1259 /* 1260 * Helper to prepare DMA descriptors for configuring registers 1261 * before writing each codeword in NAND page. 1262 */ 1263 static void config_nand_cw_write(struct nand_chip *chip) 1264 { 1265 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1266 1267 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL); 1268 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1269 1270 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL); 1271 1272 write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0); 1273 write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL); 1274 } 1275 1276 /* 1277 * the following functions are used within chip->legacy.cmdfunc() to 1278 * perform different NAND_CMD_* commands 1279 */ 1280 1281 /* sets up descriptors for NAND_CMD_PARAM */ 1282 static int nandc_param(struct qcom_nand_host *host) 1283 { 1284 struct nand_chip *chip = &host->chip; 1285 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1286 1287 /* 1288 * NAND_CMD_PARAM is called before we know much about the FLASH chip 1289 * in use. we configure the controller to perform a raw read of 512 1290 * bytes to read onfi params 1291 */ 1292 if (nandc->props->qpic_v2) 1293 nandc_set_reg(chip, NAND_FLASH_CMD, OP_PAGE_READ_ONFI_READ | 1294 PAGE_ACC | LAST_PAGE); 1295 else 1296 nandc_set_reg(chip, NAND_FLASH_CMD, OP_PAGE_READ | 1297 PAGE_ACC | LAST_PAGE); 1298 1299 nandc_set_reg(chip, NAND_ADDR0, 0); 1300 nandc_set_reg(chip, NAND_ADDR1, 0); 1301 nandc_set_reg(chip, NAND_DEV0_CFG0, 0 << CW_PER_PAGE 1302 | 512 << UD_SIZE_BYTES 1303 | 5 << NUM_ADDR_CYCLES 1304 | 0 << SPARE_SIZE_BYTES); 1305 nandc_set_reg(chip, NAND_DEV0_CFG1, 7 << NAND_RECOVERY_CYCLES 1306 | 0 << CS_ACTIVE_BSY 1307 | 17 << BAD_BLOCK_BYTE_NUM 1308 | 1 << BAD_BLOCK_IN_SPARE_AREA 1309 | 2 << WR_RD_BSY_GAP 1310 | 0 << WIDE_FLASH 1311 | 1 << DEV0_CFG1_ECC_DISABLE); 1312 if (!nandc->props->qpic_v2) 1313 nandc_set_reg(chip, NAND_EBI2_ECC_BUF_CFG, 1 << ECC_CFG_ECC_DISABLE); 1314 1315 /* configure CMD1 and VLD for ONFI param probing in QPIC v1 */ 1316 if (!nandc->props->qpic_v2) { 1317 nandc_set_reg(chip, NAND_DEV_CMD_VLD, 1318 (nandc->vld & ~READ_START_VLD)); 1319 nandc_set_reg(chip, NAND_DEV_CMD1, 1320 (nandc->cmd1 & ~(0xFF << READ_ADDR)) 1321 | NAND_CMD_PARAM << READ_ADDR); 1322 } 1323 1324 nandc_set_reg(chip, NAND_EXEC_CMD, 1); 1325 1326 if (!nandc->props->qpic_v2) { 1327 nandc_set_reg(chip, NAND_DEV_CMD1_RESTORE, nandc->cmd1); 1328 nandc_set_reg(chip, NAND_DEV_CMD_VLD_RESTORE, nandc->vld); 1329 } 1330 1331 nandc_set_read_loc(chip, 0, 0, 0, 512, 1); 1332 1333 if (!nandc->props->qpic_v2) { 1334 write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1, 0); 1335 write_reg_dma(nandc, NAND_DEV_CMD1, 1, NAND_BAM_NEXT_SGL); 1336 } 1337 1338 nandc->buf_count = 512; 1339 memset(nandc->data_buffer, 0xff, nandc->buf_count); 1340 1341 config_nand_single_cw_page_read(chip, false, 0); 1342 1343 read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, 1344 nandc->buf_count, 0); 1345 1346 /* restore CMD1 and VLD regs */ 1347 if (!nandc->props->qpic_v2) { 1348 write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1, 0); 1349 write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1, NAND_BAM_NEXT_SGL); 1350 } 1351 1352 return 0; 1353 } 1354 1355 /* sets up descriptors for NAND_CMD_ERASE1 */ 1356 static int erase_block(struct qcom_nand_host *host, int page_addr) 1357 { 1358 struct nand_chip *chip = &host->chip; 1359 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1360 1361 nandc_set_reg(chip, NAND_FLASH_CMD, 1362 OP_BLOCK_ERASE | PAGE_ACC | LAST_PAGE); 1363 nandc_set_reg(chip, NAND_ADDR0, page_addr); 1364 nandc_set_reg(chip, NAND_ADDR1, 0); 1365 nandc_set_reg(chip, NAND_DEV0_CFG0, 1366 host->cfg0_raw & ~(7 << CW_PER_PAGE)); 1367 nandc_set_reg(chip, NAND_DEV0_CFG1, host->cfg1_raw); 1368 nandc_set_reg(chip, NAND_EXEC_CMD, 1); 1369 nandc_set_reg(chip, NAND_FLASH_STATUS, host->clrflashstatus); 1370 nandc_set_reg(chip, NAND_READ_STATUS, host->clrreadstatus); 1371 1372 write_reg_dma(nandc, NAND_FLASH_CMD, 3, NAND_BAM_NEXT_SGL); 1373 write_reg_dma(nandc, NAND_DEV0_CFG0, 2, NAND_BAM_NEXT_SGL); 1374 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1375 1376 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL); 1377 1378 write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0); 1379 write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL); 1380 1381 return 0; 1382 } 1383 1384 /* sets up descriptors for NAND_CMD_READID */ 1385 static int read_id(struct qcom_nand_host *host, int column) 1386 { 1387 struct nand_chip *chip = &host->chip; 1388 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1389 1390 if (column == -1) 1391 return 0; 1392 1393 nandc_set_reg(chip, NAND_FLASH_CMD, OP_FETCH_ID); 1394 nandc_set_reg(chip, NAND_ADDR0, column); 1395 nandc_set_reg(chip, NAND_ADDR1, 0); 1396 nandc_set_reg(chip, NAND_FLASH_CHIP_SELECT, 1397 nandc->props->is_bam ? 0 : DM_EN); 1398 nandc_set_reg(chip, NAND_EXEC_CMD, 1); 1399 1400 write_reg_dma(nandc, NAND_FLASH_CMD, 4, NAND_BAM_NEXT_SGL); 1401 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1402 1403 read_reg_dma(nandc, NAND_READ_ID, 1, NAND_BAM_NEXT_SGL); 1404 1405 return 0; 1406 } 1407 1408 /* sets up descriptors for NAND_CMD_RESET */ 1409 static int reset(struct qcom_nand_host *host) 1410 { 1411 struct nand_chip *chip = &host->chip; 1412 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1413 1414 nandc_set_reg(chip, NAND_FLASH_CMD, OP_RESET_DEVICE); 1415 nandc_set_reg(chip, NAND_EXEC_CMD, 1); 1416 1417 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL); 1418 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1419 1420 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL); 1421 1422 return 0; 1423 } 1424 1425 /* helpers to submit/free our list of dma descriptors */ 1426 static int submit_descs(struct qcom_nand_controller *nandc) 1427 { 1428 struct desc_info *desc; 1429 dma_cookie_t cookie = 0; 1430 struct bam_transaction *bam_txn = nandc->bam_txn; 1431 int r; 1432 1433 if (nandc->props->is_bam) { 1434 if (bam_txn->rx_sgl_pos > bam_txn->rx_sgl_start) { 1435 r = prepare_bam_async_desc(nandc, nandc->rx_chan, 0); 1436 if (r) 1437 return r; 1438 } 1439 1440 if (bam_txn->tx_sgl_pos > bam_txn->tx_sgl_start) { 1441 r = prepare_bam_async_desc(nandc, nandc->tx_chan, 1442 DMA_PREP_INTERRUPT); 1443 if (r) 1444 return r; 1445 } 1446 1447 if (bam_txn->cmd_sgl_pos > bam_txn->cmd_sgl_start) { 1448 r = prepare_bam_async_desc(nandc, nandc->cmd_chan, 1449 DMA_PREP_CMD); 1450 if (r) 1451 return r; 1452 } 1453 } 1454 1455 list_for_each_entry(desc, &nandc->desc_list, node) 1456 cookie = dmaengine_submit(desc->dma_desc); 1457 1458 if (nandc->props->is_bam) { 1459 bam_txn->last_cmd_desc->callback = qpic_bam_dma_done; 1460 bam_txn->last_cmd_desc->callback_param = bam_txn; 1461 if (bam_txn->last_data_desc) { 1462 bam_txn->last_data_desc->callback = qpic_bam_dma_done; 1463 bam_txn->last_data_desc->callback_param = bam_txn; 1464 bam_txn->wait_second_completion = true; 1465 } 1466 1467 dma_async_issue_pending(nandc->tx_chan); 1468 dma_async_issue_pending(nandc->rx_chan); 1469 dma_async_issue_pending(nandc->cmd_chan); 1470 1471 if (!wait_for_completion_timeout(&bam_txn->txn_done, 1472 QPIC_NAND_COMPLETION_TIMEOUT)) 1473 return -ETIMEDOUT; 1474 } else { 1475 if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE) 1476 return -ETIMEDOUT; 1477 } 1478 1479 return 0; 1480 } 1481 1482 static void free_descs(struct qcom_nand_controller *nandc) 1483 { 1484 struct desc_info *desc, *n; 1485 1486 list_for_each_entry_safe(desc, n, &nandc->desc_list, node) { 1487 list_del(&desc->node); 1488 1489 if (nandc->props->is_bam) 1490 dma_unmap_sg(nandc->dev, desc->bam_sgl, 1491 desc->sgl_cnt, desc->dir); 1492 else 1493 dma_unmap_sg(nandc->dev, &desc->adm_sgl, 1, 1494 desc->dir); 1495 1496 kfree(desc); 1497 } 1498 } 1499 1500 /* reset the register read buffer for next NAND operation */ 1501 static void clear_read_regs(struct qcom_nand_controller *nandc) 1502 { 1503 nandc->reg_read_pos = 0; 1504 nandc_read_buffer_sync(nandc, false); 1505 } 1506 1507 static void pre_command(struct qcom_nand_host *host, int command) 1508 { 1509 struct nand_chip *chip = &host->chip; 1510 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1511 1512 nandc->buf_count = 0; 1513 nandc->buf_start = 0; 1514 host->use_ecc = false; 1515 host->last_command = command; 1516 1517 clear_read_regs(nandc); 1518 1519 if (command == NAND_CMD_RESET || command == NAND_CMD_READID || 1520 command == NAND_CMD_PARAM || command == NAND_CMD_ERASE1) 1521 clear_bam_transaction(nandc); 1522 } 1523 1524 /* 1525 * this is called after NAND_CMD_PAGEPROG and NAND_CMD_ERASE1 to set our 1526 * privately maintained status byte, this status byte can be read after 1527 * NAND_CMD_STATUS is called 1528 */ 1529 static void parse_erase_write_errors(struct qcom_nand_host *host, int command) 1530 { 1531 struct nand_chip *chip = &host->chip; 1532 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1533 struct nand_ecc_ctrl *ecc = &chip->ecc; 1534 int num_cw; 1535 int i; 1536 1537 num_cw = command == NAND_CMD_PAGEPROG ? ecc->steps : 1; 1538 nandc_read_buffer_sync(nandc, true); 1539 1540 for (i = 0; i < num_cw; i++) { 1541 u32 flash_status = le32_to_cpu(nandc->reg_read_buf[i]); 1542 1543 if (flash_status & FS_MPU_ERR) 1544 host->status &= ~NAND_STATUS_WP; 1545 1546 if (flash_status & FS_OP_ERR || (i == (num_cw - 1) && 1547 (flash_status & 1548 FS_DEVICE_STS_ERR))) 1549 host->status |= NAND_STATUS_FAIL; 1550 } 1551 } 1552 1553 static void post_command(struct qcom_nand_host *host, int command) 1554 { 1555 struct nand_chip *chip = &host->chip; 1556 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1557 1558 switch (command) { 1559 case NAND_CMD_READID: 1560 nandc_read_buffer_sync(nandc, true); 1561 memcpy(nandc->data_buffer, nandc->reg_read_buf, 1562 nandc->buf_count); 1563 break; 1564 case NAND_CMD_PAGEPROG: 1565 case NAND_CMD_ERASE1: 1566 parse_erase_write_errors(host, command); 1567 break; 1568 default: 1569 break; 1570 } 1571 } 1572 1573 /* 1574 * Implements chip->legacy.cmdfunc. It's only used for a limited set of 1575 * commands. The rest of the commands wouldn't be called by upper layers. 1576 * For example, NAND_CMD_READOOB would never be called because we have our own 1577 * versions of read_oob ops for nand_ecc_ctrl. 1578 */ 1579 static void qcom_nandc_command(struct nand_chip *chip, unsigned int command, 1580 int column, int page_addr) 1581 { 1582 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1583 struct nand_ecc_ctrl *ecc = &chip->ecc; 1584 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1585 bool wait = false; 1586 int ret = 0; 1587 1588 pre_command(host, command); 1589 1590 switch (command) { 1591 case NAND_CMD_RESET: 1592 ret = reset(host); 1593 wait = true; 1594 break; 1595 1596 case NAND_CMD_READID: 1597 nandc->buf_count = 4; 1598 ret = read_id(host, column); 1599 wait = true; 1600 break; 1601 1602 case NAND_CMD_PARAM: 1603 ret = nandc_param(host); 1604 wait = true; 1605 break; 1606 1607 case NAND_CMD_ERASE1: 1608 ret = erase_block(host, page_addr); 1609 wait = true; 1610 break; 1611 1612 case NAND_CMD_READ0: 1613 /* we read the entire page for now */ 1614 WARN_ON(column != 0); 1615 1616 host->use_ecc = true; 1617 set_address(host, 0, page_addr); 1618 update_rw_regs(host, ecc->steps, true, 0); 1619 break; 1620 1621 case NAND_CMD_SEQIN: 1622 WARN_ON(column != 0); 1623 set_address(host, 0, page_addr); 1624 break; 1625 1626 case NAND_CMD_PAGEPROG: 1627 case NAND_CMD_STATUS: 1628 case NAND_CMD_NONE: 1629 default: 1630 break; 1631 } 1632 1633 if (ret) { 1634 dev_err(nandc->dev, "failure executing command %d\n", 1635 command); 1636 free_descs(nandc); 1637 return; 1638 } 1639 1640 if (wait) { 1641 ret = submit_descs(nandc); 1642 if (ret) 1643 dev_err(nandc->dev, 1644 "failure submitting descs for command %d\n", 1645 command); 1646 } 1647 1648 free_descs(nandc); 1649 1650 post_command(host, command); 1651 } 1652 1653 /* 1654 * when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read 1655 * an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS. 1656 * 1657 * when using RS ECC, the HW reports the same erros when reading an erased CW, 1658 * but it notifies that it is an erased CW by placing special characters at 1659 * certain offsets in the buffer. 1660 * 1661 * verify if the page is erased or not, and fix up the page for RS ECC by 1662 * replacing the special characters with 0xff. 1663 */ 1664 static bool erased_chunk_check_and_fixup(u8 *data_buf, int data_len) 1665 { 1666 u8 empty1, empty2; 1667 1668 /* 1669 * an erased page flags an error in NAND_FLASH_STATUS, check if the page 1670 * is erased by looking for 0x54s at offsets 3 and 175 from the 1671 * beginning of each codeword 1672 */ 1673 1674 empty1 = data_buf[3]; 1675 empty2 = data_buf[175]; 1676 1677 /* 1678 * if the erased codework markers, if they exist override them with 1679 * 0xffs 1680 */ 1681 if ((empty1 == 0x54 && empty2 == 0xff) || 1682 (empty1 == 0xff && empty2 == 0x54)) { 1683 data_buf[3] = 0xff; 1684 data_buf[175] = 0xff; 1685 } 1686 1687 /* 1688 * check if the entire chunk contains 0xffs or not. if it doesn't, then 1689 * restore the original values at the special offsets 1690 */ 1691 if (memchr_inv(data_buf, 0xff, data_len)) { 1692 data_buf[3] = empty1; 1693 data_buf[175] = empty2; 1694 1695 return false; 1696 } 1697 1698 return true; 1699 } 1700 1701 struct read_stats { 1702 __le32 flash; 1703 __le32 buffer; 1704 __le32 erased_cw; 1705 }; 1706 1707 /* reads back FLASH_STATUS register set by the controller */ 1708 static int check_flash_errors(struct qcom_nand_host *host, int cw_cnt) 1709 { 1710 struct nand_chip *chip = &host->chip; 1711 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1712 int i; 1713 1714 nandc_read_buffer_sync(nandc, true); 1715 1716 for (i = 0; i < cw_cnt; i++) { 1717 u32 flash = le32_to_cpu(nandc->reg_read_buf[i]); 1718 1719 if (flash & (FS_OP_ERR | FS_MPU_ERR)) 1720 return -EIO; 1721 } 1722 1723 return 0; 1724 } 1725 1726 /* performs raw read for one codeword */ 1727 static int 1728 qcom_nandc_read_cw_raw(struct mtd_info *mtd, struct nand_chip *chip, 1729 u8 *data_buf, u8 *oob_buf, int page, int cw) 1730 { 1731 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1732 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1733 struct nand_ecc_ctrl *ecc = &chip->ecc; 1734 int data_size1, data_size2, oob_size1, oob_size2; 1735 int ret, reg_off = FLASH_BUF_ACC, read_loc = 0; 1736 int raw_cw = cw; 1737 1738 nand_read_page_op(chip, page, 0, NULL, 0); 1739 host->use_ecc = false; 1740 1741 if (nandc->props->qpic_v2) 1742 raw_cw = ecc->steps - 1; 1743 1744 clear_bam_transaction(nandc); 1745 set_address(host, host->cw_size * cw, page); 1746 update_rw_regs(host, 1, true, raw_cw); 1747 config_nand_page_read(chip); 1748 1749 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1); 1750 oob_size1 = host->bbm_size; 1751 1752 if (qcom_nandc_is_last_cw(ecc, cw) && !host->codeword_fixup) { 1753 data_size2 = ecc->size - data_size1 - 1754 ((ecc->steps - 1) * 4); 1755 oob_size2 = (ecc->steps * 4) + host->ecc_bytes_hw + 1756 host->spare_bytes; 1757 } else { 1758 data_size2 = host->cw_data - data_size1; 1759 oob_size2 = host->ecc_bytes_hw + host->spare_bytes; 1760 } 1761 1762 if (nandc->props->is_bam) { 1763 nandc_set_read_loc(chip, cw, 0, read_loc, data_size1, 0); 1764 read_loc += data_size1; 1765 1766 nandc_set_read_loc(chip, cw, 1, read_loc, oob_size1, 0); 1767 read_loc += oob_size1; 1768 1769 nandc_set_read_loc(chip, cw, 2, read_loc, data_size2, 0); 1770 read_loc += data_size2; 1771 1772 nandc_set_read_loc(chip, cw, 3, read_loc, oob_size2, 1); 1773 } 1774 1775 config_nand_cw_read(chip, false, raw_cw); 1776 1777 read_data_dma(nandc, reg_off, data_buf, data_size1, 0); 1778 reg_off += data_size1; 1779 1780 read_data_dma(nandc, reg_off, oob_buf, oob_size1, 0); 1781 reg_off += oob_size1; 1782 1783 read_data_dma(nandc, reg_off, data_buf + data_size1, data_size2, 0); 1784 reg_off += data_size2; 1785 1786 read_data_dma(nandc, reg_off, oob_buf + oob_size1, oob_size2, 0); 1787 1788 ret = submit_descs(nandc); 1789 free_descs(nandc); 1790 if (ret) { 1791 dev_err(nandc->dev, "failure to read raw cw %d\n", cw); 1792 return ret; 1793 } 1794 1795 return check_flash_errors(host, 1); 1796 } 1797 1798 /* 1799 * Bitflips can happen in erased codewords also so this function counts the 1800 * number of 0 in each CW for which ECC engine returns the uncorrectable 1801 * error. The page will be assumed as erased if this count is less than or 1802 * equal to the ecc->strength for each CW. 1803 * 1804 * 1. Both DATA and OOB need to be checked for number of 0. The 1805 * top-level API can be called with only data buf or OOB buf so use 1806 * chip->data_buf if data buf is null and chip->oob_poi if oob buf 1807 * is null for copying the raw bytes. 1808 * 2. Perform raw read for all the CW which has uncorrectable errors. 1809 * 3. For each CW, check the number of 0 in cw_data and usable OOB bytes. 1810 * The BBM and spare bytes bit flip won’t affect the ECC so don’t check 1811 * the number of bitflips in this area. 1812 */ 1813 static int 1814 check_for_erased_page(struct qcom_nand_host *host, u8 *data_buf, 1815 u8 *oob_buf, unsigned long uncorrectable_cws, 1816 int page, unsigned int max_bitflips) 1817 { 1818 struct nand_chip *chip = &host->chip; 1819 struct mtd_info *mtd = nand_to_mtd(chip); 1820 struct nand_ecc_ctrl *ecc = &chip->ecc; 1821 u8 *cw_data_buf, *cw_oob_buf; 1822 int cw, data_size, oob_size, ret = 0; 1823 1824 if (!data_buf) 1825 data_buf = nand_get_data_buf(chip); 1826 1827 if (!oob_buf) { 1828 nand_get_data_buf(chip); 1829 oob_buf = chip->oob_poi; 1830 } 1831 1832 for_each_set_bit(cw, &uncorrectable_cws, ecc->steps) { 1833 if (qcom_nandc_is_last_cw(ecc, cw) && !host->codeword_fixup) { 1834 data_size = ecc->size - ((ecc->steps - 1) * 4); 1835 oob_size = (ecc->steps * 4) + host->ecc_bytes_hw; 1836 } else { 1837 data_size = host->cw_data; 1838 oob_size = host->ecc_bytes_hw; 1839 } 1840 1841 /* determine starting buffer address for current CW */ 1842 cw_data_buf = data_buf + (cw * host->cw_data); 1843 cw_oob_buf = oob_buf + (cw * ecc->bytes); 1844 1845 ret = qcom_nandc_read_cw_raw(mtd, chip, cw_data_buf, 1846 cw_oob_buf, page, cw); 1847 if (ret) 1848 return ret; 1849 1850 /* 1851 * make sure it isn't an erased page reported 1852 * as not-erased by HW because of a few bitflips 1853 */ 1854 ret = nand_check_erased_ecc_chunk(cw_data_buf, data_size, 1855 cw_oob_buf + host->bbm_size, 1856 oob_size, NULL, 1857 0, ecc->strength); 1858 if (ret < 0) { 1859 mtd->ecc_stats.failed++; 1860 } else { 1861 mtd->ecc_stats.corrected += ret; 1862 max_bitflips = max_t(unsigned int, max_bitflips, ret); 1863 } 1864 } 1865 1866 return max_bitflips; 1867 } 1868 1869 /* 1870 * reads back status registers set by the controller to notify page read 1871 * errors. this is equivalent to what 'ecc->correct()' would do. 1872 */ 1873 static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf, 1874 u8 *oob_buf, int page) 1875 { 1876 struct nand_chip *chip = &host->chip; 1877 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1878 struct mtd_info *mtd = nand_to_mtd(chip); 1879 struct nand_ecc_ctrl *ecc = &chip->ecc; 1880 unsigned int max_bitflips = 0, uncorrectable_cws = 0; 1881 struct read_stats *buf; 1882 bool flash_op_err = false, erased; 1883 int i; 1884 u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf; 1885 1886 buf = (struct read_stats *)nandc->reg_read_buf; 1887 nandc_read_buffer_sync(nandc, true); 1888 1889 for (i = 0; i < ecc->steps; i++, buf++) { 1890 u32 flash, buffer, erased_cw; 1891 int data_len, oob_len; 1892 1893 if (qcom_nandc_is_last_cw(ecc, i)) { 1894 data_len = ecc->size - ((ecc->steps - 1) << 2); 1895 oob_len = ecc->steps << 2; 1896 } else { 1897 data_len = host->cw_data; 1898 oob_len = 0; 1899 } 1900 1901 flash = le32_to_cpu(buf->flash); 1902 buffer = le32_to_cpu(buf->buffer); 1903 erased_cw = le32_to_cpu(buf->erased_cw); 1904 1905 /* 1906 * Check ECC failure for each codeword. ECC failure can 1907 * happen in either of the following conditions 1908 * 1. If number of bitflips are greater than ECC engine 1909 * capability. 1910 * 2. If this codeword contains all 0xff for which erased 1911 * codeword detection check will be done. 1912 */ 1913 if ((flash & FS_OP_ERR) && (buffer & BS_UNCORRECTABLE_BIT)) { 1914 /* 1915 * For BCH ECC, ignore erased codeword errors, if 1916 * ERASED_CW bits are set. 1917 */ 1918 if (host->bch_enabled) { 1919 erased = (erased_cw & ERASED_CW) == ERASED_CW; 1920 /* 1921 * For RS ECC, HW reports the erased CW by placing 1922 * special characters at certain offsets in the buffer. 1923 * These special characters will be valid only if 1924 * complete page is read i.e. data_buf is not NULL. 1925 */ 1926 } else if (data_buf) { 1927 erased = erased_chunk_check_and_fixup(data_buf, 1928 data_len); 1929 } else { 1930 erased = false; 1931 } 1932 1933 if (!erased) 1934 uncorrectable_cws |= BIT(i); 1935 /* 1936 * Check if MPU or any other operational error (timeout, 1937 * device failure, etc.) happened for this codeword and 1938 * make flash_op_err true. If flash_op_err is set, then 1939 * EIO will be returned for page read. 1940 */ 1941 } else if (flash & (FS_OP_ERR | FS_MPU_ERR)) { 1942 flash_op_err = true; 1943 /* 1944 * No ECC or operational errors happened. Check the number of 1945 * bits corrected and update the ecc_stats.corrected. 1946 */ 1947 } else { 1948 unsigned int stat; 1949 1950 stat = buffer & BS_CORRECTABLE_ERR_MSK; 1951 mtd->ecc_stats.corrected += stat; 1952 max_bitflips = max(max_bitflips, stat); 1953 } 1954 1955 if (data_buf) 1956 data_buf += data_len; 1957 if (oob_buf) 1958 oob_buf += oob_len + ecc->bytes; 1959 } 1960 1961 if (flash_op_err) 1962 return -EIO; 1963 1964 if (!uncorrectable_cws) 1965 return max_bitflips; 1966 1967 return check_for_erased_page(host, data_buf_start, oob_buf_start, 1968 uncorrectable_cws, page, 1969 max_bitflips); 1970 } 1971 1972 /* 1973 * helper to perform the actual page read operation, used by ecc->read_page(), 1974 * ecc->read_oob() 1975 */ 1976 static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf, 1977 u8 *oob_buf, int page) 1978 { 1979 struct nand_chip *chip = &host->chip; 1980 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1981 struct nand_ecc_ctrl *ecc = &chip->ecc; 1982 u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf; 1983 int i, ret; 1984 1985 config_nand_page_read(chip); 1986 1987 /* queue cmd descs for each codeword */ 1988 for (i = 0; i < ecc->steps; i++) { 1989 int data_size, oob_size; 1990 1991 if (qcom_nandc_is_last_cw(ecc, i) && !host->codeword_fixup) { 1992 data_size = ecc->size - ((ecc->steps - 1) << 2); 1993 oob_size = (ecc->steps << 2) + host->ecc_bytes_hw + 1994 host->spare_bytes; 1995 } else { 1996 data_size = host->cw_data; 1997 oob_size = host->ecc_bytes_hw + host->spare_bytes; 1998 } 1999 2000 if (nandc->props->is_bam) { 2001 if (data_buf && oob_buf) { 2002 nandc_set_read_loc(chip, i, 0, 0, data_size, 0); 2003 nandc_set_read_loc(chip, i, 1, data_size, 2004 oob_size, 1); 2005 } else if (data_buf) { 2006 nandc_set_read_loc(chip, i, 0, 0, data_size, 1); 2007 } else { 2008 nandc_set_read_loc(chip, i, 0, data_size, 2009 oob_size, 1); 2010 } 2011 } 2012 2013 config_nand_cw_read(chip, true, i); 2014 2015 if (data_buf) 2016 read_data_dma(nandc, FLASH_BUF_ACC, data_buf, 2017 data_size, 0); 2018 2019 /* 2020 * when ecc is enabled, the controller doesn't read the real 2021 * or dummy bad block markers in each chunk. To maintain a 2022 * consistent layout across RAW and ECC reads, we just 2023 * leave the real/dummy BBM offsets empty (i.e, filled with 2024 * 0xffs) 2025 */ 2026 if (oob_buf) { 2027 int j; 2028 2029 for (j = 0; j < host->bbm_size; j++) 2030 *oob_buf++ = 0xff; 2031 2032 read_data_dma(nandc, FLASH_BUF_ACC + data_size, 2033 oob_buf, oob_size, 0); 2034 } 2035 2036 if (data_buf) 2037 data_buf += data_size; 2038 if (oob_buf) 2039 oob_buf += oob_size; 2040 } 2041 2042 ret = submit_descs(nandc); 2043 free_descs(nandc); 2044 2045 if (ret) { 2046 dev_err(nandc->dev, "failure to read page/oob\n"); 2047 return ret; 2048 } 2049 2050 return parse_read_errors(host, data_buf_start, oob_buf_start, page); 2051 } 2052 2053 /* 2054 * a helper that copies the last step/codeword of a page (containing free oob) 2055 * into our local buffer 2056 */ 2057 static int copy_last_cw(struct qcom_nand_host *host, int page) 2058 { 2059 struct nand_chip *chip = &host->chip; 2060 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2061 struct nand_ecc_ctrl *ecc = &chip->ecc; 2062 int size; 2063 int ret; 2064 2065 clear_read_regs(nandc); 2066 2067 size = host->use_ecc ? host->cw_data : host->cw_size; 2068 2069 /* prepare a clean read buffer */ 2070 memset(nandc->data_buffer, 0xff, size); 2071 2072 set_address(host, host->cw_size * (ecc->steps - 1), page); 2073 update_rw_regs(host, 1, true, ecc->steps - 1); 2074 2075 config_nand_single_cw_page_read(chip, host->use_ecc, ecc->steps - 1); 2076 2077 read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size, 0); 2078 2079 ret = submit_descs(nandc); 2080 if (ret) 2081 dev_err(nandc->dev, "failed to copy last codeword\n"); 2082 2083 free_descs(nandc); 2084 2085 return ret; 2086 } 2087 2088 static bool qcom_nandc_is_boot_partition(struct qcom_nand_host *host, int page) 2089 { 2090 struct qcom_nand_boot_partition *boot_partition; 2091 u32 start, end; 2092 int i; 2093 2094 /* 2095 * Since the frequent access will be to the non-boot partitions like rootfs, 2096 * optimize the page check by: 2097 * 2098 * 1. Checking if the page lies after the last boot partition. 2099 * 2. Checking from the boot partition end. 2100 */ 2101 2102 /* First check the last boot partition */ 2103 boot_partition = &host->boot_partitions[host->nr_boot_partitions - 1]; 2104 start = boot_partition->page_offset; 2105 end = start + boot_partition->page_size; 2106 2107 /* Page is after the last boot partition end. This is NOT a boot partition */ 2108 if (page > end) 2109 return false; 2110 2111 /* Actually check if it's a boot partition */ 2112 if (page < end && page >= start) 2113 return true; 2114 2115 /* Check the other boot partitions starting from the second-last partition */ 2116 for (i = host->nr_boot_partitions - 2; i >= 0; i--) { 2117 boot_partition = &host->boot_partitions[i]; 2118 start = boot_partition->page_offset; 2119 end = start + boot_partition->page_size; 2120 2121 if (page < end && page >= start) 2122 return true; 2123 } 2124 2125 return false; 2126 } 2127 2128 static void qcom_nandc_codeword_fixup(struct qcom_nand_host *host, int page) 2129 { 2130 bool codeword_fixup = qcom_nandc_is_boot_partition(host, page); 2131 2132 /* Skip conf write if we are already in the correct mode */ 2133 if (codeword_fixup == host->codeword_fixup) 2134 return; 2135 2136 host->codeword_fixup = codeword_fixup; 2137 2138 host->cw_data = codeword_fixup ? 512 : 516; 2139 host->spare_bytes = host->cw_size - host->ecc_bytes_hw - 2140 host->bbm_size - host->cw_data; 2141 2142 host->cfg0 &= ~(SPARE_SIZE_BYTES_MASK | UD_SIZE_BYTES_MASK); 2143 host->cfg0 |= host->spare_bytes << SPARE_SIZE_BYTES | 2144 host->cw_data << UD_SIZE_BYTES; 2145 2146 host->ecc_bch_cfg &= ~ECC_NUM_DATA_BYTES_MASK; 2147 host->ecc_bch_cfg |= host->cw_data << ECC_NUM_DATA_BYTES; 2148 host->ecc_buf_cfg = (host->cw_data - 1) << NUM_STEPS; 2149 } 2150 2151 /* implements ecc->read_page() */ 2152 static int qcom_nandc_read_page(struct nand_chip *chip, uint8_t *buf, 2153 int oob_required, int page) 2154 { 2155 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2156 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2157 u8 *data_buf, *oob_buf = NULL; 2158 2159 if (host->nr_boot_partitions) 2160 qcom_nandc_codeword_fixup(host, page); 2161 2162 nand_read_page_op(chip, page, 0, NULL, 0); 2163 data_buf = buf; 2164 oob_buf = oob_required ? chip->oob_poi : NULL; 2165 2166 clear_bam_transaction(nandc); 2167 2168 return read_page_ecc(host, data_buf, oob_buf, page); 2169 } 2170 2171 /* implements ecc->read_page_raw() */ 2172 static int qcom_nandc_read_page_raw(struct nand_chip *chip, uint8_t *buf, 2173 int oob_required, int page) 2174 { 2175 struct mtd_info *mtd = nand_to_mtd(chip); 2176 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2177 struct nand_ecc_ctrl *ecc = &chip->ecc; 2178 int cw, ret; 2179 u8 *data_buf = buf, *oob_buf = chip->oob_poi; 2180 2181 if (host->nr_boot_partitions) 2182 qcom_nandc_codeword_fixup(host, page); 2183 2184 for (cw = 0; cw < ecc->steps; cw++) { 2185 ret = qcom_nandc_read_cw_raw(mtd, chip, data_buf, oob_buf, 2186 page, cw); 2187 if (ret) 2188 return ret; 2189 2190 data_buf += host->cw_data; 2191 oob_buf += ecc->bytes; 2192 } 2193 2194 return 0; 2195 } 2196 2197 /* implements ecc->read_oob() */ 2198 static int qcom_nandc_read_oob(struct nand_chip *chip, int page) 2199 { 2200 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2201 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2202 struct nand_ecc_ctrl *ecc = &chip->ecc; 2203 2204 if (host->nr_boot_partitions) 2205 qcom_nandc_codeword_fixup(host, page); 2206 2207 clear_read_regs(nandc); 2208 clear_bam_transaction(nandc); 2209 2210 host->use_ecc = true; 2211 set_address(host, 0, page); 2212 update_rw_regs(host, ecc->steps, true, 0); 2213 2214 return read_page_ecc(host, NULL, chip->oob_poi, page); 2215 } 2216 2217 /* implements ecc->write_page() */ 2218 static int qcom_nandc_write_page(struct nand_chip *chip, const uint8_t *buf, 2219 int oob_required, int page) 2220 { 2221 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2222 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2223 struct nand_ecc_ctrl *ecc = &chip->ecc; 2224 u8 *data_buf, *oob_buf; 2225 int i, ret; 2226 2227 if (host->nr_boot_partitions) 2228 qcom_nandc_codeword_fixup(host, page); 2229 2230 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 2231 2232 clear_read_regs(nandc); 2233 clear_bam_transaction(nandc); 2234 2235 data_buf = (u8 *)buf; 2236 oob_buf = chip->oob_poi; 2237 2238 host->use_ecc = true; 2239 update_rw_regs(host, ecc->steps, false, 0); 2240 config_nand_page_write(chip); 2241 2242 for (i = 0; i < ecc->steps; i++) { 2243 int data_size, oob_size; 2244 2245 if (qcom_nandc_is_last_cw(ecc, i) && !host->codeword_fixup) { 2246 data_size = ecc->size - ((ecc->steps - 1) << 2); 2247 oob_size = (ecc->steps << 2) + host->ecc_bytes_hw + 2248 host->spare_bytes; 2249 } else { 2250 data_size = host->cw_data; 2251 oob_size = ecc->bytes; 2252 } 2253 2254 2255 write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size, 2256 i == (ecc->steps - 1) ? NAND_BAM_NO_EOT : 0); 2257 2258 /* 2259 * when ECC is enabled, we don't really need to write anything 2260 * to oob for the first n - 1 codewords since these oob regions 2261 * just contain ECC bytes that's written by the controller 2262 * itself. For the last codeword, we skip the bbm positions and 2263 * write to the free oob area. 2264 */ 2265 if (qcom_nandc_is_last_cw(ecc, i)) { 2266 oob_buf += host->bbm_size; 2267 2268 write_data_dma(nandc, FLASH_BUF_ACC + data_size, 2269 oob_buf, oob_size, 0); 2270 } 2271 2272 config_nand_cw_write(chip); 2273 2274 data_buf += data_size; 2275 oob_buf += oob_size; 2276 } 2277 2278 ret = submit_descs(nandc); 2279 if (ret) 2280 dev_err(nandc->dev, "failure to write page\n"); 2281 2282 free_descs(nandc); 2283 2284 if (!ret) 2285 ret = nand_prog_page_end_op(chip); 2286 2287 return ret; 2288 } 2289 2290 /* implements ecc->write_page_raw() */ 2291 static int qcom_nandc_write_page_raw(struct nand_chip *chip, 2292 const uint8_t *buf, int oob_required, 2293 int page) 2294 { 2295 struct mtd_info *mtd = nand_to_mtd(chip); 2296 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2297 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2298 struct nand_ecc_ctrl *ecc = &chip->ecc; 2299 u8 *data_buf, *oob_buf; 2300 int i, ret; 2301 2302 if (host->nr_boot_partitions) 2303 qcom_nandc_codeword_fixup(host, page); 2304 2305 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 2306 clear_read_regs(nandc); 2307 clear_bam_transaction(nandc); 2308 2309 data_buf = (u8 *)buf; 2310 oob_buf = chip->oob_poi; 2311 2312 host->use_ecc = false; 2313 update_rw_regs(host, ecc->steps, false, 0); 2314 config_nand_page_write(chip); 2315 2316 for (i = 0; i < ecc->steps; i++) { 2317 int data_size1, data_size2, oob_size1, oob_size2; 2318 int reg_off = FLASH_BUF_ACC; 2319 2320 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1); 2321 oob_size1 = host->bbm_size; 2322 2323 if (qcom_nandc_is_last_cw(ecc, i) && !host->codeword_fixup) { 2324 data_size2 = ecc->size - data_size1 - 2325 ((ecc->steps - 1) << 2); 2326 oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw + 2327 host->spare_bytes; 2328 } else { 2329 data_size2 = host->cw_data - data_size1; 2330 oob_size2 = host->ecc_bytes_hw + host->spare_bytes; 2331 } 2332 2333 write_data_dma(nandc, reg_off, data_buf, data_size1, 2334 NAND_BAM_NO_EOT); 2335 reg_off += data_size1; 2336 data_buf += data_size1; 2337 2338 write_data_dma(nandc, reg_off, oob_buf, oob_size1, 2339 NAND_BAM_NO_EOT); 2340 reg_off += oob_size1; 2341 oob_buf += oob_size1; 2342 2343 write_data_dma(nandc, reg_off, data_buf, data_size2, 2344 NAND_BAM_NO_EOT); 2345 reg_off += data_size2; 2346 data_buf += data_size2; 2347 2348 write_data_dma(nandc, reg_off, oob_buf, oob_size2, 0); 2349 oob_buf += oob_size2; 2350 2351 config_nand_cw_write(chip); 2352 } 2353 2354 ret = submit_descs(nandc); 2355 if (ret) 2356 dev_err(nandc->dev, "failure to write raw page\n"); 2357 2358 free_descs(nandc); 2359 2360 if (!ret) 2361 ret = nand_prog_page_end_op(chip); 2362 2363 return ret; 2364 } 2365 2366 /* 2367 * implements ecc->write_oob() 2368 * 2369 * the NAND controller cannot write only data or only OOB within a codeword 2370 * since ECC is calculated for the combined codeword. So update the OOB from 2371 * chip->oob_poi, and pad the data area with OxFF before writing. 2372 */ 2373 static int qcom_nandc_write_oob(struct nand_chip *chip, int page) 2374 { 2375 struct mtd_info *mtd = nand_to_mtd(chip); 2376 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2377 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2378 struct nand_ecc_ctrl *ecc = &chip->ecc; 2379 u8 *oob = chip->oob_poi; 2380 int data_size, oob_size; 2381 int ret; 2382 2383 if (host->nr_boot_partitions) 2384 qcom_nandc_codeword_fixup(host, page); 2385 2386 host->use_ecc = true; 2387 clear_bam_transaction(nandc); 2388 2389 /* calculate the data and oob size for the last codeword/step */ 2390 data_size = ecc->size - ((ecc->steps - 1) << 2); 2391 oob_size = mtd->oobavail; 2392 2393 memset(nandc->data_buffer, 0xff, host->cw_data); 2394 /* override new oob content to last codeword */ 2395 mtd_ooblayout_get_databytes(mtd, nandc->data_buffer + data_size, oob, 2396 0, mtd->oobavail); 2397 2398 set_address(host, host->cw_size * (ecc->steps - 1), page); 2399 update_rw_regs(host, 1, false, 0); 2400 2401 config_nand_page_write(chip); 2402 write_data_dma(nandc, FLASH_BUF_ACC, 2403 nandc->data_buffer, data_size + oob_size, 0); 2404 config_nand_cw_write(chip); 2405 2406 ret = submit_descs(nandc); 2407 2408 free_descs(nandc); 2409 2410 if (ret) { 2411 dev_err(nandc->dev, "failure to write oob\n"); 2412 return -EIO; 2413 } 2414 2415 return nand_prog_page_end_op(chip); 2416 } 2417 2418 static int qcom_nandc_block_bad(struct nand_chip *chip, loff_t ofs) 2419 { 2420 struct mtd_info *mtd = nand_to_mtd(chip); 2421 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2422 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2423 struct nand_ecc_ctrl *ecc = &chip->ecc; 2424 int page, ret, bbpos, bad = 0; 2425 2426 page = (int)(ofs >> chip->page_shift) & chip->pagemask; 2427 2428 /* 2429 * configure registers for a raw sub page read, the address is set to 2430 * the beginning of the last codeword, we don't care about reading ecc 2431 * portion of oob. we just want the first few bytes from this codeword 2432 * that contains the BBM 2433 */ 2434 host->use_ecc = false; 2435 2436 clear_bam_transaction(nandc); 2437 ret = copy_last_cw(host, page); 2438 if (ret) 2439 goto err; 2440 2441 if (check_flash_errors(host, 1)) { 2442 dev_warn(nandc->dev, "error when trying to read BBM\n"); 2443 goto err; 2444 } 2445 2446 bbpos = mtd->writesize - host->cw_size * (ecc->steps - 1); 2447 2448 bad = nandc->data_buffer[bbpos] != 0xff; 2449 2450 if (chip->options & NAND_BUSWIDTH_16) 2451 bad = bad || (nandc->data_buffer[bbpos + 1] != 0xff); 2452 err: 2453 return bad; 2454 } 2455 2456 static int qcom_nandc_block_markbad(struct nand_chip *chip, loff_t ofs) 2457 { 2458 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2459 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2460 struct nand_ecc_ctrl *ecc = &chip->ecc; 2461 int page, ret; 2462 2463 clear_read_regs(nandc); 2464 clear_bam_transaction(nandc); 2465 2466 /* 2467 * to mark the BBM as bad, we flash the entire last codeword with 0s. 2468 * we don't care about the rest of the content in the codeword since 2469 * we aren't going to use this block again 2470 */ 2471 memset(nandc->data_buffer, 0x00, host->cw_size); 2472 2473 page = (int)(ofs >> chip->page_shift) & chip->pagemask; 2474 2475 /* prepare write */ 2476 host->use_ecc = false; 2477 set_address(host, host->cw_size * (ecc->steps - 1), page); 2478 update_rw_regs(host, 1, false, ecc->steps - 1); 2479 2480 config_nand_page_write(chip); 2481 write_data_dma(nandc, FLASH_BUF_ACC, 2482 nandc->data_buffer, host->cw_size, 0); 2483 config_nand_cw_write(chip); 2484 2485 ret = submit_descs(nandc); 2486 2487 free_descs(nandc); 2488 2489 if (ret) { 2490 dev_err(nandc->dev, "failure to update BBM\n"); 2491 return -EIO; 2492 } 2493 2494 return nand_prog_page_end_op(chip); 2495 } 2496 2497 /* 2498 * the three functions below implement chip->legacy.read_byte(), 2499 * chip->legacy.read_buf() and chip->legacy.write_buf() respectively. these 2500 * aren't used for reading/writing page data, they are used for smaller data 2501 * like reading id, status etc 2502 */ 2503 static uint8_t qcom_nandc_read_byte(struct nand_chip *chip) 2504 { 2505 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2506 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2507 u8 *buf = nandc->data_buffer; 2508 u8 ret = 0x0; 2509 2510 if (host->last_command == NAND_CMD_STATUS) { 2511 ret = host->status; 2512 2513 host->status = NAND_STATUS_READY | NAND_STATUS_WP; 2514 2515 return ret; 2516 } 2517 2518 if (nandc->buf_start < nandc->buf_count) 2519 ret = buf[nandc->buf_start++]; 2520 2521 return ret; 2522 } 2523 2524 static void qcom_nandc_read_buf(struct nand_chip *chip, uint8_t *buf, int len) 2525 { 2526 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2527 int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start); 2528 2529 memcpy(buf, nandc->data_buffer + nandc->buf_start, real_len); 2530 nandc->buf_start += real_len; 2531 } 2532 2533 static void qcom_nandc_write_buf(struct nand_chip *chip, const uint8_t *buf, 2534 int len) 2535 { 2536 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2537 int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start); 2538 2539 memcpy(nandc->data_buffer + nandc->buf_start, buf, real_len); 2540 2541 nandc->buf_start += real_len; 2542 } 2543 2544 /* we support only one external chip for now */ 2545 static void qcom_nandc_select_chip(struct nand_chip *chip, int chipnr) 2546 { 2547 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2548 2549 if (chipnr <= 0) 2550 return; 2551 2552 dev_warn(nandc->dev, "invalid chip select\n"); 2553 } 2554 2555 /* 2556 * NAND controller page layout info 2557 * 2558 * Layout with ECC enabled: 2559 * 2560 * |----------------------| |---------------------------------| 2561 * | xx.......yy| | *********xx.......yy| 2562 * | DATA xx..ECC..yy| | DATA **SPARE**xx..ECC..yy| 2563 * | (516) xx.......yy| | (516-n*4) **(n*4)**xx.......yy| 2564 * | xx.......yy| | *********xx.......yy| 2565 * |----------------------| |---------------------------------| 2566 * codeword 1,2..n-1 codeword n 2567 * <---(528/532 Bytes)--> <-------(528/532 Bytes)---------> 2568 * 2569 * n = Number of codewords in the page 2570 * . = ECC bytes 2571 * * = Spare/free bytes 2572 * x = Unused byte(s) 2573 * y = Reserved byte(s) 2574 * 2575 * 2K page: n = 4, spare = 16 bytes 2576 * 4K page: n = 8, spare = 32 bytes 2577 * 8K page: n = 16, spare = 64 bytes 2578 * 2579 * the qcom nand controller operates at a sub page/codeword level. each 2580 * codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively. 2581 * the number of ECC bytes vary based on the ECC strength and the bus width. 2582 * 2583 * the first n - 1 codewords contains 516 bytes of user data, the remaining 2584 * 12/16 bytes consist of ECC and reserved data. The nth codeword contains 2585 * both user data and spare(oobavail) bytes that sum up to 516 bytes. 2586 * 2587 * When we access a page with ECC enabled, the reserved bytes(s) are not 2588 * accessible at all. When reading, we fill up these unreadable positions 2589 * with 0xffs. When writing, the controller skips writing the inaccessible 2590 * bytes. 2591 * 2592 * Layout with ECC disabled: 2593 * 2594 * |------------------------------| |---------------------------------------| 2595 * | yy xx.......| | bb *********xx.......| 2596 * | DATA1 yy DATA2 xx..ECC..| | DATA1 bb DATA2 **SPARE**xx..ECC..| 2597 * | (size1) yy (size2) xx.......| | (size1) bb (size2) **(n*4)**xx.......| 2598 * | yy xx.......| | bb *********xx.......| 2599 * |------------------------------| |---------------------------------------| 2600 * codeword 1,2..n-1 codeword n 2601 * <-------(528/532 Bytes)------> <-----------(528/532 Bytes)-----------> 2602 * 2603 * n = Number of codewords in the page 2604 * . = ECC bytes 2605 * * = Spare/free bytes 2606 * x = Unused byte(s) 2607 * y = Dummy Bad Bock byte(s) 2608 * b = Real Bad Block byte(s) 2609 * size1/size2 = function of codeword size and 'n' 2610 * 2611 * when the ECC block is disabled, one reserved byte (or two for 16 bit bus 2612 * width) is now accessible. For the first n - 1 codewords, these are dummy Bad 2613 * Block Markers. In the last codeword, this position contains the real BBM 2614 * 2615 * In order to have a consistent layout between RAW and ECC modes, we assume 2616 * the following OOB layout arrangement: 2617 * 2618 * |-----------| |--------------------| 2619 * |yyxx.......| |bb*********xx.......| 2620 * |yyxx..ECC..| |bb*FREEOOB*xx..ECC..| 2621 * |yyxx.......| |bb*********xx.......| 2622 * |yyxx.......| |bb*********xx.......| 2623 * |-----------| |--------------------| 2624 * first n - 1 nth OOB region 2625 * OOB regions 2626 * 2627 * n = Number of codewords in the page 2628 * . = ECC bytes 2629 * * = FREE OOB bytes 2630 * y = Dummy bad block byte(s) (inaccessible when ECC enabled) 2631 * x = Unused byte(s) 2632 * b = Real bad block byte(s) (inaccessible when ECC enabled) 2633 * 2634 * This layout is read as is when ECC is disabled. When ECC is enabled, the 2635 * inaccessible Bad Block byte(s) are ignored when we write to a page/oob, 2636 * and assumed as 0xffs when we read a page/oob. The ECC, unused and 2637 * dummy/real bad block bytes are grouped as ecc bytes (i.e, ecc->bytes is 2638 * the sum of the three). 2639 */ 2640 static int qcom_nand_ooblayout_ecc(struct mtd_info *mtd, int section, 2641 struct mtd_oob_region *oobregion) 2642 { 2643 struct nand_chip *chip = mtd_to_nand(mtd); 2644 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2645 struct nand_ecc_ctrl *ecc = &chip->ecc; 2646 2647 if (section > 1) 2648 return -ERANGE; 2649 2650 if (!section) { 2651 oobregion->length = (ecc->bytes * (ecc->steps - 1)) + 2652 host->bbm_size; 2653 oobregion->offset = 0; 2654 } else { 2655 oobregion->length = host->ecc_bytes_hw + host->spare_bytes; 2656 oobregion->offset = mtd->oobsize - oobregion->length; 2657 } 2658 2659 return 0; 2660 } 2661 2662 static int qcom_nand_ooblayout_free(struct mtd_info *mtd, int section, 2663 struct mtd_oob_region *oobregion) 2664 { 2665 struct nand_chip *chip = mtd_to_nand(mtd); 2666 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2667 struct nand_ecc_ctrl *ecc = &chip->ecc; 2668 2669 if (section) 2670 return -ERANGE; 2671 2672 oobregion->length = ecc->steps * 4; 2673 oobregion->offset = ((ecc->steps - 1) * ecc->bytes) + host->bbm_size; 2674 2675 return 0; 2676 } 2677 2678 static const struct mtd_ooblayout_ops qcom_nand_ooblayout_ops = { 2679 .ecc = qcom_nand_ooblayout_ecc, 2680 .free = qcom_nand_ooblayout_free, 2681 }; 2682 2683 static int 2684 qcom_nandc_calc_ecc_bytes(int step_size, int strength) 2685 { 2686 return strength == 4 ? 12 : 16; 2687 } 2688 NAND_ECC_CAPS_SINGLE(qcom_nandc_ecc_caps, qcom_nandc_calc_ecc_bytes, 2689 NANDC_STEP_SIZE, 4, 8); 2690 2691 static int qcom_nand_attach_chip(struct nand_chip *chip) 2692 { 2693 struct mtd_info *mtd = nand_to_mtd(chip); 2694 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2695 struct nand_ecc_ctrl *ecc = &chip->ecc; 2696 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2697 int cwperpage, bad_block_byte, ret; 2698 bool wide_bus; 2699 int ecc_mode = 1; 2700 2701 /* controller only supports 512 bytes data steps */ 2702 ecc->size = NANDC_STEP_SIZE; 2703 wide_bus = chip->options & NAND_BUSWIDTH_16 ? true : false; 2704 cwperpage = mtd->writesize / NANDC_STEP_SIZE; 2705 2706 /* 2707 * Each CW has 4 available OOB bytes which will be protected with ECC 2708 * so remaining bytes can be used for ECC. 2709 */ 2710 ret = nand_ecc_choose_conf(chip, &qcom_nandc_ecc_caps, 2711 mtd->oobsize - (cwperpage * 4)); 2712 if (ret) { 2713 dev_err(nandc->dev, "No valid ECC settings possible\n"); 2714 return ret; 2715 } 2716 2717 if (ecc->strength >= 8) { 2718 /* 8 bit ECC defaults to BCH ECC on all platforms */ 2719 host->bch_enabled = true; 2720 ecc_mode = 1; 2721 2722 if (wide_bus) { 2723 host->ecc_bytes_hw = 14; 2724 host->spare_bytes = 0; 2725 host->bbm_size = 2; 2726 } else { 2727 host->ecc_bytes_hw = 13; 2728 host->spare_bytes = 2; 2729 host->bbm_size = 1; 2730 } 2731 } else { 2732 /* 2733 * if the controller supports BCH for 4 bit ECC, the controller 2734 * uses lesser bytes for ECC. If RS is used, the ECC bytes is 2735 * always 10 bytes 2736 */ 2737 if (nandc->props->ecc_modes & ECC_BCH_4BIT) { 2738 /* BCH */ 2739 host->bch_enabled = true; 2740 ecc_mode = 0; 2741 2742 if (wide_bus) { 2743 host->ecc_bytes_hw = 8; 2744 host->spare_bytes = 2; 2745 host->bbm_size = 2; 2746 } else { 2747 host->ecc_bytes_hw = 7; 2748 host->spare_bytes = 4; 2749 host->bbm_size = 1; 2750 } 2751 } else { 2752 /* RS */ 2753 host->ecc_bytes_hw = 10; 2754 2755 if (wide_bus) { 2756 host->spare_bytes = 0; 2757 host->bbm_size = 2; 2758 } else { 2759 host->spare_bytes = 1; 2760 host->bbm_size = 1; 2761 } 2762 } 2763 } 2764 2765 /* 2766 * we consider ecc->bytes as the sum of all the non-data content in a 2767 * step. It gives us a clean representation of the oob area (even if 2768 * all the bytes aren't used for ECC).It is always 16 bytes for 8 bit 2769 * ECC and 12 bytes for 4 bit ECC 2770 */ 2771 ecc->bytes = host->ecc_bytes_hw + host->spare_bytes + host->bbm_size; 2772 2773 ecc->read_page = qcom_nandc_read_page; 2774 ecc->read_page_raw = qcom_nandc_read_page_raw; 2775 ecc->read_oob = qcom_nandc_read_oob; 2776 ecc->write_page = qcom_nandc_write_page; 2777 ecc->write_page_raw = qcom_nandc_write_page_raw; 2778 ecc->write_oob = qcom_nandc_write_oob; 2779 2780 ecc->engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; 2781 2782 mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops); 2783 /* Free the initially allocated BAM transaction for reading the ONFI params */ 2784 if (nandc->props->is_bam) 2785 free_bam_transaction(nandc); 2786 2787 nandc->max_cwperpage = max_t(unsigned int, nandc->max_cwperpage, 2788 cwperpage); 2789 2790 /* Now allocate the BAM transaction based on updated max_cwperpage */ 2791 if (nandc->props->is_bam) { 2792 nandc->bam_txn = alloc_bam_transaction(nandc); 2793 if (!nandc->bam_txn) { 2794 dev_err(nandc->dev, 2795 "failed to allocate bam transaction\n"); 2796 return -ENOMEM; 2797 } 2798 } 2799 2800 /* 2801 * DATA_UD_BYTES varies based on whether the read/write command protects 2802 * spare data with ECC too. We protect spare data by default, so we set 2803 * it to main + spare data, which are 512 and 4 bytes respectively. 2804 */ 2805 host->cw_data = 516; 2806 2807 /* 2808 * total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes 2809 * for 8 bit ECC 2810 */ 2811 host->cw_size = host->cw_data + ecc->bytes; 2812 bad_block_byte = mtd->writesize - host->cw_size * (cwperpage - 1) + 1; 2813 2814 host->cfg0 = (cwperpage - 1) << CW_PER_PAGE 2815 | host->cw_data << UD_SIZE_BYTES 2816 | 0 << DISABLE_STATUS_AFTER_WRITE 2817 | 5 << NUM_ADDR_CYCLES 2818 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_RS 2819 | 0 << STATUS_BFR_READ 2820 | 1 << SET_RD_MODE_AFTER_STATUS 2821 | host->spare_bytes << SPARE_SIZE_BYTES; 2822 2823 host->cfg1 = 7 << NAND_RECOVERY_CYCLES 2824 | 0 << CS_ACTIVE_BSY 2825 | bad_block_byte << BAD_BLOCK_BYTE_NUM 2826 | 0 << BAD_BLOCK_IN_SPARE_AREA 2827 | 2 << WR_RD_BSY_GAP 2828 | wide_bus << WIDE_FLASH 2829 | host->bch_enabled << ENABLE_BCH_ECC; 2830 2831 host->cfg0_raw = (cwperpage - 1) << CW_PER_PAGE 2832 | host->cw_size << UD_SIZE_BYTES 2833 | 5 << NUM_ADDR_CYCLES 2834 | 0 << SPARE_SIZE_BYTES; 2835 2836 host->cfg1_raw = 7 << NAND_RECOVERY_CYCLES 2837 | 0 << CS_ACTIVE_BSY 2838 | 17 << BAD_BLOCK_BYTE_NUM 2839 | 1 << BAD_BLOCK_IN_SPARE_AREA 2840 | 2 << WR_RD_BSY_GAP 2841 | wide_bus << WIDE_FLASH 2842 | 1 << DEV0_CFG1_ECC_DISABLE; 2843 2844 host->ecc_bch_cfg = !host->bch_enabled << ECC_CFG_ECC_DISABLE 2845 | 0 << ECC_SW_RESET 2846 | host->cw_data << ECC_NUM_DATA_BYTES 2847 | 1 << ECC_FORCE_CLK_OPEN 2848 | ecc_mode << ECC_MODE 2849 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_BCH; 2850 2851 if (!nandc->props->qpic_v2) 2852 host->ecc_buf_cfg = 0x203 << NUM_STEPS; 2853 2854 host->clrflashstatus = FS_READY_BSY_N; 2855 host->clrreadstatus = 0xc0; 2856 nandc->regs->erased_cw_detect_cfg_clr = 2857 cpu_to_le32(CLR_ERASED_PAGE_DET); 2858 nandc->regs->erased_cw_detect_cfg_set = 2859 cpu_to_le32(SET_ERASED_PAGE_DET); 2860 2861 dev_dbg(nandc->dev, 2862 "cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n", 2863 host->cfg0, host->cfg1, host->ecc_buf_cfg, host->ecc_bch_cfg, 2864 host->cw_size, host->cw_data, ecc->strength, ecc->bytes, 2865 cwperpage); 2866 2867 return 0; 2868 } 2869 2870 static const struct nand_controller_ops qcom_nandc_ops = { 2871 .attach_chip = qcom_nand_attach_chip, 2872 }; 2873 2874 static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc) 2875 { 2876 if (nandc->props->is_bam) { 2877 if (!dma_mapping_error(nandc->dev, nandc->reg_read_dma)) 2878 dma_unmap_single(nandc->dev, nandc->reg_read_dma, 2879 MAX_REG_RD * 2880 sizeof(*nandc->reg_read_buf), 2881 DMA_FROM_DEVICE); 2882 2883 if (nandc->tx_chan) 2884 dma_release_channel(nandc->tx_chan); 2885 2886 if (nandc->rx_chan) 2887 dma_release_channel(nandc->rx_chan); 2888 2889 if (nandc->cmd_chan) 2890 dma_release_channel(nandc->cmd_chan); 2891 } else { 2892 if (nandc->chan) 2893 dma_release_channel(nandc->chan); 2894 } 2895 } 2896 2897 static int qcom_nandc_alloc(struct qcom_nand_controller *nandc) 2898 { 2899 int ret; 2900 2901 ret = dma_set_coherent_mask(nandc->dev, DMA_BIT_MASK(32)); 2902 if (ret) { 2903 dev_err(nandc->dev, "failed to set DMA mask\n"); 2904 return ret; 2905 } 2906 2907 /* 2908 * we use the internal buffer for reading ONFI params, reading small 2909 * data like ID and status, and preforming read-copy-write operations 2910 * when writing to a codeword partially. 532 is the maximum possible 2911 * size of a codeword for our nand controller 2912 */ 2913 nandc->buf_size = 532; 2914 2915 nandc->data_buffer = devm_kzalloc(nandc->dev, nandc->buf_size, 2916 GFP_KERNEL); 2917 if (!nandc->data_buffer) 2918 return -ENOMEM; 2919 2920 nandc->regs = devm_kzalloc(nandc->dev, sizeof(*nandc->regs), 2921 GFP_KERNEL); 2922 if (!nandc->regs) 2923 return -ENOMEM; 2924 2925 nandc->reg_read_buf = devm_kcalloc(nandc->dev, 2926 MAX_REG_RD, sizeof(*nandc->reg_read_buf), 2927 GFP_KERNEL); 2928 if (!nandc->reg_read_buf) 2929 return -ENOMEM; 2930 2931 if (nandc->props->is_bam) { 2932 nandc->reg_read_dma = 2933 dma_map_single(nandc->dev, nandc->reg_read_buf, 2934 MAX_REG_RD * 2935 sizeof(*nandc->reg_read_buf), 2936 DMA_FROM_DEVICE); 2937 if (dma_mapping_error(nandc->dev, nandc->reg_read_dma)) { 2938 dev_err(nandc->dev, "failed to DMA MAP reg buffer\n"); 2939 return -EIO; 2940 } 2941 2942 nandc->tx_chan = dma_request_chan(nandc->dev, "tx"); 2943 if (IS_ERR(nandc->tx_chan)) { 2944 ret = PTR_ERR(nandc->tx_chan); 2945 nandc->tx_chan = NULL; 2946 dev_err_probe(nandc->dev, ret, 2947 "tx DMA channel request failed\n"); 2948 goto unalloc; 2949 } 2950 2951 nandc->rx_chan = dma_request_chan(nandc->dev, "rx"); 2952 if (IS_ERR(nandc->rx_chan)) { 2953 ret = PTR_ERR(nandc->rx_chan); 2954 nandc->rx_chan = NULL; 2955 dev_err_probe(nandc->dev, ret, 2956 "rx DMA channel request failed\n"); 2957 goto unalloc; 2958 } 2959 2960 nandc->cmd_chan = dma_request_chan(nandc->dev, "cmd"); 2961 if (IS_ERR(nandc->cmd_chan)) { 2962 ret = PTR_ERR(nandc->cmd_chan); 2963 nandc->cmd_chan = NULL; 2964 dev_err_probe(nandc->dev, ret, 2965 "cmd DMA channel request failed\n"); 2966 goto unalloc; 2967 } 2968 2969 /* 2970 * Initially allocate BAM transaction to read ONFI param page. 2971 * After detecting all the devices, this BAM transaction will 2972 * be freed and the next BAM tranasction will be allocated with 2973 * maximum codeword size 2974 */ 2975 nandc->max_cwperpage = 1; 2976 nandc->bam_txn = alloc_bam_transaction(nandc); 2977 if (!nandc->bam_txn) { 2978 dev_err(nandc->dev, 2979 "failed to allocate bam transaction\n"); 2980 ret = -ENOMEM; 2981 goto unalloc; 2982 } 2983 } else { 2984 nandc->chan = dma_request_chan(nandc->dev, "rxtx"); 2985 if (IS_ERR(nandc->chan)) { 2986 ret = PTR_ERR(nandc->chan); 2987 nandc->chan = NULL; 2988 dev_err_probe(nandc->dev, ret, 2989 "rxtx DMA channel request failed\n"); 2990 return ret; 2991 } 2992 } 2993 2994 INIT_LIST_HEAD(&nandc->desc_list); 2995 INIT_LIST_HEAD(&nandc->host_list); 2996 2997 nand_controller_init(&nandc->controller); 2998 nandc->controller.ops = &qcom_nandc_ops; 2999 3000 return 0; 3001 unalloc: 3002 qcom_nandc_unalloc(nandc); 3003 return ret; 3004 } 3005 3006 /* one time setup of a few nand controller registers */ 3007 static int qcom_nandc_setup(struct qcom_nand_controller *nandc) 3008 { 3009 u32 nand_ctrl; 3010 3011 /* kill onenand */ 3012 if (!nandc->props->is_qpic) 3013 nandc_write(nandc, SFLASHC_BURST_CFG, 0); 3014 3015 if (!nandc->props->qpic_v2) 3016 nandc_write(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD), 3017 NAND_DEV_CMD_VLD_VAL); 3018 3019 /* enable ADM or BAM DMA */ 3020 if (nandc->props->is_bam) { 3021 nand_ctrl = nandc_read(nandc, NAND_CTRL); 3022 3023 /* 3024 *NAND_CTRL is an operational registers, and CPU 3025 * access to operational registers are read only 3026 * in BAM mode. So update the NAND_CTRL register 3027 * only if it is not in BAM mode. In most cases BAM 3028 * mode will be enabled in bootloader 3029 */ 3030 if (!(nand_ctrl & BAM_MODE_EN)) 3031 nandc_write(nandc, NAND_CTRL, nand_ctrl | BAM_MODE_EN); 3032 } else { 3033 nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN); 3034 } 3035 3036 /* save the original values of these registers */ 3037 if (!nandc->props->qpic_v2) { 3038 nandc->cmd1 = nandc_read(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD1)); 3039 nandc->vld = NAND_DEV_CMD_VLD_VAL; 3040 } 3041 3042 return 0; 3043 } 3044 3045 static const char * const probes[] = { "cmdlinepart", "ofpart", "qcomsmem", NULL }; 3046 3047 static int qcom_nand_host_parse_boot_partitions(struct qcom_nand_controller *nandc, 3048 struct qcom_nand_host *host, 3049 struct device_node *dn) 3050 { 3051 struct nand_chip *chip = &host->chip; 3052 struct mtd_info *mtd = nand_to_mtd(chip); 3053 struct qcom_nand_boot_partition *boot_partition; 3054 struct device *dev = nandc->dev; 3055 int partitions_count, i, j, ret; 3056 3057 if (!of_property_present(dn, "qcom,boot-partitions")) 3058 return 0; 3059 3060 partitions_count = of_property_count_u32_elems(dn, "qcom,boot-partitions"); 3061 if (partitions_count <= 0) { 3062 dev_err(dev, "Error parsing boot partition\n"); 3063 return partitions_count ? partitions_count : -EINVAL; 3064 } 3065 3066 host->nr_boot_partitions = partitions_count / 2; 3067 host->boot_partitions = devm_kcalloc(dev, host->nr_boot_partitions, 3068 sizeof(*host->boot_partitions), GFP_KERNEL); 3069 if (!host->boot_partitions) { 3070 host->nr_boot_partitions = 0; 3071 return -ENOMEM; 3072 } 3073 3074 for (i = 0, j = 0; i < host->nr_boot_partitions; i++, j += 2) { 3075 boot_partition = &host->boot_partitions[i]; 3076 3077 ret = of_property_read_u32_index(dn, "qcom,boot-partitions", j, 3078 &boot_partition->page_offset); 3079 if (ret) { 3080 dev_err(dev, "Error parsing boot partition offset at index %d\n", i); 3081 host->nr_boot_partitions = 0; 3082 return ret; 3083 } 3084 3085 if (boot_partition->page_offset % mtd->writesize) { 3086 dev_err(dev, "Boot partition offset not multiple of writesize at index %i\n", 3087 i); 3088 host->nr_boot_partitions = 0; 3089 return -EINVAL; 3090 } 3091 /* Convert offset to nand pages */ 3092 boot_partition->page_offset /= mtd->writesize; 3093 3094 ret = of_property_read_u32_index(dn, "qcom,boot-partitions", j + 1, 3095 &boot_partition->page_size); 3096 if (ret) { 3097 dev_err(dev, "Error parsing boot partition size at index %d\n", i); 3098 host->nr_boot_partitions = 0; 3099 return ret; 3100 } 3101 3102 if (boot_partition->page_size % mtd->writesize) { 3103 dev_err(dev, "Boot partition size not multiple of writesize at index %i\n", 3104 i); 3105 host->nr_boot_partitions = 0; 3106 return -EINVAL; 3107 } 3108 /* Convert size to nand pages */ 3109 boot_partition->page_size /= mtd->writesize; 3110 } 3111 3112 return 0; 3113 } 3114 3115 static int qcom_nand_host_init_and_register(struct qcom_nand_controller *nandc, 3116 struct qcom_nand_host *host, 3117 struct device_node *dn) 3118 { 3119 struct nand_chip *chip = &host->chip; 3120 struct mtd_info *mtd = nand_to_mtd(chip); 3121 struct device *dev = nandc->dev; 3122 int ret; 3123 3124 ret = of_property_read_u32(dn, "reg", &host->cs); 3125 if (ret) { 3126 dev_err(dev, "can't get chip-select\n"); 3127 return -ENXIO; 3128 } 3129 3130 nand_set_flash_node(chip, dn); 3131 mtd->name = devm_kasprintf(dev, GFP_KERNEL, "qcom_nand.%d", host->cs); 3132 if (!mtd->name) 3133 return -ENOMEM; 3134 3135 mtd->owner = THIS_MODULE; 3136 mtd->dev.parent = dev; 3137 3138 chip->legacy.cmdfunc = qcom_nandc_command; 3139 chip->legacy.select_chip = qcom_nandc_select_chip; 3140 chip->legacy.read_byte = qcom_nandc_read_byte; 3141 chip->legacy.read_buf = qcom_nandc_read_buf; 3142 chip->legacy.write_buf = qcom_nandc_write_buf; 3143 chip->legacy.set_features = nand_get_set_features_notsupp; 3144 chip->legacy.get_features = nand_get_set_features_notsupp; 3145 3146 /* 3147 * the bad block marker is readable only when we read the last codeword 3148 * of a page with ECC disabled. currently, the nand_base and nand_bbt 3149 * helpers don't allow us to read BB from a nand chip with ECC 3150 * disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad 3151 * and block_markbad helpers until we permanently switch to using 3152 * MTD_OPS_RAW for all drivers (with the help of badblockbits) 3153 */ 3154 chip->legacy.block_bad = qcom_nandc_block_bad; 3155 chip->legacy.block_markbad = qcom_nandc_block_markbad; 3156 3157 chip->controller = &nandc->controller; 3158 chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USES_DMA | 3159 NAND_SKIP_BBTSCAN; 3160 3161 /* set up initial status value */ 3162 host->status = NAND_STATUS_READY | NAND_STATUS_WP; 3163 3164 ret = nand_scan(chip, 1); 3165 if (ret) 3166 return ret; 3167 3168 ret = mtd_device_parse_register(mtd, probes, NULL, NULL, 0); 3169 if (ret) 3170 goto err; 3171 3172 if (nandc->props->use_codeword_fixup) { 3173 ret = qcom_nand_host_parse_boot_partitions(nandc, host, dn); 3174 if (ret) 3175 goto err; 3176 } 3177 3178 return 0; 3179 3180 err: 3181 nand_cleanup(chip); 3182 return ret; 3183 } 3184 3185 static int qcom_probe_nand_devices(struct qcom_nand_controller *nandc) 3186 { 3187 struct device *dev = nandc->dev; 3188 struct device_node *dn = dev->of_node, *child; 3189 struct qcom_nand_host *host; 3190 int ret = -ENODEV; 3191 3192 for_each_available_child_of_node(dn, child) { 3193 host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL); 3194 if (!host) { 3195 of_node_put(child); 3196 return -ENOMEM; 3197 } 3198 3199 ret = qcom_nand_host_init_and_register(nandc, host, child); 3200 if (ret) { 3201 devm_kfree(dev, host); 3202 continue; 3203 } 3204 3205 list_add_tail(&host->node, &nandc->host_list); 3206 } 3207 3208 return ret; 3209 } 3210 3211 /* parse custom DT properties here */ 3212 static int qcom_nandc_parse_dt(struct platform_device *pdev) 3213 { 3214 struct qcom_nand_controller *nandc = platform_get_drvdata(pdev); 3215 struct device_node *np = nandc->dev->of_node; 3216 int ret; 3217 3218 if (!nandc->props->is_bam) { 3219 ret = of_property_read_u32(np, "qcom,cmd-crci", 3220 &nandc->cmd_crci); 3221 if (ret) { 3222 dev_err(nandc->dev, "command CRCI unspecified\n"); 3223 return ret; 3224 } 3225 3226 ret = of_property_read_u32(np, "qcom,data-crci", 3227 &nandc->data_crci); 3228 if (ret) { 3229 dev_err(nandc->dev, "data CRCI unspecified\n"); 3230 return ret; 3231 } 3232 } 3233 3234 return 0; 3235 } 3236 3237 static int qcom_nandc_probe(struct platform_device *pdev) 3238 { 3239 struct qcom_nand_controller *nandc; 3240 const void *dev_data; 3241 struct device *dev = &pdev->dev; 3242 struct resource *res; 3243 int ret; 3244 3245 nandc = devm_kzalloc(&pdev->dev, sizeof(*nandc), GFP_KERNEL); 3246 if (!nandc) 3247 return -ENOMEM; 3248 3249 platform_set_drvdata(pdev, nandc); 3250 nandc->dev = dev; 3251 3252 dev_data = of_device_get_match_data(dev); 3253 if (!dev_data) { 3254 dev_err(&pdev->dev, "failed to get device data\n"); 3255 return -ENODEV; 3256 } 3257 3258 nandc->props = dev_data; 3259 3260 nandc->core_clk = devm_clk_get(dev, "core"); 3261 if (IS_ERR(nandc->core_clk)) 3262 return PTR_ERR(nandc->core_clk); 3263 3264 nandc->aon_clk = devm_clk_get(dev, "aon"); 3265 if (IS_ERR(nandc->aon_clk)) 3266 return PTR_ERR(nandc->aon_clk); 3267 3268 ret = qcom_nandc_parse_dt(pdev); 3269 if (ret) 3270 return ret; 3271 3272 nandc->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); 3273 if (IS_ERR(nandc->base)) 3274 return PTR_ERR(nandc->base); 3275 3276 nandc->base_phys = res->start; 3277 nandc->base_dma = dma_map_resource(dev, res->start, 3278 resource_size(res), 3279 DMA_BIDIRECTIONAL, 0); 3280 if (dma_mapping_error(dev, nandc->base_dma)) 3281 return -ENXIO; 3282 3283 ret = clk_prepare_enable(nandc->core_clk); 3284 if (ret) 3285 goto err_core_clk; 3286 3287 ret = clk_prepare_enable(nandc->aon_clk); 3288 if (ret) 3289 goto err_aon_clk; 3290 3291 ret = qcom_nandc_alloc(nandc); 3292 if (ret) 3293 goto err_nandc_alloc; 3294 3295 ret = qcom_nandc_setup(nandc); 3296 if (ret) 3297 goto err_setup; 3298 3299 ret = qcom_probe_nand_devices(nandc); 3300 if (ret) 3301 goto err_setup; 3302 3303 return 0; 3304 3305 err_setup: 3306 qcom_nandc_unalloc(nandc); 3307 err_nandc_alloc: 3308 clk_disable_unprepare(nandc->aon_clk); 3309 err_aon_clk: 3310 clk_disable_unprepare(nandc->core_clk); 3311 err_core_clk: 3312 dma_unmap_resource(dev, res->start, resource_size(res), 3313 DMA_BIDIRECTIONAL, 0); 3314 return ret; 3315 } 3316 3317 static void qcom_nandc_remove(struct platform_device *pdev) 3318 { 3319 struct qcom_nand_controller *nandc = platform_get_drvdata(pdev); 3320 struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 3321 struct qcom_nand_host *host; 3322 struct nand_chip *chip; 3323 int ret; 3324 3325 list_for_each_entry(host, &nandc->host_list, node) { 3326 chip = &host->chip; 3327 ret = mtd_device_unregister(nand_to_mtd(chip)); 3328 WARN_ON(ret); 3329 nand_cleanup(chip); 3330 } 3331 3332 qcom_nandc_unalloc(nandc); 3333 3334 clk_disable_unprepare(nandc->aon_clk); 3335 clk_disable_unprepare(nandc->core_clk); 3336 3337 dma_unmap_resource(&pdev->dev, nandc->base_dma, resource_size(res), 3338 DMA_BIDIRECTIONAL, 0); 3339 } 3340 3341 static const struct qcom_nandc_props ipq806x_nandc_props = { 3342 .ecc_modes = (ECC_RS_4BIT | ECC_BCH_8BIT), 3343 .is_bam = false, 3344 .use_codeword_fixup = true, 3345 .dev_cmd_reg_start = 0x0, 3346 }; 3347 3348 static const struct qcom_nandc_props ipq4019_nandc_props = { 3349 .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT), 3350 .is_bam = true, 3351 .is_qpic = true, 3352 .dev_cmd_reg_start = 0x0, 3353 }; 3354 3355 static const struct qcom_nandc_props ipq8074_nandc_props = { 3356 .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT), 3357 .is_bam = true, 3358 .is_qpic = true, 3359 .dev_cmd_reg_start = 0x7000, 3360 }; 3361 3362 static const struct qcom_nandc_props sdx55_nandc_props = { 3363 .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT), 3364 .is_bam = true, 3365 .is_qpic = true, 3366 .qpic_v2 = true, 3367 .dev_cmd_reg_start = 0x7000, 3368 }; 3369 3370 /* 3371 * data will hold a struct pointer containing more differences once we support 3372 * more controller variants 3373 */ 3374 static const struct of_device_id qcom_nandc_of_match[] = { 3375 { 3376 .compatible = "qcom,ipq806x-nand", 3377 .data = &ipq806x_nandc_props, 3378 }, 3379 { 3380 .compatible = "qcom,ipq4019-nand", 3381 .data = &ipq4019_nandc_props, 3382 }, 3383 { 3384 .compatible = "qcom,ipq6018-nand", 3385 .data = &ipq8074_nandc_props, 3386 }, 3387 { 3388 .compatible = "qcom,ipq8074-nand", 3389 .data = &ipq8074_nandc_props, 3390 }, 3391 { 3392 .compatible = "qcom,sdx55-nand", 3393 .data = &sdx55_nandc_props, 3394 }, 3395 {} 3396 }; 3397 MODULE_DEVICE_TABLE(of, qcom_nandc_of_match); 3398 3399 static struct platform_driver qcom_nandc_driver = { 3400 .driver = { 3401 .name = "qcom-nandc", 3402 .of_match_table = qcom_nandc_of_match, 3403 }, 3404 .probe = qcom_nandc_probe, 3405 .remove_new = qcom_nandc_remove, 3406 }; 3407 module_platform_driver(qcom_nandc_driver); 3408 3409 MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>"); 3410 MODULE_DESCRIPTION("Qualcomm NAND Controller driver"); 3411 MODULE_LICENSE("GPL v2"); 3412