101 #define cafe_readl(cafe, addr)			readl((cafe)->mmio + CAFE_##addr)  argument
102 #define cafe_writel(cafe, datum, addr)		writel(datum, (cafe)->mmio + CAFE_##addr)  argument
106 	struct cafe_priv *cafe = nand_get_controller_data(chip);  in cafe_device_ready()  local
107 	int result = !!(cafe_readl(cafe, NAND_STATUS) & 0x40000000);  in cafe_device_ready()
108 	uint32_t irqs = cafe_readl(cafe, NAND_IRQ);  in cafe_device_ready()
110 	cafe_writel(cafe, irqs, NAND_IRQ);  in cafe_device_ready()
112 	cafe_dev_dbg(&cafe->pdev->dev, "NAND device is%s ready, IRQ %x (%x) (%x,%x)\n",  in cafe_device_ready()
113 		result?"":" not", irqs, cafe_readl(cafe, NAND_IRQ),  in cafe_device_ready()
114 		cafe_readl(cafe, GLOBAL_IRQ), cafe_readl(cafe, GLOBAL_IRQ_MASK));  in cafe_device_ready()
122 	struct cafe_priv *cafe = nand_get_controller_data(chip);  in cafe_write_buf()  local
124 	if (cafe->usedma)  in cafe_write_buf()
125 		memcpy(cafe->dmabuf + cafe->datalen, buf, len);  in cafe_write_buf()
127 		memcpy_toio(cafe->mmio + CAFE_NAND_WRITE_DATA + cafe->datalen, buf, len);  in cafe_write_buf()
129 	cafe->datalen += len;  in cafe_write_buf()
131 	cafe_dev_dbg(&cafe->pdev->dev, "Copy 0x%x bytes to write buffer. datalen 0x%x\n",  in cafe_write_buf()
132 		len, cafe->datalen);  in cafe_write_buf()
137 	struct cafe_priv *cafe = nand_get_controller_data(chip);  in cafe_read_buf()  local
139 	if (cafe->usedma)  in cafe_read_buf()
140 		memcpy(buf, cafe->dmabuf + cafe->datalen, len);  in cafe_read_buf()
142 		memcpy_fromio(buf, cafe->mmio + CAFE_NAND_READ_DATA + cafe->datalen, len);  in cafe_read_buf()
144 	cafe_dev_dbg(&cafe->pdev->dev, "Copy 0x%x bytes from position 0x%x in read buffer.\n",  in cafe_read_buf()
145 		  len, cafe->datalen);  in cafe_read_buf()
146 	cafe->datalen += len;  in cafe_read_buf()
151 	struct cafe_priv *cafe = nand_get_controller_data(chip);  in cafe_read_byte()  local
155 	cafe_dev_dbg(&cafe->pdev->dev, "Read %02x\n", d);  in cafe_read_byte()
164 	struct cafe_priv *cafe = nand_get_controller_data(chip);  in cafe_nand_cmdfunc()  local
169 	cafe_dev_dbg(&cafe->pdev->dev, "cmdfunc %02x, 0x%x, 0x%x\n",  in cafe_nand_cmdfunc()
174 		cafe_writel(cafe, cafe->ctl2 | 0x100 | command, NAND_CTRL2);  in cafe_nand_cmdfunc()
175 		ctl1 = cafe->ctl1;  in cafe_nand_cmdfunc()
176 		cafe->ctl2 &= ~(1<<30);  in cafe_nand_cmdfunc()
177 		cafe_dev_dbg(&cafe->pdev->dev, "Continue command, ctl1 %08x, #data %d\n",  in cafe_nand_cmdfunc()
178 			  cafe->ctl1, cafe->nr_data);  in cafe_nand_cmdfunc()
182 	cafe_writel(cafe, 0, NAND_CTRL2);  in cafe_nand_cmdfunc()
195 		cafe_writel(cafe, column, NAND_ADDR1);  in cafe_nand_cmdfunc()
200 		cafe_writel(cafe, page_addr & 0xffff, NAND_ADDR1);  in cafe_nand_cmdfunc()
203 		cafe_writel(cafe, page_addr, NAND_ADDR2);  in cafe_nand_cmdfunc()
209 	cafe->data_pos = cafe->datalen = 0;  in cafe_nand_cmdfunc()
212 	ctl1 = 0x80000000 | command | (cafe->ctl1 & CTRL1_CHIPSELECT);  in cafe_nand_cmdfunc()
218 		cafe->datalen = 4;  in cafe_nand_cmdfunc()
225 		cafe->datalen = mtd->writesize + mtd->oobsize - column;  in cafe_nand_cmdfunc()
236 		cafe->ctl1 = ctl1;  in cafe_nand_cmdfunc()
237 		cafe_dev_dbg(&cafe->pdev->dev, "Setup for delayed command, ctl1 %08x, dlen %x\n",  in cafe_nand_cmdfunc()
238 			  cafe->ctl1, cafe->datalen);  in cafe_nand_cmdfunc()
243 		cafe_writel(cafe, cafe->ctl2 | 0x100 | NAND_CMD_RNDOUTSTART, NAND_CTRL2);  in cafe_nand_cmdfunc()
245 		cafe_writel(cafe, cafe->ctl2 | 0x100 | NAND_CMD_READSTART, NAND_CTRL2);  in cafe_nand_cmdfunc()
248 	cafe_dev_dbg(&cafe->pdev->dev, "dlen %x, ctl1 %x, ctl2 %x\n",  in cafe_nand_cmdfunc()
249 		cafe->datalen, ctl1, cafe_readl(cafe, NAND_CTRL2));  in cafe_nand_cmdfunc()
252 	cafe_writel(cafe, cafe->datalen, NAND_DATA_LEN);  in cafe_nand_cmdfunc()
253 	cafe_writel(cafe, 0x90000000, NAND_IRQ);  in cafe_nand_cmdfunc()
254 	if (cafe->usedma && (ctl1 & (3<<25))) {  in cafe_nand_cmdfunc()
255 		uint32_t dmactl = 0xc0000000 + cafe->datalen;  in cafe_nand_cmdfunc()
264 		cafe_writel(cafe, dmactl, NAND_DMA_CTRL);  in cafe_nand_cmdfunc()
266 	cafe->datalen = 0;  in cafe_nand_cmdfunc()
272 			printk("Register %x: %08x\n", i, readl(cafe->mmio + i));  in cafe_nand_cmdfunc()
275 	cafe_writel(cafe, ctl1, NAND_CTRL1);  in cafe_nand_cmdfunc()
285 			irqs = cafe_readl(cafe, NAND_IRQ);  in cafe_nand_cmdfunc()
290 				cafe_dev_dbg(&cafe->pdev->dev, "Wait for ready, IRQ %x\n", irqs);  in cafe_nand_cmdfunc()
293 		cafe_writel(cafe, doneint, NAND_IRQ);  in cafe_nand_cmdfunc()
294 		cafe_dev_dbg(&cafe->pdev->dev, "Command %x completed after %d usec, irqs %x (%x)\n",  in cafe_nand_cmdfunc()
295 			     command, 500000-c, irqs, cafe_readl(cafe, NAND_IRQ));  in cafe_nand_cmdfunc()
298 	WARN_ON(cafe->ctl2 & (1<<30));  in cafe_nand_cmdfunc()
310 		cafe_writel(cafe, cafe->ctl2, NAND_CTRL2);  in cafe_nand_cmdfunc()
314 	cafe_writel(cafe, cafe->ctl2, NAND_CTRL2);  in cafe_nand_cmdfunc()
319 	struct cafe_priv *cafe = nand_get_controller_data(chip);  in cafe_select_chip()  local
321 	cafe_dev_dbg(&cafe->pdev->dev, "select_chip %d\n", chipnr);  in cafe_select_chip()
326 		cafe->ctl1 |= CTRL1_CHIPSELECT;  in cafe_select_chip()
328 		cafe->ctl1 &= ~CTRL1_CHIPSELECT;  in cafe_select_chip()
335 	struct cafe_priv *cafe = nand_get_controller_data(chip);  in cafe_nand_interrupt()  local
336 	uint32_t irqs = cafe_readl(cafe, NAND_IRQ);  in cafe_nand_interrupt()
337 	cafe_writel(cafe, irqs & ~0x90000000, NAND_IRQ);  in cafe_nand_interrupt()
341 	cafe_dev_dbg(&cafe->pdev->dev, "irq, bits %x (%x)\n", irqs, cafe_readl(cafe, NAND_IRQ));  in cafe_nand_interrupt()
374 	struct cafe_priv *cafe = nand_get_controller_data(chip);  in cafe_nand_read_page()  local
377 	cafe_dev_dbg(&cafe->pdev->dev, "ECC result %08x SYN1,2 %08x\n",  in cafe_nand_read_page()
378 		     cafe_readl(cafe, NAND_ECC_RESULT),  in cafe_nand_read_page()
379 		     cafe_readl(cafe, NAND_ECC_SYN01));  in cafe_nand_read_page()
384 	if (checkecc && cafe_readl(cafe, NAND_ECC_RESULT) & (1<<18)) {  in cafe_nand_read_page()
391 			uint32_t tmp = cafe_readl(cafe, NAND_ECC_SYN01 + (i*2));  in cafe_nand_read_page()
393 			syn[i] = cafe->rs->codec->index_of[tmp & 0xfff];  in cafe_nand_read_page()
394 			syn[i+1] = cafe->rs->codec->index_of[(tmp >> 16) & 0xfff];  in cafe_nand_read_page()
397 		n = decode_rs16(cafe->rs, NULL, NULL, 1367, syn, 0, pos, 0,  in cafe_nand_read_page()
435 			dev_dbg(&cafe->pdev->dev, "Failed to correct ECC at %08x\n",  in cafe_nand_read_page()
436 				cafe_readl(cafe, NAND_ADDR2) * 2048);  in cafe_nand_read_page()
438 				printk("Register %x: %08x\n", i, readl(cafe->mmio + i));  in cafe_nand_read_page()
441 			dev_dbg(&cafe->pdev->dev, "Corrected %d symbol errors\n", n);  in cafe_nand_read_page()
538 	struct cafe_priv *cafe = nand_get_controller_data(chip);  in cafe_nand_write_page_lowlevel()  local
544 	cafe->ctl2 |= (1<<30);  in cafe_nand_write_page_lowlevel()
594 	struct cafe_priv *cafe = nand_get_controller_data(chip);  in cafe_nand_attach_chip()  local
597 	cafe->dmabuf = dma_alloc_coherent(&cafe->pdev->dev, 2112,  in cafe_nand_attach_chip()
598 					  &cafe->dmaaddr, GFP_KERNEL);  in cafe_nand_attach_chip()
599 	if (!cafe->dmabuf)  in cafe_nand_attach_chip()
603 	cafe_writel(cafe, lower_32_bits(cafe->dmaaddr), NAND_DMA_ADDR0);  in cafe_nand_attach_chip()
604 	cafe_writel(cafe, upper_32_bits(cafe->dmaaddr), NAND_DMA_ADDR1);  in cafe_nand_attach_chip()
606 	cafe_dev_dbg(&cafe->pdev->dev, "Set DMA address to %x (virt %p)\n",  in cafe_nand_attach_chip()
607 		     cafe_readl(cafe, NAND_DMA_ADDR0), cafe->dmabuf);  in cafe_nand_attach_chip()
610 	cafe->usedma = usedma;  in cafe_nand_attach_chip()
612 	cafe->ctl2 = BIT(27); /* Reed-Solomon ECC */  in cafe_nand_attach_chip()
614 		cafe->ctl2 |= BIT(29); /* 2KiB page size */  in cafe_nand_attach_chip()
619 		cafe->nand.bbt_td = &cafe_bbt_main_descr_2048;  in cafe_nand_attach_chip()
620 		cafe->nand.bbt_md = &cafe_bbt_mirror_descr_2048;  in cafe_nand_attach_chip()
622 		cafe->nand.bbt_td = &cafe_bbt_main_descr_512;  in cafe_nand_attach_chip()
623 		cafe->nand.bbt_md = &cafe_bbt_mirror_descr_512;  in cafe_nand_attach_chip()
625 		dev_warn(&cafe->pdev->dev,  in cafe_nand_attach_chip()
632 	cafe->nand.ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;  in cafe_nand_attach_chip()
633 	cafe->nand.ecc.placement = NAND_ECC_PLACEMENT_INTERLEAVED;  in cafe_nand_attach_chip()
634 	cafe->nand.ecc.size = mtd->writesize;  in cafe_nand_attach_chip()
635 	cafe->nand.ecc.bytes = 14;  in cafe_nand_attach_chip()
636 	cafe->nand.ecc.strength = 4;  in cafe_nand_attach_chip()
637 	cafe->nand.ecc.write_page = cafe_nand_write_page_lowlevel;  in cafe_nand_attach_chip()
638 	cafe->nand.ecc.write_oob = cafe_nand_write_oob;  in cafe_nand_attach_chip()
639 	cafe->nand.ecc.read_page = cafe_nand_read_page;  in cafe_nand_attach_chip()
640 	cafe->nand.ecc.read_oob = cafe_nand_read_oob;  in cafe_nand_attach_chip()
645 	dma_free_coherent(&cafe->pdev->dev, 2112, cafe->dmabuf, cafe->dmaaddr);  in cafe_nand_attach_chip()
652 	struct cafe_priv *cafe = nand_get_controller_data(chip);  in cafe_nand_detach_chip()  local
654 	dma_free_coherent(&cafe->pdev->dev, 2112, cafe->dmabuf, cafe->dmaaddr);  in cafe_nand_detach_chip()
666 	struct cafe_priv *cafe;  in cafe_nand_probe()  local
681 	cafe = kzalloc(sizeof(*cafe), GFP_KERNEL);  in cafe_nand_probe()
682 	if (!cafe) {  in cafe_nand_probe()
687 	mtd = nand_to_mtd(&cafe->nand);  in cafe_nand_probe()
689 	nand_set_controller_data(&cafe->nand, cafe);  in cafe_nand_probe()
691 	cafe->pdev = pdev;  in cafe_nand_probe()
692 	cafe->mmio = pci_iomap(pdev, 0, 0);  in cafe_nand_probe()
693 	if (!cafe->mmio) {  in cafe_nand_probe()
699 	cafe->rs = init_rs_non_canonical(12, &cafe_mul, 0, 1, 8);  in cafe_nand_probe()
700 	if (!cafe->rs) {  in cafe_nand_probe()
705 	cafe->nand.legacy.cmdfunc = cafe_nand_cmdfunc;  in cafe_nand_probe()
706 	cafe->nand.legacy.dev_ready = cafe_device_ready;  in cafe_nand_probe()
707 	cafe->nand.legacy.read_byte = cafe_read_byte;  in cafe_nand_probe()
708 	cafe->nand.legacy.read_buf = cafe_read_buf;  in cafe_nand_probe()
709 	cafe->nand.legacy.write_buf = cafe_write_buf;  in cafe_nand_probe()
710 	cafe->nand.legacy.select_chip = cafe_select_chip;  in cafe_nand_probe()
711 	cafe->nand.legacy.set_features = nand_get_set_features_notsupp;  in cafe_nand_probe()
712 	cafe->nand.legacy.get_features = nand_get_set_features_notsupp;  in cafe_nand_probe()
714 	cafe->nand.legacy.chip_delay = 0;  in cafe_nand_probe()
717 	cafe->nand.bbt_options = NAND_BBT_USE_FLASH;  in cafe_nand_probe()
720 		cafe->nand.options |= NAND_SKIP_BBTSCAN | NAND_NO_BBM_QUIRK;  in cafe_nand_probe()
723 …dev_warn(&cafe->pdev->dev, "%d timing register values ignored; precisely three are required\n", nu…  in cafe_nand_probe()
727 		cafe_dev_dbg(&cafe->pdev->dev, "Using provided timings (%08x %08x %08x)\n",  in cafe_nand_probe()
730 		timing[0] = cafe_readl(cafe, NAND_TIMING1);  in cafe_nand_probe()
731 		timing[1] = cafe_readl(cafe, NAND_TIMING2);  in cafe_nand_probe()
732 		timing[2] = cafe_readl(cafe, NAND_TIMING3);  in cafe_nand_probe()
735 			cafe_dev_dbg(&cafe->pdev->dev, "Timing registers already set (%08x %08x %08x)\n",  in cafe_nand_probe()
738 			dev_warn(&cafe->pdev->dev, "Timing registers unset; using most conservative defaults\n");  in cafe_nand_probe()
744 	cafe_writel(cafe, 1, NAND_RESET);  in cafe_nand_probe()
745 	cafe_writel(cafe, 0, NAND_RESET);  in cafe_nand_probe()
747 	cafe_writel(cafe, timing[0], NAND_TIMING1);  in cafe_nand_probe()
748 	cafe_writel(cafe, timing[1], NAND_TIMING2);  in cafe_nand_probe()
749 	cafe_writel(cafe, timing[2], NAND_TIMING3);  in cafe_nand_probe()
751 	cafe_writel(cafe, 0xffffffff, NAND_IRQ_MASK);  in cafe_nand_probe()
760 	ctrl = cafe_readl(cafe, GLOBAL_CTRL);  in cafe_nand_probe()
763 	cafe_writel(cafe, ctrl | 0x05, GLOBAL_CTRL);  in cafe_nand_probe()
764 	cafe_writel(cafe, ctrl | 0x0a, GLOBAL_CTRL);  in cafe_nand_probe()
765 	cafe_writel(cafe, 0, NAND_DMA_CTRL);  in cafe_nand_probe()
767 	cafe_writel(cafe, 0x7006, GLOBAL_CTRL);  in cafe_nand_probe()
768 	cafe_writel(cafe, 0x700a, GLOBAL_CTRL);  in cafe_nand_probe()
771 	cafe_writel(cafe, 0x80000007, GLOBAL_IRQ_MASK);  in cafe_nand_probe()
772 	cafe_dev_dbg(&cafe->pdev->dev, "Control %x, IRQ mask %x\n",  in cafe_nand_probe()
773 		cafe_readl(cafe, GLOBAL_CTRL),  in cafe_nand_probe()
774 		cafe_readl(cafe, GLOBAL_IRQ_MASK));  in cafe_nand_probe()
777 	cafe->usedma = 0;  in cafe_nand_probe()
780 	cafe->nand.legacy.dummy_controller.ops = &cafe_nand_controller_ops;  in cafe_nand_probe()
781 	err = nand_scan(&cafe->nand, 2);  in cafe_nand_probe()
795 	nand_cleanup(&cafe->nand);  in cafe_nand_probe()
798 	cafe_writel(cafe, ~1 & cafe_readl(cafe, GLOBAL_IRQ_MASK), GLOBAL_IRQ_MASK);  in cafe_nand_probe()
801 	free_rs(cafe->rs);  in cafe_nand_probe()
803 	pci_iounmap(pdev, cafe->mmio);  in cafe_nand_probe()
805 	kfree(cafe);  in cafe_nand_probe()
816 	struct cafe_priv *cafe = nand_get_controller_data(chip);  in cafe_nand_remove()  local
820 	cafe_writel(cafe, ~1 & cafe_readl(cafe, GLOBAL_IRQ_MASK), GLOBAL_IRQ_MASK);  in cafe_nand_remove()
825 	free_rs(cafe->rs);  in cafe_nand_remove()
826 	pci_iounmap(pdev, cafe->mmio);  in cafe_nand_remove()
827 	dma_free_coherent(&cafe->pdev->dev, 2112, cafe->dmabuf, cafe->dmaaddr);  in cafe_nand_remove()
828 	kfree(cafe);  in cafe_nand_remove()
845 	struct cafe_priv *cafe = nand_get_controller_data(chip);  in cafe_nand_resume()  local
848 	cafe_writel(cafe, 1, NAND_RESET);  in cafe_nand_resume()
849 	cafe_writel(cafe, 0, NAND_RESET);  in cafe_nand_resume()
850 	cafe_writel(cafe, 0xffffffff, NAND_IRQ_MASK);  in cafe_nand_resume()
853 	cafe_writel(cafe, timing[0], NAND_TIMING1);  in cafe_nand_resume()
854 	cafe_writel(cafe, timing[1], NAND_TIMING2);  in cafe_nand_resume()
855 	cafe_writel(cafe, timing[2], NAND_TIMING3);  in cafe_nand_resume()
858 	ctrl = cafe_readl(cafe, GLOBAL_CTRL);  in cafe_nand_resume()
861 	cafe_writel(cafe, ctrl | 0x05, GLOBAL_CTRL);  in cafe_nand_resume()
862 	cafe_writel(cafe, ctrl | 0x0a, GLOBAL_CTRL);  in cafe_nand_resume()
863 	cafe_writel(cafe, 0, NAND_DMA_CTRL);  in cafe_nand_resume()
864 	cafe_writel(cafe, 0x7006, GLOBAL_CTRL);  in cafe_nand_resume()
865 	cafe_writel(cafe, 0x700a, GLOBAL_CTRL);  in cafe_nand_resume()
868 	cafe_writel(cafe, cafe->dmaaddr & 0xffffffff, NAND_DMA_ADDR0);  in cafe_nand_resume()
869 	if (sizeof(cafe->dmaaddr) > 4)  in cafe_nand_resume()
871 		cafe_writel(cafe, (cafe->dmaaddr >> 16) >> 16, NAND_DMA_ADDR1);  in cafe_nand_resume()
873 		cafe_writel(cafe, 0, NAND_DMA_ADDR1);  in cafe_nand_resume()
876 	cafe_writel(cafe, 0x80000007, GLOBAL_IRQ_MASK);  in cafe_nand_resume()