1 // SPDX-License-Identifier: GPL-2.0-only
23 #include <linux/dma-mapping.h>
92 static int timing[3];  variable
93 module_param_array(timing, int, &numtimings, 0644);
101 #define cafe_readl(cafe, addr)			readl((cafe)->mmio + CAFE_##addr)
102 #define cafe_writel(cafe, datum, addr)		writel(datum, (cafe)->mmio + CAFE_##addr)
112 	cafe_dev_dbg(&cafe->pdev->dev, "NAND device is%s ready, IRQ %x (%x) (%x,%x)\n",  in cafe_device_ready()
124 	if (cafe->usedma)  in cafe_write_buf()
125 		memcpy(cafe->dmabuf + cafe->datalen, buf, len);  in cafe_write_buf()
127 		memcpy_toio(cafe->mmio + CAFE_NAND_WRITE_DATA + cafe->datalen, buf, len);  in cafe_write_buf()
129 	cafe->datalen += len;  in cafe_write_buf()
131 	cafe_dev_dbg(&cafe->pdev->dev, "Copy 0x%x bytes to write buffer. datalen 0x%x\n",  in cafe_write_buf()
132 		len, cafe->datalen);  in cafe_write_buf()
139 	if (cafe->usedma)  in cafe_read_buf()
140 		memcpy(buf, cafe->dmabuf + cafe->datalen, len);  in cafe_read_buf()
142 		memcpy_fromio(buf, cafe->mmio + CAFE_NAND_READ_DATA + cafe->datalen, len);  in cafe_read_buf()
144 	cafe_dev_dbg(&cafe->pdev->dev, "Copy 0x%x bytes from position 0x%x in read buffer.\n",  in cafe_read_buf()
145 		  len, cafe->datalen);  in cafe_read_buf()
146 	cafe->datalen += len;  in cafe_read_buf()
155 	cafe_dev_dbg(&cafe->pdev->dev, "Read %02x\n", d);  in cafe_read_byte()
169 	cafe_dev_dbg(&cafe->pdev->dev, "cmdfunc %02x, 0x%x, 0x%x\n",  in cafe_nand_cmdfunc()
174 		cafe_writel(cafe, cafe->ctl2 | 0x100 | command, NAND_CTRL2);  in cafe_nand_cmdfunc()
175 		ctl1 = cafe->ctl1;  in cafe_nand_cmdfunc()
176 		cafe->ctl2 &= ~(1<<30);  in cafe_nand_cmdfunc()
177 		cafe_dev_dbg(&cafe->pdev->dev, "Continue command, ctl1 %08x, #data %d\n",  in cafe_nand_cmdfunc()
178 			  cafe->ctl1, cafe->nr_data);  in cafe_nand_cmdfunc()
184 	/* Emulate NAND_CMD_READOOB on large-page chips */  in cafe_nand_cmdfunc()
185 	if (mtd->writesize > 512 &&  in cafe_nand_cmdfunc()
187 		column += mtd->writesize;  in cafe_nand_cmdfunc()
192 	   for small-page chips, to position the buffer correctly? */  in cafe_nand_cmdfunc()
194 	if (column != -1) {  in cafe_nand_cmdfunc()
197 		if (page_addr != -1)  in cafe_nand_cmdfunc()
199 	} else if (page_addr != -1) {  in cafe_nand_cmdfunc()
205 		if (mtd->size > mtd->writesize << 16)  in cafe_nand_cmdfunc()
209 	cafe->data_pos = cafe->datalen = 0;  in cafe_nand_cmdfunc()
212 	ctl1 = 0x80000000 | command | (cafe->ctl1 & CTRL1_CHIPSELECT);  in cafe_nand_cmdfunc()
218 		cafe->datalen = 4;  in cafe_nand_cmdfunc()
219 		/* And one address cycle -- even for STATUS, since the controller doesn't work without */  in cafe_nand_cmdfunc()
225 		cafe->datalen = mtd->writesize + mtd->oobsize - column;  in cafe_nand_cmdfunc()
231 		ctl1 |= ((adrbytes-1)|8) << 27;  in cafe_nand_cmdfunc()
236 		cafe->ctl1 = ctl1;  in cafe_nand_cmdfunc()
237 		cafe_dev_dbg(&cafe->pdev->dev, "Setup for delayed command, ctl1 %08x, dlen %x\n",  in cafe_nand_cmdfunc()
238 			  cafe->ctl1, cafe->datalen);  in cafe_nand_cmdfunc()
243 		cafe_writel(cafe, cafe->ctl2 | 0x100 | NAND_CMD_RNDOUTSTART, NAND_CTRL2);  in cafe_nand_cmdfunc()
244 	else if (command == NAND_CMD_READ0 && mtd->writesize > 512)  in cafe_nand_cmdfunc()
245 		cafe_writel(cafe, cafe->ctl2 | 0x100 | NAND_CMD_READSTART, NAND_CTRL2);  in cafe_nand_cmdfunc()
248 	cafe_dev_dbg(&cafe->pdev->dev, "dlen %x, ctl1 %x, ctl2 %x\n",  in cafe_nand_cmdfunc()
249 		cafe->datalen, ctl1, cafe_readl(cafe, NAND_CTRL2));  in cafe_nand_cmdfunc()
251 	/* NB: The datasheet lies -- we really should be subtracting 1 here */  in cafe_nand_cmdfunc()
252 	cafe_writel(cafe, cafe->datalen, NAND_DATA_LEN);  in cafe_nand_cmdfunc()
254 	if (cafe->usedma && (ctl1 & (3<<25))) {  in cafe_nand_cmdfunc()
255 		uint32_t dmactl = 0xc0000000 + cafe->datalen;  in cafe_nand_cmdfunc()
266 	cafe->datalen = 0;  in cafe_nand_cmdfunc()
272 			printk("Register %x: %08x\n", i, readl(cafe->mmio + i));  in cafe_nand_cmdfunc()
284 		for (c = 500000; c != 0; c--) {  in cafe_nand_cmdfunc()
290 				cafe_dev_dbg(&cafe->pdev->dev, "Wait for ready, IRQ %x\n", irqs);  in cafe_nand_cmdfunc()
294 		cafe_dev_dbg(&cafe->pdev->dev, "Command %x completed after %d usec, irqs %x (%x)\n",  in cafe_nand_cmdfunc()
295 			     command, 500000-c, irqs, cafe_readl(cafe, NAND_IRQ));  in cafe_nand_cmdfunc()
298 	WARN_ON(cafe->ctl2 & (1<<30));  in cafe_nand_cmdfunc()
310 		cafe_writel(cafe, cafe->ctl2, NAND_CTRL2);  in cafe_nand_cmdfunc()
314 	cafe_writel(cafe, cafe->ctl2, NAND_CTRL2);  in cafe_nand_cmdfunc()
321 	cafe_dev_dbg(&cafe->pdev->dev, "select_chip %d\n", chipnr);  in cafe_select_chip()
326 		cafe->ctl1 |= CTRL1_CHIPSELECT;  in cafe_select_chip()
328 		cafe->ctl1 &= ~CTRL1_CHIPSELECT;  in cafe_select_chip()
341 	cafe_dev_dbg(&cafe->pdev->dev, "irq, bits %x (%x)\n", irqs, cafe_readl(cafe, NAND_IRQ));  in cafe_nand_interrupt()
349 	return nand_prog_page_op(chip, page, mtd->writesize, chip->oob_poi,  in cafe_nand_write_oob()
350 				 mtd->oobsize);  in cafe_nand_write_oob()
353 /* Don't use -- use nand_read_oob_std for now */
358 	return nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize);  in cafe_nand_read_oob()
361  * cafe_nand_read_page - [REPLACEABLE] hardware ecc syndrome based page read
364  * @oob_required:	caller expects OOB data read to chip->oob_poi
367  * The hw generator calculates the error syndrome automatically. Therefore
377 	cafe_dev_dbg(&cafe->pdev->dev, "ECC result %08x SYN1,2 %08x\n",  in cafe_nand_read_page()
381 	nand_read_page_op(chip, page, 0, buf, mtd->writesize);  in cafe_nand_read_page()
382 	chip->legacy.read_buf(chip, chip->oob_poi, mtd->oobsize);  in cafe_nand_read_page()
387 		u8 *oob = chip->oob_poi;  in cafe_nand_read_page()
393 			syn[i] = cafe->rs->codec->index_of[tmp & 0xfff];  in cafe_nand_read_page()
394 			syn[i+1] = cafe->rs->codec->index_of[(tmp >> 16) & 0xfff];  in cafe_nand_read_page()
397 		n = decode_rs16(cafe->rs, NULL, NULL, 1367, syn, 0, pos, 0,  in cafe_nand_read_page()
403 			/* The 12-bit symbols are mapped to bytes here */  in cafe_nand_read_page()
407 				n = -1374;  in cafe_nand_read_page()
411 					n = -2048;  in cafe_nand_read_page()
419 					oob[3*p/2 - 2048] ^= pat[i] >> 4;  in cafe_nand_read_page()
420 					oob[3*p/2 - 2047] ^= pat[i] << 4;  in cafe_nand_read_page()
422 					oob[3*p/2 - 2049] ^= pat[i] >> 8;  in cafe_nand_read_page()
423 					oob[3*p/2 - 2048] ^= pat[i];  in cafe_nand_read_page()
429 				buf[3*p/2 - 1] ^= pat[i] >> 8;  in cafe_nand_read_page()
435 			dev_dbg(&cafe->pdev->dev, "Failed to correct ECC at %08x\n",  in cafe_nand_read_page()
438 				printk("Register %x: %08x\n", i, readl(cafe->mmio + i));  in cafe_nand_read_page()
439 			mtd->ecc_stats.failed++;  in cafe_nand_read_page()
441 			dev_dbg(&cafe->pdev->dev, "Corrected %d symbol errors\n", n);  in cafe_nand_read_page()
442 			mtd->ecc_stats.corrected += n;  in cafe_nand_read_page()
456 		return -ERANGE;  in cafe_ooblayout_ecc()
458 	oobregion->offset = 0;  in cafe_ooblayout_ecc()
459 	oobregion->length = chip->ecc.total;  in cafe_ooblayout_ecc()
470 		return -ERANGE;  in cafe_ooblayout_free()
472 	oobregion->offset = chip->ecc.total;  in cafe_ooblayout_free()
473 	oobregion->length = mtd->oobsize - chip->ecc.total;  in cafe_ooblayout_free()
540 	nand_prog_page_begin_op(chip, page, 0, buf, mtd->writesize);  in cafe_nand_write_page_lowlevel()
541 	chip->legacy.write_buf(chip, chip->oob_poi, mtd->oobsize);  in cafe_nand_write_page_lowlevel()
544 	cafe->ctl2 |= (1<<30);  in cafe_nand_write_page_lowlevel()
568 /* F_64[X]/(X**2+X+A**-1) with A the generator of F_64[X]  */
597 	cafe->dmabuf = dma_alloc_coherent(&cafe->pdev->dev, 2112,  in cafe_nand_attach_chip()
598 					  &cafe->dmaaddr, GFP_KERNEL);  in cafe_nand_attach_chip()
599 	if (!cafe->dmabuf)  in cafe_nand_attach_chip()
600 		return -ENOMEM;  in cafe_nand_attach_chip()
603 	cafe_writel(cafe, lower_32_bits(cafe->dmaaddr), NAND_DMA_ADDR0);  in cafe_nand_attach_chip()
604 	cafe_writel(cafe, upper_32_bits(cafe->dmaaddr), NAND_DMA_ADDR1);  in cafe_nand_attach_chip()
606 	cafe_dev_dbg(&cafe->pdev->dev, "Set DMA address to %x (virt %p)\n",  in cafe_nand_attach_chip()
607 		     cafe_readl(cafe, NAND_DMA_ADDR0), cafe->dmabuf);  in cafe_nand_attach_chip()
610 	cafe->usedma = usedma;  in cafe_nand_attach_chip()
612 	cafe->ctl2 = BIT(27); /* Reed-Solomon ECC */  in cafe_nand_attach_chip()
613 	if (mtd->writesize == 2048)  in cafe_nand_attach_chip()
614 		cafe->ctl2 |= BIT(29); /* 2KiB page size */  in cafe_nand_attach_chip()
618 	if (mtd->writesize == 2048) {  in cafe_nand_attach_chip()
619 		cafe->nand.bbt_td = &cafe_bbt_main_descr_2048;  in cafe_nand_attach_chip()
620 		cafe->nand.bbt_md = &cafe_bbt_mirror_descr_2048;  in cafe_nand_attach_chip()
621 	} else if (mtd->writesize == 512) {  in cafe_nand_attach_chip()
622 		cafe->nand.bbt_td = &cafe_bbt_main_descr_512;  in cafe_nand_attach_chip()
623 		cafe->nand.bbt_md = &cafe_bbt_mirror_descr_512;  in cafe_nand_attach_chip()
625 		dev_warn(&cafe->pdev->dev,  in cafe_nand_attach_chip()
627 			 mtd->writesize);  in cafe_nand_attach_chip()
628 		err = -ENOTSUPP;  in cafe_nand_attach_chip()
632 	cafe->nand.ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;  in cafe_nand_attach_chip()
633 	cafe->nand.ecc.placement = NAND_ECC_PLACEMENT_INTERLEAVED;  in cafe_nand_attach_chip()
634 	cafe->nand.ecc.size = mtd->writesize;  in cafe_nand_attach_chip()
635 	cafe->nand.ecc.bytes = 14;  in cafe_nand_attach_chip()
636 	cafe->nand.ecc.strength = 4;  in cafe_nand_attach_chip()
637 	cafe->nand.ecc.write_page = cafe_nand_write_page_lowlevel;  in cafe_nand_attach_chip()
638 	cafe->nand.ecc.write_oob = cafe_nand_write_oob;  in cafe_nand_attach_chip()
639 	cafe->nand.ecc.read_page = cafe_nand_read_page;  in cafe_nand_attach_chip()
640 	cafe->nand.ecc.read_oob = cafe_nand_read_oob;  in cafe_nand_attach_chip()
645 	dma_free_coherent(&cafe->pdev->dev, 2112, cafe->dmabuf, cafe->dmaaddr);  in cafe_nand_attach_chip()
654 	dma_free_coherent(&cafe->pdev->dev, 2112, cafe->dmabuf, cafe->dmaaddr);  in cafe_nand_detach_chip()
672 	if ((pdev->class >> 8) != PCI_CLASS_MEMORY_FLASH)  in cafe_nand_probe()
673 		return -ENODEV;  in cafe_nand_probe()
683 		err = -ENOMEM;  in cafe_nand_probe()
687 	mtd = nand_to_mtd(&cafe->nand);  in cafe_nand_probe()
688 	mtd->dev.parent = &pdev->dev;  in cafe_nand_probe()
689 	nand_set_controller_data(&cafe->nand, cafe);  in cafe_nand_probe()
691 	cafe->pdev = pdev;  in cafe_nand_probe()
692 	cafe->mmio = pci_iomap(pdev, 0, 0);  in cafe_nand_probe()
693 	if (!cafe->mmio) {  in cafe_nand_probe()
694 		dev_warn(&pdev->dev, "failed to iomap\n");  in cafe_nand_probe()
695 		err = -ENOMEM;  in cafe_nand_probe()
699 	cafe->rs = init_rs_non_canonical(12, &cafe_mul, 0, 1, 8);  in cafe_nand_probe()
700 	if (!cafe->rs) {  in cafe_nand_probe()
701 		err = -ENOMEM;  in cafe_nand_probe()
705 	cafe->nand.legacy.cmdfunc = cafe_nand_cmdfunc;  in cafe_nand_probe()
706 	cafe->nand.legacy.dev_ready = cafe_device_ready;  in cafe_nand_probe()
707 	cafe->nand.legacy.read_byte = cafe_read_byte;  in cafe_nand_probe()
708 	cafe->nand.legacy.read_buf = cafe_read_buf;  in cafe_nand_probe()
709 	cafe->nand.legacy.write_buf = cafe_write_buf;  in cafe_nand_probe()
710 	cafe->nand.legacy.select_chip = cafe_select_chip;  in cafe_nand_probe()
711 	cafe->nand.legacy.set_features = nand_get_set_features_notsupp;  in cafe_nand_probe()
712 	cafe->nand.legacy.get_features = nand_get_set_features_notsupp;  in cafe_nand_probe()
714 	cafe->nand.legacy.chip_delay = 0;  in cafe_nand_probe()
717 	cafe->nand.bbt_options = NAND_BBT_USE_FLASH;  in cafe_nand_probe()
720 		cafe->nand.options |= NAND_SKIP_BBTSCAN | NAND_NO_BBM_QUIRK;  in cafe_nand_probe()
723 …dev_warn(&cafe->pdev->dev, "%d timing register values ignored; precisely three are required\n", nu…  in cafe_nand_probe()
727 		cafe_dev_dbg(&cafe->pdev->dev, "Using provided timings (%08x %08x %08x)\n",  in cafe_nand_probe()
728 			     timing[0], timing[1], timing[2]);  in cafe_nand_probe()
730 		timing[0] = cafe_readl(cafe, NAND_TIMING1);  in cafe_nand_probe()
731 		timing[1] = cafe_readl(cafe, NAND_TIMING2);  in cafe_nand_probe()
732 		timing[2] = cafe_readl(cafe, NAND_TIMING3);  in cafe_nand_probe()
734 		if (timing[0] | timing[1] | timing[2]) {  in cafe_nand_probe()
735 			cafe_dev_dbg(&cafe->pdev->dev, "Timing registers already set (%08x %08x %08x)\n",  in cafe_nand_probe()
736 				     timing[0], timing[1], timing[2]);  in cafe_nand_probe()
738 			dev_warn(&cafe->pdev->dev, "Timing registers unset; using most conservative defaults\n");  in cafe_nand_probe()
739 			timing[0] = timing[1] = timing[2] = 0xffffffff;  in cafe_nand_probe()
747 	cafe_writel(cafe, timing[0], NAND_TIMING1);  in cafe_nand_probe()
748 	cafe_writel(cafe, timing[1], NAND_TIMING2);  in cafe_nand_probe()
749 	cafe_writel(cafe, timing[2], NAND_TIMING3);  in cafe_nand_probe()
752 	err = request_irq(pdev->irq, &cafe_nand_interrupt, IRQF_SHARED,  in cafe_nand_probe()
755 		dev_warn(&pdev->dev, "Could not register IRQ %d\n", pdev->irq);  in cafe_nand_probe()
759 	/* Disable master reset, enable NAND clock */  in cafe_nand_probe()
772 	cafe_dev_dbg(&cafe->pdev->dev, "Control %x, IRQ mask %x\n",  in cafe_nand_probe()
777 	cafe->usedma = 0;  in cafe_nand_probe()
780 	cafe->nand.legacy.dummy_controller.ops = &cafe_nand_controller_ops;  in cafe_nand_probe()
781 	err = nand_scan(&cafe->nand, 2);  in cafe_nand_probe()
787 	mtd->name = "cafe_nand";  in cafe_nand_probe()
795 	nand_cleanup(&cafe->nand);  in cafe_nand_probe()
797 	/* Disable NAND IRQ in global IRQ mask register */  in cafe_nand_probe()
799 	free_irq(pdev->irq, mtd);  in cafe_nand_probe()
801 	free_rs(cafe->rs);  in cafe_nand_probe()
803 	pci_iounmap(pdev, cafe->mmio);  in cafe_nand_probe()
819 	/* Disable NAND IRQ in global IRQ mask register */  in cafe_nand_remove()
821 	free_irq(pdev->irq, mtd);  in cafe_nand_remove()
825 	free_rs(cafe->rs);  in cafe_nand_remove()
826 	pci_iounmap(pdev, cafe->mmio);  in cafe_nand_remove()
827 	dma_free_coherent(&cafe->pdev->dev, 2112, cafe->dmabuf, cafe->dmaaddr);  in cafe_nand_remove()
852 	/* Restore timing configuration */  in cafe_nand_resume()
853 	cafe_writel(cafe, timing[0], NAND_TIMING1);  in cafe_nand_resume()
854 	cafe_writel(cafe, timing[1], NAND_TIMING2);  in cafe_nand_resume()
855 	cafe_writel(cafe, timing[2], NAND_TIMING3);  in cafe_nand_resume()
857         /* Disable master reset, enable NAND clock */  in cafe_nand_resume()
868 	cafe_writel(cafe, cafe->dmaaddr & 0xffffffff, NAND_DMA_ADDR0);  in cafe_nand_resume()
869 	if (sizeof(cafe->dmaaddr) > 4)  in cafe_nand_resume()
871 		cafe_writel(cafe, (cafe->dmaaddr >> 16) >> 16, NAND_DMA_ADDR1);  in cafe_nand_resume()