427 	struct pdc_state *pdcs;  in pdc_debugfs_read()  local
437 	pdcs = filp->private_data;  in pdc_debugfs_read()
440 			       "SPU %u stats:\n", pdcs->pdc_idx);  in pdc_debugfs_read()
443 			       pdcs->pdc_requests);  in pdc_debugfs_read()
446 			       pdcs->pdc_replies);  in pdc_debugfs_read()
449 			       pdcs->last_tx_not_done);  in pdc_debugfs_read()
452 			       pdcs->tx_ring_full);  in pdc_debugfs_read()
455 			       pdcs->rx_ring_full);  in pdc_debugfs_read()
458 			       pdcs->txnobuf);  in pdc_debugfs_read()
461 			       pdcs->rxnobuf);  in pdc_debugfs_read()
464 			       pdcs->rx_oflow);  in pdc_debugfs_read()
467 			       NRXDACTIVE(pdcs->rxin, pdcs->last_rx_curr,  in pdc_debugfs_read()
468 					  pdcs->nrxpost));  in pdc_debugfs_read()
490 static void pdc_setup_debugfs(struct pdc_state *pdcs)  in pdc_setup_debugfs()  argument
497 	snprintf(spu_stats_name, 16, "pdc%d_stats", pdcs->pdc_idx);  in pdc_setup_debugfs()
502 	debugfs_create_file(spu_stats_name, 0400, debugfs_dir, pdcs,  in pdc_setup_debugfs()
520 pdc_build_rxd(struct pdc_state *pdcs, dma_addr_t dma_addr,  in pdc_build_rxd()  argument
523 	struct device *dev = &pdcs->pdev->dev;  in pdc_build_rxd()
524 	struct dma64dd *rxd = &pdcs->rxd_64[pdcs->rxout];  in pdc_build_rxd()
528 		pdcs->pdc_idx, pdcs->rxout, buf_len, flags);  in pdc_build_rxd()
536 	pdcs->rxout = NEXTRXD(pdcs->rxout, pdcs->nrxpost);  in pdc_build_rxd()
548 pdc_build_txd(struct pdc_state *pdcs, dma_addr_t dma_addr, u32 buf_len,  in pdc_build_txd()  argument
551 	struct device *dev = &pdcs->pdev->dev;  in pdc_build_txd()
552 	struct dma64dd *txd = &pdcs->txd_64[pdcs->txout];  in pdc_build_txd()
556 		pdcs->pdc_idx, pdcs->txout, buf_len, flags);  in pdc_build_txd()
564 	pdcs->txout = NEXTTXD(pdcs->txout, pdcs->ntxpost);  in pdc_build_txd()
579 pdc_receive_one(struct pdc_state *pdcs)  in pdc_receive_one()  argument
581 	struct device *dev = &pdcs->pdev->dev;  in pdc_receive_one()
593 	mbc = &pdcs->mbc;  in pdc_receive_one()
602 	frags_rdy = NRXDACTIVE(pdcs->rxin, pdcs->last_rx_curr, pdcs->nrxpost);  in pdc_receive_one()
604 	    (frags_rdy < pdcs->rx_ctx[pdcs->rxin].rxin_numd))  in pdc_receive_one()
608 	num_frags = pdcs->txin_numd[pdcs->txin];  in pdc_receive_one()
611 	dma_unmap_sg(dev, pdcs->src_sg[pdcs->txin],  in pdc_receive_one()
612 		     sg_nents(pdcs->src_sg[pdcs->txin]), DMA_TO_DEVICE);  in pdc_receive_one()
614 	pdcs->txin = (pdcs->txin + num_frags) & pdcs->ntxpost;  in pdc_receive_one()
617 		pdcs->pdc_idx, num_frags);  in pdc_receive_one()
619 	rx_idx = pdcs->rxin;  in pdc_receive_one()
620 	rx_ctx = &pdcs->rx_ctx[rx_idx];  in pdc_receive_one()
630 	pdcs->rxin = (pdcs->rxin + num_frags) & pdcs->nrxpost;  in pdc_receive_one()
633 		pdcs->pdc_idx, num_frags);  in pdc_receive_one()
637 		pdcs->pdc_idx, pdcs->txin, pdcs->txout, pdcs->rxin,  in pdc_receive_one()
638 		pdcs->rxout, pdcs->last_rx_curr);  in pdc_receive_one()
640 	if (pdcs->pdc_resp_hdr_len == PDC_SPUM_RESP_HDR_LEN) {  in pdc_receive_one()
652 				pdcs->rx_oflow++;  in pdc_receive_one()
660 	dma_pool_free(pdcs->rx_buf_pool, resp_hdr, resp_hdr_daddr);  in pdc_receive_one()
664 	pdcs->pdc_replies++;  in pdc_receive_one()
676 pdc_receive(struct pdc_state *pdcs)  in pdc_receive()  argument
681 	pdcs->last_rx_curr =  in pdc_receive()
682 	    (ioread32((const void __iomem *)&pdcs->rxregs_64->status0) &  in pdc_receive()
687 		rx_status = pdc_receive_one(pdcs);  in pdc_receive()
706 static int pdc_tx_list_sg_add(struct pdc_state *pdcs, struct scatterlist *sg)  in pdc_tx_list_sg_add()  argument
724 	tx_avail = pdcs->ntxpost - NTXDACTIVE(pdcs->txin, pdcs->txout,  in pdc_tx_list_sg_add()
725 					      pdcs->ntxpost);  in pdc_tx_list_sg_add()
727 		pdcs->txnobuf++;  in pdc_tx_list_sg_add()
732 	if (pdcs->tx_msg_start == pdcs->txout) {  in pdc_tx_list_sg_add()
734 		pdcs->txin_numd[pdcs->tx_msg_start] = 0;  in pdc_tx_list_sg_add()
735 		pdcs->src_sg[pdcs->txout] = sg;  in pdc_tx_list_sg_add()
740 		if (unlikely(pdcs->txout == (pdcs->ntxd - 1)))  in pdc_tx_list_sg_add()
752 			pdc_build_txd(pdcs, databufptr, PDC_DMA_BUF_MAX,  in pdc_tx_list_sg_add()
757 			if (unlikely(pdcs->txout == (pdcs->ntxd - 1)))  in pdc_tx_list_sg_add()
766 		pdc_build_txd(pdcs, databufptr, bufcnt, flags | eot);  in pdc_tx_list_sg_add()
771 	pdcs->txin_numd[pdcs->tx_msg_start] += desc_w;  in pdc_tx_list_sg_add()
785 static int pdc_tx_list_final(struct pdc_state *pdcs)  in pdc_tx_list_final()  argument
792 	iowrite32(pdcs->rxout << 4, &pdcs->rxregs_64->ptr);  in pdc_tx_list_final()
793 	iowrite32(pdcs->txout << 4, &pdcs->txregs_64->ptr);  in pdc_tx_list_final()
794 	pdcs->pdc_requests++;  in pdc_tx_list_final()
814 static int pdc_rx_list_init(struct pdc_state *pdcs, struct scatterlist *dst_sg,  in pdc_rx_list_init()  argument
824 	rx_avail = pdcs->nrxpost - NRXDACTIVE(pdcs->rxin, pdcs->rxout,  in pdc_rx_list_init()
825 					      pdcs->nrxpost);  in pdc_rx_list_init()
827 		pdcs->rxnobuf++;  in pdc_rx_list_init()
832 	vaddr = dma_pool_zalloc(pdcs->rx_buf_pool, GFP_ATOMIC, &daddr);  in pdc_rx_list_init()
841 	pdcs->rx_msg_start = pdcs->rxout;  in pdc_rx_list_init()
842 	pdcs->tx_msg_start = pdcs->txout;  in pdc_rx_list_init()
846 	pdcs->rx_ctx[pdcs->rx_msg_start].rxin_numd = 1;  in pdc_rx_list_init()
848 	if (unlikely(pdcs->rxout == (pdcs->nrxd - 1)))  in pdc_rx_list_init()
851 	rx_ctx = &pdcs->rx_ctx[pdcs->rxout];  in pdc_rx_list_init()
856 	pdc_build_rxd(pdcs, daddr, pdcs->pdc_resp_hdr_len, flags);  in pdc_rx_list_init()
874 static int pdc_rx_list_sg_add(struct pdc_state *pdcs, struct scatterlist *sg)  in pdc_rx_list_sg_add()  argument
890 	rx_avail = pdcs->nrxpost - NRXDACTIVE(pdcs->rxin, pdcs->rxout,  in pdc_rx_list_sg_add()
891 					      pdcs->nrxpost);  in pdc_rx_list_sg_add()
893 		pdcs->rxnobuf++;  in pdc_rx_list_sg_add()
898 		if (unlikely(pdcs->rxout == (pdcs->nrxd - 1)))  in pdc_rx_list_sg_add()
910 			pdc_build_rxd(pdcs, databufptr, PDC_DMA_BUF_MAX, flags);  in pdc_rx_list_sg_add()
914 			if (unlikely(pdcs->rxout == (pdcs->nrxd - 1)))  in pdc_rx_list_sg_add()
919 		pdc_build_rxd(pdcs, databufptr, bufcnt, flags);  in pdc_rx_list_sg_add()
923 	pdcs->rx_ctx[pdcs->rx_msg_start].rxin_numd += desc_w;  in pdc_rx_list_sg_add()
943 	struct pdc_state *pdcs = dev_get_drvdata(dev);  in pdc_irq_handler()  local
944 	u32 intstatus = ioread32(pdcs->pdc_reg_vbase + PDC_INTSTATUS_OFFSET);  in pdc_irq_handler()
950 	iowrite32(0, pdcs->pdc_reg_vbase + PDC_INTMASK_OFFSET);  in pdc_irq_handler()
953 	iowrite32(intstatus, pdcs->pdc_reg_vbase + PDC_INTSTATUS_OFFSET);  in pdc_irq_handler()
956 	tasklet_schedule(&pdcs->rx_tasklet);  in pdc_irq_handler()
967 	struct pdc_state *pdcs = from_tasklet(pdcs, t, rx_tasklet);  in pdc_tasklet_cb()  local
969 	pdc_receive(pdcs);  in pdc_tasklet_cb()
972 	iowrite32(PDC_INTMASK, pdcs->pdc_reg_vbase + PDC_INTMASK_OFFSET);  in pdc_tasklet_cb()
984 static int pdc_ring_init(struct pdc_state *pdcs, int ringset)  in pdc_ring_init()  argument
989 	struct device *dev = &pdcs->pdev->dev;  in pdc_ring_init()
994 	tx.vbase = dma_pool_zalloc(pdcs->ring_pool, GFP_KERNEL, &tx.dmabase);  in pdc_ring_init()
1001 	rx.vbase = dma_pool_zalloc(pdcs->ring_pool, GFP_KERNEL, &rx.dmabase);  in pdc_ring_init()
1012 	memcpy(&pdcs->tx_ring_alloc, &tx, sizeof(tx));  in pdc_ring_init()
1013 	memcpy(&pdcs->rx_ring_alloc, &rx, sizeof(rx));  in pdc_ring_init()
1015 	pdcs->rxin = 0;  in pdc_ring_init()
1016 	pdcs->rx_msg_start = 0;  in pdc_ring_init()
1017 	pdcs->last_rx_curr = 0;  in pdc_ring_init()
1018 	pdcs->rxout = 0;  in pdc_ring_init()
1019 	pdcs->txin = 0;  in pdc_ring_init()
1020 	pdcs->tx_msg_start = 0;  in pdc_ring_init()
1021 	pdcs->txout = 0;  in pdc_ring_init()
1024 	pdcs->txd_64 = (struct dma64dd *)pdcs->tx_ring_alloc.vbase;  in pdc_ring_init()
1025 	pdcs->rxd_64 = (struct dma64dd *)pdcs->rx_ring_alloc.vbase;  in pdc_ring_init()
1028 	dma_reg = &pdcs->regs->dmaregs[ringset];  in pdc_ring_init()
1032 	iowrite32((PDC_RX_CTL + (pdcs->rx_status_len << 1)),  in pdc_ring_init()
1038 	iowrite32(lower_32_bits(pdcs->tx_ring_alloc.dmabase),  in pdc_ring_init()
1040 	iowrite32(upper_32_bits(pdcs->tx_ring_alloc.dmabase),  in pdc_ring_init()
1043 	iowrite32(lower_32_bits(pdcs->rx_ring_alloc.dmabase),  in pdc_ring_init()
1045 	iowrite32(upper_32_bits(pdcs->rx_ring_alloc.dmabase),  in pdc_ring_init()
1050 	iowrite32((PDC_RX_CTL | PDC_RX_ENABLE | (pdcs->rx_status_len << 1)),  in pdc_ring_init()
1056 		if (i != pdcs->ntxpost) {  in pdc_ring_init()
1058 				  &pdcs->txd_64[i].ctrl1);  in pdc_ring_init()
1062 				  D64_CTRL1_EOT, &pdcs->txd_64[i].ctrl1);  in pdc_ring_init()
1066 		if (i != pdcs->nrxpost) {  in pdc_ring_init()
1068 				  &pdcs->rxd_64[i].ctrl1);  in pdc_ring_init()
1072 				  &pdcs->rxd_64[i].ctrl1);  in pdc_ring_init()
1078 	dma_pool_free(pdcs->ring_pool, tx.vbase, tx.dmabase);  in pdc_ring_init()
1083 static void pdc_ring_free(struct pdc_state *pdcs)  in pdc_ring_free()  argument
1085 	if (pdcs->tx_ring_alloc.vbase) {  in pdc_ring_free()
1086 		dma_pool_free(pdcs->ring_pool, pdcs->tx_ring_alloc.vbase,  in pdc_ring_free()
1087 			      pdcs->tx_ring_alloc.dmabase);  in pdc_ring_free()
1088 		pdcs->tx_ring_alloc.vbase = NULL;  in pdc_ring_free()
1091 	if (pdcs->rx_ring_alloc.vbase) {  in pdc_ring_free()
1092 		dma_pool_free(pdcs->ring_pool, pdcs->rx_ring_alloc.vbase,  in pdc_ring_free()
1093 			      pdcs->rx_ring_alloc.dmabase);  in pdc_ring_free()
1094 		pdcs->rx_ring_alloc.vbase = NULL;  in pdc_ring_free()
1125 static bool pdc_rings_full(struct pdc_state *pdcs, int tx_cnt, int rx_cnt)  in pdc_rings_full()  argument
1132 	rx_avail = pdcs->nrxpost - NRXDACTIVE(pdcs->rxin, pdcs->rxout,  in pdc_rings_full()
1133 					      pdcs->nrxpost);  in pdc_rings_full()
1135 		pdcs->rx_ring_full++;  in pdc_rings_full()
1140 		tx_avail = pdcs->ntxpost - NTXDACTIVE(pdcs->txin, pdcs->txout,  in pdc_rings_full()
1141 						      pdcs->ntxpost);  in pdc_rings_full()
1143 			pdcs->tx_ring_full++;  in pdc_rings_full()
1159 	struct pdc_state *pdcs = chan->con_priv;  in pdc_last_tx_done()  local
1162 	if (unlikely(pdc_rings_full(pdcs, PDC_RING_SPACE_MIN,  in pdc_last_tx_done()
1164 		pdcs->last_tx_not_done++;  in pdc_last_tx_done()
1196 	struct pdc_state *pdcs = chan->con_priv;  in pdc_send_data()  local
1197 	struct device *dev = &pdcs->pdev->dev;  in pdc_send_data()
1238 	if (unlikely(pdc_rings_full(pdcs, tx_desc_req, rx_desc_req + 1)))  in pdc_send_data()
1242 	err = pdc_rx_list_init(pdcs, mssg->spu.dst, mssg->ctx);  in pdc_send_data()
1243 	err |= pdc_rx_list_sg_add(pdcs, mssg->spu.dst);  in pdc_send_data()
1246 	err |= pdc_tx_list_sg_add(pdcs, mssg->spu.src);  in pdc_send_data()
1247 	err |= pdc_tx_list_final(pdcs);	/* initiate transfer */  in pdc_send_data()
1250 		dev_err(&pdcs->pdev->dev,  in pdc_send_data()
1263 	struct pdc_state *pdcs = chan->con_priv;  in pdc_shutdown()  local
1265 	if (!pdcs)  in pdc_shutdown()
1268 	dev_dbg(&pdcs->pdev->dev,  in pdc_shutdown()
1269 		"Shutdown mailbox channel for PDC %u", pdcs->pdc_idx);  in pdc_shutdown()
1270 	pdc_ring_free(pdcs);  in pdc_shutdown()
1279 void pdc_hw_init(struct pdc_state *pdcs)  in pdc_hw_init()  argument
1286 	pdev = pdcs->pdev;  in pdc_hw_init()
1289 	dev_dbg(dev, "PDC %u initial values:", pdcs->pdc_idx);  in pdc_hw_init()
1291 		pdcs);  in pdc_hw_init()
1293 		pdcs->pdc_reg_vbase);  in pdc_hw_init()
1296 	pdcs->regs = (struct pdc_regs *)pdcs->pdc_reg_vbase;  in pdc_hw_init()
1297 	pdcs->txregs_64 = (struct dma64_regs *)  in pdc_hw_init()
1298 	    (((u8 *)pdcs->pdc_reg_vbase) +  in pdc_hw_init()
1300 	pdcs->rxregs_64 = (struct dma64_regs *)  in pdc_hw_init()
1301 	    (((u8 *)pdcs->pdc_reg_vbase) +  in pdc_hw_init()
1304 	pdcs->ntxd = PDC_RING_ENTRIES;  in pdc_hw_init()
1305 	pdcs->nrxd = PDC_RING_ENTRIES;  in pdc_hw_init()
1306 	pdcs->ntxpost = PDC_RING_ENTRIES - 1;  in pdc_hw_init()
1307 	pdcs->nrxpost = PDC_RING_ENTRIES - 1;  in pdc_hw_init()
1308 	iowrite32(0, &pdcs->regs->intmask);  in pdc_hw_init()
1310 	dma_reg = &pdcs->regs->dmaregs[ringset];  in pdc_hw_init()
1315 	iowrite32(PDC_RX_CTL + (pdcs->rx_status_len << 1),  in pdc_hw_init()
1322 	if (pdcs->pdc_resp_hdr_len == PDC_SPU2_RESP_HDR_LEN)  in pdc_hw_init()
1324 			  pdcs->pdc_reg_vbase + PDC_CKSUM_CTRL_OFFSET);  in pdc_hw_init()
1332 static void pdc_hw_disable(struct pdc_state *pdcs)  in pdc_hw_disable()  argument
1336 	dma_reg = &pdcs->regs->dmaregs[PDC_RINGSET];  in pdc_hw_disable()
1338 	iowrite32(PDC_RX_CTL + (pdcs->rx_status_len << 1),  in pdc_hw_disable()
1353 static int pdc_rx_buf_pool_create(struct pdc_state *pdcs)  in pdc_rx_buf_pool_create()  argument
1358 	pdev = pdcs->pdev;  in pdc_rx_buf_pool_create()
1361 	pdcs->pdc_resp_hdr_len = pdcs->rx_status_len;  in pdc_rx_buf_pool_create()
1362 	if (pdcs->use_bcm_hdr)  in pdc_rx_buf_pool_create()
1363 		pdcs->pdc_resp_hdr_len += BCM_HDR_LEN;  in pdc_rx_buf_pool_create()
1365 	pdcs->rx_buf_pool = dma_pool_create("pdc rx bufs", dev,  in pdc_rx_buf_pool_create()
1366 					    pdcs->pdc_resp_hdr_len,  in pdc_rx_buf_pool_create()
1368 	if (!pdcs->rx_buf_pool)  in pdc_rx_buf_pool_create()
1386 static int pdc_interrupts_init(struct pdc_state *pdcs)  in pdc_interrupts_init()  argument
1388 	struct platform_device *pdev = pdcs->pdev;  in pdc_interrupts_init()
1394 	iowrite32(PDC_INTMASK, pdcs->pdc_reg_vbase + PDC_INTMASK_OFFSET);  in pdc_interrupts_init()
1396 	if (pdcs->hw_type == FA_HW)  in pdc_interrupts_init()
1397 		iowrite32(PDC_LAZY_INT, pdcs->pdc_reg_vbase +  in pdc_interrupts_init()
1400 		iowrite32(PDC_LAZY_INT, pdcs->pdc_reg_vbase +  in pdc_interrupts_init()
1404 	pdcs->pdc_irq = irq_of_parse_and_map(dn, 0);  in pdc_interrupts_init()
1406 		dev_name(dev), pdcs->pdc_irq, pdcs);  in pdc_interrupts_init()
1408 	err = devm_request_irq(dev, pdcs->pdc_irq, pdc_irq_handler, 0,  in pdc_interrupts_init()
1412 			pdcs->pdc_irq, err);  in pdc_interrupts_init()
1437 static int pdc_mb_init(struct pdc_state *pdcs)  in pdc_mb_init()  argument
1439 	struct device *dev = &pdcs->pdev->dev;  in pdc_mb_init()
1444 	mbc = &pdcs->mbc;  in pdc_mb_init()
1457 		mbc->chans[chan_index].con_priv = pdcs;  in pdc_mb_init()
1493 static int pdc_dt_read(struct platform_device *pdev, struct pdc_state *pdcs)  in pdc_dt_read()  argument
1502 				   &pdcs->rx_status_len);  in pdc_dt_read()
1508 	pdcs->use_bcm_hdr = of_property_read_bool(dn, "brcm,use-bcm-hdr");  in pdc_dt_read()
1510 	pdcs->hw_type = PDC_HW;  in pdc_dt_read()
1515 		pdcs->hw_type = *hw_type;  in pdc_dt_read()
1537 	struct pdc_state *pdcs;  in pdc_probe()  local
1540 	pdcs = devm_kzalloc(dev, sizeof(*pdcs), GFP_KERNEL);  in pdc_probe()
1541 	if (!pdcs) {  in pdc_probe()
1546 	pdcs->pdev = pdev;  in pdc_probe()
1547 	platform_set_drvdata(pdev, pdcs);  in pdc_probe()
1548 	pdcs->pdc_idx = pdcg.num_spu;  in pdc_probe()
1558 	pdcs->ring_pool = dma_pool_create("pdc rings", dev, PDC_RING_SIZE,  in pdc_probe()
1560 	if (!pdcs->ring_pool) {  in pdc_probe()
1565 	err = pdc_dt_read(pdev, pdcs);  in pdc_probe()
1569 	pdcs->pdc_reg_vbase = devm_platform_get_and_ioremap_resource(pdev, 0, &pdc_regs);  in pdc_probe()
1570 	if (IS_ERR(pdcs->pdc_reg_vbase)) {  in pdc_probe()
1571 		err = PTR_ERR(pdcs->pdc_reg_vbase);  in pdc_probe()
1578 	err = pdc_rx_buf_pool_create(pdcs);  in pdc_probe()
1582 	pdc_hw_init(pdcs);  in pdc_probe()
1585 	tasklet_setup(&pdcs->rx_tasklet, pdc_tasklet_cb);  in pdc_probe()
1587 	err = pdc_interrupts_init(pdcs);  in pdc_probe()
1592 	err = pdc_mb_init(pdcs);  in pdc_probe()
1596 	pdc_setup_debugfs(pdcs);  in pdc_probe()
1602 	tasklet_kill(&pdcs->rx_tasklet);  in pdc_probe()
1603 	dma_pool_destroy(pdcs->rx_buf_pool);  in pdc_probe()
1606 	dma_pool_destroy(pdcs->ring_pool);  in pdc_probe()
1614 	struct pdc_state *pdcs = platform_get_drvdata(pdev);  in pdc_remove()  local
1618 	tasklet_kill(&pdcs->rx_tasklet);  in pdc_remove()
1620 	pdc_hw_disable(pdcs);  in pdc_remove()
1622 	dma_pool_destroy(pdcs->rx_buf_pool);  in pdc_remove()
1623 	dma_pool_destroy(pdcs->ring_pool);  in pdc_remove()