Lines Matching +full:pl330 +full:- +full:periph +full:- +full:burst
1 // SPDX-License-Identifier: GPL-2.0-or-later
19 #include <linux/dma-mapping.h>
46 CCTRL6, /* Cacheable write-through, allocate on writes only */
47 CCTRL7, /* Cacheable write-back, allocate on writes only */
245 * at 1byte/burst for P<->M and M<->M respectively.
246 * For typical scenario, at 1word/burst, 10MB and 20MB xfers per req
247 * should be enough for P<->M and M<->M respectively.
274 /* Populated by the PL330 core driver for DMA API driver's info */
290 * The PL330 core does not modify this and uses the last
303 * and burst size/length are assumed same.
351 BURST, enumerator
382 /* Index of the last submitted request or -1 if the DMA is stopped */
402 * to the PL330 core. Not more than two descriptors
414 * by PL330 core
423 /* DMA-Engine Channel */
444 * Hardware channel thread of PL330 DMAC. NULL if the channel is
449 /* For D-to-M and M-to-D channels */
451 int burst_len; /* the number of burst */
453 /* DMA-mapped view of the FIFO; may differ if an IOMMU is present */
466 /* DMA-Engine Device */
476 /* ioremap'ed address of PL330 registers. */
478 /* Populated by the PL330 core driver during pl330_add */
514 .quirk = "arm,pl330-broken-no-flushp",
518 .quirk = "arm,pl330-periph-burst",
530 /* Xfer for PL330 core */
561 return thrd->req[0].desc != NULL && thrd->req[1].desc != NULL; in _queue_full()
566 return thrd->dmac->manager == thrd; in is_manager()
569 /* If manager of the thread is in Non-Secure mode */
572 return (thrd->dmac->pcfg.mode & DMAC_MODE_NS) ? true : false; in _manager_ns()
617 else if (cond == BURST) in _emit_LD()
621 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A')); in _emit_LD()
634 if (cond == BURST) in _emit_LDP()
658 cnt--; /* DMAC increments by 1 internally */ in _emit_LP()
676 enum pl330_cond cond = arg->cond; in _emit_LPEND()
677 bool forever = arg->forever; in _emit_LPEND()
678 unsigned loop = arg->loop; in _emit_LPEND()
679 u8 bjump = arg->bjump; in _emit_LPEND()
694 else if (cond == BURST) in _emit_LPEND()
701 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'), in _emit_LPEND()
774 else if (cond == BURST) in _emit_ST()
778 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A')); in _emit_ST()
791 if (cond == BURST) in _emit_STP()
814 else if (cond == BURST) in _emit_WFP()
824 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'P'), peri >> 3); in _emit_WFP()
850 u8 chan = arg->chan; in _emit_GO()
851 u32 addr = arg->addr; in _emit_GO()
852 unsigned ns = arg->ns; in _emit_GO()
870 /* Returns Time-Out */
873 void __iomem *regs = thrd->dmac->base; in _until_dmac_idle()
882 } while (--loops); in _until_dmac_idle()
893 void __iomem *regs = thrd->dmac->base; in _execute_DBGINSN()
896 /* If timed out due to halted state-machine */ in _execute_DBGINSN()
898 dev_err(thrd->dmac->ddma.dev, "DMAC halted!\n"); in _execute_DBGINSN()
905 val |= (thrd->id << 8); /* Channel Number */ in _execute_DBGINSN()
918 void __iomem *regs = thrd->dmac->base; in _state()
924 val = readl(regs + CS(thrd->id)) & 0xf; in _state()
976 void __iomem *regs = thrd->dmac->base; in _stop()
994 if (inten & (1 << thrd->ev)) in _stop()
995 writel(1 << thrd->ev, regs + INTCLR); in _stop()
997 writel(inten & ~(1 << thrd->ev), regs + INTEN); in _stop()
1003 void __iomem *regs = thrd->dmac->base; in _trigger()
1015 idx = 1 - thrd->lstenq; in _trigger()
1016 if (thrd->req[idx].desc != NULL) { in _trigger()
1017 req = &thrd->req[idx]; in _trigger()
1019 idx = thrd->lstenq; in _trigger()
1020 if (thrd->req[idx].desc != NULL) in _trigger()
1021 req = &thrd->req[idx]; in _trigger()
1031 if (idx == thrd->req_running) in _trigger()
1034 desc = req->desc; in _trigger()
1036 ns = desc->rqcfg.nonsecure ? 1 : 0; in _trigger()
1038 /* See 'Abort Sources' point-4 at Page 2-25 */ in _trigger()
1040 dev_info(thrd->dmac->ddma.dev, "%s:%d Recipe for ABORT!\n", in _trigger()
1043 go.chan = thrd->id; in _trigger()
1044 go.addr = req->mc_bus; in _trigger()
1049 writel(readl(regs + INTEN) | (1 << thrd->ev), regs + INTEN); in _trigger()
1054 thrd->req_running = idx; in _trigger()
1099 struct pl330_config *pcfg = pxs->desc->rqcfg.pcfg; in _ldst_memtomem()
1101 /* check lock-up free version */ in _ldst_memtomem()
1102 if (get_revision(pcfg->periph_id) >= PERIPH_REV_R1P0) { in _ldst_memtomem()
1103 while (cyc--) { in _ldst_memtomem()
1108 while (cyc--) { in _ldst_memtomem()
1135 off += _emit_LDP(dry_run, &buf[off], BURST, in _emit_load()
1168 off += _emit_STP(dry_run, &buf[off], BURST, in _emit_store()
1185 static inline int _ldst_peripheral(struct pl330_dmac *pl330, in _ldst_peripheral() argument
1196 if (!(pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP)) in _ldst_peripheral()
1197 off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri); in _ldst_peripheral()
1198 while (cyc--) { in _ldst_peripheral()
1199 off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri); in _ldst_peripheral()
1200 off += _emit_load(dry_run, &buf[off], cond, pxs->desc->rqtype, in _ldst_peripheral()
1201 pxs->desc->peri); in _ldst_peripheral()
1202 off += _emit_store(dry_run, &buf[off], cond, pxs->desc->rqtype, in _ldst_peripheral()
1203 pxs->desc->peri); in _ldst_peripheral()
1209 static int _bursts(struct pl330_dmac *pl330, unsigned dry_run, u8 buf[], in _bursts() argument
1213 enum pl330_cond cond = BRST_LEN(pxs->ccr) > 1 ? BURST : SINGLE; in _bursts()
1215 if (pl330->quirks & PL330_QUIRK_PERIPH_BURST) in _bursts()
1216 cond = BURST; in _bursts()
1218 switch (pxs->desc->rqtype) { in _bursts()
1221 off += _ldst_peripheral(pl330, dry_run, &buf[off], pxs, cyc, in _bursts()
1239 * only the unaligned burst transfers have the dregs.
1240 * so, still transfer dregs with a reduced size burst
1241 * for mem-to-mem, mem-to-dev or dev-to-mem.
1243 static int _dregs(struct pl330_dmac *pl330, unsigned int dry_run, u8 buf[], in _dregs() argument
1253 * dregs_len = (total bytes - BURST_TO_BYTE(bursts, ccr)) / in _dregs()
1255 * the dregs len must be smaller than burst len, in _dregs()
1257 * to use a reduced size burst len for the dregs. in _dregs()
1259 dregs_ccr = pxs->ccr; in _dregs()
1262 dregs_ccr |= (((transfer_length - 1) & 0xf) << in _dregs()
1264 dregs_ccr |= (((transfer_length - 1) & 0xf) << in _dregs()
1267 switch (pxs->desc->rqtype) { in _dregs()
1271 off += _ldst_peripheral(pl330, dry_run, &buf[off], pxs, 1, in _dregs()
1272 BURST); in _dregs()
1290 static inline int _loop(struct pl330_dmac *pl330, unsigned dry_run, u8 buf[], in _loop() argument
1298 return _bursts(pl330, dry_run, buf, pxs, 1); in _loop()
1316 szbrst = _bursts(pl330, 1, buf, pxs, 1); in _loop()
1332 * which is 8-bits and hence 255 in _loop()
1334 cycmax = (255 - (szlp + szlpend)) / szbrst; in _loop()
1348 off += _bursts(pl330, dry_run, &buf[off], pxs, cyc); in _loop()
1353 lpend.bjump = off - ljmp1; in _loop()
1360 lpend.bjump = off - ljmp0; in _loop()
1371 static inline int _setup_loops(struct pl330_dmac *pl330, in _setup_loops() argument
1375 struct pl330_xfer *x = &pxs->desc->px; in _setup_loops()
1376 u32 ccr = pxs->ccr; in _setup_loops()
1377 unsigned long c, bursts = BYTE_TO_BURST(x->bytes, ccr); in _setup_loops()
1378 int num_dregs = (x->bytes - BURST_TO_BYTE(bursts, ccr)) / in _setup_loops()
1384 off += _loop(pl330, dry_run, &buf[off], &c, pxs); in _setup_loops()
1385 bursts -= c; in _setup_loops()
1387 off += _dregs(pl330, dry_run, &buf[off], pxs, num_dregs); in _setup_loops()
1392 static inline int _setup_xfer(struct pl330_dmac *pl330, in _setup_xfer() argument
1396 struct pl330_xfer *x = &pxs->desc->px; in _setup_xfer()
1399 /* DMAMOV SAR, x->src_addr */ in _setup_xfer()
1400 off += _emit_MOV(dry_run, &buf[off], SAR, x->src_addr); in _setup_xfer()
1401 /* DMAMOV DAR, x->dst_addr */ in _setup_xfer()
1402 off += _emit_MOV(dry_run, &buf[off], DAR, x->dst_addr); in _setup_xfer()
1405 off += _setup_loops(pl330, dry_run, &buf[off], pxs); in _setup_xfer()
1414 static int _setup_req(struct pl330_dmac *pl330, unsigned dry_run, in _setup_req() argument
1418 struct _pl330_req *req = &thrd->req[index]; in _setup_req()
1419 u8 *buf = req->mc_cpu; in _setup_req()
1422 PL330_DBGMC_START(req->mc_bus); in _setup_req()
1425 off += _emit_MOV(dry_run, &buf[off], CCR, pxs->ccr); in _setup_req()
1427 off += _setup_xfer(pl330, dry_run, &buf[off], pxs); in _setup_req()
1430 off += _emit_SEV(dry_run, &buf[off], thrd->ev); in _setup_req()
1441 if (rqc->src_inc) in _prepare_ccr()
1444 if (rqc->dst_inc) in _prepare_ccr()
1448 if (rqc->privileged) in _prepare_ccr()
1450 if (rqc->nonsecure) in _prepare_ccr()
1452 if (rqc->insnaccess) in _prepare_ccr()
1455 ccr |= (((rqc->brst_len - 1) & 0xf) << CC_SRCBRSTLEN_SHFT); in _prepare_ccr()
1456 ccr |= (((rqc->brst_len - 1) & 0xf) << CC_DSTBRSTLEN_SHFT); in _prepare_ccr()
1458 ccr |= (rqc->brst_size << CC_SRCBRSTSIZE_SHFT); in _prepare_ccr()
1459 ccr |= (rqc->brst_size << CC_DSTBRSTSIZE_SHFT); in _prepare_ccr()
1461 ccr |= (rqc->scctl << CC_SRCCCTRL_SHFT); in _prepare_ccr()
1462 ccr |= (rqc->dcctl << CC_DSTCCTRL_SHFT); in _prepare_ccr()
1464 ccr |= (rqc->swap << CC_SWAP_SHFT); in _prepare_ccr()
1477 struct pl330_dmac *pl330 = thrd->dmac; in pl330_submit_req() local
1484 switch (desc->rqtype) { in pl330_submit_req()
1495 return -ENOTSUPP; in pl330_submit_req()
1498 if (pl330->state == DYING in pl330_submit_req()
1499 || pl330->dmac_tbd.reset_chan & (1 << thrd->id)) { in pl330_submit_req()
1500 dev_info(thrd->dmac->ddma.dev, "%s:%d\n", in pl330_submit_req()
1502 return -EAGAIN; in pl330_submit_req()
1505 /* If request for non-existing peripheral */ in pl330_submit_req()
1506 if (desc->rqtype != DMA_MEM_TO_MEM && in pl330_submit_req()
1507 desc->peri >= pl330->pcfg.num_peri) { in pl330_submit_req()
1508 dev_info(thrd->dmac->ddma.dev, in pl330_submit_req()
1510 __func__, __LINE__, desc->peri); in pl330_submit_req()
1511 return -EINVAL; in pl330_submit_req()
1514 spin_lock_irqsave(&pl330->lock, flags); in pl330_submit_req()
1517 ret = -EAGAIN; in pl330_submit_req()
1523 desc->rqcfg.nonsecure = 0; in pl330_submit_req()
1525 desc->rqcfg.nonsecure = 1; in pl330_submit_req()
1527 ccr = _prepare_ccr(&desc->rqcfg); in pl330_submit_req()
1529 idx = thrd->req[0].desc == NULL ? 0 : 1; in pl330_submit_req()
1535 ret = _setup_req(pl330, 1, thrd, idx, &xs); in pl330_submit_req()
1537 if (ret > pl330->mcbufsz / 2) { in pl330_submit_req()
1538 dev_info(pl330->ddma.dev, "%s:%d Try increasing mcbufsz (%i/%i)\n", in pl330_submit_req()
1539 __func__, __LINE__, ret, pl330->mcbufsz / 2); in pl330_submit_req()
1540 ret = -ENOMEM; in pl330_submit_req()
1545 thrd->lstenq = idx; in pl330_submit_req()
1546 thrd->req[idx].desc = desc; in pl330_submit_req()
1547 _setup_req(pl330, 0, thrd, idx, &xs); in pl330_submit_req()
1552 spin_unlock_irqrestore(&pl330->lock, flags); in pl330_submit_req()
1565 pch = desc->pchan; in dma_pl330_rqcb()
1571 spin_lock_irqsave(&pch->lock, flags); in dma_pl330_rqcb()
1573 desc->status = DONE; in dma_pl330_rqcb()
1575 spin_unlock_irqrestore(&pch->lock, flags); in dma_pl330_rqcb()
1577 tasklet_schedule(&pch->task); in dma_pl330_rqcb()
1582 struct pl330_dmac *pl330 = from_tasklet(pl330, t, tasks); in pl330_dotask() local
1586 spin_lock_irqsave(&pl330->lock, flags); in pl330_dotask()
1589 if (pl330->dmac_tbd.reset_dmac) { in pl330_dotask()
1590 pl330->state = DYING; in pl330_dotask()
1592 pl330->dmac_tbd.reset_mngr = true; in pl330_dotask()
1594 pl330->dmac_tbd.reset_dmac = false; in pl330_dotask()
1597 if (pl330->dmac_tbd.reset_mngr) { in pl330_dotask()
1598 _stop(pl330->manager); in pl330_dotask()
1600 pl330->dmac_tbd.reset_chan = (1 << pl330->pcfg.num_chan) - 1; in pl330_dotask()
1602 pl330->dmac_tbd.reset_mngr = false; in pl330_dotask()
1605 for (i = 0; i < pl330->pcfg.num_chan; i++) { in pl330_dotask()
1607 if (pl330->dmac_tbd.reset_chan & (1 << i)) { in pl330_dotask()
1608 struct pl330_thread *thrd = &pl330->channels[i]; in pl330_dotask()
1609 void __iomem *regs = pl330->base; in pl330_dotask()
1614 if (readl(regs + FSC) & (1 << thrd->id)) in pl330_dotask()
1619 spin_unlock_irqrestore(&pl330->lock, flags); in pl330_dotask()
1620 dma_pl330_rqcb(thrd->req[1 - thrd->lstenq].desc, err); in pl330_dotask()
1621 dma_pl330_rqcb(thrd->req[thrd->lstenq].desc, err); in pl330_dotask()
1622 spin_lock_irqsave(&pl330->lock, flags); in pl330_dotask()
1624 thrd->req[0].desc = NULL; in pl330_dotask()
1625 thrd->req[1].desc = NULL; in pl330_dotask()
1626 thrd->req_running = -1; in pl330_dotask()
1629 pl330->dmac_tbd.reset_chan &= ~(1 << i); in pl330_dotask()
1633 spin_unlock_irqrestore(&pl330->lock, flags); in pl330_dotask()
1639 static int pl330_update(struct pl330_dmac *pl330) in pl330_update() argument
1647 regs = pl330->base; in pl330_update()
1649 spin_lock_irqsave(&pl330->lock, flags); in pl330_update()
1653 pl330->dmac_tbd.reset_mngr = true; in pl330_update()
1655 pl330->dmac_tbd.reset_mngr = false; in pl330_update()
1657 val = readl(regs + FSC) & ((1 << pl330->pcfg.num_chan) - 1); in pl330_update()
1658 pl330->dmac_tbd.reset_chan |= val; in pl330_update()
1661 while (i < pl330->pcfg.num_chan) { in pl330_update()
1663 dev_info(pl330->ddma.dev, in pl330_update()
1664 "Reset Channel-%d\t CS-%x FTC-%x\n", in pl330_update()
1667 _stop(&pl330->channels[i]); in pl330_update()
1675 if (pl330->pcfg.num_events < 32 in pl330_update()
1676 && val & ~((1 << pl330->pcfg.num_events) - 1)) { in pl330_update()
1677 pl330->dmac_tbd.reset_dmac = true; in pl330_update()
1678 dev_err(pl330->ddma.dev, "%s:%d Unexpected!\n", __func__, in pl330_update()
1684 for (ev = 0; ev < pl330->pcfg.num_events; ev++) { in pl330_update()
1696 id = pl330->events[ev]; in pl330_update()
1698 thrd = &pl330->channels[id]; in pl330_update()
1700 active = thrd->req_running; in pl330_update()
1701 if (active == -1) /* Aborted */ in pl330_update()
1705 descdone = thrd->req[active].desc; in pl330_update()
1706 thrd->req[active].desc = NULL; in pl330_update()
1708 thrd->req_running = -1; in pl330_update()
1714 list_add_tail(&descdone->rqd, &pl330->req_done); in pl330_update()
1719 while (!list_empty(&pl330->req_done)) { in pl330_update()
1720 descdone = list_first_entry(&pl330->req_done, in pl330_update()
1722 list_del(&descdone->rqd); in pl330_update()
1723 spin_unlock_irqrestore(&pl330->lock, flags); in pl330_update()
1725 spin_lock_irqsave(&pl330->lock, flags); in pl330_update()
1729 spin_unlock_irqrestore(&pl330->lock, flags); in pl330_update()
1731 if (pl330->dmac_tbd.reset_dmac in pl330_update()
1732 || pl330->dmac_tbd.reset_mngr in pl330_update()
1733 || pl330->dmac_tbd.reset_chan) { in pl330_update()
1735 tasklet_schedule(&pl330->tasks); in pl330_update()
1744 struct pl330_dmac *pl330 = thrd->dmac; in _alloc_event() local
1747 for (ev = 0; ev < pl330->pcfg.num_events; ev++) in _alloc_event()
1748 if (pl330->events[ev] == -1) { in _alloc_event()
1749 pl330->events[ev] = thrd->id; in _alloc_event()
1753 return -1; in _alloc_event()
1756 static bool _chan_ns(const struct pl330_dmac *pl330, int i) in _chan_ns() argument
1758 return pl330->pcfg.irq_ns & (1 << i); in _chan_ns()
1764 static struct pl330_thread *pl330_request_channel(struct pl330_dmac *pl330) in pl330_request_channel() argument
1769 if (pl330->state == DYING) in pl330_request_channel()
1772 chans = pl330->pcfg.num_chan; in pl330_request_channel()
1775 thrd = &pl330->channels[i]; in pl330_request_channel()
1776 if ((thrd->free) && (!_manager_ns(thrd) || in pl330_request_channel()
1777 _chan_ns(pl330, i))) { in pl330_request_channel()
1778 thrd->ev = _alloc_event(thrd); in pl330_request_channel()
1779 if (thrd->ev >= 0) { in pl330_request_channel()
1780 thrd->free = false; in pl330_request_channel()
1781 thrd->lstenq = 1; in pl330_request_channel()
1782 thrd->req[0].desc = NULL; in pl330_request_channel()
1783 thrd->req[1].desc = NULL; in pl330_request_channel()
1784 thrd->req_running = -1; in pl330_request_channel()
1797 struct pl330_dmac *pl330 = thrd->dmac; in _free_event() local
1800 if (ev >= 0 && ev < pl330->pcfg.num_events in _free_event()
1801 && pl330->events[ev] == thrd->id) in _free_event()
1802 pl330->events[ev] = -1; in _free_event()
1807 if (!thrd || thrd->free) in pl330_release_channel()
1812 dma_pl330_rqcb(thrd->req[1 - thrd->lstenq].desc, PL330_ERR_ABORT); in pl330_release_channel()
1813 dma_pl330_rqcb(thrd->req[thrd->lstenq].desc, PL330_ERR_ABORT); in pl330_release_channel()
1815 _free_event(thrd, thrd->ev); in pl330_release_channel()
1816 thrd->free = true; in pl330_release_channel()
1819 /* Initialize the structure for PL330 configuration, that can be used
1822 static void read_dmac_config(struct pl330_dmac *pl330) in read_dmac_config() argument
1824 void __iomem *regs = pl330->base; in read_dmac_config()
1829 pl330->pcfg.data_bus_width = 8 * (1 << val); in read_dmac_config()
1833 pl330->pcfg.data_buf_dep = val + 1; in read_dmac_config()
1838 pl330->pcfg.num_chan = val; in read_dmac_config()
1844 pl330->pcfg.num_peri = val; in read_dmac_config()
1845 pl330->pcfg.peri_ns = readl(regs + CR4); in read_dmac_config()
1847 pl330->pcfg.num_peri = 0; in read_dmac_config()
1852 pl330->pcfg.mode |= DMAC_MODE_NS; in read_dmac_config()
1854 pl330->pcfg.mode &= ~DMAC_MODE_NS; in read_dmac_config()
1859 pl330->pcfg.num_events = val; in read_dmac_config()
1861 pl330->pcfg.irq_ns = readl(regs + CR3); in read_dmac_config()
1866 struct pl330_dmac *pl330 = thrd->dmac; in _reset_thread() local
1868 thrd->req[0].mc_cpu = pl330->mcode_cpu in _reset_thread()
1869 + (thrd->id * pl330->mcbufsz); in _reset_thread()
1870 thrd->req[0].mc_bus = pl330->mcode_bus in _reset_thread()
1871 + (thrd->id * pl330->mcbufsz); in _reset_thread()
1872 thrd->req[0].desc = NULL; in _reset_thread()
1874 thrd->req[1].mc_cpu = thrd->req[0].mc_cpu in _reset_thread()
1875 + pl330->mcbufsz / 2; in _reset_thread()
1876 thrd->req[1].mc_bus = thrd->req[0].mc_bus in _reset_thread()
1877 + pl330->mcbufsz / 2; in _reset_thread()
1878 thrd->req[1].desc = NULL; in _reset_thread()
1880 thrd->req_running = -1; in _reset_thread()
1883 static int dmac_alloc_threads(struct pl330_dmac *pl330) in dmac_alloc_threads() argument
1885 int chans = pl330->pcfg.num_chan; in dmac_alloc_threads()
1890 pl330->channels = kcalloc(1 + chans, sizeof(*thrd), in dmac_alloc_threads()
1892 if (!pl330->channels) in dmac_alloc_threads()
1893 return -ENOMEM; in dmac_alloc_threads()
1897 thrd = &pl330->channels[i]; in dmac_alloc_threads()
1898 thrd->id = i; in dmac_alloc_threads()
1899 thrd->dmac = pl330; in dmac_alloc_threads()
1901 thrd->free = true; in dmac_alloc_threads()
1905 thrd = &pl330->channels[chans]; in dmac_alloc_threads()
1906 thrd->id = chans; in dmac_alloc_threads()
1907 thrd->dmac = pl330; in dmac_alloc_threads()
1908 thrd->free = false; in dmac_alloc_threads()
1909 pl330->manager = thrd; in dmac_alloc_threads()
1914 static int dmac_alloc_resources(struct pl330_dmac *pl330) in dmac_alloc_resources() argument
1916 int chans = pl330->pcfg.num_chan; in dmac_alloc_resources()
1923 pl330->mcode_cpu = dma_alloc_attrs(pl330->ddma.dev, in dmac_alloc_resources()
1924 chans * pl330->mcbufsz, in dmac_alloc_resources()
1925 &pl330->mcode_bus, GFP_KERNEL, in dmac_alloc_resources()
1927 if (!pl330->mcode_cpu) { in dmac_alloc_resources()
1928 dev_err(pl330->ddma.dev, "%s:%d Can't allocate memory!\n", in dmac_alloc_resources()
1930 return -ENOMEM; in dmac_alloc_resources()
1933 ret = dmac_alloc_threads(pl330); in dmac_alloc_resources()
1935 dev_err(pl330->ddma.dev, "%s:%d Can't to create channels for DMAC!\n", in dmac_alloc_resources()
1937 dma_free_attrs(pl330->ddma.dev, in dmac_alloc_resources()
1938 chans * pl330->mcbufsz, in dmac_alloc_resources()
1939 pl330->mcode_cpu, pl330->mcode_bus, in dmac_alloc_resources()
1947 static int pl330_add(struct pl330_dmac *pl330) in pl330_add() argument
1952 if ((pl330->pcfg.periph_id & 0xfffff) != PERIPH_ID_VAL) { in pl330_add()
1953 dev_err(pl330->ddma.dev, "PERIPH_ID 0x%x !\n", in pl330_add()
1954 pl330->pcfg.periph_id); in pl330_add()
1955 return -EINVAL; in pl330_add()
1959 read_dmac_config(pl330); in pl330_add()
1961 if (pl330->pcfg.num_events == 0) { in pl330_add()
1962 dev_err(pl330->ddma.dev, "%s:%d Can't work without events!\n", in pl330_add()
1964 return -EINVAL; in pl330_add()
1967 spin_lock_init(&pl330->lock); in pl330_add()
1969 INIT_LIST_HEAD(&pl330->req_done); in pl330_add()
1972 if (!pl330->mcbufsz) in pl330_add()
1973 pl330->mcbufsz = MCODE_BUFF_PER_REQ * 2; in pl330_add()
1976 for (i = 0; i < pl330->pcfg.num_events; i++) in pl330_add()
1977 pl330->events[i] = -1; in pl330_add()
1980 ret = dmac_alloc_resources(pl330); in pl330_add()
1982 dev_err(pl330->ddma.dev, "Unable to create channels for DMAC\n"); in pl330_add()
1986 tasklet_setup(&pl330->tasks, pl330_dotask); in pl330_add()
1988 pl330->state = INIT; in pl330_add()
1993 static int dmac_free_threads(struct pl330_dmac *pl330) in dmac_free_threads() argument
1999 for (i = 0; i < pl330->pcfg.num_chan; i++) { in dmac_free_threads()
2000 thrd = &pl330->channels[i]; in dmac_free_threads()
2005 kfree(pl330->channels); in dmac_free_threads()
2010 static void pl330_del(struct pl330_dmac *pl330) in pl330_del() argument
2012 pl330->state = UNINIT; in pl330_del()
2014 tasklet_kill(&pl330->tasks); in pl330_del()
2017 dmac_free_threads(pl330); in pl330_del()
2019 dma_free_attrs(pl330->ddma.dev, in pl330_del()
2020 pl330->pcfg.num_chan * pl330->mcbufsz, pl330->mcode_cpu, in pl330_del()
2021 pl330->mcode_bus, DMA_ATTR_PRIVILEGED); in pl330_del()
2047 list_for_each_entry(desc, &pch->work_list, node) { in fill_queue()
2050 if (desc->status == BUSY || desc->status == PAUSED) in fill_queue()
2053 ret = pl330_submit_req(pch->thread, desc); in fill_queue()
2055 desc->status = BUSY; in fill_queue()
2056 } else if (ret == -EAGAIN) { in fill_queue()
2061 desc->status = DONE; in fill_queue()
2062 dev_err(pch->dmac->ddma.dev, "%s:%d Bad Desc(%d)\n", in fill_queue()
2063 __func__, __LINE__, desc->txd.cookie); in fill_queue()
2064 tasklet_schedule(&pch->task); in fill_queue()
2076 spin_lock_irqsave(&pch->lock, flags); in pl330_tasklet()
2079 list_for_each_entry_safe(desc, _dt, &pch->work_list, node) in pl330_tasklet()
2080 if (desc->status == DONE) { in pl330_tasklet()
2081 if (!pch->cyclic) in pl330_tasklet()
2082 dma_cookie_complete(&desc->txd); in pl330_tasklet()
2083 list_move_tail(&desc->node, &pch->completed_list); in pl330_tasklet()
2089 if (list_empty(&pch->work_list)) { in pl330_tasklet()
2090 spin_lock(&pch->thread->dmac->lock); in pl330_tasklet()
2091 _stop(pch->thread); in pl330_tasklet()
2092 spin_unlock(&pch->thread->dmac->lock); in pl330_tasklet()
2094 pch->active = false; in pl330_tasklet()
2096 /* Make sure the PL330 Channel thread is active */ in pl330_tasklet()
2097 spin_lock(&pch->thread->dmac->lock); in pl330_tasklet()
2098 pl330_start_thread(pch->thread); in pl330_tasklet()
2099 spin_unlock(&pch->thread->dmac->lock); in pl330_tasklet()
2102 while (!list_empty(&pch->completed_list)) { in pl330_tasklet()
2105 desc = list_first_entry(&pch->completed_list, in pl330_tasklet()
2108 dmaengine_desc_get_callback(&desc->txd, &cb); in pl330_tasklet()
2110 if (pch->cyclic) { in pl330_tasklet()
2111 desc->status = PREP; in pl330_tasklet()
2112 list_move_tail(&desc->node, &pch->work_list); in pl330_tasklet()
2114 pch->active = true; in pl330_tasklet()
2115 spin_lock(&pch->thread->dmac->lock); in pl330_tasklet()
2116 pl330_start_thread(pch->thread); in pl330_tasklet()
2117 spin_unlock(&pch->thread->dmac->lock); in pl330_tasklet()
2121 desc->status = FREE; in pl330_tasklet()
2122 list_move_tail(&desc->node, &pch->dmac->desc_pool); in pl330_tasklet()
2125 dma_descriptor_unmap(&desc->txd); in pl330_tasklet()
2128 spin_unlock_irqrestore(&pch->lock, flags); in pl330_tasklet()
2130 spin_lock_irqsave(&pch->lock, flags); in pl330_tasklet()
2133 spin_unlock_irqrestore(&pch->lock, flags); in pl330_tasklet()
2137 pm_runtime_mark_last_busy(pch->dmac->ddma.dev); in pl330_tasklet()
2138 pm_runtime_put_autosuspend(pch->dmac->ddma.dev); in pl330_tasklet()
2145 int count = dma_spec->args_count; in of_dma_pl330_xlate()
2146 struct pl330_dmac *pl330 = ofdma->of_dma_data; in of_dma_pl330_xlate() local
2149 if (!pl330) in of_dma_pl330_xlate()
2155 chan_id = dma_spec->args[0]; in of_dma_pl330_xlate()
2156 if (chan_id >= pl330->num_peripherals) in of_dma_pl330_xlate()
2159 return dma_get_slave_channel(&pl330->peripherals[chan_id].chan); in of_dma_pl330_xlate()
2165 struct pl330_dmac *pl330 = pch->dmac; in pl330_alloc_chan_resources() local
2168 spin_lock_irqsave(&pl330->lock, flags); in pl330_alloc_chan_resources()
2171 pch->cyclic = false; in pl330_alloc_chan_resources()
2173 pch->thread = pl330_request_channel(pl330); in pl330_alloc_chan_resources()
2174 if (!pch->thread) { in pl330_alloc_chan_resources()
2175 spin_unlock_irqrestore(&pl330->lock, flags); in pl330_alloc_chan_resources()
2176 return -ENOMEM; in pl330_alloc_chan_resources()
2179 tasklet_setup(&pch->task, pl330_tasklet); in pl330_alloc_chan_resources()
2181 spin_unlock_irqrestore(&pl330->lock, flags); in pl330_alloc_chan_resources()
2187 * We need the data direction between the DMAC (the dma-mapping "device") and
2207 if (pch->dir != DMA_NONE) in pl330_unprep_slave_fifo()
2208 dma_unmap_resource(pch->chan.device->dev, pch->fifo_dma, in pl330_unprep_slave_fifo()
2209 1 << pch->burst_sz, pch->dir, 0); in pl330_unprep_slave_fifo()
2210 pch->dir = DMA_NONE; in pl330_unprep_slave_fifo()
2217 struct device *dev = pch->chan.device->dev; in pl330_prep_slave_fifo()
2221 if (pch->dir == dma_dir) in pl330_prep_slave_fifo()
2225 pch->fifo_dma = dma_map_resource(dev, pch->fifo_addr, in pl330_prep_slave_fifo()
2226 1 << pch->burst_sz, dma_dir, 0); in pl330_prep_slave_fifo()
2227 if (dma_mapping_error(dev, pch->fifo_dma)) in pl330_prep_slave_fifo()
2230 pch->dir = dma_dir; in pl330_prep_slave_fifo()
2252 if (slave_config->dst_addr) in pl330_config_write()
2253 pch->fifo_addr = slave_config->dst_addr; in pl330_config_write()
2254 if (slave_config->dst_addr_width) in pl330_config_write()
2255 pch->burst_sz = __ffs(slave_config->dst_addr_width); in pl330_config_write()
2256 pch->burst_len = fixup_burst_len(slave_config->dst_maxburst, in pl330_config_write()
2257 pch->dmac->quirks); in pl330_config_write()
2259 if (slave_config->src_addr) in pl330_config_write()
2260 pch->fifo_addr = slave_config->src_addr; in pl330_config_write()
2261 if (slave_config->src_addr_width) in pl330_config_write()
2262 pch->burst_sz = __ffs(slave_config->src_addr_width); in pl330_config_write()
2263 pch->burst_len = fixup_burst_len(slave_config->src_maxburst, in pl330_config_write()
2264 pch->dmac->quirks); in pl330_config_write()
2275 memcpy(&pch->slave_config, slave_config, sizeof(*slave_config)); in pl330_config()
2285 struct pl330_dmac *pl330 = pch->dmac; in pl330_terminate_all() local
2288 pm_runtime_get_sync(pl330->ddma.dev); in pl330_terminate_all()
2289 spin_lock_irqsave(&pch->lock, flags); in pl330_terminate_all()
2291 spin_lock(&pl330->lock); in pl330_terminate_all()
2292 _stop(pch->thread); in pl330_terminate_all()
2293 pch->thread->req[0].desc = NULL; in pl330_terminate_all()
2294 pch->thread->req[1].desc = NULL; in pl330_terminate_all()
2295 pch->thread->req_running = -1; in pl330_terminate_all()
2296 spin_unlock(&pl330->lock); in pl330_terminate_all()
2298 power_down = pch->active; in pl330_terminate_all()
2299 pch->active = false; in pl330_terminate_all()
2302 list_for_each_entry(desc, &pch->submitted_list, node) { in pl330_terminate_all()
2303 desc->status = FREE; in pl330_terminate_all()
2304 dma_cookie_complete(&desc->txd); in pl330_terminate_all()
2307 list_for_each_entry(desc, &pch->work_list , node) { in pl330_terminate_all()
2308 desc->status = FREE; in pl330_terminate_all()
2309 dma_cookie_complete(&desc->txd); in pl330_terminate_all()
2312 list_splice_tail_init(&pch->submitted_list, &pl330->desc_pool); in pl330_terminate_all()
2313 list_splice_tail_init(&pch->work_list, &pl330->desc_pool); in pl330_terminate_all()
2314 list_splice_tail_init(&pch->completed_list, &pl330->desc_pool); in pl330_terminate_all()
2315 spin_unlock_irqrestore(&pch->lock, flags); in pl330_terminate_all()
2316 pm_runtime_mark_last_busy(pl330->ddma.dev); in pl330_terminate_all()
2318 pm_runtime_put_autosuspend(pl330->ddma.dev); in pl330_terminate_all()
2319 pm_runtime_put_autosuspend(pl330->ddma.dev); in pl330_terminate_all()
2334 struct pl330_dmac *pl330 = pch->dmac; in pl330_pause() local
2338 pm_runtime_get_sync(pl330->ddma.dev); in pl330_pause()
2339 spin_lock_irqsave(&pch->lock, flags); in pl330_pause()
2341 spin_lock(&pl330->lock); in pl330_pause()
2342 _stop(pch->thread); in pl330_pause()
2343 spin_unlock(&pl330->lock); in pl330_pause()
2345 list_for_each_entry(desc, &pch->work_list, node) { in pl330_pause()
2346 if (desc->status == BUSY) in pl330_pause()
2347 desc->status = PAUSED; in pl330_pause()
2349 spin_unlock_irqrestore(&pch->lock, flags); in pl330_pause()
2350 pm_runtime_mark_last_busy(pl330->ddma.dev); in pl330_pause()
2351 pm_runtime_put_autosuspend(pl330->ddma.dev); in pl330_pause()
2359 struct pl330_dmac *pl330 = pch->dmac; in pl330_free_chan_resources() local
2362 tasklet_kill(&pch->task); in pl330_free_chan_resources()
2364 pm_runtime_get_sync(pch->dmac->ddma.dev); in pl330_free_chan_resources()
2365 spin_lock_irqsave(&pl330->lock, flags); in pl330_free_chan_resources()
2367 pl330_release_channel(pch->thread); in pl330_free_chan_resources()
2368 pch->thread = NULL; in pl330_free_chan_resources()
2370 if (pch->cyclic) in pl330_free_chan_resources()
2371 list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool); in pl330_free_chan_resources()
2373 spin_unlock_irqrestore(&pl330->lock, flags); in pl330_free_chan_resources()
2374 pm_runtime_mark_last_busy(pch->dmac->ddma.dev); in pl330_free_chan_resources()
2375 pm_runtime_put_autosuspend(pch->dmac->ddma.dev); in pl330_free_chan_resources()
2382 struct pl330_thread *thrd = pch->thread; in pl330_get_current_xferred_count()
2383 struct pl330_dmac *pl330 = pch->dmac; in pl330_get_current_xferred_count() local
2384 void __iomem *regs = thrd->dmac->base; in pl330_get_current_xferred_count()
2387 pm_runtime_get_sync(pl330->ddma.dev); in pl330_get_current_xferred_count()
2389 if (desc->rqcfg.src_inc) { in pl330_get_current_xferred_count()
2390 val = readl(regs + SA(thrd->id)); in pl330_get_current_xferred_count()
2391 addr = desc->px.src_addr; in pl330_get_current_xferred_count()
2393 val = readl(regs + DA(thrd->id)); in pl330_get_current_xferred_count()
2394 addr = desc->px.dst_addr; in pl330_get_current_xferred_count()
2396 pm_runtime_mark_last_busy(pch->dmac->ddma.dev); in pl330_get_current_xferred_count()
2397 pm_runtime_put_autosuspend(pl330->ddma.dev); in pl330_get_current_xferred_count()
2403 return val - addr; in pl330_get_current_xferred_count()
2424 spin_lock_irqsave(&pch->lock, flags); in pl330_tx_status()
2425 spin_lock(&pch->thread->dmac->lock); in pl330_tx_status()
2427 if (pch->thread->req_running != -1) in pl330_tx_status()
2428 running = pch->thread->req[pch->thread->req_running].desc; in pl330_tx_status()
2430 last_enq = pch->thread->req[pch->thread->lstenq].desc; in pl330_tx_status()
2433 list_for_each_entry(desc, &pch->work_list, node) { in pl330_tx_status()
2434 if (desc->status == DONE) in pl330_tx_status()
2435 transferred = desc->bytes_requested; in pl330_tx_status()
2439 else if (desc->status == BUSY || desc->status == PAUSED) in pl330_tx_status()
2447 transferred = desc->bytes_requested; in pl330_tx_status()
2450 residual += desc->bytes_requested - transferred; in pl330_tx_status()
2451 if (desc->txd.cookie == cookie) { in pl330_tx_status()
2452 switch (desc->status) { in pl330_tx_status()
2468 if (desc->last) in pl330_tx_status()
2471 spin_unlock(&pch->thread->dmac->lock); in pl330_tx_status()
2472 spin_unlock_irqrestore(&pch->lock, flags); in pl330_tx_status()
2485 spin_lock_irqsave(&pch->lock, flags); in pl330_issue_pending()
2486 if (list_empty(&pch->work_list)) { in pl330_issue_pending()
2492 WARN_ON(list_empty(&pch->submitted_list)); in pl330_issue_pending()
2493 pch->active = true; in pl330_issue_pending()
2494 pm_runtime_get_sync(pch->dmac->ddma.dev); in pl330_issue_pending()
2496 list_splice_tail_init(&pch->submitted_list, &pch->work_list); in pl330_issue_pending()
2497 spin_unlock_irqrestore(&pch->lock, flags); in pl330_issue_pending()
2499 pl330_tasklet(&pch->task); in pl330_issue_pending()
2510 struct dma_pl330_chan *pch = to_pchan(tx->chan); in pl330_tx_submit()
2514 spin_lock_irqsave(&pch->lock, flags); in pl330_tx_submit()
2517 while (!list_empty(&last->node)) { in pl330_tx_submit()
2518 desc = list_entry(last->node.next, struct dma_pl330_desc, node); in pl330_tx_submit()
2519 if (pch->cyclic) { in pl330_tx_submit()
2520 desc->txd.callback = last->txd.callback; in pl330_tx_submit()
2521 desc->txd.callback_param = last->txd.callback_param; in pl330_tx_submit()
2523 desc->last = false; in pl330_tx_submit()
2525 dma_cookie_assign(&desc->txd); in pl330_tx_submit()
2527 list_move_tail(&desc->node, &pch->submitted_list); in pl330_tx_submit()
2530 last->last = true; in pl330_tx_submit()
2531 cookie = dma_cookie_assign(&last->txd); in pl330_tx_submit()
2532 list_add_tail(&last->node, &pch->submitted_list); in pl330_tx_submit()
2533 spin_unlock_irqrestore(&pch->lock, flags); in pl330_tx_submit()
2540 desc->rqcfg.swap = SWAP_NO; in _init_desc()
2541 desc->rqcfg.scctl = CCTRL0; in _init_desc()
2542 desc->rqcfg.dcctl = CCTRL0; in _init_desc()
2543 desc->txd.tx_submit = pl330_tx_submit; in _init_desc()
2545 INIT_LIST_HEAD(&desc->node); in _init_desc()
2581 desc = list_entry(pool->next, in pluck_desc()
2584 list_del_init(&desc->node); in pluck_desc()
2586 desc->status = PREP; in pluck_desc()
2587 desc->txd.callback = NULL; in pluck_desc()
2597 struct pl330_dmac *pl330 = pch->dmac; in pl330_get_desc() local
2598 u8 *peri_id = pch->chan.private; in pl330_get_desc()
2602 desc = pluck_desc(&pl330->desc_pool, &pl330->pool_lock); in pl330_get_desc()
2617 desc->pchan = pch; in pl330_get_desc()
2618 desc->txd.cookie = 0; in pl330_get_desc()
2619 async_tx_ack(&desc->txd); in pl330_get_desc()
2621 desc->peri = peri_id ? pch->chan.chan_id : 0; in pl330_get_desc()
2622 desc->rqcfg.pcfg = &pch->dmac->pcfg; in pl330_get_desc()
2624 dma_async_tx_descriptor_init(&desc->txd, &pch->chan); in pl330_get_desc()
2632 px->bytes = len; in fill_px()
2633 px->dst_addr = dst; in fill_px()
2634 px->src_addr = src; in fill_px()
2644 dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n", in __pl330_prep_dma_memcpy()
2653 * going to be word-unaligned and more than 200MB, in __pl330_prep_dma_memcpy()
2659 fill_px(&desc->px, dst, src, len); in __pl330_prep_dma_memcpy()
2664 /* Call after fixing burst size */
2667 struct dma_pl330_chan *pch = desc->pchan; in get_burst_len()
2668 struct pl330_dmac *pl330 = pch->dmac; in get_burst_len() local
2671 burst_len = pl330->pcfg.data_bus_width / 8; in get_burst_len()
2672 burst_len *= pl330->pcfg.data_buf_dep / pl330->pcfg.num_chan; in get_burst_len()
2673 burst_len >>= desc->rqcfg.brst_size; in get_burst_len()
2689 struct pl330_dmac *pl330 = pch->dmac; in pl330_prep_dma_cyclic() local
2698 dev_err(pch->dmac->ddma.dev, "%s:%d Invalid dma direction\n", in pl330_prep_dma_cyclic()
2703 pl330_config_write(chan, &pch->slave_config, direction); in pl330_prep_dma_cyclic()
2713 dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n", in pl330_prep_dma_cyclic()
2719 spin_lock_irqsave(&pl330->pool_lock, iflags); in pl330_prep_dma_cyclic()
2721 while (!list_empty(&first->node)) { in pl330_prep_dma_cyclic()
2722 desc = list_entry(first->node.next, in pl330_prep_dma_cyclic()
2724 list_move_tail(&desc->node, &pl330->desc_pool); in pl330_prep_dma_cyclic()
2727 list_move_tail(&first->node, &pl330->desc_pool); in pl330_prep_dma_cyclic()
2729 spin_unlock_irqrestore(&pl330->pool_lock, iflags); in pl330_prep_dma_cyclic()
2736 desc->rqcfg.src_inc = 1; in pl330_prep_dma_cyclic()
2737 desc->rqcfg.dst_inc = 0; in pl330_prep_dma_cyclic()
2739 dst = pch->fifo_dma; in pl330_prep_dma_cyclic()
2742 desc->rqcfg.src_inc = 0; in pl330_prep_dma_cyclic()
2743 desc->rqcfg.dst_inc = 1; in pl330_prep_dma_cyclic()
2744 src = pch->fifo_dma; in pl330_prep_dma_cyclic()
2751 desc->rqtype = direction; in pl330_prep_dma_cyclic()
2752 desc->rqcfg.brst_size = pch->burst_sz; in pl330_prep_dma_cyclic()
2753 desc->rqcfg.brst_len = pch->burst_len; in pl330_prep_dma_cyclic()
2754 desc->bytes_requested = period_len; in pl330_prep_dma_cyclic()
2755 fill_px(&desc->px, dst, src, period_len); in pl330_prep_dma_cyclic()
2760 list_add_tail(&desc->node, &first->node); in pl330_prep_dma_cyclic()
2768 pch->cyclic = true; in pl330_prep_dma_cyclic()
2770 return &desc->txd; in pl330_prep_dma_cyclic()
2779 struct pl330_dmac *pl330; in pl330_prep_dma_memcpy() local
2780 int burst; in pl330_prep_dma_memcpy() local
2785 pl330 = pch->dmac; in pl330_prep_dma_memcpy()
2791 desc->rqcfg.src_inc = 1; in pl330_prep_dma_memcpy()
2792 desc->rqcfg.dst_inc = 1; in pl330_prep_dma_memcpy()
2793 desc->rqtype = DMA_MEM_TO_MEM; in pl330_prep_dma_memcpy()
2795 /* Select max possible burst size */ in pl330_prep_dma_memcpy()
2796 burst = pl330->pcfg.data_bus_width / 8; in pl330_prep_dma_memcpy()
2799 * Make sure we use a burst size that aligns with all the memcpy in pl330_prep_dma_memcpy()
2803 while ((src | dst | len) & (burst - 1)) in pl330_prep_dma_memcpy()
2804 burst /= 2; in pl330_prep_dma_memcpy()
2806 desc->rqcfg.brst_size = 0; in pl330_prep_dma_memcpy()
2807 while (burst != (1 << desc->rqcfg.brst_size)) in pl330_prep_dma_memcpy()
2808 desc->rqcfg.brst_size++; in pl330_prep_dma_memcpy()
2810 desc->rqcfg.brst_len = get_burst_len(desc, len); in pl330_prep_dma_memcpy()
2812 * If burst size is smaller than bus width then make sure we only in pl330_prep_dma_memcpy()
2813 * transfer one at a time to avoid a burst stradling an MFIFO entry. in pl330_prep_dma_memcpy()
2815 if (burst * 8 < pl330->pcfg.data_bus_width) in pl330_prep_dma_memcpy()
2816 desc->rqcfg.brst_len = 1; in pl330_prep_dma_memcpy()
2818 desc->bytes_requested = len; in pl330_prep_dma_memcpy()
2820 return &desc->txd; in pl330_prep_dma_memcpy()
2823 static void __pl330_giveback_desc(struct pl330_dmac *pl330, in __pl330_giveback_desc() argument
2832 spin_lock_irqsave(&pl330->pool_lock, flags); in __pl330_giveback_desc()
2834 while (!list_empty(&first->node)) { in __pl330_giveback_desc()
2835 desc = list_entry(first->node.next, in __pl330_giveback_desc()
2837 list_move_tail(&desc->node, &pl330->desc_pool); in __pl330_giveback_desc()
2840 list_move_tail(&first->node, &pl330->desc_pool); in __pl330_giveback_desc()
2842 spin_unlock_irqrestore(&pl330->pool_lock, flags); in __pl330_giveback_desc()
2858 pl330_config_write(chan, &pch->slave_config, direction); in pl330_prep_slave_sg()
2869 struct pl330_dmac *pl330 = pch->dmac; in pl330_prep_slave_sg() local
2871 dev_err(pch->dmac->ddma.dev, in pl330_prep_slave_sg()
2874 __pl330_giveback_desc(pl330, first); in pl330_prep_slave_sg()
2882 list_add_tail(&desc->node, &first->node); in pl330_prep_slave_sg()
2885 desc->rqcfg.src_inc = 1; in pl330_prep_slave_sg()
2886 desc->rqcfg.dst_inc = 0; in pl330_prep_slave_sg()
2887 fill_px(&desc->px, pch->fifo_dma, sg_dma_address(sg), in pl330_prep_slave_sg()
2890 desc->rqcfg.src_inc = 0; in pl330_prep_slave_sg()
2891 desc->rqcfg.dst_inc = 1; in pl330_prep_slave_sg()
2892 fill_px(&desc->px, sg_dma_address(sg), pch->fifo_dma, in pl330_prep_slave_sg()
2896 desc->rqcfg.brst_size = pch->burst_sz; in pl330_prep_slave_sg()
2897 desc->rqcfg.brst_len = pch->burst_len; in pl330_prep_slave_sg()
2898 desc->rqtype = direction; in pl330_prep_slave_sg()
2899 desc->bytes_requested = sg_dma_len(sg); in pl330_prep_slave_sg()
2903 return &desc->txd; in pl330_prep_slave_sg()
2924 struct pl330_dmac *pl330 = s->private; in pl330_debugfs_show() local
2927 chans = pl330->pcfg.num_chan; in pl330_debugfs_show()
2928 pchs = pl330->num_peripherals; in pl330_debugfs_show()
2930 seq_puts(s, "PL330 physical channels:\n"); in pl330_debugfs_show()
2932 seq_puts(s, "--------\t-----\n"); in pl330_debugfs_show()
2934 struct pl330_thread *thrd = &pl330->channels[ch]; in pl330_debugfs_show()
2935 int found = -1; in pl330_debugfs_show()
2938 struct dma_pl330_chan *pch = &pl330->peripherals[pr]; in pl330_debugfs_show()
2940 if (!pch->thread || thrd->id != pch->thread->id) in pl330_debugfs_show()
2946 seq_printf(s, "%d\t\t", thrd->id); in pl330_debugfs_show()
2947 if (found == -1) in pl330_debugfs_show()
2948 seq_puts(s, "--\n"); in pl330_debugfs_show()
2958 static inline void init_pl330_debugfs(struct pl330_dmac *pl330) in init_pl330_debugfs() argument
2960 debugfs_create_file(dev_name(pl330->ddma.dev), in init_pl330_debugfs()
2961 S_IFREG | 0444, NULL, pl330, in init_pl330_debugfs()
2965 static inline void init_pl330_debugfs(struct pl330_dmac *pl330) in init_pl330_debugfs() argument
2981 clk_unprepare(pcdev->pclk); in pl330_suspend()
2991 ret = clk_prepare(pcdev->pclk); in pl330_resume()
3008 struct pl330_dmac *pl330; in pl330_probe() local
3014 struct device_node *np = adev->dev.of_node; in pl330_probe()
3016 ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32)); in pl330_probe()
3021 pl330 = devm_kzalloc(&adev->dev, sizeof(*pl330), GFP_KERNEL); in pl330_probe()
3022 if (!pl330) in pl330_probe()
3023 return -ENOMEM; in pl330_probe()
3025 pd = &pl330->ddma; in pl330_probe()
3026 pd->dev = &adev->dev; in pl330_probe()
3028 pl330->mcbufsz = 0; in pl330_probe()
3033 pl330->quirks |= of_quirks[i].id; in pl330_probe()
3035 res = &adev->res; in pl330_probe()
3036 pl330->base = devm_ioremap_resource(&adev->dev, res); in pl330_probe()
3037 if (IS_ERR(pl330->base)) in pl330_probe()
3038 return PTR_ERR(pl330->base); in pl330_probe()
3040 amba_set_drvdata(adev, pl330); in pl330_probe()
3042 pl330->rstc = devm_reset_control_get_optional(&adev->dev, "dma"); in pl330_probe()
3043 if (IS_ERR(pl330->rstc)) { in pl330_probe()
3044 return dev_err_probe(&adev->dev, PTR_ERR(pl330->rstc), "Failed to get reset!\n"); in pl330_probe()
3046 ret = reset_control_deassert(pl330->rstc); in pl330_probe()
3048 dev_err(&adev->dev, "Couldn't deassert the device from reset!\n"); in pl330_probe()
3053 pl330->rstc_ocp = devm_reset_control_get_optional(&adev->dev, "dma-ocp"); in pl330_probe()
3054 if (IS_ERR(pl330->rstc_ocp)) { in pl330_probe()
3055 return dev_err_probe(&adev->dev, PTR_ERR(pl330->rstc_ocp), in pl330_probe()
3058 ret = reset_control_deassert(pl330->rstc_ocp); in pl330_probe()
3060 dev_err(&adev->dev, "Couldn't deassert the device from OCP reset!\n"); in pl330_probe()
3066 irq = adev->irq[i]; in pl330_probe()
3068 ret = devm_request_irq(&adev->dev, irq, in pl330_probe()
3070 dev_name(&adev->dev), pl330); in pl330_probe()
3078 pcfg = &pl330->pcfg; in pl330_probe()
3080 pcfg->periph_id = adev->periphid; in pl330_probe()
3081 ret = pl330_add(pl330); in pl330_probe()
3085 INIT_LIST_HEAD(&pl330->desc_pool); in pl330_probe()
3086 spin_lock_init(&pl330->pool_lock); in pl330_probe()
3089 if (!add_desc(&pl330->desc_pool, &pl330->pool_lock, in pl330_probe()
3091 dev_warn(&adev->dev, "unable to allocate desc\n"); in pl330_probe()
3093 INIT_LIST_HEAD(&pd->channels); in pl330_probe()
3096 num_chan = max_t(int, pcfg->num_peri, pcfg->num_chan); in pl330_probe()
3098 pl330->num_peripherals = num_chan; in pl330_probe()
3100 pl330->peripherals = kcalloc(num_chan, sizeof(*pch), GFP_KERNEL); in pl330_probe()
3101 if (!pl330->peripherals) { in pl330_probe()
3102 ret = -ENOMEM; in pl330_probe()
3107 pch = &pl330->peripherals[i]; in pl330_probe()
3109 pch->chan.private = adev->dev.of_node; in pl330_probe()
3110 INIT_LIST_HEAD(&pch->submitted_list); in pl330_probe()
3111 INIT_LIST_HEAD(&pch->work_list); in pl330_probe()
3112 INIT_LIST_HEAD(&pch->completed_list); in pl330_probe()
3113 spin_lock_init(&pch->lock); in pl330_probe()
3114 pch->thread = NULL; in pl330_probe()
3115 pch->chan.device = pd; in pl330_probe()
3116 pch->dmac = pl330; in pl330_probe()
3117 pch->dir = DMA_NONE; in pl330_probe()
3120 list_add_tail(&pch->chan.device_node, &pd->channels); in pl330_probe()
3123 dma_cap_set(DMA_MEMCPY, pd->cap_mask); in pl330_probe()
3124 if (pcfg->num_peri) { in pl330_probe()
3125 dma_cap_set(DMA_SLAVE, pd->cap_mask); in pl330_probe()
3126 dma_cap_set(DMA_CYCLIC, pd->cap_mask); in pl330_probe()
3127 dma_cap_set(DMA_PRIVATE, pd->cap_mask); in pl330_probe()
3130 pd->device_alloc_chan_resources = pl330_alloc_chan_resources; in pl330_probe()
3131 pd->device_free_chan_resources = pl330_free_chan_resources; in pl330_probe()
3132 pd->device_prep_dma_memcpy = pl330_prep_dma_memcpy; in pl330_probe()
3133 pd->device_prep_dma_cyclic = pl330_prep_dma_cyclic; in pl330_probe()
3134 pd->device_tx_status = pl330_tx_status; in pl330_probe()
3135 pd->device_prep_slave_sg = pl330_prep_slave_sg; in pl330_probe()
3136 pd->device_config = pl330_config; in pl330_probe()
3137 pd->device_pause = pl330_pause; in pl330_probe()
3138 pd->device_terminate_all = pl330_terminate_all; in pl330_probe()
3139 pd->device_issue_pending = pl330_issue_pending; in pl330_probe()
3140 pd->src_addr_widths = PL330_DMA_BUSWIDTHS; in pl330_probe()
3141 pd->dst_addr_widths = PL330_DMA_BUSWIDTHS; in pl330_probe()
3142 pd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); in pl330_probe()
3143 pd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; in pl330_probe()
3144 pd->max_burst = PL330_MAX_BURST; in pl330_probe()
3148 dev_err(&adev->dev, "unable to register DMAC\n"); in pl330_probe()
3152 if (adev->dev.of_node) { in pl330_probe()
3153 ret = of_dma_controller_register(adev->dev.of_node, in pl330_probe()
3154 of_dma_pl330_xlate, pl330); in pl330_probe()
3156 dev_err(&adev->dev, in pl330_probe()
3165 ret = dma_set_max_seg_size(&adev->dev, 1900800); in pl330_probe()
3167 dev_err(&adev->dev, "unable to set the seg size\n"); in pl330_probe()
3170 init_pl330_debugfs(pl330); in pl330_probe()
3171 dev_info(&adev->dev, in pl330_probe()
3172 "Loaded driver for PL330 DMAC-%x\n", adev->periphid); in pl330_probe()
3173 dev_info(&adev->dev, in pl330_probe()
3174 "\tDBUFF-%ux%ubytes Num_Chans-%u Num_Peri-%u Num_Events-%u\n", in pl330_probe()
3175 pcfg->data_buf_dep, pcfg->data_bus_width / 8, pcfg->num_chan, in pl330_probe()
3176 pcfg->num_peri, pcfg->num_events); in pl330_probe()
3178 pm_runtime_irq_safe(&adev->dev); in pl330_probe()
3179 pm_runtime_use_autosuspend(&adev->dev); in pl330_probe()
3180 pm_runtime_set_autosuspend_delay(&adev->dev, PL330_AUTOSUSPEND_DELAY); in pl330_probe()
3181 pm_runtime_mark_last_busy(&adev->dev); in pl330_probe()
3182 pm_runtime_put_autosuspend(&adev->dev); in pl330_probe()
3187 list_for_each_entry_safe(pch, _p, &pl330->ddma.channels, in pl330_probe()
3191 list_del(&pch->chan.device_node); in pl330_probe()
3194 if (pch->thread) { in pl330_probe()
3195 pl330_terminate_all(&pch->chan); in pl330_probe()
3196 pl330_free_chan_resources(&pch->chan); in pl330_probe()
3200 pl330_del(pl330); in pl330_probe()
3202 if (pl330->rstc_ocp) in pl330_probe()
3203 reset_control_assert(pl330->rstc_ocp); in pl330_probe()
3205 if (pl330->rstc) in pl330_probe()
3206 reset_control_assert(pl330->rstc); in pl330_probe()
3212 struct pl330_dmac *pl330 = amba_get_drvdata(adev); in pl330_remove() local
3216 pm_runtime_get_noresume(pl330->ddma.dev); in pl330_remove()
3218 if (adev->dev.of_node) in pl330_remove()
3219 of_dma_controller_free(adev->dev.of_node); in pl330_remove()
3222 irq = adev->irq[i]; in pl330_remove()
3224 devm_free_irq(&adev->dev, irq, pl330); in pl330_remove()
3227 dma_async_device_unregister(&pl330->ddma); in pl330_remove()
3230 list_for_each_entry_safe(pch, _p, &pl330->ddma.channels, in pl330_remove()
3234 list_del(&pch->chan.device_node); in pl330_remove()
3237 if (pch->thread) { in pl330_remove()
3238 pl330_terminate_all(&pch->chan); in pl330_remove()
3239 pl330_free_chan_resources(&pch->chan); in pl330_remove()
3243 pl330_del(pl330); in pl330_remove()
3245 if (pl330->rstc_ocp) in pl330_remove()
3246 reset_control_assert(pl330->rstc_ocp); in pl330_remove()
3248 if (pl330->rstc) in pl330_remove()
3249 reset_control_assert(pl330->rstc); in pl330_remove()
3265 .name = "dma-pl330",
3276 MODULE_DESCRIPTION("API Driver for PL330 DMAC");