17fdf9b05SPeng Ma // SPDX-License-Identifier: GPL-2.0 27fdf9b05SPeng Ma // Copyright 2019 NXP 37fdf9b05SPeng Ma 47fdf9b05SPeng Ma #include <linux/init.h> 57fdf9b05SPeng Ma #include <linux/module.h> 67fdf9b05SPeng Ma #include <linux/dmapool.h> 77fdf9b05SPeng Ma #include <linux/of_irq.h> 87fdf9b05SPeng Ma #include <linux/iommu.h> 97fdf9b05SPeng Ma #include <linux/sys_soc.h> 107fdf9b05SPeng Ma #include <linux/fsl/mc.h> 117fdf9b05SPeng Ma #include <soc/fsl/dpaa2-io.h> 127fdf9b05SPeng Ma 137fdf9b05SPeng Ma #include "../virt-dma.h" 147fdf9b05SPeng Ma #include "dpdmai.h" 157fdf9b05SPeng Ma #include "dpaa2-qdma.h" 167fdf9b05SPeng Ma 177fdf9b05SPeng Ma static bool smmu_disable = true; 187fdf9b05SPeng Ma 197fdf9b05SPeng Ma static struct dpaa2_qdma_chan *to_dpaa2_qdma_chan(struct dma_chan *chan) 207fdf9b05SPeng Ma { 217fdf9b05SPeng Ma return container_of(chan, struct dpaa2_qdma_chan, vchan.chan); 227fdf9b05SPeng Ma } 237fdf9b05SPeng Ma 247fdf9b05SPeng Ma static struct dpaa2_qdma_comp *to_fsl_qdma_comp(struct virt_dma_desc *vd) 257fdf9b05SPeng Ma { 267fdf9b05SPeng Ma return container_of(vd, struct dpaa2_qdma_comp, vdesc); 277fdf9b05SPeng Ma } 287fdf9b05SPeng Ma 297fdf9b05SPeng Ma static int dpaa2_qdma_alloc_chan_resources(struct dma_chan *chan) 307fdf9b05SPeng Ma { 317fdf9b05SPeng Ma struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan); 327fdf9b05SPeng Ma struct dpaa2_qdma_engine *dpaa2_qdma = dpaa2_chan->qdma; 337fdf9b05SPeng Ma struct device *dev = &dpaa2_qdma->priv->dpdmai_dev->dev; 347fdf9b05SPeng Ma 357fdf9b05SPeng Ma dpaa2_chan->fd_pool = dma_pool_create("fd_pool", dev, 367fdf9b05SPeng Ma sizeof(struct dpaa2_fd), 377fdf9b05SPeng Ma sizeof(struct dpaa2_fd), 0); 387fdf9b05SPeng Ma if (!dpaa2_chan->fd_pool) 397fdf9b05SPeng Ma goto err; 407fdf9b05SPeng Ma 417fdf9b05SPeng Ma dpaa2_chan->fl_pool = dma_pool_create("fl_pool", dev, 427fdf9b05SPeng Ma sizeof(struct dpaa2_fl_entry), 437fdf9b05SPeng Ma sizeof(struct dpaa2_fl_entry), 0); 447fdf9b05SPeng Ma if (!dpaa2_chan->fl_pool) 457fdf9b05SPeng Ma goto err_fd; 467fdf9b05SPeng Ma 477fdf9b05SPeng Ma dpaa2_chan->sdd_pool = 487fdf9b05SPeng Ma dma_pool_create("sdd_pool", dev, 497fdf9b05SPeng Ma sizeof(struct dpaa2_qdma_sd_d), 507fdf9b05SPeng Ma sizeof(struct dpaa2_qdma_sd_d), 0); 517fdf9b05SPeng Ma if (!dpaa2_chan->sdd_pool) 527fdf9b05SPeng Ma goto err_fl; 537fdf9b05SPeng Ma 547fdf9b05SPeng Ma return dpaa2_qdma->desc_allocated++; 557fdf9b05SPeng Ma err_fl: 567fdf9b05SPeng Ma dma_pool_destroy(dpaa2_chan->fl_pool); 577fdf9b05SPeng Ma err_fd: 587fdf9b05SPeng Ma dma_pool_destroy(dpaa2_chan->fd_pool); 597fdf9b05SPeng Ma err: 607fdf9b05SPeng Ma return -ENOMEM; 617fdf9b05SPeng Ma } 627fdf9b05SPeng Ma 637fdf9b05SPeng Ma static void dpaa2_qdma_free_chan_resources(struct dma_chan *chan) 647fdf9b05SPeng Ma { 657fdf9b05SPeng Ma struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan); 667fdf9b05SPeng Ma struct dpaa2_qdma_engine *dpaa2_qdma = dpaa2_chan->qdma; 677fdf9b05SPeng Ma unsigned long flags; 687fdf9b05SPeng Ma 697fdf9b05SPeng Ma LIST_HEAD(head); 707fdf9b05SPeng Ma 717fdf9b05SPeng Ma spin_lock_irqsave(&dpaa2_chan->vchan.lock, flags); 727fdf9b05SPeng Ma vchan_get_all_descriptors(&dpaa2_chan->vchan, &head); 737fdf9b05SPeng Ma spin_unlock_irqrestore(&dpaa2_chan->vchan.lock, flags); 747fdf9b05SPeng Ma 757fdf9b05SPeng Ma vchan_dma_desc_free_list(&dpaa2_chan->vchan, &head); 767fdf9b05SPeng Ma 777fdf9b05SPeng Ma dpaa2_dpdmai_free_comp(dpaa2_chan, &dpaa2_chan->comp_used); 787fdf9b05SPeng Ma dpaa2_dpdmai_free_comp(dpaa2_chan, &dpaa2_chan->comp_free); 797fdf9b05SPeng Ma 807fdf9b05SPeng Ma dma_pool_destroy(dpaa2_chan->fd_pool); 817fdf9b05SPeng Ma dma_pool_destroy(dpaa2_chan->fl_pool); 827fdf9b05SPeng Ma dma_pool_destroy(dpaa2_chan->sdd_pool); 837fdf9b05SPeng Ma dpaa2_qdma->desc_allocated--; 847fdf9b05SPeng Ma } 857fdf9b05SPeng Ma 867fdf9b05SPeng Ma /* 877fdf9b05SPeng Ma * Request a command descriptor for enqueue. 887fdf9b05SPeng Ma */ 897fdf9b05SPeng Ma static struct dpaa2_qdma_comp * 907fdf9b05SPeng Ma dpaa2_qdma_request_desc(struct dpaa2_qdma_chan *dpaa2_chan) 917fdf9b05SPeng Ma { 927fdf9b05SPeng Ma struct dpaa2_qdma_priv *qdma_priv = dpaa2_chan->qdma->priv; 937fdf9b05SPeng Ma struct device *dev = &qdma_priv->dpdmai_dev->dev; 947fdf9b05SPeng Ma struct dpaa2_qdma_comp *comp_temp = NULL; 957fdf9b05SPeng Ma unsigned long flags; 967fdf9b05SPeng Ma 977fdf9b05SPeng Ma spin_lock_irqsave(&dpaa2_chan->queue_lock, flags); 987fdf9b05SPeng Ma if (list_empty(&dpaa2_chan->comp_free)) { 997fdf9b05SPeng Ma spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags); 1007fdf9b05SPeng Ma comp_temp = kzalloc(sizeof(*comp_temp), GFP_NOWAIT); 1017fdf9b05SPeng Ma if (!comp_temp) 1027fdf9b05SPeng Ma goto err; 1037fdf9b05SPeng Ma comp_temp->fd_virt_addr = 1047fdf9b05SPeng Ma dma_pool_alloc(dpaa2_chan->fd_pool, GFP_NOWAIT, 1057fdf9b05SPeng Ma &comp_temp->fd_bus_addr); 1067fdf9b05SPeng Ma if (!comp_temp->fd_virt_addr) 1077fdf9b05SPeng Ma goto err_comp; 1087fdf9b05SPeng Ma 1097fdf9b05SPeng Ma comp_temp->fl_virt_addr = 1107fdf9b05SPeng Ma dma_pool_alloc(dpaa2_chan->fl_pool, GFP_NOWAIT, 1117fdf9b05SPeng Ma &comp_temp->fl_bus_addr); 1127fdf9b05SPeng Ma if (!comp_temp->fl_virt_addr) 1137fdf9b05SPeng Ma goto err_fd_virt; 1147fdf9b05SPeng Ma 1157fdf9b05SPeng Ma comp_temp->desc_virt_addr = 1167fdf9b05SPeng Ma dma_pool_alloc(dpaa2_chan->sdd_pool, GFP_NOWAIT, 1177fdf9b05SPeng Ma &comp_temp->desc_bus_addr); 1187fdf9b05SPeng Ma if (!comp_temp->desc_virt_addr) 1197fdf9b05SPeng Ma goto err_fl_virt; 1207fdf9b05SPeng Ma 1217fdf9b05SPeng Ma comp_temp->qchan = dpaa2_chan; 1227fdf9b05SPeng Ma return comp_temp; 1237fdf9b05SPeng Ma } 1247fdf9b05SPeng Ma 1257fdf9b05SPeng Ma comp_temp = list_first_entry(&dpaa2_chan->comp_free, 1267fdf9b05SPeng Ma struct dpaa2_qdma_comp, list); 1277fdf9b05SPeng Ma list_del(&comp_temp->list); 1287fdf9b05SPeng Ma spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags); 1297fdf9b05SPeng Ma 1307fdf9b05SPeng Ma comp_temp->qchan = dpaa2_chan; 1317fdf9b05SPeng Ma 1327fdf9b05SPeng Ma return comp_temp; 1337fdf9b05SPeng Ma 1347fdf9b05SPeng Ma err_fl_virt: 1357fdf9b05SPeng Ma dma_pool_free(dpaa2_chan->fl_pool, 1367fdf9b05SPeng Ma comp_temp->fl_virt_addr, 1377fdf9b05SPeng Ma comp_temp->fl_bus_addr); 1387fdf9b05SPeng Ma err_fd_virt: 1397fdf9b05SPeng Ma dma_pool_free(dpaa2_chan->fd_pool, 1407fdf9b05SPeng Ma comp_temp->fd_virt_addr, 1417fdf9b05SPeng Ma comp_temp->fd_bus_addr); 1427fdf9b05SPeng Ma err_comp: 1437fdf9b05SPeng Ma kfree(comp_temp); 1447fdf9b05SPeng Ma err: 1457fdf9b05SPeng Ma dev_err(dev, "Failed to request descriptor\n"); 1467fdf9b05SPeng Ma return NULL; 1477fdf9b05SPeng Ma } 1487fdf9b05SPeng Ma 1497fdf9b05SPeng Ma static void 1507fdf9b05SPeng Ma dpaa2_qdma_populate_fd(u32 format, struct dpaa2_qdma_comp *dpaa2_comp) 1517fdf9b05SPeng Ma { 1527fdf9b05SPeng Ma struct dpaa2_fd *fd; 1537fdf9b05SPeng Ma 1547fdf9b05SPeng Ma fd = dpaa2_comp->fd_virt_addr; 1557fdf9b05SPeng Ma memset(fd, 0, sizeof(struct dpaa2_fd)); 1567fdf9b05SPeng Ma 1577fdf9b05SPeng Ma /* fd populated */ 1587fdf9b05SPeng Ma dpaa2_fd_set_addr(fd, dpaa2_comp->fl_bus_addr); 1597fdf9b05SPeng Ma 1607fdf9b05SPeng Ma /* 1617fdf9b05SPeng Ma * Bypass memory translation, Frame list format, short length disable 1627fdf9b05SPeng Ma * we need to disable BMT if fsl-mc use iova addr 1637fdf9b05SPeng Ma */ 1647fdf9b05SPeng Ma if (smmu_disable) 1657fdf9b05SPeng Ma dpaa2_fd_set_bpid(fd, QMAN_FD_BMT_ENABLE); 1667fdf9b05SPeng Ma dpaa2_fd_set_format(fd, QMAN_FD_FMT_ENABLE | QMAN_FD_SL_DISABLE); 1677fdf9b05SPeng Ma 1687fdf9b05SPeng Ma dpaa2_fd_set_frc(fd, format | QDMA_SER_CTX); 1697fdf9b05SPeng Ma } 1707fdf9b05SPeng Ma 1717fdf9b05SPeng Ma /* first frame list for descriptor buffer */ 1727fdf9b05SPeng Ma static void 1737fdf9b05SPeng Ma dpaa2_qdma_populate_first_framel(struct dpaa2_fl_entry *f_list, 1747fdf9b05SPeng Ma struct dpaa2_qdma_comp *dpaa2_comp, 1757fdf9b05SPeng Ma bool wrt_changed) 1767fdf9b05SPeng Ma { 1777fdf9b05SPeng Ma struct dpaa2_qdma_sd_d *sdd; 1787fdf9b05SPeng Ma 1797fdf9b05SPeng Ma sdd = dpaa2_comp->desc_virt_addr; 1807fdf9b05SPeng Ma memset(sdd, 0, 2 * (sizeof(*sdd))); 1817fdf9b05SPeng Ma 1827fdf9b05SPeng Ma /* source descriptor CMD */ 1837fdf9b05SPeng Ma sdd->cmd = cpu_to_le32(QDMA_SD_CMD_RDTTYPE_COHERENT); 1847fdf9b05SPeng Ma sdd++; 1857fdf9b05SPeng Ma 1867fdf9b05SPeng Ma /* dest descriptor CMD */ 1877fdf9b05SPeng Ma if (wrt_changed) 1887fdf9b05SPeng Ma sdd->cmd = cpu_to_le32(LX2160_QDMA_DD_CMD_WRTTYPE_COHERENT); 1897fdf9b05SPeng Ma else 1907fdf9b05SPeng Ma sdd->cmd = cpu_to_le32(QDMA_DD_CMD_WRTTYPE_COHERENT); 1917fdf9b05SPeng Ma 1927fdf9b05SPeng Ma memset(f_list, 0, sizeof(struct dpaa2_fl_entry)); 1937fdf9b05SPeng Ma 1947fdf9b05SPeng Ma /* first frame list to source descriptor */ 1957fdf9b05SPeng Ma dpaa2_fl_set_addr(f_list, dpaa2_comp->desc_bus_addr); 1967fdf9b05SPeng Ma dpaa2_fl_set_len(f_list, 0x20); 1977fdf9b05SPeng Ma dpaa2_fl_set_format(f_list, QDMA_FL_FMT_SBF | QDMA_FL_SL_LONG); 1987fdf9b05SPeng Ma 1997fdf9b05SPeng Ma /* bypass memory translation */ 2007fdf9b05SPeng Ma if (smmu_disable) 2017fdf9b05SPeng Ma f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); 2027fdf9b05SPeng Ma } 2037fdf9b05SPeng Ma 2047fdf9b05SPeng Ma /* source and destination frame list */ 2057fdf9b05SPeng Ma static void 2067fdf9b05SPeng Ma dpaa2_qdma_populate_frames(struct dpaa2_fl_entry *f_list, 2077fdf9b05SPeng Ma dma_addr_t dst, dma_addr_t src, 2087fdf9b05SPeng Ma size_t len, uint8_t fmt) 2097fdf9b05SPeng Ma { 2107fdf9b05SPeng Ma /* source frame list to source buffer */ 2117fdf9b05SPeng Ma memset(f_list, 0, sizeof(struct dpaa2_fl_entry)); 2127fdf9b05SPeng Ma 2137fdf9b05SPeng Ma dpaa2_fl_set_addr(f_list, src); 2147fdf9b05SPeng Ma dpaa2_fl_set_len(f_list, len); 2157fdf9b05SPeng Ma 2167fdf9b05SPeng Ma /* single buffer frame or scatter gather frame */ 2177fdf9b05SPeng Ma dpaa2_fl_set_format(f_list, (fmt | QDMA_FL_SL_LONG)); 2187fdf9b05SPeng Ma 2197fdf9b05SPeng Ma /* bypass memory translation */ 2207fdf9b05SPeng Ma if (smmu_disable) 2217fdf9b05SPeng Ma f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); 2227fdf9b05SPeng Ma 2237fdf9b05SPeng Ma f_list++; 2247fdf9b05SPeng Ma 2257fdf9b05SPeng Ma /* destination frame list to destination buffer */ 2267fdf9b05SPeng Ma memset(f_list, 0, sizeof(struct dpaa2_fl_entry)); 2277fdf9b05SPeng Ma 2287fdf9b05SPeng Ma dpaa2_fl_set_addr(f_list, dst); 2297fdf9b05SPeng Ma dpaa2_fl_set_len(f_list, len); 2307fdf9b05SPeng Ma dpaa2_fl_set_format(f_list, (fmt | QDMA_FL_SL_LONG)); 2317fdf9b05SPeng Ma /* single buffer frame or scatter gather frame */ 2327fdf9b05SPeng Ma dpaa2_fl_set_final(f_list, QDMA_FL_F); 2337fdf9b05SPeng Ma /* bypass memory translation */ 2347fdf9b05SPeng Ma if (smmu_disable) 2357fdf9b05SPeng Ma f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); 2367fdf9b05SPeng Ma } 2377fdf9b05SPeng Ma 2387fdf9b05SPeng Ma static struct dma_async_tx_descriptor 2397fdf9b05SPeng Ma *dpaa2_qdma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, 2407fdf9b05SPeng Ma dma_addr_t src, size_t len, ulong flags) 2417fdf9b05SPeng Ma { 2427fdf9b05SPeng Ma struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan); 2437fdf9b05SPeng Ma struct dpaa2_qdma_engine *dpaa2_qdma; 2447fdf9b05SPeng Ma struct dpaa2_qdma_comp *dpaa2_comp; 2457fdf9b05SPeng Ma struct dpaa2_fl_entry *f_list; 2467fdf9b05SPeng Ma bool wrt_changed; 2477fdf9b05SPeng Ma 2487fdf9b05SPeng Ma dpaa2_qdma = dpaa2_chan->qdma; 2497fdf9b05SPeng Ma dpaa2_comp = dpaa2_qdma_request_desc(dpaa2_chan); 2507fdf9b05SPeng Ma if (!dpaa2_comp) 2517fdf9b05SPeng Ma return NULL; 2527fdf9b05SPeng Ma 2537fdf9b05SPeng Ma wrt_changed = (bool)dpaa2_qdma->qdma_wrtype_fixup; 2547fdf9b05SPeng Ma 2557fdf9b05SPeng Ma /* populate Frame descriptor */ 2567fdf9b05SPeng Ma dpaa2_qdma_populate_fd(QDMA_FD_LONG_FORMAT, dpaa2_comp); 2577fdf9b05SPeng Ma 2587fdf9b05SPeng Ma f_list = dpaa2_comp->fl_virt_addr; 2597fdf9b05SPeng Ma 2607fdf9b05SPeng Ma /* first frame list for descriptor buffer (logn format) */ 2617fdf9b05SPeng Ma dpaa2_qdma_populate_first_framel(f_list, dpaa2_comp, wrt_changed); 2627fdf9b05SPeng Ma 2637fdf9b05SPeng Ma f_list++; 2647fdf9b05SPeng Ma 2657fdf9b05SPeng Ma dpaa2_qdma_populate_frames(f_list, dst, src, len, QDMA_FL_FMT_SBF); 2667fdf9b05SPeng Ma 2677fdf9b05SPeng Ma return vchan_tx_prep(&dpaa2_chan->vchan, &dpaa2_comp->vdesc, flags); 2687fdf9b05SPeng Ma } 2697fdf9b05SPeng Ma 2707fdf9b05SPeng Ma static void dpaa2_qdma_issue_pending(struct dma_chan *chan) 2717fdf9b05SPeng Ma { 2727fdf9b05SPeng Ma struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan); 2737fdf9b05SPeng Ma struct dpaa2_qdma_comp *dpaa2_comp; 2747fdf9b05SPeng Ma struct virt_dma_desc *vdesc; 2757fdf9b05SPeng Ma struct dpaa2_fd *fd; 2767fdf9b05SPeng Ma unsigned long flags; 2777fdf9b05SPeng Ma int err; 2787fdf9b05SPeng Ma 2797fdf9b05SPeng Ma spin_lock_irqsave(&dpaa2_chan->queue_lock, flags); 2807fdf9b05SPeng Ma spin_lock(&dpaa2_chan->vchan.lock); 2817fdf9b05SPeng Ma if (vchan_issue_pending(&dpaa2_chan->vchan)) { 2827fdf9b05SPeng Ma vdesc = vchan_next_desc(&dpaa2_chan->vchan); 2837fdf9b05SPeng Ma if (!vdesc) 2847fdf9b05SPeng Ma goto err_enqueue; 2857fdf9b05SPeng Ma dpaa2_comp = to_fsl_qdma_comp(vdesc); 2867fdf9b05SPeng Ma 2877fdf9b05SPeng Ma fd = dpaa2_comp->fd_virt_addr; 2887fdf9b05SPeng Ma 2897fdf9b05SPeng Ma list_del(&vdesc->node); 2907fdf9b05SPeng Ma list_add_tail(&dpaa2_comp->list, &dpaa2_chan->comp_used); 2917fdf9b05SPeng Ma 2927fdf9b05SPeng Ma err = dpaa2_io_service_enqueue_fq(NULL, dpaa2_chan->fqid, fd); 2937fdf9b05SPeng Ma if (err) { 294df208d63SBaokun Li list_move_tail(&dpaa2_comp->list, 2957fdf9b05SPeng Ma &dpaa2_chan->comp_free); 2967fdf9b05SPeng Ma } 2977fdf9b05SPeng Ma } 2987fdf9b05SPeng Ma err_enqueue: 2997fdf9b05SPeng Ma spin_unlock(&dpaa2_chan->vchan.lock); 3007fdf9b05SPeng Ma spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags); 3017fdf9b05SPeng Ma } 3027fdf9b05SPeng Ma 3037fdf9b05SPeng Ma static int __cold dpaa2_qdma_setup(struct fsl_mc_device *ls_dev) 3047fdf9b05SPeng Ma { 3057fdf9b05SPeng Ma struct dpaa2_qdma_priv_per_prio *ppriv; 3067fdf9b05SPeng Ma struct device *dev = &ls_dev->dev; 3077fdf9b05SPeng Ma struct dpaa2_qdma_priv *priv; 3087fdf9b05SPeng Ma u8 prio_def = DPDMAI_PRIO_NUM; 3097fdf9b05SPeng Ma int err = -EINVAL; 3107fdf9b05SPeng Ma int i; 3117fdf9b05SPeng Ma 3127fdf9b05SPeng Ma priv = dev_get_drvdata(dev); 3137fdf9b05SPeng Ma 3147fdf9b05SPeng Ma priv->dev = dev; 3157fdf9b05SPeng Ma priv->dpqdma_id = ls_dev->obj_desc.id; 3167fdf9b05SPeng Ma 3177fdf9b05SPeng Ma /* Get the handle for the DPDMAI this interface is associate with */ 3187fdf9b05SPeng Ma err = dpdmai_open(priv->mc_io, 0, priv->dpqdma_id, &ls_dev->mc_handle); 3197fdf9b05SPeng Ma if (err) { 3207fdf9b05SPeng Ma dev_err(dev, "dpdmai_open() failed\n"); 3217fdf9b05SPeng Ma return err; 3227fdf9b05SPeng Ma } 3237fdf9b05SPeng Ma 3247fdf9b05SPeng Ma dev_dbg(dev, "Opened dpdmai object successfully\n"); 3257fdf9b05SPeng Ma 3267fdf9b05SPeng Ma err = dpdmai_get_attributes(priv->mc_io, 0, ls_dev->mc_handle, 3277fdf9b05SPeng Ma &priv->dpdmai_attr); 3287fdf9b05SPeng Ma if (err) { 3297fdf9b05SPeng Ma dev_err(dev, "dpdmai_get_attributes() failed\n"); 3307fdf9b05SPeng Ma goto exit; 3317fdf9b05SPeng Ma } 3327fdf9b05SPeng Ma 3337fdf9b05SPeng Ma if (priv->dpdmai_attr.version.major > DPDMAI_VER_MAJOR) { 33417866bc6SZhen Lei err = -EINVAL; 3357fdf9b05SPeng Ma dev_err(dev, "DPDMAI major version mismatch\n" 3367fdf9b05SPeng Ma "Found %u.%u, supported version is %u.%u\n", 3377fdf9b05SPeng Ma priv->dpdmai_attr.version.major, 3387fdf9b05SPeng Ma priv->dpdmai_attr.version.minor, 3397fdf9b05SPeng Ma DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR); 3407fdf9b05SPeng Ma goto exit; 3417fdf9b05SPeng Ma } 3427fdf9b05SPeng Ma 3437fdf9b05SPeng Ma if (priv->dpdmai_attr.version.minor > DPDMAI_VER_MINOR) { 34417866bc6SZhen Lei err = -EINVAL; 3457fdf9b05SPeng Ma dev_err(dev, "DPDMAI minor version mismatch\n" 3467fdf9b05SPeng Ma "Found %u.%u, supported version is %u.%u\n", 3477fdf9b05SPeng Ma priv->dpdmai_attr.version.major, 3487fdf9b05SPeng Ma priv->dpdmai_attr.version.minor, 3497fdf9b05SPeng Ma DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR); 3507fdf9b05SPeng Ma goto exit; 3517fdf9b05SPeng Ma } 3527fdf9b05SPeng Ma 3537fdf9b05SPeng Ma priv->num_pairs = min(priv->dpdmai_attr.num_of_priorities, prio_def); 3547fdf9b05SPeng Ma ppriv = kcalloc(priv->num_pairs, sizeof(*ppriv), GFP_KERNEL); 3557fdf9b05SPeng Ma if (!ppriv) { 3567fdf9b05SPeng Ma err = -ENOMEM; 3577fdf9b05SPeng Ma goto exit; 3587fdf9b05SPeng Ma } 3597fdf9b05SPeng Ma priv->ppriv = ppriv; 3607fdf9b05SPeng Ma 3617fdf9b05SPeng Ma for (i = 0; i < priv->num_pairs; i++) { 3627fdf9b05SPeng Ma err = dpdmai_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, 3637fdf9b05SPeng Ma i, &priv->rx_queue_attr[i]); 3647fdf9b05SPeng Ma if (err) { 3657fdf9b05SPeng Ma dev_err(dev, "dpdmai_get_rx_queue() failed\n"); 3667fdf9b05SPeng Ma goto exit; 3677fdf9b05SPeng Ma } 3687fdf9b05SPeng Ma ppriv->rsp_fqid = priv->rx_queue_attr[i].fqid; 3697fdf9b05SPeng Ma 3707fdf9b05SPeng Ma err = dpdmai_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, 3717fdf9b05SPeng Ma i, &priv->tx_fqid[i]); 3727fdf9b05SPeng Ma if (err) { 3737fdf9b05SPeng Ma dev_err(dev, "dpdmai_get_tx_queue() failed\n"); 3747fdf9b05SPeng Ma goto exit; 3757fdf9b05SPeng Ma } 3767fdf9b05SPeng Ma ppriv->req_fqid = priv->tx_fqid[i]; 3777fdf9b05SPeng Ma ppriv->prio = i; 3787fdf9b05SPeng Ma ppriv->priv = priv; 3797fdf9b05SPeng Ma ppriv++; 3807fdf9b05SPeng Ma } 3817fdf9b05SPeng Ma 3827fdf9b05SPeng Ma return 0; 3837fdf9b05SPeng Ma exit: 3847fdf9b05SPeng Ma dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle); 3857fdf9b05SPeng Ma return err; 3867fdf9b05SPeng Ma } 3877fdf9b05SPeng Ma 3887fdf9b05SPeng Ma static void dpaa2_qdma_fqdan_cb(struct dpaa2_io_notification_ctx *ctx) 3897fdf9b05SPeng Ma { 3907fdf9b05SPeng Ma struct dpaa2_qdma_priv_per_prio *ppriv = container_of(ctx, 3917fdf9b05SPeng Ma struct dpaa2_qdma_priv_per_prio, nctx); 3927fdf9b05SPeng Ma struct dpaa2_qdma_comp *dpaa2_comp, *_comp_tmp; 3937fdf9b05SPeng Ma struct dpaa2_qdma_priv *priv = ppriv->priv; 3947fdf9b05SPeng Ma u32 n_chans = priv->dpaa2_qdma->n_chans; 3957fdf9b05SPeng Ma struct dpaa2_qdma_chan *qchan; 3967fdf9b05SPeng Ma const struct dpaa2_fd *fd_eq; 3977fdf9b05SPeng Ma const struct dpaa2_fd *fd; 3987fdf9b05SPeng Ma struct dpaa2_dq *dq; 3997fdf9b05SPeng Ma int is_last = 0; 4007fdf9b05SPeng Ma int found; 4017fdf9b05SPeng Ma u8 status; 4027fdf9b05SPeng Ma int err; 4037fdf9b05SPeng Ma int i; 4047fdf9b05SPeng Ma 4057fdf9b05SPeng Ma do { 4067fdf9b05SPeng Ma err = dpaa2_io_service_pull_fq(NULL, ppriv->rsp_fqid, 4077fdf9b05SPeng Ma ppriv->store); 4087fdf9b05SPeng Ma } while (err); 4097fdf9b05SPeng Ma 4107fdf9b05SPeng Ma while (!is_last) { 4117fdf9b05SPeng Ma do { 4127fdf9b05SPeng Ma dq = dpaa2_io_store_next(ppriv->store, &is_last); 4137fdf9b05SPeng Ma } while (!is_last && !dq); 4147fdf9b05SPeng Ma if (!dq) { 4157fdf9b05SPeng Ma dev_err(priv->dev, "FQID returned no valid frames!\n"); 4167fdf9b05SPeng Ma continue; 4177fdf9b05SPeng Ma } 4187fdf9b05SPeng Ma 4197fdf9b05SPeng Ma /* obtain FD and process the error */ 4207fdf9b05SPeng Ma fd = dpaa2_dq_fd(dq); 4217fdf9b05SPeng Ma 4227fdf9b05SPeng Ma status = dpaa2_fd_get_ctrl(fd) & 0xff; 4237fdf9b05SPeng Ma if (status) 4247fdf9b05SPeng Ma dev_err(priv->dev, "FD error occurred\n"); 4257fdf9b05SPeng Ma found = 0; 4267fdf9b05SPeng Ma for (i = 0; i < n_chans; i++) { 4277fdf9b05SPeng Ma qchan = &priv->dpaa2_qdma->chans[i]; 4287fdf9b05SPeng Ma spin_lock(&qchan->queue_lock); 4297fdf9b05SPeng Ma if (list_empty(&qchan->comp_used)) { 4307fdf9b05SPeng Ma spin_unlock(&qchan->queue_lock); 4317fdf9b05SPeng Ma continue; 4327fdf9b05SPeng Ma } 4337fdf9b05SPeng Ma list_for_each_entry_safe(dpaa2_comp, _comp_tmp, 4347fdf9b05SPeng Ma &qchan->comp_used, list) { 4357fdf9b05SPeng Ma fd_eq = dpaa2_comp->fd_virt_addr; 4367fdf9b05SPeng Ma 4377fdf9b05SPeng Ma if (le64_to_cpu(fd_eq->simple.addr) == 4387fdf9b05SPeng Ma le64_to_cpu(fd->simple.addr)) { 4397fdf9b05SPeng Ma spin_lock(&qchan->vchan.lock); 4407fdf9b05SPeng Ma vchan_cookie_complete(& 4417fdf9b05SPeng Ma dpaa2_comp->vdesc); 4427fdf9b05SPeng Ma spin_unlock(&qchan->vchan.lock); 4437fdf9b05SPeng Ma found = 1; 4447fdf9b05SPeng Ma break; 4457fdf9b05SPeng Ma } 4467fdf9b05SPeng Ma } 4477fdf9b05SPeng Ma spin_unlock(&qchan->queue_lock); 4487fdf9b05SPeng Ma if (found) 4497fdf9b05SPeng Ma break; 4507fdf9b05SPeng Ma } 4517fdf9b05SPeng Ma } 4527fdf9b05SPeng Ma 4537fdf9b05SPeng Ma dpaa2_io_service_rearm(NULL, ctx); 4547fdf9b05SPeng Ma } 4557fdf9b05SPeng Ma 4567fdf9b05SPeng Ma static int __cold dpaa2_qdma_dpio_setup(struct dpaa2_qdma_priv *priv) 4577fdf9b05SPeng Ma { 4587fdf9b05SPeng Ma struct dpaa2_qdma_priv_per_prio *ppriv; 4597fdf9b05SPeng Ma struct device *dev = priv->dev; 4607fdf9b05SPeng Ma int err = -EINVAL; 4617fdf9b05SPeng Ma int i, num; 4627fdf9b05SPeng Ma 4637fdf9b05SPeng Ma num = priv->num_pairs; 4647fdf9b05SPeng Ma ppriv = priv->ppriv; 4657fdf9b05SPeng Ma for (i = 0; i < num; i++) { 4667fdf9b05SPeng Ma ppriv->nctx.is_cdan = 0; 4677fdf9b05SPeng Ma ppriv->nctx.desired_cpu = DPAA2_IO_ANY_CPU; 4687fdf9b05SPeng Ma ppriv->nctx.id = ppriv->rsp_fqid; 4697fdf9b05SPeng Ma ppriv->nctx.cb = dpaa2_qdma_fqdan_cb; 4707fdf9b05SPeng Ma err = dpaa2_io_service_register(NULL, &ppriv->nctx, dev); 4717fdf9b05SPeng Ma if (err) { 4727fdf9b05SPeng Ma dev_err(dev, "Notification register failed\n"); 4737fdf9b05SPeng Ma goto err_service; 4747fdf9b05SPeng Ma } 4757fdf9b05SPeng Ma 4767fdf9b05SPeng Ma ppriv->store = 4777fdf9b05SPeng Ma dpaa2_io_store_create(DPAA2_QDMA_STORE_SIZE, dev); 4787fdf9b05SPeng Ma if (!ppriv->store) { 47917866bc6SZhen Lei err = -ENOMEM; 4807fdf9b05SPeng Ma dev_err(dev, "dpaa2_io_store_create() failed\n"); 4817fdf9b05SPeng Ma goto err_store; 4827fdf9b05SPeng Ma } 4837fdf9b05SPeng Ma 4847fdf9b05SPeng Ma ppriv++; 4857fdf9b05SPeng Ma } 4867fdf9b05SPeng Ma return 0; 4877fdf9b05SPeng Ma 4887fdf9b05SPeng Ma err_store: 4897fdf9b05SPeng Ma dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev); 4907fdf9b05SPeng Ma err_service: 4917fdf9b05SPeng Ma ppriv--; 4927fdf9b05SPeng Ma while (ppriv >= priv->ppriv) { 4937fdf9b05SPeng Ma dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev); 4947fdf9b05SPeng Ma dpaa2_io_store_destroy(ppriv->store); 4957fdf9b05SPeng Ma ppriv--; 4967fdf9b05SPeng Ma } 4977fdf9b05SPeng Ma return err; 4987fdf9b05SPeng Ma } 4997fdf9b05SPeng Ma 5007fdf9b05SPeng Ma static void dpaa2_dpmai_store_free(struct dpaa2_qdma_priv *priv) 5017fdf9b05SPeng Ma { 5027fdf9b05SPeng Ma struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv; 5037fdf9b05SPeng Ma int i; 5047fdf9b05SPeng Ma 5057fdf9b05SPeng Ma for (i = 0; i < priv->num_pairs; i++) { 5067fdf9b05SPeng Ma dpaa2_io_store_destroy(ppriv->store); 5077fdf9b05SPeng Ma ppriv++; 5087fdf9b05SPeng Ma } 5097fdf9b05SPeng Ma } 5107fdf9b05SPeng Ma 5117fdf9b05SPeng Ma static void dpaa2_dpdmai_dpio_free(struct dpaa2_qdma_priv *priv) 5127fdf9b05SPeng Ma { 5137fdf9b05SPeng Ma struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv; 5147fdf9b05SPeng Ma struct device *dev = priv->dev; 5157fdf9b05SPeng Ma int i; 5167fdf9b05SPeng Ma 5177fdf9b05SPeng Ma for (i = 0; i < priv->num_pairs; i++) { 5187fdf9b05SPeng Ma dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev); 5197fdf9b05SPeng Ma ppriv++; 5207fdf9b05SPeng Ma } 5217fdf9b05SPeng Ma } 5227fdf9b05SPeng Ma 5237fdf9b05SPeng Ma static int __cold dpaa2_dpdmai_bind(struct dpaa2_qdma_priv *priv) 5247fdf9b05SPeng Ma { 5257fdf9b05SPeng Ma struct dpdmai_rx_queue_cfg rx_queue_cfg; 5267fdf9b05SPeng Ma struct dpaa2_qdma_priv_per_prio *ppriv; 5277fdf9b05SPeng Ma struct device *dev = priv->dev; 5287fdf9b05SPeng Ma struct fsl_mc_device *ls_dev; 5297fdf9b05SPeng Ma int i, num; 5307fdf9b05SPeng Ma int err; 5317fdf9b05SPeng Ma 5327fdf9b05SPeng Ma ls_dev = to_fsl_mc_device(dev); 5337fdf9b05SPeng Ma num = priv->num_pairs; 5347fdf9b05SPeng Ma ppriv = priv->ppriv; 5357fdf9b05SPeng Ma for (i = 0; i < num; i++) { 5367fdf9b05SPeng Ma rx_queue_cfg.options = DPDMAI_QUEUE_OPT_USER_CTX | 5377fdf9b05SPeng Ma DPDMAI_QUEUE_OPT_DEST; 5387fdf9b05SPeng Ma rx_queue_cfg.user_ctx = ppriv->nctx.qman64; 5397fdf9b05SPeng Ma rx_queue_cfg.dest_cfg.dest_type = DPDMAI_DEST_DPIO; 5407fdf9b05SPeng Ma rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; 5417fdf9b05SPeng Ma rx_queue_cfg.dest_cfg.priority = ppriv->prio; 5427fdf9b05SPeng Ma err = dpdmai_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, 5437fdf9b05SPeng Ma rx_queue_cfg.dest_cfg.priority, 5447fdf9b05SPeng Ma &rx_queue_cfg); 5457fdf9b05SPeng Ma if (err) { 5467fdf9b05SPeng Ma dev_err(dev, "dpdmai_set_rx_queue() failed\n"); 5477fdf9b05SPeng Ma return err; 5487fdf9b05SPeng Ma } 5497fdf9b05SPeng Ma 5507fdf9b05SPeng Ma ppriv++; 5517fdf9b05SPeng Ma } 5527fdf9b05SPeng Ma 5537fdf9b05SPeng Ma return 0; 5547fdf9b05SPeng Ma } 5557fdf9b05SPeng Ma 5567fdf9b05SPeng Ma static int __cold dpaa2_dpdmai_dpio_unbind(struct dpaa2_qdma_priv *priv) 5577fdf9b05SPeng Ma { 5587fdf9b05SPeng Ma struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv; 5597fdf9b05SPeng Ma struct device *dev = priv->dev; 5607fdf9b05SPeng Ma struct fsl_mc_device *ls_dev; 5617fdf9b05SPeng Ma int err = 0; 5627fdf9b05SPeng Ma int i; 5637fdf9b05SPeng Ma 5647fdf9b05SPeng Ma ls_dev = to_fsl_mc_device(dev); 5657fdf9b05SPeng Ma 5667fdf9b05SPeng Ma for (i = 0; i < priv->num_pairs; i++) { 5677fdf9b05SPeng Ma ppriv->nctx.qman64 = 0; 5687fdf9b05SPeng Ma ppriv->nctx.dpio_id = 0; 5697fdf9b05SPeng Ma ppriv++; 5707fdf9b05SPeng Ma } 5717fdf9b05SPeng Ma 5727fdf9b05SPeng Ma err = dpdmai_reset(priv->mc_io, 0, ls_dev->mc_handle); 5737fdf9b05SPeng Ma if (err) 5747fdf9b05SPeng Ma dev_err(dev, "dpdmai_reset() failed\n"); 5757fdf9b05SPeng Ma 5767fdf9b05SPeng Ma return err; 5777fdf9b05SPeng Ma } 5787fdf9b05SPeng Ma 5797fdf9b05SPeng Ma static void dpaa2_dpdmai_free_comp(struct dpaa2_qdma_chan *qchan, 5807fdf9b05SPeng Ma struct list_head *head) 5817fdf9b05SPeng Ma { 5827fdf9b05SPeng Ma struct dpaa2_qdma_comp *comp_tmp, *_comp_tmp; 5837fdf9b05SPeng Ma unsigned long flags; 5847fdf9b05SPeng Ma 5857fdf9b05SPeng Ma list_for_each_entry_safe(comp_tmp, _comp_tmp, 5867fdf9b05SPeng Ma head, list) { 5877fdf9b05SPeng Ma spin_lock_irqsave(&qchan->queue_lock, flags); 5887fdf9b05SPeng Ma list_del(&comp_tmp->list); 5897fdf9b05SPeng Ma spin_unlock_irqrestore(&qchan->queue_lock, flags); 5907fdf9b05SPeng Ma dma_pool_free(qchan->fd_pool, 5917fdf9b05SPeng Ma comp_tmp->fd_virt_addr, 5927fdf9b05SPeng Ma comp_tmp->fd_bus_addr); 5937fdf9b05SPeng Ma dma_pool_free(qchan->fl_pool, 5947fdf9b05SPeng Ma comp_tmp->fl_virt_addr, 5957fdf9b05SPeng Ma comp_tmp->fl_bus_addr); 5967fdf9b05SPeng Ma dma_pool_free(qchan->sdd_pool, 5977fdf9b05SPeng Ma comp_tmp->desc_virt_addr, 5987fdf9b05SPeng Ma comp_tmp->desc_bus_addr); 5997fdf9b05SPeng Ma kfree(comp_tmp); 6007fdf9b05SPeng Ma } 6017fdf9b05SPeng Ma } 6027fdf9b05SPeng Ma 6037fdf9b05SPeng Ma static void dpaa2_dpdmai_free_channels(struct dpaa2_qdma_engine *dpaa2_qdma) 6047fdf9b05SPeng Ma { 6057fdf9b05SPeng Ma struct dpaa2_qdma_chan *qchan; 6067fdf9b05SPeng Ma int num, i; 6077fdf9b05SPeng Ma 6087fdf9b05SPeng Ma num = dpaa2_qdma->n_chans; 6097fdf9b05SPeng Ma for (i = 0; i < num; i++) { 6107fdf9b05SPeng Ma qchan = &dpaa2_qdma->chans[i]; 6117fdf9b05SPeng Ma dpaa2_dpdmai_free_comp(qchan, &qchan->comp_used); 6127fdf9b05SPeng Ma dpaa2_dpdmai_free_comp(qchan, &qchan->comp_free); 6137fdf9b05SPeng Ma dma_pool_destroy(qchan->fd_pool); 6147fdf9b05SPeng Ma dma_pool_destroy(qchan->fl_pool); 6157fdf9b05SPeng Ma dma_pool_destroy(qchan->sdd_pool); 6167fdf9b05SPeng Ma } 6177fdf9b05SPeng Ma } 6187fdf9b05SPeng Ma 6197fdf9b05SPeng Ma static void dpaa2_qdma_free_desc(struct virt_dma_desc *vdesc) 6207fdf9b05SPeng Ma { 6217fdf9b05SPeng Ma struct dpaa2_qdma_comp *dpaa2_comp; 6227fdf9b05SPeng Ma struct dpaa2_qdma_chan *qchan; 6237fdf9b05SPeng Ma unsigned long flags; 6247fdf9b05SPeng Ma 6257fdf9b05SPeng Ma dpaa2_comp = to_fsl_qdma_comp(vdesc); 6267fdf9b05SPeng Ma qchan = dpaa2_comp->qchan; 6277fdf9b05SPeng Ma spin_lock_irqsave(&qchan->queue_lock, flags); 628df208d63SBaokun Li list_move_tail(&dpaa2_comp->list, &qchan->comp_free); 6297fdf9b05SPeng Ma spin_unlock_irqrestore(&qchan->queue_lock, flags); 6307fdf9b05SPeng Ma } 6317fdf9b05SPeng Ma 6327fdf9b05SPeng Ma static int dpaa2_dpdmai_init_channels(struct dpaa2_qdma_engine *dpaa2_qdma) 6337fdf9b05SPeng Ma { 6347fdf9b05SPeng Ma struct dpaa2_qdma_priv *priv = dpaa2_qdma->priv; 6357fdf9b05SPeng Ma struct dpaa2_qdma_chan *dpaa2_chan; 6367fdf9b05SPeng Ma int num = priv->num_pairs; 6377fdf9b05SPeng Ma int i; 6387fdf9b05SPeng Ma 6397fdf9b05SPeng Ma INIT_LIST_HEAD(&dpaa2_qdma->dma_dev.channels); 6407fdf9b05SPeng Ma for (i = 0; i < dpaa2_qdma->n_chans; i++) { 6417fdf9b05SPeng Ma dpaa2_chan = &dpaa2_qdma->chans[i]; 6427fdf9b05SPeng Ma dpaa2_chan->qdma = dpaa2_qdma; 6437fdf9b05SPeng Ma dpaa2_chan->fqid = priv->tx_fqid[i % num]; 6447fdf9b05SPeng Ma dpaa2_chan->vchan.desc_free = dpaa2_qdma_free_desc; 6457fdf9b05SPeng Ma vchan_init(&dpaa2_chan->vchan, &dpaa2_qdma->dma_dev); 6467fdf9b05SPeng Ma spin_lock_init(&dpaa2_chan->queue_lock); 6477fdf9b05SPeng Ma INIT_LIST_HEAD(&dpaa2_chan->comp_used); 6487fdf9b05SPeng Ma INIT_LIST_HEAD(&dpaa2_chan->comp_free); 6497fdf9b05SPeng Ma } 6507fdf9b05SPeng Ma return 0; 6517fdf9b05SPeng Ma } 6527fdf9b05SPeng Ma 6537fdf9b05SPeng Ma static int dpaa2_qdma_probe(struct fsl_mc_device *dpdmai_dev) 6547fdf9b05SPeng Ma { 6557fdf9b05SPeng Ma struct device *dev = &dpdmai_dev->dev; 6567fdf9b05SPeng Ma struct dpaa2_qdma_engine *dpaa2_qdma; 6577fdf9b05SPeng Ma struct dpaa2_qdma_priv *priv; 6587fdf9b05SPeng Ma int err; 6597fdf9b05SPeng Ma 6607fdf9b05SPeng Ma priv = kzalloc(sizeof(*priv), GFP_KERNEL); 6617fdf9b05SPeng Ma if (!priv) 6627fdf9b05SPeng Ma return -ENOMEM; 6637fdf9b05SPeng Ma dev_set_drvdata(dev, priv); 6647fdf9b05SPeng Ma priv->dpdmai_dev = dpdmai_dev; 6657fdf9b05SPeng Ma 6667fdf9b05SPeng Ma priv->iommu_domain = iommu_get_domain_for_dev(dev); 6677fdf9b05SPeng Ma if (priv->iommu_domain) 6687fdf9b05SPeng Ma smmu_disable = false; 6697fdf9b05SPeng Ma 6707fdf9b05SPeng Ma /* obtain a MC portal */ 6717fdf9b05SPeng Ma err = fsl_mc_portal_allocate(dpdmai_dev, 0, &priv->mc_io); 6727fdf9b05SPeng Ma if (err) { 6737fdf9b05SPeng Ma if (err == -ENXIO) 6747fdf9b05SPeng Ma err = -EPROBE_DEFER; 6757fdf9b05SPeng Ma else 6767fdf9b05SPeng Ma dev_err(dev, "MC portal allocation failed\n"); 6777fdf9b05SPeng Ma goto err_mcportal; 6787fdf9b05SPeng Ma } 6797fdf9b05SPeng Ma 6807fdf9b05SPeng Ma /* DPDMAI initialization */ 6817fdf9b05SPeng Ma err = dpaa2_qdma_setup(dpdmai_dev); 6827fdf9b05SPeng Ma if (err) { 6837fdf9b05SPeng Ma dev_err(dev, "dpaa2_dpdmai_setup() failed\n"); 6847fdf9b05SPeng Ma goto err_dpdmai_setup; 6857fdf9b05SPeng Ma } 6867fdf9b05SPeng Ma 6877fdf9b05SPeng Ma /* DPIO */ 6887fdf9b05SPeng Ma err = dpaa2_qdma_dpio_setup(priv); 6897fdf9b05SPeng Ma if (err) { 6907fdf9b05SPeng Ma dev_err(dev, "dpaa2_dpdmai_dpio_setup() failed\n"); 6917fdf9b05SPeng Ma goto err_dpio_setup; 6927fdf9b05SPeng Ma } 6937fdf9b05SPeng Ma 6947fdf9b05SPeng Ma /* DPDMAI binding to DPIO */ 6957fdf9b05SPeng Ma err = dpaa2_dpdmai_bind(priv); 6967fdf9b05SPeng Ma if (err) { 6977fdf9b05SPeng Ma dev_err(dev, "dpaa2_dpdmai_bind() failed\n"); 6987fdf9b05SPeng Ma goto err_bind; 6997fdf9b05SPeng Ma } 7007fdf9b05SPeng Ma 7017fdf9b05SPeng Ma /* DPDMAI enable */ 7027fdf9b05SPeng Ma err = dpdmai_enable(priv->mc_io, 0, dpdmai_dev->mc_handle); 7037fdf9b05SPeng Ma if (err) { 704*64d57d2cSColin Ian King dev_err(dev, "dpdmai_enable() failed\n"); 7057fdf9b05SPeng Ma goto err_enable; 7067fdf9b05SPeng Ma } 7077fdf9b05SPeng Ma 7087fdf9b05SPeng Ma dpaa2_qdma = kzalloc(sizeof(*dpaa2_qdma), GFP_KERNEL); 7097fdf9b05SPeng Ma if (!dpaa2_qdma) { 7107fdf9b05SPeng Ma err = -ENOMEM; 7117fdf9b05SPeng Ma goto err_eng; 7127fdf9b05SPeng Ma } 7137fdf9b05SPeng Ma 7147fdf9b05SPeng Ma priv->dpaa2_qdma = dpaa2_qdma; 7157fdf9b05SPeng Ma dpaa2_qdma->priv = priv; 7167fdf9b05SPeng Ma 7177fdf9b05SPeng Ma dpaa2_qdma->desc_allocated = 0; 7187fdf9b05SPeng Ma dpaa2_qdma->n_chans = NUM_CH; 7197fdf9b05SPeng Ma 7207fdf9b05SPeng Ma dpaa2_dpdmai_init_channels(dpaa2_qdma); 7217fdf9b05SPeng Ma 7227fdf9b05SPeng Ma if (soc_device_match(soc_fixup_tuning)) 7237fdf9b05SPeng Ma dpaa2_qdma->qdma_wrtype_fixup = true; 7247fdf9b05SPeng Ma else 7257fdf9b05SPeng Ma dpaa2_qdma->qdma_wrtype_fixup = false; 7267fdf9b05SPeng Ma 7277fdf9b05SPeng Ma dma_cap_set(DMA_PRIVATE, dpaa2_qdma->dma_dev.cap_mask); 7287fdf9b05SPeng Ma dma_cap_set(DMA_SLAVE, dpaa2_qdma->dma_dev.cap_mask); 7297fdf9b05SPeng Ma dma_cap_set(DMA_MEMCPY, dpaa2_qdma->dma_dev.cap_mask); 7307fdf9b05SPeng Ma 7317fdf9b05SPeng Ma dpaa2_qdma->dma_dev.dev = dev; 7327fdf9b05SPeng Ma dpaa2_qdma->dma_dev.device_alloc_chan_resources = 7337fdf9b05SPeng Ma dpaa2_qdma_alloc_chan_resources; 7347fdf9b05SPeng Ma dpaa2_qdma->dma_dev.device_free_chan_resources = 7357fdf9b05SPeng Ma dpaa2_qdma_free_chan_resources; 7367fdf9b05SPeng Ma dpaa2_qdma->dma_dev.device_tx_status = dma_cookie_status; 7377fdf9b05SPeng Ma dpaa2_qdma->dma_dev.device_prep_dma_memcpy = dpaa2_qdma_prep_memcpy; 7387fdf9b05SPeng Ma dpaa2_qdma->dma_dev.device_issue_pending = dpaa2_qdma_issue_pending; 7397fdf9b05SPeng Ma 7407fdf9b05SPeng Ma err = dma_async_device_register(&dpaa2_qdma->dma_dev); 7417fdf9b05SPeng Ma if (err) { 7427fdf9b05SPeng Ma dev_err(dev, "Can't register NXP QDMA engine.\n"); 7437fdf9b05SPeng Ma goto err_dpaa2_qdma; 7447fdf9b05SPeng Ma } 7457fdf9b05SPeng Ma 7467fdf9b05SPeng Ma return 0; 7477fdf9b05SPeng Ma 7487fdf9b05SPeng Ma err_dpaa2_qdma: 7497fdf9b05SPeng Ma kfree(dpaa2_qdma); 7507fdf9b05SPeng Ma err_eng: 7517fdf9b05SPeng Ma dpdmai_disable(priv->mc_io, 0, dpdmai_dev->mc_handle); 7527fdf9b05SPeng Ma err_enable: 7537fdf9b05SPeng Ma dpaa2_dpdmai_dpio_unbind(priv); 7547fdf9b05SPeng Ma err_bind: 7557fdf9b05SPeng Ma dpaa2_dpmai_store_free(priv); 7567fdf9b05SPeng Ma dpaa2_dpdmai_dpio_free(priv); 7577fdf9b05SPeng Ma err_dpio_setup: 7587fdf9b05SPeng Ma kfree(priv->ppriv); 7597fdf9b05SPeng Ma dpdmai_close(priv->mc_io, 0, dpdmai_dev->mc_handle); 7607fdf9b05SPeng Ma err_dpdmai_setup: 7617fdf9b05SPeng Ma fsl_mc_portal_free(priv->mc_io); 7627fdf9b05SPeng Ma err_mcportal: 7637fdf9b05SPeng Ma kfree(priv); 7647fdf9b05SPeng Ma dev_set_drvdata(dev, NULL); 7657fdf9b05SPeng Ma return err; 7667fdf9b05SPeng Ma } 7677fdf9b05SPeng Ma 7687fdf9b05SPeng Ma static int dpaa2_qdma_remove(struct fsl_mc_device *ls_dev) 7697fdf9b05SPeng Ma { 7707fdf9b05SPeng Ma struct dpaa2_qdma_engine *dpaa2_qdma; 7717fdf9b05SPeng Ma struct dpaa2_qdma_priv *priv; 7727fdf9b05SPeng Ma struct device *dev; 7737fdf9b05SPeng Ma 7747fdf9b05SPeng Ma dev = &ls_dev->dev; 7757fdf9b05SPeng Ma priv = dev_get_drvdata(dev); 7767fdf9b05SPeng Ma dpaa2_qdma = priv->dpaa2_qdma; 7777fdf9b05SPeng Ma 7787fdf9b05SPeng Ma dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle); 7797fdf9b05SPeng Ma dpaa2_dpdmai_dpio_unbind(priv); 7807fdf9b05SPeng Ma dpaa2_dpmai_store_free(priv); 7817fdf9b05SPeng Ma dpaa2_dpdmai_dpio_free(priv); 7827fdf9b05SPeng Ma dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle); 7837fdf9b05SPeng Ma fsl_mc_portal_free(priv->mc_io); 7847fdf9b05SPeng Ma dev_set_drvdata(dev, NULL); 7857fdf9b05SPeng Ma dpaa2_dpdmai_free_channels(dpaa2_qdma); 7867fdf9b05SPeng Ma 7877fdf9b05SPeng Ma dma_async_device_unregister(&dpaa2_qdma->dma_dev); 7887fdf9b05SPeng Ma kfree(priv); 7897fdf9b05SPeng Ma kfree(dpaa2_qdma); 7907fdf9b05SPeng Ma 7917fdf9b05SPeng Ma return 0; 7927fdf9b05SPeng Ma } 7937fdf9b05SPeng Ma 7943e0ca3c3SPeng Ma static void dpaa2_qdma_shutdown(struct fsl_mc_device *ls_dev) 7953e0ca3c3SPeng Ma { 7963e0ca3c3SPeng Ma struct dpaa2_qdma_priv *priv; 7973e0ca3c3SPeng Ma struct device *dev; 7983e0ca3c3SPeng Ma 7993e0ca3c3SPeng Ma dev = &ls_dev->dev; 8003e0ca3c3SPeng Ma priv = dev_get_drvdata(dev); 8013e0ca3c3SPeng Ma 8023e0ca3c3SPeng Ma dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle); 8033e0ca3c3SPeng Ma dpaa2_dpdmai_dpio_unbind(priv); 8043e0ca3c3SPeng Ma dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle); 8053e0ca3c3SPeng Ma dpdmai_destroy(priv->mc_io, 0, ls_dev->mc_handle); 8063e0ca3c3SPeng Ma } 8073e0ca3c3SPeng Ma 8087fdf9b05SPeng Ma static const struct fsl_mc_device_id dpaa2_qdma_id_table[] = { 8097fdf9b05SPeng Ma { 8107fdf9b05SPeng Ma .vendor = FSL_MC_VENDOR_FREESCALE, 8117fdf9b05SPeng Ma .obj_type = "dpdmai", 8127fdf9b05SPeng Ma }, 8137fdf9b05SPeng Ma { .vendor = 0x0 } 8147fdf9b05SPeng Ma }; 8157fdf9b05SPeng Ma 8167fdf9b05SPeng Ma static struct fsl_mc_driver dpaa2_qdma_driver = { 8177fdf9b05SPeng Ma .driver = { 8187fdf9b05SPeng Ma .name = "dpaa2-qdma", 8197fdf9b05SPeng Ma .owner = THIS_MODULE, 8207fdf9b05SPeng Ma }, 8217fdf9b05SPeng Ma .probe = dpaa2_qdma_probe, 8227fdf9b05SPeng Ma .remove = dpaa2_qdma_remove, 8233e0ca3c3SPeng Ma .shutdown = dpaa2_qdma_shutdown, 8247fdf9b05SPeng Ma .match_id_table = dpaa2_qdma_id_table 8257fdf9b05SPeng Ma }; 8267fdf9b05SPeng Ma 8277fdf9b05SPeng Ma static int __init dpaa2_qdma_driver_init(void) 8287fdf9b05SPeng Ma { 8297fdf9b05SPeng Ma return fsl_mc_driver_register(&(dpaa2_qdma_driver)); 8307fdf9b05SPeng Ma } 8317fdf9b05SPeng Ma late_initcall(dpaa2_qdma_driver_init); 8327fdf9b05SPeng Ma 8337fdf9b05SPeng Ma static void __exit fsl_qdma_exit(void) 8347fdf9b05SPeng Ma { 8357fdf9b05SPeng Ma fsl_mc_driver_unregister(&(dpaa2_qdma_driver)); 8367fdf9b05SPeng Ma } 8377fdf9b05SPeng Ma module_exit(fsl_qdma_exit); 8387fdf9b05SPeng Ma 8397fdf9b05SPeng Ma MODULE_ALIAS("platform:fsl-dpaa2-qdma"); 8407fdf9b05SPeng Ma MODULE_LICENSE("GPL v2"); 8417fdf9b05SPeng Ma MODULE_DESCRIPTION("NXP Layerscape DPAA2 qDMA engine driver"); 842