1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
258ea8abfSGary R Hook /*
358ea8abfSGary R Hook * AMD Cryptographic Coprocessor (CCP) driver
458ea8abfSGary R Hook *
5a7c26470SHook, Gary * Copyright (C) 2016,2019 Advanced Micro Devices, Inc.
658ea8abfSGary R Hook *
758ea8abfSGary R Hook * Author: Gary R Hook <gary.hook@amd.com>
858ea8abfSGary R Hook */
958ea8abfSGary R Hook
10efc989fcSGary R Hook #include <linux/module.h>
1158ea8abfSGary R Hook #include <linux/kernel.h>
12f6b0b78aSBjorn Helgaas #include <linux/dma-mapping.h>
1358ea8abfSGary R Hook #include <linux/dmaengine.h>
1458ea8abfSGary R Hook #include <linux/spinlock.h>
1558ea8abfSGary R Hook #include <linux/mutex.h>
1658ea8abfSGary R Hook #include <linux/ccp.h>
1758ea8abfSGary R Hook
1858ea8abfSGary R Hook #include "ccp-dev.h"
1958ea8abfSGary R Hook #include "../../dma/dmaengine.h"
2058ea8abfSGary R Hook
2158ea8abfSGary R Hook #define CCP_DMA_WIDTH(_mask) \
2258ea8abfSGary R Hook ({ \
2358ea8abfSGary R Hook u64 mask = _mask + 1; \
2458ea8abfSGary R Hook (mask == 0) ? 64 : fls64(mask); \
2558ea8abfSGary R Hook })
2658ea8abfSGary R Hook
27efc989fcSGary R Hook /* The CCP as a DMA provider can be configured for public or private
28efc989fcSGary R Hook * channels. Default is specified in the vdata for the device (PCI ID).
29efc989fcSGary R Hook * This module parameter will override for all channels on all devices:
30efc989fcSGary R Hook * dma_chan_attr = 0x2 to force all channels public
31efc989fcSGary R Hook * = 0x1 to force all channels private
32efc989fcSGary R Hook * = 0x0 to defer to the vdata setting
33efc989fcSGary R Hook * = any other value: warning, revert to 0x0
34efc989fcSGary R Hook */
35efc989fcSGary R Hook static unsigned int dma_chan_attr = CCP_DMA_DFLT;
36efc989fcSGary R Hook module_param(dma_chan_attr, uint, 0444);
37efc989fcSGary R Hook MODULE_PARM_DESC(dma_chan_attr, "Set DMA channel visibility: 0 (default) = device defaults, 1 = make private, 2 = make public");
38efc989fcSGary R Hook
39a7c26470SHook, Gary static unsigned int dmaengine = 1;
40a7c26470SHook, Gary module_param(dmaengine, uint, 0444);
41a7c26470SHook, Gary MODULE_PARM_DESC(dmaengine, "Register services with the DMA subsystem (any non-zero value, default: 1)");
42a7c26470SHook, Gary
ccp_get_dma_chan_attr(struct ccp_device * ccp)43404a36a7SColin Ian King static unsigned int ccp_get_dma_chan_attr(struct ccp_device *ccp)
44efc989fcSGary R Hook {
45efc989fcSGary R Hook switch (dma_chan_attr) {
46efc989fcSGary R Hook case CCP_DMA_DFLT:
47efc989fcSGary R Hook return ccp->vdata->dma_chan_attr;
48efc989fcSGary R Hook
49efc989fcSGary R Hook case CCP_DMA_PRIV:
50efc989fcSGary R Hook return DMA_PRIVATE;
51efc989fcSGary R Hook
52efc989fcSGary R Hook case CCP_DMA_PUB:
53efc989fcSGary R Hook return 0;
54efc989fcSGary R Hook
55efc989fcSGary R Hook default:
56efc989fcSGary R Hook dev_info_once(ccp->dev, "Invalid value for dma_chan_attr: %d\n",
57efc989fcSGary R Hook dma_chan_attr);
58efc989fcSGary R Hook return ccp->vdata->dma_chan_attr;
59efc989fcSGary R Hook }
60efc989fcSGary R Hook }
61efc989fcSGary R Hook
ccp_free_cmd_resources(struct ccp_device * ccp,struct list_head * list)6258ea8abfSGary R Hook static void ccp_free_cmd_resources(struct ccp_device *ccp,
6358ea8abfSGary R Hook struct list_head *list)
6458ea8abfSGary R Hook {
6558ea8abfSGary R Hook struct ccp_dma_cmd *cmd, *ctmp;
6658ea8abfSGary R Hook
6758ea8abfSGary R Hook list_for_each_entry_safe(cmd, ctmp, list, entry) {
6858ea8abfSGary R Hook list_del(&cmd->entry);
6958ea8abfSGary R Hook kmem_cache_free(ccp->dma_cmd_cache, cmd);
7058ea8abfSGary R Hook }
7158ea8abfSGary R Hook }
7258ea8abfSGary R Hook
ccp_free_desc_resources(struct ccp_device * ccp,struct list_head * list)7358ea8abfSGary R Hook static void ccp_free_desc_resources(struct ccp_device *ccp,
7458ea8abfSGary R Hook struct list_head *list)
7558ea8abfSGary R Hook {
7658ea8abfSGary R Hook struct ccp_dma_desc *desc, *dtmp;
7758ea8abfSGary R Hook
7858ea8abfSGary R Hook list_for_each_entry_safe(desc, dtmp, list, entry) {
7958ea8abfSGary R Hook ccp_free_cmd_resources(ccp, &desc->active);
8058ea8abfSGary R Hook ccp_free_cmd_resources(ccp, &desc->pending);
8158ea8abfSGary R Hook
8258ea8abfSGary R Hook list_del(&desc->entry);
8358ea8abfSGary R Hook kmem_cache_free(ccp->dma_desc_cache, desc);
8458ea8abfSGary R Hook }
8558ea8abfSGary R Hook }
8658ea8abfSGary R Hook
ccp_free_chan_resources(struct dma_chan * dma_chan)8758ea8abfSGary R Hook static void ccp_free_chan_resources(struct dma_chan *dma_chan)
8858ea8abfSGary R Hook {
8958ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
9058ea8abfSGary R Hook dma_chan);
9158ea8abfSGary R Hook unsigned long flags;
9258ea8abfSGary R Hook
9358ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s - chan=%p\n", __func__, chan);
9458ea8abfSGary R Hook
9558ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags);
9658ea8abfSGary R Hook
9758ea8abfSGary R Hook ccp_free_desc_resources(chan->ccp, &chan->complete);
9858ea8abfSGary R Hook ccp_free_desc_resources(chan->ccp, &chan->active);
9958ea8abfSGary R Hook ccp_free_desc_resources(chan->ccp, &chan->pending);
100e5da5c56SGary R Hook ccp_free_desc_resources(chan->ccp, &chan->created);
10158ea8abfSGary R Hook
10258ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags);
10358ea8abfSGary R Hook }
10458ea8abfSGary R Hook
ccp_cleanup_desc_resources(struct ccp_device * ccp,struct list_head * list)10558ea8abfSGary R Hook static void ccp_cleanup_desc_resources(struct ccp_device *ccp,
10658ea8abfSGary R Hook struct list_head *list)
10758ea8abfSGary R Hook {
10858ea8abfSGary R Hook struct ccp_dma_desc *desc, *dtmp;
10958ea8abfSGary R Hook
11058ea8abfSGary R Hook list_for_each_entry_safe_reverse(desc, dtmp, list, entry) {
11158ea8abfSGary R Hook if (!async_tx_test_ack(&desc->tx_desc))
11258ea8abfSGary R Hook continue;
11358ea8abfSGary R Hook
11458ea8abfSGary R Hook dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc);
11558ea8abfSGary R Hook
11658ea8abfSGary R Hook ccp_free_cmd_resources(ccp, &desc->active);
11758ea8abfSGary R Hook ccp_free_cmd_resources(ccp, &desc->pending);
11858ea8abfSGary R Hook
11958ea8abfSGary R Hook list_del(&desc->entry);
12058ea8abfSGary R Hook kmem_cache_free(ccp->dma_desc_cache, desc);
12158ea8abfSGary R Hook }
12258ea8abfSGary R Hook }
12358ea8abfSGary R Hook
ccp_do_cleanup(unsigned long data)12458ea8abfSGary R Hook static void ccp_do_cleanup(unsigned long data)
12558ea8abfSGary R Hook {
12658ea8abfSGary R Hook struct ccp_dma_chan *chan = (struct ccp_dma_chan *)data;
12758ea8abfSGary R Hook unsigned long flags;
12858ea8abfSGary R Hook
12958ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s - chan=%s\n", __func__,
13058ea8abfSGary R Hook dma_chan_name(&chan->dma_chan));
13158ea8abfSGary R Hook
13258ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags);
13358ea8abfSGary R Hook
13458ea8abfSGary R Hook ccp_cleanup_desc_resources(chan->ccp, &chan->complete);
13558ea8abfSGary R Hook
13658ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags);
13758ea8abfSGary R Hook }
13858ea8abfSGary R Hook
ccp_issue_next_cmd(struct ccp_dma_desc * desc)13958ea8abfSGary R Hook static int ccp_issue_next_cmd(struct ccp_dma_desc *desc)
14058ea8abfSGary R Hook {
14158ea8abfSGary R Hook struct ccp_dma_cmd *cmd;
14258ea8abfSGary R Hook int ret;
14358ea8abfSGary R Hook
14458ea8abfSGary R Hook cmd = list_first_entry(&desc->pending, struct ccp_dma_cmd, entry);
14558ea8abfSGary R Hook list_move(&cmd->entry, &desc->active);
14658ea8abfSGary R Hook
14758ea8abfSGary R Hook dev_dbg(desc->ccp->dev, "%s - tx %d, cmd=%p\n", __func__,
14858ea8abfSGary R Hook desc->tx_desc.cookie, cmd);
14958ea8abfSGary R Hook
15058ea8abfSGary R Hook ret = ccp_enqueue_cmd(&cmd->ccp_cmd);
15158ea8abfSGary R Hook if (!ret || (ret == -EINPROGRESS) || (ret == -EBUSY))
15258ea8abfSGary R Hook return 0;
15358ea8abfSGary R Hook
15458ea8abfSGary R Hook dev_dbg(desc->ccp->dev, "%s - error: ret=%d, tx %d, cmd=%p\n", __func__,
15558ea8abfSGary R Hook ret, desc->tx_desc.cookie, cmd);
15658ea8abfSGary R Hook
15758ea8abfSGary R Hook return ret;
15858ea8abfSGary R Hook }
15958ea8abfSGary R Hook
ccp_free_active_cmd(struct ccp_dma_desc * desc)16058ea8abfSGary R Hook static void ccp_free_active_cmd(struct ccp_dma_desc *desc)
16158ea8abfSGary R Hook {
16258ea8abfSGary R Hook struct ccp_dma_cmd *cmd;
16358ea8abfSGary R Hook
16458ea8abfSGary R Hook cmd = list_first_entry_or_null(&desc->active, struct ccp_dma_cmd,
16558ea8abfSGary R Hook entry);
16658ea8abfSGary R Hook if (!cmd)
16758ea8abfSGary R Hook return;
16858ea8abfSGary R Hook
16958ea8abfSGary R Hook dev_dbg(desc->ccp->dev, "%s - freeing tx %d cmd=%p\n",
17058ea8abfSGary R Hook __func__, desc->tx_desc.cookie, cmd);
17158ea8abfSGary R Hook
17258ea8abfSGary R Hook list_del(&cmd->entry);
17358ea8abfSGary R Hook kmem_cache_free(desc->ccp->dma_cmd_cache, cmd);
17458ea8abfSGary R Hook }
17558ea8abfSGary R Hook
__ccp_next_dma_desc(struct ccp_dma_chan * chan,struct ccp_dma_desc * desc)17658ea8abfSGary R Hook static struct ccp_dma_desc *__ccp_next_dma_desc(struct ccp_dma_chan *chan,
17758ea8abfSGary R Hook struct ccp_dma_desc *desc)
17858ea8abfSGary R Hook {
17958ea8abfSGary R Hook /* Move current DMA descriptor to the complete list */
18058ea8abfSGary R Hook if (desc)
18158ea8abfSGary R Hook list_move(&desc->entry, &chan->complete);
18258ea8abfSGary R Hook
18358ea8abfSGary R Hook /* Get the next DMA descriptor on the active list */
18458ea8abfSGary R Hook desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc,
18558ea8abfSGary R Hook entry);
18658ea8abfSGary R Hook
18758ea8abfSGary R Hook return desc;
18858ea8abfSGary R Hook }
18958ea8abfSGary R Hook
ccp_handle_active_desc(struct ccp_dma_chan * chan,struct ccp_dma_desc * desc)19058ea8abfSGary R Hook static struct ccp_dma_desc *ccp_handle_active_desc(struct ccp_dma_chan *chan,
19158ea8abfSGary R Hook struct ccp_dma_desc *desc)
19258ea8abfSGary R Hook {
19358ea8abfSGary R Hook struct dma_async_tx_descriptor *tx_desc;
19458ea8abfSGary R Hook unsigned long flags;
19558ea8abfSGary R Hook
19658ea8abfSGary R Hook /* Loop over descriptors until one is found with commands */
19758ea8abfSGary R Hook do {
19858ea8abfSGary R Hook if (desc) {
19958ea8abfSGary R Hook /* Remove the DMA command from the list and free it */
20058ea8abfSGary R Hook ccp_free_active_cmd(desc);
20158ea8abfSGary R Hook
20258ea8abfSGary R Hook if (!list_empty(&desc->pending)) {
20358ea8abfSGary R Hook /* No errors, keep going */
20458ea8abfSGary R Hook if (desc->status != DMA_ERROR)
20558ea8abfSGary R Hook return desc;
20658ea8abfSGary R Hook
20758ea8abfSGary R Hook /* Error, free remaining commands and move on */
20858ea8abfSGary R Hook ccp_free_cmd_resources(desc->ccp,
20958ea8abfSGary R Hook &desc->pending);
21058ea8abfSGary R Hook }
21158ea8abfSGary R Hook
21258ea8abfSGary R Hook tx_desc = &desc->tx_desc;
21358ea8abfSGary R Hook } else {
21458ea8abfSGary R Hook tx_desc = NULL;
21558ea8abfSGary R Hook }
21658ea8abfSGary R Hook
21758ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags);
21858ea8abfSGary R Hook
21958ea8abfSGary R Hook if (desc) {
22058ea8abfSGary R Hook if (desc->status != DMA_ERROR)
22158ea8abfSGary R Hook desc->status = DMA_COMPLETE;
22258ea8abfSGary R Hook
22358ea8abfSGary R Hook dev_dbg(desc->ccp->dev,
22458ea8abfSGary R Hook "%s - tx %d complete, status=%u\n", __func__,
22558ea8abfSGary R Hook desc->tx_desc.cookie, desc->status);
22658ea8abfSGary R Hook
22758ea8abfSGary R Hook dma_cookie_complete(tx_desc);
22801c4c097Samd dma_descriptor_unmap(tx_desc);
22958ea8abfSGary R Hook }
23058ea8abfSGary R Hook
23158ea8abfSGary R Hook desc = __ccp_next_dma_desc(chan, desc);
23258ea8abfSGary R Hook
23358ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags);
23458ea8abfSGary R Hook
23558ea8abfSGary R Hook if (tx_desc) {
236c07f7c29Samd dmaengine_desc_get_callback_invoke(tx_desc, NULL);
23758ea8abfSGary R Hook
23858ea8abfSGary R Hook dma_run_dependencies(tx_desc);
23958ea8abfSGary R Hook }
24058ea8abfSGary R Hook } while (desc);
24158ea8abfSGary R Hook
24258ea8abfSGary R Hook return NULL;
24358ea8abfSGary R Hook }
24458ea8abfSGary R Hook
__ccp_pending_to_active(struct ccp_dma_chan * chan)24558ea8abfSGary R Hook static struct ccp_dma_desc *__ccp_pending_to_active(struct ccp_dma_chan *chan)
24658ea8abfSGary R Hook {
24758ea8abfSGary R Hook struct ccp_dma_desc *desc;
24858ea8abfSGary R Hook
24958ea8abfSGary R Hook if (list_empty(&chan->pending))
25058ea8abfSGary R Hook return NULL;
25158ea8abfSGary R Hook
25258ea8abfSGary R Hook desc = list_empty(&chan->active)
25358ea8abfSGary R Hook ? list_first_entry(&chan->pending, struct ccp_dma_desc, entry)
25458ea8abfSGary R Hook : NULL;
25558ea8abfSGary R Hook
25658ea8abfSGary R Hook list_splice_tail_init(&chan->pending, &chan->active);
25758ea8abfSGary R Hook
25858ea8abfSGary R Hook return desc;
25958ea8abfSGary R Hook }
26058ea8abfSGary R Hook
ccp_cmd_callback(void * data,int err)26158ea8abfSGary R Hook static void ccp_cmd_callback(void *data, int err)
26258ea8abfSGary R Hook {
26358ea8abfSGary R Hook struct ccp_dma_desc *desc = data;
26458ea8abfSGary R Hook struct ccp_dma_chan *chan;
26558ea8abfSGary R Hook int ret;
26658ea8abfSGary R Hook
26758ea8abfSGary R Hook if (err == -EINPROGRESS)
26858ea8abfSGary R Hook return;
26958ea8abfSGary R Hook
27058ea8abfSGary R Hook chan = container_of(desc->tx_desc.chan, struct ccp_dma_chan,
27158ea8abfSGary R Hook dma_chan);
27258ea8abfSGary R Hook
27358ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s - tx %d callback, err=%d\n",
27458ea8abfSGary R Hook __func__, desc->tx_desc.cookie, err);
27558ea8abfSGary R Hook
27658ea8abfSGary R Hook if (err)
27758ea8abfSGary R Hook desc->status = DMA_ERROR;
27858ea8abfSGary R Hook
27958ea8abfSGary R Hook while (true) {
28058ea8abfSGary R Hook /* Check for DMA descriptor completion */
28158ea8abfSGary R Hook desc = ccp_handle_active_desc(chan, desc);
28258ea8abfSGary R Hook
28358ea8abfSGary R Hook /* Don't submit cmd if no descriptor or DMA is paused */
28458ea8abfSGary R Hook if (!desc || (chan->status == DMA_PAUSED))
28558ea8abfSGary R Hook break;
28658ea8abfSGary R Hook
28758ea8abfSGary R Hook ret = ccp_issue_next_cmd(desc);
28858ea8abfSGary R Hook if (!ret)
28958ea8abfSGary R Hook break;
29058ea8abfSGary R Hook
29158ea8abfSGary R Hook desc->status = DMA_ERROR;
29258ea8abfSGary R Hook }
29358ea8abfSGary R Hook
29458ea8abfSGary R Hook tasklet_schedule(&chan->cleanup_tasklet);
29558ea8abfSGary R Hook }
29658ea8abfSGary R Hook
ccp_tx_submit(struct dma_async_tx_descriptor * tx_desc)29758ea8abfSGary R Hook static dma_cookie_t ccp_tx_submit(struct dma_async_tx_descriptor *tx_desc)
29858ea8abfSGary R Hook {
29958ea8abfSGary R Hook struct ccp_dma_desc *desc = container_of(tx_desc, struct ccp_dma_desc,
30058ea8abfSGary R Hook tx_desc);
30158ea8abfSGary R Hook struct ccp_dma_chan *chan;
30258ea8abfSGary R Hook dma_cookie_t cookie;
30358ea8abfSGary R Hook unsigned long flags;
30458ea8abfSGary R Hook
30558ea8abfSGary R Hook chan = container_of(tx_desc->chan, struct ccp_dma_chan, dma_chan);
30658ea8abfSGary R Hook
30758ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags);
30858ea8abfSGary R Hook
30958ea8abfSGary R Hook cookie = dma_cookie_assign(tx_desc);
31087c8ba5cSBaokun Li list_move_tail(&desc->entry, &chan->pending);
31158ea8abfSGary R Hook
31258ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags);
31358ea8abfSGary R Hook
31458ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s - added tx descriptor %d to pending list\n",
31558ea8abfSGary R Hook __func__, cookie);
31658ea8abfSGary R Hook
31758ea8abfSGary R Hook return cookie;
31858ea8abfSGary R Hook }
31958ea8abfSGary R Hook
ccp_alloc_dma_cmd(struct ccp_dma_chan * chan)32058ea8abfSGary R Hook static struct ccp_dma_cmd *ccp_alloc_dma_cmd(struct ccp_dma_chan *chan)
32158ea8abfSGary R Hook {
32258ea8abfSGary R Hook struct ccp_dma_cmd *cmd;
32358ea8abfSGary R Hook
32458ea8abfSGary R Hook cmd = kmem_cache_alloc(chan->ccp->dma_cmd_cache, GFP_NOWAIT);
32558ea8abfSGary R Hook if (cmd)
32658ea8abfSGary R Hook memset(cmd, 0, sizeof(*cmd));
32758ea8abfSGary R Hook
32858ea8abfSGary R Hook return cmd;
32958ea8abfSGary R Hook }
33058ea8abfSGary R Hook
ccp_alloc_dma_desc(struct ccp_dma_chan * chan,unsigned long flags)33158ea8abfSGary R Hook static struct ccp_dma_desc *ccp_alloc_dma_desc(struct ccp_dma_chan *chan,
33258ea8abfSGary R Hook unsigned long flags)
33358ea8abfSGary R Hook {
33458ea8abfSGary R Hook struct ccp_dma_desc *desc;
33558ea8abfSGary R Hook
336664f570aSWei Yongjun desc = kmem_cache_zalloc(chan->ccp->dma_desc_cache, GFP_NOWAIT);
33758ea8abfSGary R Hook if (!desc)
33858ea8abfSGary R Hook return NULL;
33958ea8abfSGary R Hook
34058ea8abfSGary R Hook dma_async_tx_descriptor_init(&desc->tx_desc, &chan->dma_chan);
34158ea8abfSGary R Hook desc->tx_desc.flags = flags;
34258ea8abfSGary R Hook desc->tx_desc.tx_submit = ccp_tx_submit;
34358ea8abfSGary R Hook desc->ccp = chan->ccp;
344691505a8SMark Salter INIT_LIST_HEAD(&desc->entry);
34558ea8abfSGary R Hook INIT_LIST_HEAD(&desc->pending);
34658ea8abfSGary R Hook INIT_LIST_HEAD(&desc->active);
34758ea8abfSGary R Hook desc->status = DMA_IN_PROGRESS;
34858ea8abfSGary R Hook
34958ea8abfSGary R Hook return desc;
35058ea8abfSGary R Hook }
35158ea8abfSGary R Hook
ccp_create_desc(struct dma_chan * dma_chan,struct scatterlist * dst_sg,unsigned int dst_nents,struct scatterlist * src_sg,unsigned int src_nents,unsigned long flags)35258ea8abfSGary R Hook static struct ccp_dma_desc *ccp_create_desc(struct dma_chan *dma_chan,
35358ea8abfSGary R Hook struct scatterlist *dst_sg,
35458ea8abfSGary R Hook unsigned int dst_nents,
35558ea8abfSGary R Hook struct scatterlist *src_sg,
35658ea8abfSGary R Hook unsigned int src_nents,
35758ea8abfSGary R Hook unsigned long flags)
35858ea8abfSGary R Hook {
35958ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
36058ea8abfSGary R Hook dma_chan);
36158ea8abfSGary R Hook struct ccp_device *ccp = chan->ccp;
36258ea8abfSGary R Hook struct ccp_dma_desc *desc;
36358ea8abfSGary R Hook struct ccp_dma_cmd *cmd;
36458ea8abfSGary R Hook struct ccp_cmd *ccp_cmd;
36558ea8abfSGary R Hook struct ccp_passthru_nomap_engine *ccp_pt;
36658ea8abfSGary R Hook unsigned int src_offset, src_len;
36758ea8abfSGary R Hook unsigned int dst_offset, dst_len;
36858ea8abfSGary R Hook unsigned int len;
36958ea8abfSGary R Hook unsigned long sflags;
37058ea8abfSGary R Hook size_t total_len;
37158ea8abfSGary R Hook
37258ea8abfSGary R Hook if (!dst_sg || !src_sg)
37358ea8abfSGary R Hook return NULL;
37458ea8abfSGary R Hook
37558ea8abfSGary R Hook if (!dst_nents || !src_nents)
37658ea8abfSGary R Hook return NULL;
37758ea8abfSGary R Hook
37858ea8abfSGary R Hook desc = ccp_alloc_dma_desc(chan, flags);
37958ea8abfSGary R Hook if (!desc)
38058ea8abfSGary R Hook return NULL;
38158ea8abfSGary R Hook
38258ea8abfSGary R Hook total_len = 0;
38358ea8abfSGary R Hook
38458ea8abfSGary R Hook src_len = sg_dma_len(src_sg);
38558ea8abfSGary R Hook src_offset = 0;
38658ea8abfSGary R Hook
38758ea8abfSGary R Hook dst_len = sg_dma_len(dst_sg);
38858ea8abfSGary R Hook dst_offset = 0;
38958ea8abfSGary R Hook
39058ea8abfSGary R Hook while (true) {
39158ea8abfSGary R Hook if (!src_len) {
39258ea8abfSGary R Hook src_nents--;
39358ea8abfSGary R Hook if (!src_nents)
39458ea8abfSGary R Hook break;
39558ea8abfSGary R Hook
39658ea8abfSGary R Hook src_sg = sg_next(src_sg);
39758ea8abfSGary R Hook if (!src_sg)
39858ea8abfSGary R Hook break;
39958ea8abfSGary R Hook
40058ea8abfSGary R Hook src_len = sg_dma_len(src_sg);
40158ea8abfSGary R Hook src_offset = 0;
40258ea8abfSGary R Hook continue;
40358ea8abfSGary R Hook }
40458ea8abfSGary R Hook
40558ea8abfSGary R Hook if (!dst_len) {
40658ea8abfSGary R Hook dst_nents--;
40758ea8abfSGary R Hook if (!dst_nents)
40858ea8abfSGary R Hook break;
40958ea8abfSGary R Hook
41058ea8abfSGary R Hook dst_sg = sg_next(dst_sg);
41158ea8abfSGary R Hook if (!dst_sg)
41258ea8abfSGary R Hook break;
41358ea8abfSGary R Hook
41458ea8abfSGary R Hook dst_len = sg_dma_len(dst_sg);
41558ea8abfSGary R Hook dst_offset = 0;
41658ea8abfSGary R Hook continue;
41758ea8abfSGary R Hook }
41858ea8abfSGary R Hook
41958ea8abfSGary R Hook len = min(dst_len, src_len);
42058ea8abfSGary R Hook
42158ea8abfSGary R Hook cmd = ccp_alloc_dma_cmd(chan);
42258ea8abfSGary R Hook if (!cmd)
42358ea8abfSGary R Hook goto err;
42458ea8abfSGary R Hook
42558ea8abfSGary R Hook ccp_cmd = &cmd->ccp_cmd;
4267c468447SGary R Hook ccp_cmd->ccp = chan->ccp;
42758ea8abfSGary R Hook ccp_pt = &ccp_cmd->u.passthru_nomap;
42858ea8abfSGary R Hook ccp_cmd->flags = CCP_CMD_MAY_BACKLOG;
42958ea8abfSGary R Hook ccp_cmd->flags |= CCP_CMD_PASSTHRU_NO_DMA_MAP;
43058ea8abfSGary R Hook ccp_cmd->engine = CCP_ENGINE_PASSTHRU;
43158ea8abfSGary R Hook ccp_pt->bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
43258ea8abfSGary R Hook ccp_pt->byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
43358ea8abfSGary R Hook ccp_pt->src_dma = sg_dma_address(src_sg) + src_offset;
43458ea8abfSGary R Hook ccp_pt->dst_dma = sg_dma_address(dst_sg) + dst_offset;
43558ea8abfSGary R Hook ccp_pt->src_len = len;
43658ea8abfSGary R Hook ccp_pt->final = 1;
43758ea8abfSGary R Hook ccp_cmd->callback = ccp_cmd_callback;
43858ea8abfSGary R Hook ccp_cmd->data = desc;
43958ea8abfSGary R Hook
44058ea8abfSGary R Hook list_add_tail(&cmd->entry, &desc->pending);
44158ea8abfSGary R Hook
44258ea8abfSGary R Hook dev_dbg(ccp->dev,
44358ea8abfSGary R Hook "%s - cmd=%p, src=%pad, dst=%pad, len=%llu\n", __func__,
44458ea8abfSGary R Hook cmd, &ccp_pt->src_dma,
44558ea8abfSGary R Hook &ccp_pt->dst_dma, ccp_pt->src_len);
44658ea8abfSGary R Hook
44758ea8abfSGary R Hook total_len += len;
44858ea8abfSGary R Hook
44958ea8abfSGary R Hook src_len -= len;
45058ea8abfSGary R Hook src_offset += len;
45158ea8abfSGary R Hook
45258ea8abfSGary R Hook dst_len -= len;
45358ea8abfSGary R Hook dst_offset += len;
45458ea8abfSGary R Hook }
45558ea8abfSGary R Hook
45658ea8abfSGary R Hook desc->len = total_len;
45758ea8abfSGary R Hook
45858ea8abfSGary R Hook if (list_empty(&desc->pending))
45958ea8abfSGary R Hook goto err;
46058ea8abfSGary R Hook
46158ea8abfSGary R Hook dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc);
46258ea8abfSGary R Hook
46358ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, sflags);
46458ea8abfSGary R Hook
465e5da5c56SGary R Hook list_add_tail(&desc->entry, &chan->created);
46658ea8abfSGary R Hook
46758ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, sflags);
46858ea8abfSGary R Hook
46958ea8abfSGary R Hook return desc;
47058ea8abfSGary R Hook
47158ea8abfSGary R Hook err:
47258ea8abfSGary R Hook ccp_free_cmd_resources(ccp, &desc->pending);
47358ea8abfSGary R Hook kmem_cache_free(ccp->dma_desc_cache, desc);
47458ea8abfSGary R Hook
47558ea8abfSGary R Hook return NULL;
47658ea8abfSGary R Hook }
47758ea8abfSGary R Hook
ccp_prep_dma_memcpy(struct dma_chan * dma_chan,dma_addr_t dst,dma_addr_t src,size_t len,unsigned long flags)47858ea8abfSGary R Hook static struct dma_async_tx_descriptor *ccp_prep_dma_memcpy(
47958ea8abfSGary R Hook struct dma_chan *dma_chan, dma_addr_t dst, dma_addr_t src, size_t len,
48058ea8abfSGary R Hook unsigned long flags)
48158ea8abfSGary R Hook {
48258ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
48358ea8abfSGary R Hook dma_chan);
48458ea8abfSGary R Hook struct ccp_dma_desc *desc;
48558ea8abfSGary R Hook struct scatterlist dst_sg, src_sg;
48658ea8abfSGary R Hook
48758ea8abfSGary R Hook dev_dbg(chan->ccp->dev,
48858ea8abfSGary R Hook "%s - src=%pad, dst=%pad, len=%zu, flags=%#lx\n",
48958ea8abfSGary R Hook __func__, &src, &dst, len, flags);
49058ea8abfSGary R Hook
49158ea8abfSGary R Hook sg_init_table(&dst_sg, 1);
49258ea8abfSGary R Hook sg_dma_address(&dst_sg) = dst;
49358ea8abfSGary R Hook sg_dma_len(&dst_sg) = len;
49458ea8abfSGary R Hook
49558ea8abfSGary R Hook sg_init_table(&src_sg, 1);
49658ea8abfSGary R Hook sg_dma_address(&src_sg) = src;
49758ea8abfSGary R Hook sg_dma_len(&src_sg) = len;
49858ea8abfSGary R Hook
49958ea8abfSGary R Hook desc = ccp_create_desc(dma_chan, &dst_sg, 1, &src_sg, 1, flags);
50058ea8abfSGary R Hook if (!desc)
50158ea8abfSGary R Hook return NULL;
50258ea8abfSGary R Hook
50358ea8abfSGary R Hook return &desc->tx_desc;
50458ea8abfSGary R Hook }
50558ea8abfSGary R Hook
ccp_prep_dma_interrupt(struct dma_chan * dma_chan,unsigned long flags)50658ea8abfSGary R Hook static struct dma_async_tx_descriptor *ccp_prep_dma_interrupt(
50758ea8abfSGary R Hook struct dma_chan *dma_chan, unsigned long flags)
50858ea8abfSGary R Hook {
50958ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
51058ea8abfSGary R Hook dma_chan);
51158ea8abfSGary R Hook struct ccp_dma_desc *desc;
51258ea8abfSGary R Hook
51358ea8abfSGary R Hook desc = ccp_alloc_dma_desc(chan, flags);
51458ea8abfSGary R Hook if (!desc)
51558ea8abfSGary R Hook return NULL;
51658ea8abfSGary R Hook
51758ea8abfSGary R Hook return &desc->tx_desc;
51858ea8abfSGary R Hook }
51958ea8abfSGary R Hook
ccp_issue_pending(struct dma_chan * dma_chan)52058ea8abfSGary R Hook static void ccp_issue_pending(struct dma_chan *dma_chan)
52158ea8abfSGary R Hook {
52258ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
52358ea8abfSGary R Hook dma_chan);
52458ea8abfSGary R Hook struct ccp_dma_desc *desc;
52558ea8abfSGary R Hook unsigned long flags;
52658ea8abfSGary R Hook
52758ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s\n", __func__);
52858ea8abfSGary R Hook
52958ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags);
53058ea8abfSGary R Hook
53158ea8abfSGary R Hook desc = __ccp_pending_to_active(chan);
53258ea8abfSGary R Hook
53358ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags);
53458ea8abfSGary R Hook
53558ea8abfSGary R Hook /* If there was nothing active, start processing */
53658ea8abfSGary R Hook if (desc)
53758ea8abfSGary R Hook ccp_cmd_callback(desc, 0);
53858ea8abfSGary R Hook }
53958ea8abfSGary R Hook
ccp_tx_status(struct dma_chan * dma_chan,dma_cookie_t cookie,struct dma_tx_state * state)54058ea8abfSGary R Hook static enum dma_status ccp_tx_status(struct dma_chan *dma_chan,
54158ea8abfSGary R Hook dma_cookie_t cookie,
54258ea8abfSGary R Hook struct dma_tx_state *state)
54358ea8abfSGary R Hook {
54458ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
54558ea8abfSGary R Hook dma_chan);
54658ea8abfSGary R Hook struct ccp_dma_desc *desc;
54758ea8abfSGary R Hook enum dma_status ret;
54858ea8abfSGary R Hook unsigned long flags;
54958ea8abfSGary R Hook
55058ea8abfSGary R Hook if (chan->status == DMA_PAUSED) {
55158ea8abfSGary R Hook ret = DMA_PAUSED;
55258ea8abfSGary R Hook goto out;
55358ea8abfSGary R Hook }
55458ea8abfSGary R Hook
55558ea8abfSGary R Hook ret = dma_cookie_status(dma_chan, cookie, state);
55658ea8abfSGary R Hook if (ret == DMA_COMPLETE) {
55758ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags);
55858ea8abfSGary R Hook
55958ea8abfSGary R Hook /* Get status from complete chain, if still there */
56058ea8abfSGary R Hook list_for_each_entry(desc, &chan->complete, entry) {
56158ea8abfSGary R Hook if (desc->tx_desc.cookie != cookie)
56258ea8abfSGary R Hook continue;
56358ea8abfSGary R Hook
56458ea8abfSGary R Hook ret = desc->status;
56558ea8abfSGary R Hook break;
56658ea8abfSGary R Hook }
56758ea8abfSGary R Hook
56858ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags);
56958ea8abfSGary R Hook }
57058ea8abfSGary R Hook
57158ea8abfSGary R Hook out:
57258ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s - %u\n", __func__, ret);
57358ea8abfSGary R Hook
57458ea8abfSGary R Hook return ret;
57558ea8abfSGary R Hook }
57658ea8abfSGary R Hook
ccp_pause(struct dma_chan * dma_chan)57758ea8abfSGary R Hook static int ccp_pause(struct dma_chan *dma_chan)
57858ea8abfSGary R Hook {
57958ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
58058ea8abfSGary R Hook dma_chan);
58158ea8abfSGary R Hook
58258ea8abfSGary R Hook chan->status = DMA_PAUSED;
58358ea8abfSGary R Hook
58458ea8abfSGary R Hook /*TODO: Wait for active DMA to complete before returning? */
58558ea8abfSGary R Hook
58658ea8abfSGary R Hook return 0;
58758ea8abfSGary R Hook }
58858ea8abfSGary R Hook
ccp_resume(struct dma_chan * dma_chan)58958ea8abfSGary R Hook static int ccp_resume(struct dma_chan *dma_chan)
59058ea8abfSGary R Hook {
59158ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
59258ea8abfSGary R Hook dma_chan);
59358ea8abfSGary R Hook struct ccp_dma_desc *desc;
59458ea8abfSGary R Hook unsigned long flags;
59558ea8abfSGary R Hook
59658ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags);
59758ea8abfSGary R Hook
59858ea8abfSGary R Hook desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc,
59958ea8abfSGary R Hook entry);
60058ea8abfSGary R Hook
60158ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags);
60258ea8abfSGary R Hook
60358ea8abfSGary R Hook /* Indicate the channel is running again */
60458ea8abfSGary R Hook chan->status = DMA_IN_PROGRESS;
60558ea8abfSGary R Hook
60658ea8abfSGary R Hook /* If there was something active, re-start */
60758ea8abfSGary R Hook if (desc)
60858ea8abfSGary R Hook ccp_cmd_callback(desc, 0);
60958ea8abfSGary R Hook
61058ea8abfSGary R Hook return 0;
61158ea8abfSGary R Hook }
61258ea8abfSGary R Hook
ccp_terminate_all(struct dma_chan * dma_chan)61358ea8abfSGary R Hook static int ccp_terminate_all(struct dma_chan *dma_chan)
61458ea8abfSGary R Hook {
61558ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
61658ea8abfSGary R Hook dma_chan);
61758ea8abfSGary R Hook unsigned long flags;
61858ea8abfSGary R Hook
61958ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s\n", __func__);
62058ea8abfSGary R Hook
62158ea8abfSGary R Hook /*TODO: Wait for active DMA to complete before continuing */
62258ea8abfSGary R Hook
62358ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags);
62458ea8abfSGary R Hook
62558ea8abfSGary R Hook /*TODO: Purge the complete list? */
62658ea8abfSGary R Hook ccp_free_desc_resources(chan->ccp, &chan->active);
62758ea8abfSGary R Hook ccp_free_desc_resources(chan->ccp, &chan->pending);
628e5da5c56SGary R Hook ccp_free_desc_resources(chan->ccp, &chan->created);
62958ea8abfSGary R Hook
63058ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags);
63158ea8abfSGary R Hook
63258ea8abfSGary R Hook return 0;
63358ea8abfSGary R Hook }
63458ea8abfSGary R Hook
ccp_dma_release(struct ccp_device * ccp)63554cce8ecSDāvis Mosāns static void ccp_dma_release(struct ccp_device *ccp)
63654cce8ecSDāvis Mosāns {
63754cce8ecSDāvis Mosāns struct ccp_dma_chan *chan;
63854cce8ecSDāvis Mosāns struct dma_chan *dma_chan;
63954cce8ecSDāvis Mosāns unsigned int i;
64054cce8ecSDāvis Mosāns
64154cce8ecSDāvis Mosāns for (i = 0; i < ccp->cmd_q_count; i++) {
64254cce8ecSDāvis Mosāns chan = ccp->ccp_dma_chan + i;
64354cce8ecSDāvis Mosāns dma_chan = &chan->dma_chan;
64468dbe80fSKoba Ko
64554cce8ecSDāvis Mosāns tasklet_kill(&chan->cleanup_tasklet);
64654cce8ecSDāvis Mosāns list_del_rcu(&dma_chan->device_node);
64754cce8ecSDāvis Mosāns }
64854cce8ecSDāvis Mosāns }
64954cce8ecSDāvis Mosāns
ccp_dma_release_channels(struct ccp_device * ccp)650*299bf602SKoba Ko static void ccp_dma_release_channels(struct ccp_device *ccp)
651*299bf602SKoba Ko {
652*299bf602SKoba Ko struct ccp_dma_chan *chan;
653*299bf602SKoba Ko struct dma_chan *dma_chan;
654*299bf602SKoba Ko unsigned int i;
655*299bf602SKoba Ko
656*299bf602SKoba Ko for (i = 0; i < ccp->cmd_q_count; i++) {
657*299bf602SKoba Ko chan = ccp->ccp_dma_chan + i;
658*299bf602SKoba Ko dma_chan = &chan->dma_chan;
659*299bf602SKoba Ko
660*299bf602SKoba Ko if (dma_chan->client_count)
661*299bf602SKoba Ko dma_release_channel(dma_chan);
662*299bf602SKoba Ko }
663*299bf602SKoba Ko }
664*299bf602SKoba Ko
ccp_dmaengine_register(struct ccp_device * ccp)66558ea8abfSGary R Hook int ccp_dmaengine_register(struct ccp_device *ccp)
66658ea8abfSGary R Hook {
66758ea8abfSGary R Hook struct ccp_dma_chan *chan;
66858ea8abfSGary R Hook struct dma_device *dma_dev = &ccp->dma_dev;
66958ea8abfSGary R Hook struct dma_chan *dma_chan;
67058ea8abfSGary R Hook char *dma_cmd_cache_name;
67158ea8abfSGary R Hook char *dma_desc_cache_name;
67258ea8abfSGary R Hook unsigned int i;
67358ea8abfSGary R Hook int ret;
67458ea8abfSGary R Hook
675a7c26470SHook, Gary if (!dmaengine)
676a7c26470SHook, Gary return 0;
677a7c26470SHook, Gary
67858ea8abfSGary R Hook ccp->ccp_dma_chan = devm_kcalloc(ccp->dev, ccp->cmd_q_count,
67958ea8abfSGary R Hook sizeof(*(ccp->ccp_dma_chan)),
68058ea8abfSGary R Hook GFP_KERNEL);
68158ea8abfSGary R Hook if (!ccp->ccp_dma_chan)
68258ea8abfSGary R Hook return -ENOMEM;
68358ea8abfSGary R Hook
68458ea8abfSGary R Hook dma_cmd_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL,
68558ea8abfSGary R Hook "%s-dmaengine-cmd-cache",
68658ea8abfSGary R Hook ccp->name);
68758ea8abfSGary R Hook if (!dma_cmd_cache_name)
68858ea8abfSGary R Hook return -ENOMEM;
68958ea8abfSGary R Hook
69058ea8abfSGary R Hook ccp->dma_cmd_cache = kmem_cache_create(dma_cmd_cache_name,
69158ea8abfSGary R Hook sizeof(struct ccp_dma_cmd),
69258ea8abfSGary R Hook sizeof(void *),
69358ea8abfSGary R Hook SLAB_HWCACHE_ALIGN, NULL);
69458ea8abfSGary R Hook if (!ccp->dma_cmd_cache)
69558ea8abfSGary R Hook return -ENOMEM;
69658ea8abfSGary R Hook
69758ea8abfSGary R Hook dma_desc_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL,
69858ea8abfSGary R Hook "%s-dmaengine-desc-cache",
69958ea8abfSGary R Hook ccp->name);
7007514e368SWei Yongjun if (!dma_desc_cache_name) {
701ba22a1e2SQuentin Lambert ret = -ENOMEM;
702ba22a1e2SQuentin Lambert goto err_cache;
703ba22a1e2SQuentin Lambert }
704ba22a1e2SQuentin Lambert
70558ea8abfSGary R Hook ccp->dma_desc_cache = kmem_cache_create(dma_desc_cache_name,
70658ea8abfSGary R Hook sizeof(struct ccp_dma_desc),
70758ea8abfSGary R Hook sizeof(void *),
70858ea8abfSGary R Hook SLAB_HWCACHE_ALIGN, NULL);
70958ea8abfSGary R Hook if (!ccp->dma_desc_cache) {
71058ea8abfSGary R Hook ret = -ENOMEM;
71158ea8abfSGary R Hook goto err_cache;
71258ea8abfSGary R Hook }
71358ea8abfSGary R Hook
71458ea8abfSGary R Hook dma_dev->dev = ccp->dev;
71558ea8abfSGary R Hook dma_dev->src_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev));
71658ea8abfSGary R Hook dma_dev->dst_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev));
71758ea8abfSGary R Hook dma_dev->directions = DMA_MEM_TO_MEM;
71858ea8abfSGary R Hook dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
71958ea8abfSGary R Hook dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
72058ea8abfSGary R Hook dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
72158ea8abfSGary R Hook
722efc989fcSGary R Hook /* The DMA channels for this device can be set to public or private,
723efc989fcSGary R Hook * and overridden by the module parameter dma_chan_attr.
724efc989fcSGary R Hook * Default: according to the value in vdata (dma_chan_attr=0)
725efc989fcSGary R Hook * dma_chan_attr=0x1: all channels private (override vdata)
726efc989fcSGary R Hook * dma_chan_attr=0x2: all channels public (override vdata)
727efc989fcSGary R Hook */
728efc989fcSGary R Hook if (ccp_get_dma_chan_attr(ccp) == DMA_PRIVATE)
729efc989fcSGary R Hook dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
730efc989fcSGary R Hook
73158ea8abfSGary R Hook INIT_LIST_HEAD(&dma_dev->channels);
73258ea8abfSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) {
73358ea8abfSGary R Hook chan = ccp->ccp_dma_chan + i;
73458ea8abfSGary R Hook dma_chan = &chan->dma_chan;
73558ea8abfSGary R Hook
73658ea8abfSGary R Hook chan->ccp = ccp;
73758ea8abfSGary R Hook
73858ea8abfSGary R Hook spin_lock_init(&chan->lock);
739e5da5c56SGary R Hook INIT_LIST_HEAD(&chan->created);
74058ea8abfSGary R Hook INIT_LIST_HEAD(&chan->pending);
74158ea8abfSGary R Hook INIT_LIST_HEAD(&chan->active);
74258ea8abfSGary R Hook INIT_LIST_HEAD(&chan->complete);
74358ea8abfSGary R Hook
74458ea8abfSGary R Hook tasklet_init(&chan->cleanup_tasklet, ccp_do_cleanup,
74558ea8abfSGary R Hook (unsigned long)chan);
74658ea8abfSGary R Hook
74758ea8abfSGary R Hook dma_chan->device = dma_dev;
74858ea8abfSGary R Hook dma_cookie_init(dma_chan);
74958ea8abfSGary R Hook
75058ea8abfSGary R Hook list_add_tail(&dma_chan->device_node, &dma_dev->channels);
75158ea8abfSGary R Hook }
75258ea8abfSGary R Hook
75358ea8abfSGary R Hook dma_dev->device_free_chan_resources = ccp_free_chan_resources;
75458ea8abfSGary R Hook dma_dev->device_prep_dma_memcpy = ccp_prep_dma_memcpy;
75558ea8abfSGary R Hook dma_dev->device_prep_dma_interrupt = ccp_prep_dma_interrupt;
75658ea8abfSGary R Hook dma_dev->device_issue_pending = ccp_issue_pending;
75758ea8abfSGary R Hook dma_dev->device_tx_status = ccp_tx_status;
75858ea8abfSGary R Hook dma_dev->device_pause = ccp_pause;
75958ea8abfSGary R Hook dma_dev->device_resume = ccp_resume;
76058ea8abfSGary R Hook dma_dev->device_terminate_all = ccp_terminate_all;
76158ea8abfSGary R Hook
76258ea8abfSGary R Hook ret = dma_async_device_register(dma_dev);
76358ea8abfSGary R Hook if (ret)
76458ea8abfSGary R Hook goto err_reg;
76558ea8abfSGary R Hook
76658ea8abfSGary R Hook return 0;
76758ea8abfSGary R Hook
76858ea8abfSGary R Hook err_reg:
76954cce8ecSDāvis Mosāns ccp_dma_release(ccp);
77058ea8abfSGary R Hook kmem_cache_destroy(ccp->dma_desc_cache);
77158ea8abfSGary R Hook
77258ea8abfSGary R Hook err_cache:
77358ea8abfSGary R Hook kmem_cache_destroy(ccp->dma_cmd_cache);
77458ea8abfSGary R Hook
77558ea8abfSGary R Hook return ret;
77658ea8abfSGary R Hook }
77758ea8abfSGary R Hook
ccp_dmaengine_unregister(struct ccp_device * ccp)77858ea8abfSGary R Hook void ccp_dmaengine_unregister(struct ccp_device *ccp)
77958ea8abfSGary R Hook {
78058ea8abfSGary R Hook struct dma_device *dma_dev = &ccp->dma_dev;
78158ea8abfSGary R Hook
782a7c26470SHook, Gary if (!dmaengine)
783a7c26470SHook, Gary return;
784a7c26470SHook, Gary
785*299bf602SKoba Ko ccp_dma_release_channels(ccp);
78668dbe80fSKoba Ko dma_async_device_unregister(dma_dev);
787*299bf602SKoba Ko ccp_dma_release(ccp);
78858ea8abfSGary R Hook
78958ea8abfSGary R Hook kmem_cache_destroy(ccp->dma_desc_cache);
79058ea8abfSGary R Hook kmem_cache_destroy(ccp->dma_cmd_cache);
79158ea8abfSGary R Hook }
792