158ea8abfSGary R Hook /* 258ea8abfSGary R Hook * AMD Cryptographic Coprocessor (CCP) driver 358ea8abfSGary R Hook * 468cc652fSGary R Hook * Copyright (C) 2016,2017 Advanced Micro Devices, Inc. 558ea8abfSGary R Hook * 658ea8abfSGary R Hook * Author: Gary R Hook <gary.hook@amd.com> 758ea8abfSGary R Hook * 858ea8abfSGary R Hook * This program is free software; you can redistribute it and/or modify 958ea8abfSGary R Hook * it under the terms of the GNU General Public License version 2 as 1058ea8abfSGary R Hook * published by the Free Software Foundation. 1158ea8abfSGary R Hook */ 1258ea8abfSGary R Hook 13efc989fcSGary R Hook #include <linux/module.h> 1458ea8abfSGary R Hook #include <linux/kernel.h> 1558ea8abfSGary R Hook #include <linux/dmaengine.h> 1658ea8abfSGary R Hook #include <linux/spinlock.h> 1758ea8abfSGary R Hook #include <linux/mutex.h> 1858ea8abfSGary R Hook #include <linux/ccp.h> 1958ea8abfSGary R Hook 2058ea8abfSGary R Hook #include "ccp-dev.h" 2158ea8abfSGary R Hook #include "../../dma/dmaengine.h" 2258ea8abfSGary R Hook 2358ea8abfSGary R Hook #define CCP_DMA_WIDTH(_mask) \ 2458ea8abfSGary R Hook ({ \ 2558ea8abfSGary R Hook u64 mask = _mask + 1; \ 2658ea8abfSGary R Hook (mask == 0) ? 64 : fls64(mask); \ 2758ea8abfSGary R Hook }) 2858ea8abfSGary R Hook 29efc989fcSGary R Hook /* The CCP as a DMA provider can be configured for public or private 30efc989fcSGary R Hook * channels. Default is specified in the vdata for the device (PCI ID). 31efc989fcSGary R Hook * This module parameter will override for all channels on all devices: 32efc989fcSGary R Hook * dma_chan_attr = 0x2 to force all channels public 33efc989fcSGary R Hook * = 0x1 to force all channels private 34efc989fcSGary R Hook * = 0x0 to defer to the vdata setting 35efc989fcSGary R Hook * = any other value: warning, revert to 0x0 36efc989fcSGary R Hook */ 37efc989fcSGary R Hook static unsigned int dma_chan_attr = CCP_DMA_DFLT; 38efc989fcSGary R Hook module_param(dma_chan_attr, uint, 0444); 39efc989fcSGary R Hook MODULE_PARM_DESC(dma_chan_attr, "Set DMA channel visibility: 0 (default) = device defaults, 1 = make private, 2 = make public"); 40efc989fcSGary R Hook 41efc989fcSGary R Hook unsigned int ccp_get_dma_chan_attr(struct ccp_device *ccp) 42efc989fcSGary R Hook { 43efc989fcSGary R Hook switch (dma_chan_attr) { 44efc989fcSGary R Hook case CCP_DMA_DFLT: 45efc989fcSGary R Hook return ccp->vdata->dma_chan_attr; 46efc989fcSGary R Hook 47efc989fcSGary R Hook case CCP_DMA_PRIV: 48efc989fcSGary R Hook return DMA_PRIVATE; 49efc989fcSGary R Hook 50efc989fcSGary R Hook case CCP_DMA_PUB: 51efc989fcSGary R Hook return 0; 52efc989fcSGary R Hook 53efc989fcSGary R Hook default: 54efc989fcSGary R Hook dev_info_once(ccp->dev, "Invalid value for dma_chan_attr: %d\n", 55efc989fcSGary R Hook dma_chan_attr); 56efc989fcSGary R Hook return ccp->vdata->dma_chan_attr; 57efc989fcSGary R Hook } 58efc989fcSGary R Hook } 59efc989fcSGary R Hook 6058ea8abfSGary R Hook static void ccp_free_cmd_resources(struct ccp_device *ccp, 6158ea8abfSGary R Hook struct list_head *list) 6258ea8abfSGary R Hook { 6358ea8abfSGary R Hook struct ccp_dma_cmd *cmd, *ctmp; 6458ea8abfSGary R Hook 6558ea8abfSGary R Hook list_for_each_entry_safe(cmd, ctmp, list, entry) { 6658ea8abfSGary R Hook list_del(&cmd->entry); 6758ea8abfSGary R Hook kmem_cache_free(ccp->dma_cmd_cache, cmd); 6858ea8abfSGary R Hook } 6958ea8abfSGary R Hook } 7058ea8abfSGary R Hook 7158ea8abfSGary R Hook static void ccp_free_desc_resources(struct ccp_device *ccp, 7258ea8abfSGary R Hook struct list_head *list) 7358ea8abfSGary R Hook { 7458ea8abfSGary R Hook struct ccp_dma_desc *desc, *dtmp; 7558ea8abfSGary R Hook 7658ea8abfSGary R Hook list_for_each_entry_safe(desc, dtmp, list, entry) { 7758ea8abfSGary R Hook ccp_free_cmd_resources(ccp, &desc->active); 7858ea8abfSGary R Hook ccp_free_cmd_resources(ccp, &desc->pending); 7958ea8abfSGary R Hook 8058ea8abfSGary R Hook list_del(&desc->entry); 8158ea8abfSGary R Hook kmem_cache_free(ccp->dma_desc_cache, desc); 8258ea8abfSGary R Hook } 8358ea8abfSGary R Hook } 8458ea8abfSGary R Hook 8558ea8abfSGary R Hook static void ccp_free_chan_resources(struct dma_chan *dma_chan) 8658ea8abfSGary R Hook { 8758ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 8858ea8abfSGary R Hook dma_chan); 8958ea8abfSGary R Hook unsigned long flags; 9058ea8abfSGary R Hook 9158ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s - chan=%p\n", __func__, chan); 9258ea8abfSGary R Hook 9358ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 9458ea8abfSGary R Hook 9558ea8abfSGary R Hook ccp_free_desc_resources(chan->ccp, &chan->complete); 9658ea8abfSGary R Hook ccp_free_desc_resources(chan->ccp, &chan->active); 9758ea8abfSGary R Hook ccp_free_desc_resources(chan->ccp, &chan->pending); 98e5da5c56SGary R Hook ccp_free_desc_resources(chan->ccp, &chan->created); 9958ea8abfSGary R Hook 10058ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 10158ea8abfSGary R Hook } 10258ea8abfSGary R Hook 10358ea8abfSGary R Hook static void ccp_cleanup_desc_resources(struct ccp_device *ccp, 10458ea8abfSGary R Hook struct list_head *list) 10558ea8abfSGary R Hook { 10658ea8abfSGary R Hook struct ccp_dma_desc *desc, *dtmp; 10758ea8abfSGary R Hook 10858ea8abfSGary R Hook list_for_each_entry_safe_reverse(desc, dtmp, list, entry) { 10958ea8abfSGary R Hook if (!async_tx_test_ack(&desc->tx_desc)) 11058ea8abfSGary R Hook continue; 11158ea8abfSGary R Hook 11258ea8abfSGary R Hook dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc); 11358ea8abfSGary R Hook 11458ea8abfSGary R Hook ccp_free_cmd_resources(ccp, &desc->active); 11558ea8abfSGary R Hook ccp_free_cmd_resources(ccp, &desc->pending); 11658ea8abfSGary R Hook 11758ea8abfSGary R Hook list_del(&desc->entry); 11858ea8abfSGary R Hook kmem_cache_free(ccp->dma_desc_cache, desc); 11958ea8abfSGary R Hook } 12058ea8abfSGary R Hook } 12158ea8abfSGary R Hook 12258ea8abfSGary R Hook static void ccp_do_cleanup(unsigned long data) 12358ea8abfSGary R Hook { 12458ea8abfSGary R Hook struct ccp_dma_chan *chan = (struct ccp_dma_chan *)data; 12558ea8abfSGary R Hook unsigned long flags; 12658ea8abfSGary R Hook 12758ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s - chan=%s\n", __func__, 12858ea8abfSGary R Hook dma_chan_name(&chan->dma_chan)); 12958ea8abfSGary R Hook 13058ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 13158ea8abfSGary R Hook 13258ea8abfSGary R Hook ccp_cleanup_desc_resources(chan->ccp, &chan->complete); 13358ea8abfSGary R Hook 13458ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 13558ea8abfSGary R Hook } 13658ea8abfSGary R Hook 13758ea8abfSGary R Hook static int ccp_issue_next_cmd(struct ccp_dma_desc *desc) 13858ea8abfSGary R Hook { 13958ea8abfSGary R Hook struct ccp_dma_cmd *cmd; 14058ea8abfSGary R Hook int ret; 14158ea8abfSGary R Hook 14258ea8abfSGary R Hook cmd = list_first_entry(&desc->pending, struct ccp_dma_cmd, entry); 14358ea8abfSGary R Hook list_move(&cmd->entry, &desc->active); 14458ea8abfSGary R Hook 14558ea8abfSGary R Hook dev_dbg(desc->ccp->dev, "%s - tx %d, cmd=%p\n", __func__, 14658ea8abfSGary R Hook desc->tx_desc.cookie, cmd); 14758ea8abfSGary R Hook 14858ea8abfSGary R Hook ret = ccp_enqueue_cmd(&cmd->ccp_cmd); 14958ea8abfSGary R Hook if (!ret || (ret == -EINPROGRESS) || (ret == -EBUSY)) 15058ea8abfSGary R Hook return 0; 15158ea8abfSGary R Hook 15258ea8abfSGary R Hook dev_dbg(desc->ccp->dev, "%s - error: ret=%d, tx %d, cmd=%p\n", __func__, 15358ea8abfSGary R Hook ret, desc->tx_desc.cookie, cmd); 15458ea8abfSGary R Hook 15558ea8abfSGary R Hook return ret; 15658ea8abfSGary R Hook } 15758ea8abfSGary R Hook 15858ea8abfSGary R Hook static void ccp_free_active_cmd(struct ccp_dma_desc *desc) 15958ea8abfSGary R Hook { 16058ea8abfSGary R Hook struct ccp_dma_cmd *cmd; 16158ea8abfSGary R Hook 16258ea8abfSGary R Hook cmd = list_first_entry_or_null(&desc->active, struct ccp_dma_cmd, 16358ea8abfSGary R Hook entry); 16458ea8abfSGary R Hook if (!cmd) 16558ea8abfSGary R Hook return; 16658ea8abfSGary R Hook 16758ea8abfSGary R Hook dev_dbg(desc->ccp->dev, "%s - freeing tx %d cmd=%p\n", 16858ea8abfSGary R Hook __func__, desc->tx_desc.cookie, cmd); 16958ea8abfSGary R Hook 17058ea8abfSGary R Hook list_del(&cmd->entry); 17158ea8abfSGary R Hook kmem_cache_free(desc->ccp->dma_cmd_cache, cmd); 17258ea8abfSGary R Hook } 17358ea8abfSGary R Hook 17458ea8abfSGary R Hook static struct ccp_dma_desc *__ccp_next_dma_desc(struct ccp_dma_chan *chan, 17558ea8abfSGary R Hook struct ccp_dma_desc *desc) 17658ea8abfSGary R Hook { 17758ea8abfSGary R Hook /* Move current DMA descriptor to the complete list */ 17858ea8abfSGary R Hook if (desc) 17958ea8abfSGary R Hook list_move(&desc->entry, &chan->complete); 18058ea8abfSGary R Hook 18158ea8abfSGary R Hook /* Get the next DMA descriptor on the active list */ 18258ea8abfSGary R Hook desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc, 18358ea8abfSGary R Hook entry); 18458ea8abfSGary R Hook 18558ea8abfSGary R Hook return desc; 18658ea8abfSGary R Hook } 18758ea8abfSGary R Hook 18858ea8abfSGary R Hook static struct ccp_dma_desc *ccp_handle_active_desc(struct ccp_dma_chan *chan, 18958ea8abfSGary R Hook struct ccp_dma_desc *desc) 19058ea8abfSGary R Hook { 19158ea8abfSGary R Hook struct dma_async_tx_descriptor *tx_desc; 19258ea8abfSGary R Hook unsigned long flags; 19358ea8abfSGary R Hook 19458ea8abfSGary R Hook /* Loop over descriptors until one is found with commands */ 19558ea8abfSGary R Hook do { 19658ea8abfSGary R Hook if (desc) { 19758ea8abfSGary R Hook /* Remove the DMA command from the list and free it */ 19858ea8abfSGary R Hook ccp_free_active_cmd(desc); 19958ea8abfSGary R Hook 20058ea8abfSGary R Hook if (!list_empty(&desc->pending)) { 20158ea8abfSGary R Hook /* No errors, keep going */ 20258ea8abfSGary R Hook if (desc->status != DMA_ERROR) 20358ea8abfSGary R Hook return desc; 20458ea8abfSGary R Hook 20558ea8abfSGary R Hook /* Error, free remaining commands and move on */ 20658ea8abfSGary R Hook ccp_free_cmd_resources(desc->ccp, 20758ea8abfSGary R Hook &desc->pending); 20858ea8abfSGary R Hook } 20958ea8abfSGary R Hook 21058ea8abfSGary R Hook tx_desc = &desc->tx_desc; 21158ea8abfSGary R Hook } else { 21258ea8abfSGary R Hook tx_desc = NULL; 21358ea8abfSGary R Hook } 21458ea8abfSGary R Hook 21558ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 21658ea8abfSGary R Hook 21758ea8abfSGary R Hook if (desc) { 21858ea8abfSGary R Hook if (desc->status != DMA_ERROR) 21958ea8abfSGary R Hook desc->status = DMA_COMPLETE; 22058ea8abfSGary R Hook 22158ea8abfSGary R Hook dev_dbg(desc->ccp->dev, 22258ea8abfSGary R Hook "%s - tx %d complete, status=%u\n", __func__, 22358ea8abfSGary R Hook desc->tx_desc.cookie, desc->status); 22458ea8abfSGary R Hook 22558ea8abfSGary R Hook dma_cookie_complete(tx_desc); 22658ea8abfSGary R Hook } 22758ea8abfSGary R Hook 22858ea8abfSGary R Hook desc = __ccp_next_dma_desc(chan, desc); 22958ea8abfSGary R Hook 23058ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 23158ea8abfSGary R Hook 23258ea8abfSGary R Hook if (tx_desc) { 23358ea8abfSGary R Hook if (tx_desc->callback && 23458ea8abfSGary R Hook (tx_desc->flags & DMA_PREP_INTERRUPT)) 23558ea8abfSGary R Hook tx_desc->callback(tx_desc->callback_param); 23658ea8abfSGary R Hook 23758ea8abfSGary R Hook dma_run_dependencies(tx_desc); 23858ea8abfSGary R Hook } 23958ea8abfSGary R Hook } while (desc); 24058ea8abfSGary R Hook 24158ea8abfSGary R Hook return NULL; 24258ea8abfSGary R Hook } 24358ea8abfSGary R Hook 24458ea8abfSGary R Hook static struct ccp_dma_desc *__ccp_pending_to_active(struct ccp_dma_chan *chan) 24558ea8abfSGary R Hook { 24658ea8abfSGary R Hook struct ccp_dma_desc *desc; 24758ea8abfSGary R Hook 24858ea8abfSGary R Hook if (list_empty(&chan->pending)) 24958ea8abfSGary R Hook return NULL; 25058ea8abfSGary R Hook 25158ea8abfSGary R Hook desc = list_empty(&chan->active) 25258ea8abfSGary R Hook ? list_first_entry(&chan->pending, struct ccp_dma_desc, entry) 25358ea8abfSGary R Hook : NULL; 25458ea8abfSGary R Hook 25558ea8abfSGary R Hook list_splice_tail_init(&chan->pending, &chan->active); 25658ea8abfSGary R Hook 25758ea8abfSGary R Hook return desc; 25858ea8abfSGary R Hook } 25958ea8abfSGary R Hook 26058ea8abfSGary R Hook static void ccp_cmd_callback(void *data, int err) 26158ea8abfSGary R Hook { 26258ea8abfSGary R Hook struct ccp_dma_desc *desc = data; 26358ea8abfSGary R Hook struct ccp_dma_chan *chan; 26458ea8abfSGary R Hook int ret; 26558ea8abfSGary R Hook 26658ea8abfSGary R Hook if (err == -EINPROGRESS) 26758ea8abfSGary R Hook return; 26858ea8abfSGary R Hook 26958ea8abfSGary R Hook chan = container_of(desc->tx_desc.chan, struct ccp_dma_chan, 27058ea8abfSGary R Hook dma_chan); 27158ea8abfSGary R Hook 27258ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s - tx %d callback, err=%d\n", 27358ea8abfSGary R Hook __func__, desc->tx_desc.cookie, err); 27458ea8abfSGary R Hook 27558ea8abfSGary R Hook if (err) 27658ea8abfSGary R Hook desc->status = DMA_ERROR; 27758ea8abfSGary R Hook 27858ea8abfSGary R Hook while (true) { 27958ea8abfSGary R Hook /* Check for DMA descriptor completion */ 28058ea8abfSGary R Hook desc = ccp_handle_active_desc(chan, desc); 28158ea8abfSGary R Hook 28258ea8abfSGary R Hook /* Don't submit cmd if no descriptor or DMA is paused */ 28358ea8abfSGary R Hook if (!desc || (chan->status == DMA_PAUSED)) 28458ea8abfSGary R Hook break; 28558ea8abfSGary R Hook 28658ea8abfSGary R Hook ret = ccp_issue_next_cmd(desc); 28758ea8abfSGary R Hook if (!ret) 28858ea8abfSGary R Hook break; 28958ea8abfSGary R Hook 29058ea8abfSGary R Hook desc->status = DMA_ERROR; 29158ea8abfSGary R Hook } 29258ea8abfSGary R Hook 29358ea8abfSGary R Hook tasklet_schedule(&chan->cleanup_tasklet); 29458ea8abfSGary R Hook } 29558ea8abfSGary R Hook 29658ea8abfSGary R Hook static dma_cookie_t ccp_tx_submit(struct dma_async_tx_descriptor *tx_desc) 29758ea8abfSGary R Hook { 29858ea8abfSGary R Hook struct ccp_dma_desc *desc = container_of(tx_desc, struct ccp_dma_desc, 29958ea8abfSGary R Hook tx_desc); 30058ea8abfSGary R Hook struct ccp_dma_chan *chan; 30158ea8abfSGary R Hook dma_cookie_t cookie; 30258ea8abfSGary R Hook unsigned long flags; 30358ea8abfSGary R Hook 30458ea8abfSGary R Hook chan = container_of(tx_desc->chan, struct ccp_dma_chan, dma_chan); 30558ea8abfSGary R Hook 30658ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 30758ea8abfSGary R Hook 30858ea8abfSGary R Hook cookie = dma_cookie_assign(tx_desc); 309e5da5c56SGary R Hook list_del(&desc->entry); 31058ea8abfSGary R Hook list_add_tail(&desc->entry, &chan->pending); 31158ea8abfSGary R Hook 31258ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 31358ea8abfSGary R Hook 31458ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s - added tx descriptor %d to pending list\n", 31558ea8abfSGary R Hook __func__, cookie); 31658ea8abfSGary R Hook 31758ea8abfSGary R Hook return cookie; 31858ea8abfSGary R Hook } 31958ea8abfSGary R Hook 32058ea8abfSGary R Hook static struct ccp_dma_cmd *ccp_alloc_dma_cmd(struct ccp_dma_chan *chan) 32158ea8abfSGary R Hook { 32258ea8abfSGary R Hook struct ccp_dma_cmd *cmd; 32358ea8abfSGary R Hook 32458ea8abfSGary R Hook cmd = kmem_cache_alloc(chan->ccp->dma_cmd_cache, GFP_NOWAIT); 32558ea8abfSGary R Hook if (cmd) 32658ea8abfSGary R Hook memset(cmd, 0, sizeof(*cmd)); 32758ea8abfSGary R Hook 32858ea8abfSGary R Hook return cmd; 32958ea8abfSGary R Hook } 33058ea8abfSGary R Hook 33158ea8abfSGary R Hook static struct ccp_dma_desc *ccp_alloc_dma_desc(struct ccp_dma_chan *chan, 33258ea8abfSGary R Hook unsigned long flags) 33358ea8abfSGary R Hook { 33458ea8abfSGary R Hook struct ccp_dma_desc *desc; 33558ea8abfSGary R Hook 336664f570aSWei Yongjun desc = kmem_cache_zalloc(chan->ccp->dma_desc_cache, GFP_NOWAIT); 33758ea8abfSGary R Hook if (!desc) 33858ea8abfSGary R Hook return NULL; 33958ea8abfSGary R Hook 34058ea8abfSGary R Hook dma_async_tx_descriptor_init(&desc->tx_desc, &chan->dma_chan); 34158ea8abfSGary R Hook desc->tx_desc.flags = flags; 34258ea8abfSGary R Hook desc->tx_desc.tx_submit = ccp_tx_submit; 34358ea8abfSGary R Hook desc->ccp = chan->ccp; 34458ea8abfSGary R Hook INIT_LIST_HEAD(&desc->pending); 34558ea8abfSGary R Hook INIT_LIST_HEAD(&desc->active); 34658ea8abfSGary R Hook desc->status = DMA_IN_PROGRESS; 34758ea8abfSGary R Hook 34858ea8abfSGary R Hook return desc; 34958ea8abfSGary R Hook } 35058ea8abfSGary R Hook 35158ea8abfSGary R Hook static struct ccp_dma_desc *ccp_create_desc(struct dma_chan *dma_chan, 35258ea8abfSGary R Hook struct scatterlist *dst_sg, 35358ea8abfSGary R Hook unsigned int dst_nents, 35458ea8abfSGary R Hook struct scatterlist *src_sg, 35558ea8abfSGary R Hook unsigned int src_nents, 35658ea8abfSGary R Hook unsigned long flags) 35758ea8abfSGary R Hook { 35858ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 35958ea8abfSGary R Hook dma_chan); 36058ea8abfSGary R Hook struct ccp_device *ccp = chan->ccp; 36158ea8abfSGary R Hook struct ccp_dma_desc *desc; 36258ea8abfSGary R Hook struct ccp_dma_cmd *cmd; 36358ea8abfSGary R Hook struct ccp_cmd *ccp_cmd; 36458ea8abfSGary R Hook struct ccp_passthru_nomap_engine *ccp_pt; 36558ea8abfSGary R Hook unsigned int src_offset, src_len; 36658ea8abfSGary R Hook unsigned int dst_offset, dst_len; 36758ea8abfSGary R Hook unsigned int len; 36858ea8abfSGary R Hook unsigned long sflags; 36958ea8abfSGary R Hook size_t total_len; 37058ea8abfSGary R Hook 37158ea8abfSGary R Hook if (!dst_sg || !src_sg) 37258ea8abfSGary R Hook return NULL; 37358ea8abfSGary R Hook 37458ea8abfSGary R Hook if (!dst_nents || !src_nents) 37558ea8abfSGary R Hook return NULL; 37658ea8abfSGary R Hook 37758ea8abfSGary R Hook desc = ccp_alloc_dma_desc(chan, flags); 37858ea8abfSGary R Hook if (!desc) 37958ea8abfSGary R Hook return NULL; 38058ea8abfSGary R Hook 38158ea8abfSGary R Hook total_len = 0; 38258ea8abfSGary R Hook 38358ea8abfSGary R Hook src_len = sg_dma_len(src_sg); 38458ea8abfSGary R Hook src_offset = 0; 38558ea8abfSGary R Hook 38658ea8abfSGary R Hook dst_len = sg_dma_len(dst_sg); 38758ea8abfSGary R Hook dst_offset = 0; 38858ea8abfSGary R Hook 38958ea8abfSGary R Hook while (true) { 39058ea8abfSGary R Hook if (!src_len) { 39158ea8abfSGary R Hook src_nents--; 39258ea8abfSGary R Hook if (!src_nents) 39358ea8abfSGary R Hook break; 39458ea8abfSGary R Hook 39558ea8abfSGary R Hook src_sg = sg_next(src_sg); 39658ea8abfSGary R Hook if (!src_sg) 39758ea8abfSGary R Hook break; 39858ea8abfSGary R Hook 39958ea8abfSGary R Hook src_len = sg_dma_len(src_sg); 40058ea8abfSGary R Hook src_offset = 0; 40158ea8abfSGary R Hook continue; 40258ea8abfSGary R Hook } 40358ea8abfSGary R Hook 40458ea8abfSGary R Hook if (!dst_len) { 40558ea8abfSGary R Hook dst_nents--; 40658ea8abfSGary R Hook if (!dst_nents) 40758ea8abfSGary R Hook break; 40858ea8abfSGary R Hook 40958ea8abfSGary R Hook dst_sg = sg_next(dst_sg); 41058ea8abfSGary R Hook if (!dst_sg) 41158ea8abfSGary R Hook break; 41258ea8abfSGary R Hook 41358ea8abfSGary R Hook dst_len = sg_dma_len(dst_sg); 41458ea8abfSGary R Hook dst_offset = 0; 41558ea8abfSGary R Hook continue; 41658ea8abfSGary R Hook } 41758ea8abfSGary R Hook 41858ea8abfSGary R Hook len = min(dst_len, src_len); 41958ea8abfSGary R Hook 42058ea8abfSGary R Hook cmd = ccp_alloc_dma_cmd(chan); 42158ea8abfSGary R Hook if (!cmd) 42258ea8abfSGary R Hook goto err; 42358ea8abfSGary R Hook 42458ea8abfSGary R Hook ccp_cmd = &cmd->ccp_cmd; 4257c468447SGary R Hook ccp_cmd->ccp = chan->ccp; 42658ea8abfSGary R Hook ccp_pt = &ccp_cmd->u.passthru_nomap; 42758ea8abfSGary R Hook ccp_cmd->flags = CCP_CMD_MAY_BACKLOG; 42858ea8abfSGary R Hook ccp_cmd->flags |= CCP_CMD_PASSTHRU_NO_DMA_MAP; 42958ea8abfSGary R Hook ccp_cmd->engine = CCP_ENGINE_PASSTHRU; 43058ea8abfSGary R Hook ccp_pt->bit_mod = CCP_PASSTHRU_BITWISE_NOOP; 43158ea8abfSGary R Hook ccp_pt->byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP; 43258ea8abfSGary R Hook ccp_pt->src_dma = sg_dma_address(src_sg) + src_offset; 43358ea8abfSGary R Hook ccp_pt->dst_dma = sg_dma_address(dst_sg) + dst_offset; 43458ea8abfSGary R Hook ccp_pt->src_len = len; 43558ea8abfSGary R Hook ccp_pt->final = 1; 43658ea8abfSGary R Hook ccp_cmd->callback = ccp_cmd_callback; 43758ea8abfSGary R Hook ccp_cmd->data = desc; 43858ea8abfSGary R Hook 43958ea8abfSGary R Hook list_add_tail(&cmd->entry, &desc->pending); 44058ea8abfSGary R Hook 44158ea8abfSGary R Hook dev_dbg(ccp->dev, 44258ea8abfSGary R Hook "%s - cmd=%p, src=%pad, dst=%pad, len=%llu\n", __func__, 44358ea8abfSGary R Hook cmd, &ccp_pt->src_dma, 44458ea8abfSGary R Hook &ccp_pt->dst_dma, ccp_pt->src_len); 44558ea8abfSGary R Hook 44658ea8abfSGary R Hook total_len += len; 44758ea8abfSGary R Hook 44858ea8abfSGary R Hook src_len -= len; 44958ea8abfSGary R Hook src_offset += len; 45058ea8abfSGary R Hook 45158ea8abfSGary R Hook dst_len -= len; 45258ea8abfSGary R Hook dst_offset += len; 45358ea8abfSGary R Hook } 45458ea8abfSGary R Hook 45558ea8abfSGary R Hook desc->len = total_len; 45658ea8abfSGary R Hook 45758ea8abfSGary R Hook if (list_empty(&desc->pending)) 45858ea8abfSGary R Hook goto err; 45958ea8abfSGary R Hook 46058ea8abfSGary R Hook dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc); 46158ea8abfSGary R Hook 46258ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, sflags); 46358ea8abfSGary R Hook 464e5da5c56SGary R Hook list_add_tail(&desc->entry, &chan->created); 46558ea8abfSGary R Hook 46658ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, sflags); 46758ea8abfSGary R Hook 46858ea8abfSGary R Hook return desc; 46958ea8abfSGary R Hook 47058ea8abfSGary R Hook err: 47158ea8abfSGary R Hook ccp_free_cmd_resources(ccp, &desc->pending); 47258ea8abfSGary R Hook kmem_cache_free(ccp->dma_desc_cache, desc); 47358ea8abfSGary R Hook 47458ea8abfSGary R Hook return NULL; 47558ea8abfSGary R Hook } 47658ea8abfSGary R Hook 47758ea8abfSGary R Hook static struct dma_async_tx_descriptor *ccp_prep_dma_memcpy( 47858ea8abfSGary R Hook struct dma_chan *dma_chan, dma_addr_t dst, dma_addr_t src, size_t len, 47958ea8abfSGary R Hook unsigned long flags) 48058ea8abfSGary R Hook { 48158ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 48258ea8abfSGary R Hook dma_chan); 48358ea8abfSGary R Hook struct ccp_dma_desc *desc; 48458ea8abfSGary R Hook struct scatterlist dst_sg, src_sg; 48558ea8abfSGary R Hook 48658ea8abfSGary R Hook dev_dbg(chan->ccp->dev, 48758ea8abfSGary R Hook "%s - src=%pad, dst=%pad, len=%zu, flags=%#lx\n", 48858ea8abfSGary R Hook __func__, &src, &dst, len, flags); 48958ea8abfSGary R Hook 49058ea8abfSGary R Hook sg_init_table(&dst_sg, 1); 49158ea8abfSGary R Hook sg_dma_address(&dst_sg) = dst; 49258ea8abfSGary R Hook sg_dma_len(&dst_sg) = len; 49358ea8abfSGary R Hook 49458ea8abfSGary R Hook sg_init_table(&src_sg, 1); 49558ea8abfSGary R Hook sg_dma_address(&src_sg) = src; 49658ea8abfSGary R Hook sg_dma_len(&src_sg) = len; 49758ea8abfSGary R Hook 49858ea8abfSGary R Hook desc = ccp_create_desc(dma_chan, &dst_sg, 1, &src_sg, 1, flags); 49958ea8abfSGary R Hook if (!desc) 50058ea8abfSGary R Hook return NULL; 50158ea8abfSGary R Hook 50258ea8abfSGary R Hook return &desc->tx_desc; 50358ea8abfSGary R Hook } 50458ea8abfSGary R Hook 50558ea8abfSGary R Hook static struct dma_async_tx_descriptor *ccp_prep_dma_sg( 50658ea8abfSGary R Hook struct dma_chan *dma_chan, struct scatterlist *dst_sg, 50758ea8abfSGary R Hook unsigned int dst_nents, struct scatterlist *src_sg, 50858ea8abfSGary R Hook unsigned int src_nents, unsigned long flags) 50958ea8abfSGary R Hook { 51058ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 51158ea8abfSGary R Hook dma_chan); 51258ea8abfSGary R Hook struct ccp_dma_desc *desc; 51358ea8abfSGary R Hook 51458ea8abfSGary R Hook dev_dbg(chan->ccp->dev, 51558ea8abfSGary R Hook "%s - src=%p, src_nents=%u dst=%p, dst_nents=%u, flags=%#lx\n", 51658ea8abfSGary R Hook __func__, src_sg, src_nents, dst_sg, dst_nents, flags); 51758ea8abfSGary R Hook 51858ea8abfSGary R Hook desc = ccp_create_desc(dma_chan, dst_sg, dst_nents, src_sg, src_nents, 51958ea8abfSGary R Hook flags); 52058ea8abfSGary R Hook if (!desc) 52158ea8abfSGary R Hook return NULL; 52258ea8abfSGary R Hook 52358ea8abfSGary R Hook return &desc->tx_desc; 52458ea8abfSGary R Hook } 52558ea8abfSGary R Hook 52658ea8abfSGary R Hook static struct dma_async_tx_descriptor *ccp_prep_dma_interrupt( 52758ea8abfSGary R Hook struct dma_chan *dma_chan, unsigned long flags) 52858ea8abfSGary R Hook { 52958ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 53058ea8abfSGary R Hook dma_chan); 53158ea8abfSGary R Hook struct ccp_dma_desc *desc; 53258ea8abfSGary R Hook 53358ea8abfSGary R Hook desc = ccp_alloc_dma_desc(chan, flags); 53458ea8abfSGary R Hook if (!desc) 53558ea8abfSGary R Hook return NULL; 53658ea8abfSGary R Hook 53758ea8abfSGary R Hook return &desc->tx_desc; 53858ea8abfSGary R Hook } 53958ea8abfSGary R Hook 54058ea8abfSGary R Hook static void ccp_issue_pending(struct dma_chan *dma_chan) 54158ea8abfSGary R Hook { 54258ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 54358ea8abfSGary R Hook dma_chan); 54458ea8abfSGary R Hook struct ccp_dma_desc *desc; 54558ea8abfSGary R Hook unsigned long flags; 54658ea8abfSGary R Hook 54758ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s\n", __func__); 54858ea8abfSGary R Hook 54958ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 55058ea8abfSGary R Hook 55158ea8abfSGary R Hook desc = __ccp_pending_to_active(chan); 55258ea8abfSGary R Hook 55358ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 55458ea8abfSGary R Hook 55558ea8abfSGary R Hook /* If there was nothing active, start processing */ 55658ea8abfSGary R Hook if (desc) 55758ea8abfSGary R Hook ccp_cmd_callback(desc, 0); 55858ea8abfSGary R Hook } 55958ea8abfSGary R Hook 56058ea8abfSGary R Hook static enum dma_status ccp_tx_status(struct dma_chan *dma_chan, 56158ea8abfSGary R Hook dma_cookie_t cookie, 56258ea8abfSGary R Hook struct dma_tx_state *state) 56358ea8abfSGary R Hook { 56458ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 56558ea8abfSGary R Hook dma_chan); 56658ea8abfSGary R Hook struct ccp_dma_desc *desc; 56758ea8abfSGary R Hook enum dma_status ret; 56858ea8abfSGary R Hook unsigned long flags; 56958ea8abfSGary R Hook 57058ea8abfSGary R Hook if (chan->status == DMA_PAUSED) { 57158ea8abfSGary R Hook ret = DMA_PAUSED; 57258ea8abfSGary R Hook goto out; 57358ea8abfSGary R Hook } 57458ea8abfSGary R Hook 57558ea8abfSGary R Hook ret = dma_cookie_status(dma_chan, cookie, state); 57658ea8abfSGary R Hook if (ret == DMA_COMPLETE) { 57758ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 57858ea8abfSGary R Hook 57958ea8abfSGary R Hook /* Get status from complete chain, if still there */ 58058ea8abfSGary R Hook list_for_each_entry(desc, &chan->complete, entry) { 58158ea8abfSGary R Hook if (desc->tx_desc.cookie != cookie) 58258ea8abfSGary R Hook continue; 58358ea8abfSGary R Hook 58458ea8abfSGary R Hook ret = desc->status; 58558ea8abfSGary R Hook break; 58658ea8abfSGary R Hook } 58758ea8abfSGary R Hook 58858ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 58958ea8abfSGary R Hook } 59058ea8abfSGary R Hook 59158ea8abfSGary R Hook out: 59258ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s - %u\n", __func__, ret); 59358ea8abfSGary R Hook 59458ea8abfSGary R Hook return ret; 59558ea8abfSGary R Hook } 59658ea8abfSGary R Hook 59758ea8abfSGary R Hook static int ccp_pause(struct dma_chan *dma_chan) 59858ea8abfSGary R Hook { 59958ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 60058ea8abfSGary R Hook dma_chan); 60158ea8abfSGary R Hook 60258ea8abfSGary R Hook chan->status = DMA_PAUSED; 60358ea8abfSGary R Hook 60458ea8abfSGary R Hook /*TODO: Wait for active DMA to complete before returning? */ 60558ea8abfSGary R Hook 60658ea8abfSGary R Hook return 0; 60758ea8abfSGary R Hook } 60858ea8abfSGary R Hook 60958ea8abfSGary R Hook static int ccp_resume(struct dma_chan *dma_chan) 61058ea8abfSGary R Hook { 61158ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 61258ea8abfSGary R Hook dma_chan); 61358ea8abfSGary R Hook struct ccp_dma_desc *desc; 61458ea8abfSGary R Hook unsigned long flags; 61558ea8abfSGary R Hook 61658ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 61758ea8abfSGary R Hook 61858ea8abfSGary R Hook desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc, 61958ea8abfSGary R Hook entry); 62058ea8abfSGary R Hook 62158ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 62258ea8abfSGary R Hook 62358ea8abfSGary R Hook /* Indicate the channel is running again */ 62458ea8abfSGary R Hook chan->status = DMA_IN_PROGRESS; 62558ea8abfSGary R Hook 62658ea8abfSGary R Hook /* If there was something active, re-start */ 62758ea8abfSGary R Hook if (desc) 62858ea8abfSGary R Hook ccp_cmd_callback(desc, 0); 62958ea8abfSGary R Hook 63058ea8abfSGary R Hook return 0; 63158ea8abfSGary R Hook } 63258ea8abfSGary R Hook 63358ea8abfSGary R Hook static int ccp_terminate_all(struct dma_chan *dma_chan) 63458ea8abfSGary R Hook { 63558ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 63658ea8abfSGary R Hook dma_chan); 63758ea8abfSGary R Hook unsigned long flags; 63858ea8abfSGary R Hook 63958ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s\n", __func__); 64058ea8abfSGary R Hook 64158ea8abfSGary R Hook /*TODO: Wait for active DMA to complete before continuing */ 64258ea8abfSGary R Hook 64358ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 64458ea8abfSGary R Hook 64558ea8abfSGary R Hook /*TODO: Purge the complete list? */ 64658ea8abfSGary R Hook ccp_free_desc_resources(chan->ccp, &chan->active); 64758ea8abfSGary R Hook ccp_free_desc_resources(chan->ccp, &chan->pending); 648e5da5c56SGary R Hook ccp_free_desc_resources(chan->ccp, &chan->created); 64958ea8abfSGary R Hook 65058ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 65158ea8abfSGary R Hook 65258ea8abfSGary R Hook return 0; 65358ea8abfSGary R Hook } 65458ea8abfSGary R Hook 65558ea8abfSGary R Hook int ccp_dmaengine_register(struct ccp_device *ccp) 65658ea8abfSGary R Hook { 65758ea8abfSGary R Hook struct ccp_dma_chan *chan; 65858ea8abfSGary R Hook struct dma_device *dma_dev = &ccp->dma_dev; 65958ea8abfSGary R Hook struct dma_chan *dma_chan; 66058ea8abfSGary R Hook char *dma_cmd_cache_name; 66158ea8abfSGary R Hook char *dma_desc_cache_name; 66258ea8abfSGary R Hook unsigned int i; 66358ea8abfSGary R Hook int ret; 66458ea8abfSGary R Hook 66558ea8abfSGary R Hook ccp->ccp_dma_chan = devm_kcalloc(ccp->dev, ccp->cmd_q_count, 66658ea8abfSGary R Hook sizeof(*(ccp->ccp_dma_chan)), 66758ea8abfSGary R Hook GFP_KERNEL); 66858ea8abfSGary R Hook if (!ccp->ccp_dma_chan) 66958ea8abfSGary R Hook return -ENOMEM; 67058ea8abfSGary R Hook 67158ea8abfSGary R Hook dma_cmd_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL, 67258ea8abfSGary R Hook "%s-dmaengine-cmd-cache", 67358ea8abfSGary R Hook ccp->name); 67458ea8abfSGary R Hook if (!dma_cmd_cache_name) 67558ea8abfSGary R Hook return -ENOMEM; 67658ea8abfSGary R Hook 67758ea8abfSGary R Hook ccp->dma_cmd_cache = kmem_cache_create(dma_cmd_cache_name, 67858ea8abfSGary R Hook sizeof(struct ccp_dma_cmd), 67958ea8abfSGary R Hook sizeof(void *), 68058ea8abfSGary R Hook SLAB_HWCACHE_ALIGN, NULL); 68158ea8abfSGary R Hook if (!ccp->dma_cmd_cache) 68258ea8abfSGary R Hook return -ENOMEM; 68358ea8abfSGary R Hook 68458ea8abfSGary R Hook dma_desc_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL, 68558ea8abfSGary R Hook "%s-dmaengine-desc-cache", 68658ea8abfSGary R Hook ccp->name); 6877514e368SWei Yongjun if (!dma_desc_cache_name) { 688ba22a1e2SQuentin Lambert ret = -ENOMEM; 689ba22a1e2SQuentin Lambert goto err_cache; 690ba22a1e2SQuentin Lambert } 691ba22a1e2SQuentin Lambert 69258ea8abfSGary R Hook ccp->dma_desc_cache = kmem_cache_create(dma_desc_cache_name, 69358ea8abfSGary R Hook sizeof(struct ccp_dma_desc), 69458ea8abfSGary R Hook sizeof(void *), 69558ea8abfSGary R Hook SLAB_HWCACHE_ALIGN, NULL); 69658ea8abfSGary R Hook if (!ccp->dma_desc_cache) { 69758ea8abfSGary R Hook ret = -ENOMEM; 69858ea8abfSGary R Hook goto err_cache; 69958ea8abfSGary R Hook } 70058ea8abfSGary R Hook 70158ea8abfSGary R Hook dma_dev->dev = ccp->dev; 70258ea8abfSGary R Hook dma_dev->src_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev)); 70358ea8abfSGary R Hook dma_dev->dst_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev)); 70458ea8abfSGary R Hook dma_dev->directions = DMA_MEM_TO_MEM; 70558ea8abfSGary R Hook dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 70658ea8abfSGary R Hook dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); 70758ea8abfSGary R Hook dma_cap_set(DMA_SG, dma_dev->cap_mask); 70858ea8abfSGary R Hook dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); 70958ea8abfSGary R Hook 710efc989fcSGary R Hook /* The DMA channels for this device can be set to public or private, 711efc989fcSGary R Hook * and overridden by the module parameter dma_chan_attr. 712efc989fcSGary R Hook * Default: according to the value in vdata (dma_chan_attr=0) 713efc989fcSGary R Hook * dma_chan_attr=0x1: all channels private (override vdata) 714efc989fcSGary R Hook * dma_chan_attr=0x2: all channels public (override vdata) 715efc989fcSGary R Hook */ 716efc989fcSGary R Hook if (ccp_get_dma_chan_attr(ccp) == DMA_PRIVATE) 717efc989fcSGary R Hook dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); 718efc989fcSGary R Hook 71958ea8abfSGary R Hook INIT_LIST_HEAD(&dma_dev->channels); 72058ea8abfSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) { 72158ea8abfSGary R Hook chan = ccp->ccp_dma_chan + i; 72258ea8abfSGary R Hook dma_chan = &chan->dma_chan; 72358ea8abfSGary R Hook 72458ea8abfSGary R Hook chan->ccp = ccp; 72558ea8abfSGary R Hook 72658ea8abfSGary R Hook spin_lock_init(&chan->lock); 727e5da5c56SGary R Hook INIT_LIST_HEAD(&chan->created); 72858ea8abfSGary R Hook INIT_LIST_HEAD(&chan->pending); 72958ea8abfSGary R Hook INIT_LIST_HEAD(&chan->active); 73058ea8abfSGary R Hook INIT_LIST_HEAD(&chan->complete); 73158ea8abfSGary R Hook 73258ea8abfSGary R Hook tasklet_init(&chan->cleanup_tasklet, ccp_do_cleanup, 73358ea8abfSGary R Hook (unsigned long)chan); 73458ea8abfSGary R Hook 73558ea8abfSGary R Hook dma_chan->device = dma_dev; 73658ea8abfSGary R Hook dma_cookie_init(dma_chan); 73758ea8abfSGary R Hook 73858ea8abfSGary R Hook list_add_tail(&dma_chan->device_node, &dma_dev->channels); 73958ea8abfSGary R Hook } 74058ea8abfSGary R Hook 74158ea8abfSGary R Hook dma_dev->device_free_chan_resources = ccp_free_chan_resources; 74258ea8abfSGary R Hook dma_dev->device_prep_dma_memcpy = ccp_prep_dma_memcpy; 74358ea8abfSGary R Hook dma_dev->device_prep_dma_sg = ccp_prep_dma_sg; 74458ea8abfSGary R Hook dma_dev->device_prep_dma_interrupt = ccp_prep_dma_interrupt; 74558ea8abfSGary R Hook dma_dev->device_issue_pending = ccp_issue_pending; 74658ea8abfSGary R Hook dma_dev->device_tx_status = ccp_tx_status; 74758ea8abfSGary R Hook dma_dev->device_pause = ccp_pause; 74858ea8abfSGary R Hook dma_dev->device_resume = ccp_resume; 74958ea8abfSGary R Hook dma_dev->device_terminate_all = ccp_terminate_all; 75058ea8abfSGary R Hook 75158ea8abfSGary R Hook ret = dma_async_device_register(dma_dev); 75258ea8abfSGary R Hook if (ret) 75358ea8abfSGary R Hook goto err_reg; 75458ea8abfSGary R Hook 75558ea8abfSGary R Hook return 0; 75658ea8abfSGary R Hook 75758ea8abfSGary R Hook err_reg: 75858ea8abfSGary R Hook kmem_cache_destroy(ccp->dma_desc_cache); 75958ea8abfSGary R Hook 76058ea8abfSGary R Hook err_cache: 76158ea8abfSGary R Hook kmem_cache_destroy(ccp->dma_cmd_cache); 76258ea8abfSGary R Hook 76358ea8abfSGary R Hook return ret; 76458ea8abfSGary R Hook } 76558ea8abfSGary R Hook 76658ea8abfSGary R Hook void ccp_dmaengine_unregister(struct ccp_device *ccp) 76758ea8abfSGary R Hook { 76858ea8abfSGary R Hook struct dma_device *dma_dev = &ccp->dma_dev; 76958ea8abfSGary R Hook 77058ea8abfSGary R Hook dma_async_device_unregister(dma_dev); 77158ea8abfSGary R Hook 77258ea8abfSGary R Hook kmem_cache_destroy(ccp->dma_desc_cache); 77358ea8abfSGary R Hook kmem_cache_destroy(ccp->dma_cmd_cache); 77458ea8abfSGary R Hook } 775