1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 258ea8abfSGary R Hook /* 358ea8abfSGary R Hook * AMD Cryptographic Coprocessor (CCP) driver 458ea8abfSGary R Hook * 568cc652fSGary R Hook * Copyright (C) 2016,2017 Advanced Micro Devices, Inc. 658ea8abfSGary R Hook * 758ea8abfSGary R Hook * Author: Gary R Hook <gary.hook@amd.com> 858ea8abfSGary R Hook */ 958ea8abfSGary R Hook 10efc989fcSGary R Hook #include <linux/module.h> 1158ea8abfSGary R Hook #include <linux/kernel.h> 1258ea8abfSGary R Hook #include <linux/dmaengine.h> 1358ea8abfSGary R Hook #include <linux/spinlock.h> 1458ea8abfSGary R Hook #include <linux/mutex.h> 1558ea8abfSGary R Hook #include <linux/ccp.h> 1658ea8abfSGary R Hook 1758ea8abfSGary R Hook #include "ccp-dev.h" 1858ea8abfSGary R Hook #include "../../dma/dmaengine.h" 1958ea8abfSGary R Hook 2058ea8abfSGary R Hook #define CCP_DMA_WIDTH(_mask) \ 2158ea8abfSGary R Hook ({ \ 2258ea8abfSGary R Hook u64 mask = _mask + 1; \ 2358ea8abfSGary R Hook (mask == 0) ? 64 : fls64(mask); \ 2458ea8abfSGary R Hook }) 2558ea8abfSGary R Hook 26efc989fcSGary R Hook /* The CCP as a DMA provider can be configured for public or private 27efc989fcSGary R Hook * channels. Default is specified in the vdata for the device (PCI ID). 28efc989fcSGary R Hook * This module parameter will override for all channels on all devices: 29efc989fcSGary R Hook * dma_chan_attr = 0x2 to force all channels public 30efc989fcSGary R Hook * = 0x1 to force all channels private 31efc989fcSGary R Hook * = 0x0 to defer to the vdata setting 32efc989fcSGary R Hook * = any other value: warning, revert to 0x0 33efc989fcSGary R Hook */ 34efc989fcSGary R Hook static unsigned int dma_chan_attr = CCP_DMA_DFLT; 35efc989fcSGary R Hook module_param(dma_chan_attr, uint, 0444); 36efc989fcSGary R Hook MODULE_PARM_DESC(dma_chan_attr, "Set DMA channel visibility: 0 (default) = device defaults, 1 = make private, 2 = make public"); 37efc989fcSGary R Hook 38404a36a7SColin Ian King static unsigned int ccp_get_dma_chan_attr(struct ccp_device *ccp) 39efc989fcSGary R Hook { 40efc989fcSGary R Hook switch (dma_chan_attr) { 41efc989fcSGary R Hook case CCP_DMA_DFLT: 42efc989fcSGary R Hook return ccp->vdata->dma_chan_attr; 43efc989fcSGary R Hook 44efc989fcSGary R Hook case CCP_DMA_PRIV: 45efc989fcSGary R Hook return DMA_PRIVATE; 46efc989fcSGary R Hook 47efc989fcSGary R Hook case CCP_DMA_PUB: 48efc989fcSGary R Hook return 0; 49efc989fcSGary R Hook 50efc989fcSGary R Hook default: 51efc989fcSGary R Hook dev_info_once(ccp->dev, "Invalid value for dma_chan_attr: %d\n", 52efc989fcSGary R Hook dma_chan_attr); 53efc989fcSGary R Hook return ccp->vdata->dma_chan_attr; 54efc989fcSGary R Hook } 55efc989fcSGary R Hook } 56efc989fcSGary R Hook 5758ea8abfSGary R Hook static void ccp_free_cmd_resources(struct ccp_device *ccp, 5858ea8abfSGary R Hook struct list_head *list) 5958ea8abfSGary R Hook { 6058ea8abfSGary R Hook struct ccp_dma_cmd *cmd, *ctmp; 6158ea8abfSGary R Hook 6258ea8abfSGary R Hook list_for_each_entry_safe(cmd, ctmp, list, entry) { 6358ea8abfSGary R Hook list_del(&cmd->entry); 6458ea8abfSGary R Hook kmem_cache_free(ccp->dma_cmd_cache, cmd); 6558ea8abfSGary R Hook } 6658ea8abfSGary R Hook } 6758ea8abfSGary R Hook 6858ea8abfSGary R Hook static void ccp_free_desc_resources(struct ccp_device *ccp, 6958ea8abfSGary R Hook struct list_head *list) 7058ea8abfSGary R Hook { 7158ea8abfSGary R Hook struct ccp_dma_desc *desc, *dtmp; 7258ea8abfSGary R Hook 7358ea8abfSGary R Hook list_for_each_entry_safe(desc, dtmp, list, entry) { 7458ea8abfSGary R Hook ccp_free_cmd_resources(ccp, &desc->active); 7558ea8abfSGary R Hook ccp_free_cmd_resources(ccp, &desc->pending); 7658ea8abfSGary R Hook 7758ea8abfSGary R Hook list_del(&desc->entry); 7858ea8abfSGary R Hook kmem_cache_free(ccp->dma_desc_cache, desc); 7958ea8abfSGary R Hook } 8058ea8abfSGary R Hook } 8158ea8abfSGary R Hook 8258ea8abfSGary R Hook static void ccp_free_chan_resources(struct dma_chan *dma_chan) 8358ea8abfSGary R Hook { 8458ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 8558ea8abfSGary R Hook dma_chan); 8658ea8abfSGary R Hook unsigned long flags; 8758ea8abfSGary R Hook 8858ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s - chan=%p\n", __func__, chan); 8958ea8abfSGary R Hook 9058ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 9158ea8abfSGary R Hook 9258ea8abfSGary R Hook ccp_free_desc_resources(chan->ccp, &chan->complete); 9358ea8abfSGary R Hook ccp_free_desc_resources(chan->ccp, &chan->active); 9458ea8abfSGary R Hook ccp_free_desc_resources(chan->ccp, &chan->pending); 95e5da5c56SGary R Hook ccp_free_desc_resources(chan->ccp, &chan->created); 9658ea8abfSGary R Hook 9758ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 9858ea8abfSGary R Hook } 9958ea8abfSGary R Hook 10058ea8abfSGary R Hook static void ccp_cleanup_desc_resources(struct ccp_device *ccp, 10158ea8abfSGary R Hook struct list_head *list) 10258ea8abfSGary R Hook { 10358ea8abfSGary R Hook struct ccp_dma_desc *desc, *dtmp; 10458ea8abfSGary R Hook 10558ea8abfSGary R Hook list_for_each_entry_safe_reverse(desc, dtmp, list, entry) { 10658ea8abfSGary R Hook if (!async_tx_test_ack(&desc->tx_desc)) 10758ea8abfSGary R Hook continue; 10858ea8abfSGary R Hook 10958ea8abfSGary R Hook dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc); 11058ea8abfSGary R Hook 11158ea8abfSGary R Hook ccp_free_cmd_resources(ccp, &desc->active); 11258ea8abfSGary R Hook ccp_free_cmd_resources(ccp, &desc->pending); 11358ea8abfSGary R Hook 11458ea8abfSGary R Hook list_del(&desc->entry); 11558ea8abfSGary R Hook kmem_cache_free(ccp->dma_desc_cache, desc); 11658ea8abfSGary R Hook } 11758ea8abfSGary R Hook } 11858ea8abfSGary R Hook 11958ea8abfSGary R Hook static void ccp_do_cleanup(unsigned long data) 12058ea8abfSGary R Hook { 12158ea8abfSGary R Hook struct ccp_dma_chan *chan = (struct ccp_dma_chan *)data; 12258ea8abfSGary R Hook unsigned long flags; 12358ea8abfSGary R Hook 12458ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s - chan=%s\n", __func__, 12558ea8abfSGary R Hook dma_chan_name(&chan->dma_chan)); 12658ea8abfSGary R Hook 12758ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 12858ea8abfSGary R Hook 12958ea8abfSGary R Hook ccp_cleanup_desc_resources(chan->ccp, &chan->complete); 13058ea8abfSGary R Hook 13158ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 13258ea8abfSGary R Hook } 13358ea8abfSGary R Hook 13458ea8abfSGary R Hook static int ccp_issue_next_cmd(struct ccp_dma_desc *desc) 13558ea8abfSGary R Hook { 13658ea8abfSGary R Hook struct ccp_dma_cmd *cmd; 13758ea8abfSGary R Hook int ret; 13858ea8abfSGary R Hook 13958ea8abfSGary R Hook cmd = list_first_entry(&desc->pending, struct ccp_dma_cmd, entry); 14058ea8abfSGary R Hook list_move(&cmd->entry, &desc->active); 14158ea8abfSGary R Hook 14258ea8abfSGary R Hook dev_dbg(desc->ccp->dev, "%s - tx %d, cmd=%p\n", __func__, 14358ea8abfSGary R Hook desc->tx_desc.cookie, cmd); 14458ea8abfSGary R Hook 14558ea8abfSGary R Hook ret = ccp_enqueue_cmd(&cmd->ccp_cmd); 14658ea8abfSGary R Hook if (!ret || (ret == -EINPROGRESS) || (ret == -EBUSY)) 14758ea8abfSGary R Hook return 0; 14858ea8abfSGary R Hook 14958ea8abfSGary R Hook dev_dbg(desc->ccp->dev, "%s - error: ret=%d, tx %d, cmd=%p\n", __func__, 15058ea8abfSGary R Hook ret, desc->tx_desc.cookie, cmd); 15158ea8abfSGary R Hook 15258ea8abfSGary R Hook return ret; 15358ea8abfSGary R Hook } 15458ea8abfSGary R Hook 15558ea8abfSGary R Hook static void ccp_free_active_cmd(struct ccp_dma_desc *desc) 15658ea8abfSGary R Hook { 15758ea8abfSGary R Hook struct ccp_dma_cmd *cmd; 15858ea8abfSGary R Hook 15958ea8abfSGary R Hook cmd = list_first_entry_or_null(&desc->active, struct ccp_dma_cmd, 16058ea8abfSGary R Hook entry); 16158ea8abfSGary R Hook if (!cmd) 16258ea8abfSGary R Hook return; 16358ea8abfSGary R Hook 16458ea8abfSGary R Hook dev_dbg(desc->ccp->dev, "%s - freeing tx %d cmd=%p\n", 16558ea8abfSGary R Hook __func__, desc->tx_desc.cookie, cmd); 16658ea8abfSGary R Hook 16758ea8abfSGary R Hook list_del(&cmd->entry); 16858ea8abfSGary R Hook kmem_cache_free(desc->ccp->dma_cmd_cache, cmd); 16958ea8abfSGary R Hook } 17058ea8abfSGary R Hook 17158ea8abfSGary R Hook static struct ccp_dma_desc *__ccp_next_dma_desc(struct ccp_dma_chan *chan, 17258ea8abfSGary R Hook struct ccp_dma_desc *desc) 17358ea8abfSGary R Hook { 17458ea8abfSGary R Hook /* Move current DMA descriptor to the complete list */ 17558ea8abfSGary R Hook if (desc) 17658ea8abfSGary R Hook list_move(&desc->entry, &chan->complete); 17758ea8abfSGary R Hook 17858ea8abfSGary R Hook /* Get the next DMA descriptor on the active list */ 17958ea8abfSGary R Hook desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc, 18058ea8abfSGary R Hook entry); 18158ea8abfSGary R Hook 18258ea8abfSGary R Hook return desc; 18358ea8abfSGary R Hook } 18458ea8abfSGary R Hook 18558ea8abfSGary R Hook static struct ccp_dma_desc *ccp_handle_active_desc(struct ccp_dma_chan *chan, 18658ea8abfSGary R Hook struct ccp_dma_desc *desc) 18758ea8abfSGary R Hook { 18858ea8abfSGary R Hook struct dma_async_tx_descriptor *tx_desc; 18958ea8abfSGary R Hook unsigned long flags; 19058ea8abfSGary R Hook 19158ea8abfSGary R Hook /* Loop over descriptors until one is found with commands */ 19258ea8abfSGary R Hook do { 19358ea8abfSGary R Hook if (desc) { 19458ea8abfSGary R Hook /* Remove the DMA command from the list and free it */ 19558ea8abfSGary R Hook ccp_free_active_cmd(desc); 19658ea8abfSGary R Hook 19758ea8abfSGary R Hook if (!list_empty(&desc->pending)) { 19858ea8abfSGary R Hook /* No errors, keep going */ 19958ea8abfSGary R Hook if (desc->status != DMA_ERROR) 20058ea8abfSGary R Hook return desc; 20158ea8abfSGary R Hook 20258ea8abfSGary R Hook /* Error, free remaining commands and move on */ 20358ea8abfSGary R Hook ccp_free_cmd_resources(desc->ccp, 20458ea8abfSGary R Hook &desc->pending); 20558ea8abfSGary R Hook } 20658ea8abfSGary R Hook 20758ea8abfSGary R Hook tx_desc = &desc->tx_desc; 20858ea8abfSGary R Hook } else { 20958ea8abfSGary R Hook tx_desc = NULL; 21058ea8abfSGary R Hook } 21158ea8abfSGary R Hook 21258ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 21358ea8abfSGary R Hook 21458ea8abfSGary R Hook if (desc) { 21558ea8abfSGary R Hook if (desc->status != DMA_ERROR) 21658ea8abfSGary R Hook desc->status = DMA_COMPLETE; 21758ea8abfSGary R Hook 21858ea8abfSGary R Hook dev_dbg(desc->ccp->dev, 21958ea8abfSGary R Hook "%s - tx %d complete, status=%u\n", __func__, 22058ea8abfSGary R Hook desc->tx_desc.cookie, desc->status); 22158ea8abfSGary R Hook 22258ea8abfSGary R Hook dma_cookie_complete(tx_desc); 22301c4c097Samd dma_descriptor_unmap(tx_desc); 22458ea8abfSGary R Hook } 22558ea8abfSGary R Hook 22658ea8abfSGary R Hook desc = __ccp_next_dma_desc(chan, desc); 22758ea8abfSGary R Hook 22858ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 22958ea8abfSGary R Hook 23058ea8abfSGary R Hook if (tx_desc) { 231c07f7c29Samd dmaengine_desc_get_callback_invoke(tx_desc, NULL); 23258ea8abfSGary R Hook 23358ea8abfSGary R Hook dma_run_dependencies(tx_desc); 23458ea8abfSGary R Hook } 23558ea8abfSGary R Hook } while (desc); 23658ea8abfSGary R Hook 23758ea8abfSGary R Hook return NULL; 23858ea8abfSGary R Hook } 23958ea8abfSGary R Hook 24058ea8abfSGary R Hook static struct ccp_dma_desc *__ccp_pending_to_active(struct ccp_dma_chan *chan) 24158ea8abfSGary R Hook { 24258ea8abfSGary R Hook struct ccp_dma_desc *desc; 24358ea8abfSGary R Hook 24458ea8abfSGary R Hook if (list_empty(&chan->pending)) 24558ea8abfSGary R Hook return NULL; 24658ea8abfSGary R Hook 24758ea8abfSGary R Hook desc = list_empty(&chan->active) 24858ea8abfSGary R Hook ? list_first_entry(&chan->pending, struct ccp_dma_desc, entry) 24958ea8abfSGary R Hook : NULL; 25058ea8abfSGary R Hook 25158ea8abfSGary R Hook list_splice_tail_init(&chan->pending, &chan->active); 25258ea8abfSGary R Hook 25358ea8abfSGary R Hook return desc; 25458ea8abfSGary R Hook } 25558ea8abfSGary R Hook 25658ea8abfSGary R Hook static void ccp_cmd_callback(void *data, int err) 25758ea8abfSGary R Hook { 25858ea8abfSGary R Hook struct ccp_dma_desc *desc = data; 25958ea8abfSGary R Hook struct ccp_dma_chan *chan; 26058ea8abfSGary R Hook int ret; 26158ea8abfSGary R Hook 26258ea8abfSGary R Hook if (err == -EINPROGRESS) 26358ea8abfSGary R Hook return; 26458ea8abfSGary R Hook 26558ea8abfSGary R Hook chan = container_of(desc->tx_desc.chan, struct ccp_dma_chan, 26658ea8abfSGary R Hook dma_chan); 26758ea8abfSGary R Hook 26858ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s - tx %d callback, err=%d\n", 26958ea8abfSGary R Hook __func__, desc->tx_desc.cookie, err); 27058ea8abfSGary R Hook 27158ea8abfSGary R Hook if (err) 27258ea8abfSGary R Hook desc->status = DMA_ERROR; 27358ea8abfSGary R Hook 27458ea8abfSGary R Hook while (true) { 27558ea8abfSGary R Hook /* Check for DMA descriptor completion */ 27658ea8abfSGary R Hook desc = ccp_handle_active_desc(chan, desc); 27758ea8abfSGary R Hook 27858ea8abfSGary R Hook /* Don't submit cmd if no descriptor or DMA is paused */ 27958ea8abfSGary R Hook if (!desc || (chan->status == DMA_PAUSED)) 28058ea8abfSGary R Hook break; 28158ea8abfSGary R Hook 28258ea8abfSGary R Hook ret = ccp_issue_next_cmd(desc); 28358ea8abfSGary R Hook if (!ret) 28458ea8abfSGary R Hook break; 28558ea8abfSGary R Hook 28658ea8abfSGary R Hook desc->status = DMA_ERROR; 28758ea8abfSGary R Hook } 28858ea8abfSGary R Hook 28958ea8abfSGary R Hook tasklet_schedule(&chan->cleanup_tasklet); 29058ea8abfSGary R Hook } 29158ea8abfSGary R Hook 29258ea8abfSGary R Hook static dma_cookie_t ccp_tx_submit(struct dma_async_tx_descriptor *tx_desc) 29358ea8abfSGary R Hook { 29458ea8abfSGary R Hook struct ccp_dma_desc *desc = container_of(tx_desc, struct ccp_dma_desc, 29558ea8abfSGary R Hook tx_desc); 29658ea8abfSGary R Hook struct ccp_dma_chan *chan; 29758ea8abfSGary R Hook dma_cookie_t cookie; 29858ea8abfSGary R Hook unsigned long flags; 29958ea8abfSGary R Hook 30058ea8abfSGary R Hook chan = container_of(tx_desc->chan, struct ccp_dma_chan, dma_chan); 30158ea8abfSGary R Hook 30258ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 30358ea8abfSGary R Hook 30458ea8abfSGary R Hook cookie = dma_cookie_assign(tx_desc); 305e5da5c56SGary R Hook list_del(&desc->entry); 30658ea8abfSGary R Hook list_add_tail(&desc->entry, &chan->pending); 30758ea8abfSGary R Hook 30858ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 30958ea8abfSGary R Hook 31058ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s - added tx descriptor %d to pending list\n", 31158ea8abfSGary R Hook __func__, cookie); 31258ea8abfSGary R Hook 31358ea8abfSGary R Hook return cookie; 31458ea8abfSGary R Hook } 31558ea8abfSGary R Hook 31658ea8abfSGary R Hook static struct ccp_dma_cmd *ccp_alloc_dma_cmd(struct ccp_dma_chan *chan) 31758ea8abfSGary R Hook { 31858ea8abfSGary R Hook struct ccp_dma_cmd *cmd; 31958ea8abfSGary R Hook 32058ea8abfSGary R Hook cmd = kmem_cache_alloc(chan->ccp->dma_cmd_cache, GFP_NOWAIT); 32158ea8abfSGary R Hook if (cmd) 32258ea8abfSGary R Hook memset(cmd, 0, sizeof(*cmd)); 32358ea8abfSGary R Hook 32458ea8abfSGary R Hook return cmd; 32558ea8abfSGary R Hook } 32658ea8abfSGary R Hook 32758ea8abfSGary R Hook static struct ccp_dma_desc *ccp_alloc_dma_desc(struct ccp_dma_chan *chan, 32858ea8abfSGary R Hook unsigned long flags) 32958ea8abfSGary R Hook { 33058ea8abfSGary R Hook struct ccp_dma_desc *desc; 33158ea8abfSGary R Hook 332664f570aSWei Yongjun desc = kmem_cache_zalloc(chan->ccp->dma_desc_cache, GFP_NOWAIT); 33358ea8abfSGary R Hook if (!desc) 33458ea8abfSGary R Hook return NULL; 33558ea8abfSGary R Hook 33658ea8abfSGary R Hook dma_async_tx_descriptor_init(&desc->tx_desc, &chan->dma_chan); 33758ea8abfSGary R Hook desc->tx_desc.flags = flags; 33858ea8abfSGary R Hook desc->tx_desc.tx_submit = ccp_tx_submit; 33958ea8abfSGary R Hook desc->ccp = chan->ccp; 34058ea8abfSGary R Hook INIT_LIST_HEAD(&desc->pending); 34158ea8abfSGary R Hook INIT_LIST_HEAD(&desc->active); 34258ea8abfSGary R Hook desc->status = DMA_IN_PROGRESS; 34358ea8abfSGary R Hook 34458ea8abfSGary R Hook return desc; 34558ea8abfSGary R Hook } 34658ea8abfSGary R Hook 34758ea8abfSGary R Hook static struct ccp_dma_desc *ccp_create_desc(struct dma_chan *dma_chan, 34858ea8abfSGary R Hook struct scatterlist *dst_sg, 34958ea8abfSGary R Hook unsigned int dst_nents, 35058ea8abfSGary R Hook struct scatterlist *src_sg, 35158ea8abfSGary R Hook unsigned int src_nents, 35258ea8abfSGary R Hook unsigned long flags) 35358ea8abfSGary R Hook { 35458ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 35558ea8abfSGary R Hook dma_chan); 35658ea8abfSGary R Hook struct ccp_device *ccp = chan->ccp; 35758ea8abfSGary R Hook struct ccp_dma_desc *desc; 35858ea8abfSGary R Hook struct ccp_dma_cmd *cmd; 35958ea8abfSGary R Hook struct ccp_cmd *ccp_cmd; 36058ea8abfSGary R Hook struct ccp_passthru_nomap_engine *ccp_pt; 36158ea8abfSGary R Hook unsigned int src_offset, src_len; 36258ea8abfSGary R Hook unsigned int dst_offset, dst_len; 36358ea8abfSGary R Hook unsigned int len; 36458ea8abfSGary R Hook unsigned long sflags; 36558ea8abfSGary R Hook size_t total_len; 36658ea8abfSGary R Hook 36758ea8abfSGary R Hook if (!dst_sg || !src_sg) 36858ea8abfSGary R Hook return NULL; 36958ea8abfSGary R Hook 37058ea8abfSGary R Hook if (!dst_nents || !src_nents) 37158ea8abfSGary R Hook return NULL; 37258ea8abfSGary R Hook 37358ea8abfSGary R Hook desc = ccp_alloc_dma_desc(chan, flags); 37458ea8abfSGary R Hook if (!desc) 37558ea8abfSGary R Hook return NULL; 37658ea8abfSGary R Hook 37758ea8abfSGary R Hook total_len = 0; 37858ea8abfSGary R Hook 37958ea8abfSGary R Hook src_len = sg_dma_len(src_sg); 38058ea8abfSGary R Hook src_offset = 0; 38158ea8abfSGary R Hook 38258ea8abfSGary R Hook dst_len = sg_dma_len(dst_sg); 38358ea8abfSGary R Hook dst_offset = 0; 38458ea8abfSGary R Hook 38558ea8abfSGary R Hook while (true) { 38658ea8abfSGary R Hook if (!src_len) { 38758ea8abfSGary R Hook src_nents--; 38858ea8abfSGary R Hook if (!src_nents) 38958ea8abfSGary R Hook break; 39058ea8abfSGary R Hook 39158ea8abfSGary R Hook src_sg = sg_next(src_sg); 39258ea8abfSGary R Hook if (!src_sg) 39358ea8abfSGary R Hook break; 39458ea8abfSGary R Hook 39558ea8abfSGary R Hook src_len = sg_dma_len(src_sg); 39658ea8abfSGary R Hook src_offset = 0; 39758ea8abfSGary R Hook continue; 39858ea8abfSGary R Hook } 39958ea8abfSGary R Hook 40058ea8abfSGary R Hook if (!dst_len) { 40158ea8abfSGary R Hook dst_nents--; 40258ea8abfSGary R Hook if (!dst_nents) 40358ea8abfSGary R Hook break; 40458ea8abfSGary R Hook 40558ea8abfSGary R Hook dst_sg = sg_next(dst_sg); 40658ea8abfSGary R Hook if (!dst_sg) 40758ea8abfSGary R Hook break; 40858ea8abfSGary R Hook 40958ea8abfSGary R Hook dst_len = sg_dma_len(dst_sg); 41058ea8abfSGary R Hook dst_offset = 0; 41158ea8abfSGary R Hook continue; 41258ea8abfSGary R Hook } 41358ea8abfSGary R Hook 41458ea8abfSGary R Hook len = min(dst_len, src_len); 41558ea8abfSGary R Hook 41658ea8abfSGary R Hook cmd = ccp_alloc_dma_cmd(chan); 41758ea8abfSGary R Hook if (!cmd) 41858ea8abfSGary R Hook goto err; 41958ea8abfSGary R Hook 42058ea8abfSGary R Hook ccp_cmd = &cmd->ccp_cmd; 4217c468447SGary R Hook ccp_cmd->ccp = chan->ccp; 42258ea8abfSGary R Hook ccp_pt = &ccp_cmd->u.passthru_nomap; 42358ea8abfSGary R Hook ccp_cmd->flags = CCP_CMD_MAY_BACKLOG; 42458ea8abfSGary R Hook ccp_cmd->flags |= CCP_CMD_PASSTHRU_NO_DMA_MAP; 42558ea8abfSGary R Hook ccp_cmd->engine = CCP_ENGINE_PASSTHRU; 42658ea8abfSGary R Hook ccp_pt->bit_mod = CCP_PASSTHRU_BITWISE_NOOP; 42758ea8abfSGary R Hook ccp_pt->byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP; 42858ea8abfSGary R Hook ccp_pt->src_dma = sg_dma_address(src_sg) + src_offset; 42958ea8abfSGary R Hook ccp_pt->dst_dma = sg_dma_address(dst_sg) + dst_offset; 43058ea8abfSGary R Hook ccp_pt->src_len = len; 43158ea8abfSGary R Hook ccp_pt->final = 1; 43258ea8abfSGary R Hook ccp_cmd->callback = ccp_cmd_callback; 43358ea8abfSGary R Hook ccp_cmd->data = desc; 43458ea8abfSGary R Hook 43558ea8abfSGary R Hook list_add_tail(&cmd->entry, &desc->pending); 43658ea8abfSGary R Hook 43758ea8abfSGary R Hook dev_dbg(ccp->dev, 43858ea8abfSGary R Hook "%s - cmd=%p, src=%pad, dst=%pad, len=%llu\n", __func__, 43958ea8abfSGary R Hook cmd, &ccp_pt->src_dma, 44058ea8abfSGary R Hook &ccp_pt->dst_dma, ccp_pt->src_len); 44158ea8abfSGary R Hook 44258ea8abfSGary R Hook total_len += len; 44358ea8abfSGary R Hook 44458ea8abfSGary R Hook src_len -= len; 44558ea8abfSGary R Hook src_offset += len; 44658ea8abfSGary R Hook 44758ea8abfSGary R Hook dst_len -= len; 44858ea8abfSGary R Hook dst_offset += len; 44958ea8abfSGary R Hook } 45058ea8abfSGary R Hook 45158ea8abfSGary R Hook desc->len = total_len; 45258ea8abfSGary R Hook 45358ea8abfSGary R Hook if (list_empty(&desc->pending)) 45458ea8abfSGary R Hook goto err; 45558ea8abfSGary R Hook 45658ea8abfSGary R Hook dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc); 45758ea8abfSGary R Hook 45858ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, sflags); 45958ea8abfSGary R Hook 460e5da5c56SGary R Hook list_add_tail(&desc->entry, &chan->created); 46158ea8abfSGary R Hook 46258ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, sflags); 46358ea8abfSGary R Hook 46458ea8abfSGary R Hook return desc; 46558ea8abfSGary R Hook 46658ea8abfSGary R Hook err: 46758ea8abfSGary R Hook ccp_free_cmd_resources(ccp, &desc->pending); 46858ea8abfSGary R Hook kmem_cache_free(ccp->dma_desc_cache, desc); 46958ea8abfSGary R Hook 47058ea8abfSGary R Hook return NULL; 47158ea8abfSGary R Hook } 47258ea8abfSGary R Hook 47358ea8abfSGary R Hook static struct dma_async_tx_descriptor *ccp_prep_dma_memcpy( 47458ea8abfSGary R Hook struct dma_chan *dma_chan, dma_addr_t dst, dma_addr_t src, size_t len, 47558ea8abfSGary R Hook unsigned long flags) 47658ea8abfSGary R Hook { 47758ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 47858ea8abfSGary R Hook dma_chan); 47958ea8abfSGary R Hook struct ccp_dma_desc *desc; 48058ea8abfSGary R Hook struct scatterlist dst_sg, src_sg; 48158ea8abfSGary R Hook 48258ea8abfSGary R Hook dev_dbg(chan->ccp->dev, 48358ea8abfSGary R Hook "%s - src=%pad, dst=%pad, len=%zu, flags=%#lx\n", 48458ea8abfSGary R Hook __func__, &src, &dst, len, flags); 48558ea8abfSGary R Hook 48658ea8abfSGary R Hook sg_init_table(&dst_sg, 1); 48758ea8abfSGary R Hook sg_dma_address(&dst_sg) = dst; 48858ea8abfSGary R Hook sg_dma_len(&dst_sg) = len; 48958ea8abfSGary R Hook 49058ea8abfSGary R Hook sg_init_table(&src_sg, 1); 49158ea8abfSGary R Hook sg_dma_address(&src_sg) = src; 49258ea8abfSGary R Hook sg_dma_len(&src_sg) = len; 49358ea8abfSGary R Hook 49458ea8abfSGary R Hook desc = ccp_create_desc(dma_chan, &dst_sg, 1, &src_sg, 1, flags); 49558ea8abfSGary R Hook if (!desc) 49658ea8abfSGary R Hook return NULL; 49758ea8abfSGary R Hook 49858ea8abfSGary R Hook return &desc->tx_desc; 49958ea8abfSGary R Hook } 50058ea8abfSGary R Hook 50158ea8abfSGary R Hook static struct dma_async_tx_descriptor *ccp_prep_dma_interrupt( 50258ea8abfSGary R Hook struct dma_chan *dma_chan, unsigned long flags) 50358ea8abfSGary R Hook { 50458ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 50558ea8abfSGary R Hook dma_chan); 50658ea8abfSGary R Hook struct ccp_dma_desc *desc; 50758ea8abfSGary R Hook 50858ea8abfSGary R Hook desc = ccp_alloc_dma_desc(chan, flags); 50958ea8abfSGary R Hook if (!desc) 51058ea8abfSGary R Hook return NULL; 51158ea8abfSGary R Hook 51258ea8abfSGary R Hook return &desc->tx_desc; 51358ea8abfSGary R Hook } 51458ea8abfSGary R Hook 51558ea8abfSGary R Hook static void ccp_issue_pending(struct dma_chan *dma_chan) 51658ea8abfSGary R Hook { 51758ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 51858ea8abfSGary R Hook dma_chan); 51958ea8abfSGary R Hook struct ccp_dma_desc *desc; 52058ea8abfSGary R Hook unsigned long flags; 52158ea8abfSGary R Hook 52258ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s\n", __func__); 52358ea8abfSGary R Hook 52458ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 52558ea8abfSGary R Hook 52658ea8abfSGary R Hook desc = __ccp_pending_to_active(chan); 52758ea8abfSGary R Hook 52858ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 52958ea8abfSGary R Hook 53058ea8abfSGary R Hook /* If there was nothing active, start processing */ 53158ea8abfSGary R Hook if (desc) 53258ea8abfSGary R Hook ccp_cmd_callback(desc, 0); 53358ea8abfSGary R Hook } 53458ea8abfSGary R Hook 53558ea8abfSGary R Hook static enum dma_status ccp_tx_status(struct dma_chan *dma_chan, 53658ea8abfSGary R Hook dma_cookie_t cookie, 53758ea8abfSGary R Hook struct dma_tx_state *state) 53858ea8abfSGary R Hook { 53958ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 54058ea8abfSGary R Hook dma_chan); 54158ea8abfSGary R Hook struct ccp_dma_desc *desc; 54258ea8abfSGary R Hook enum dma_status ret; 54358ea8abfSGary R Hook unsigned long flags; 54458ea8abfSGary R Hook 54558ea8abfSGary R Hook if (chan->status == DMA_PAUSED) { 54658ea8abfSGary R Hook ret = DMA_PAUSED; 54758ea8abfSGary R Hook goto out; 54858ea8abfSGary R Hook } 54958ea8abfSGary R Hook 55058ea8abfSGary R Hook ret = dma_cookie_status(dma_chan, cookie, state); 55158ea8abfSGary R Hook if (ret == DMA_COMPLETE) { 55258ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 55358ea8abfSGary R Hook 55458ea8abfSGary R Hook /* Get status from complete chain, if still there */ 55558ea8abfSGary R Hook list_for_each_entry(desc, &chan->complete, entry) { 55658ea8abfSGary R Hook if (desc->tx_desc.cookie != cookie) 55758ea8abfSGary R Hook continue; 55858ea8abfSGary R Hook 55958ea8abfSGary R Hook ret = desc->status; 56058ea8abfSGary R Hook break; 56158ea8abfSGary R Hook } 56258ea8abfSGary R Hook 56358ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 56458ea8abfSGary R Hook } 56558ea8abfSGary R Hook 56658ea8abfSGary R Hook out: 56758ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s - %u\n", __func__, ret); 56858ea8abfSGary R Hook 56958ea8abfSGary R Hook return ret; 57058ea8abfSGary R Hook } 57158ea8abfSGary R Hook 57258ea8abfSGary R Hook static int ccp_pause(struct dma_chan *dma_chan) 57358ea8abfSGary R Hook { 57458ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 57558ea8abfSGary R Hook dma_chan); 57658ea8abfSGary R Hook 57758ea8abfSGary R Hook chan->status = DMA_PAUSED; 57858ea8abfSGary R Hook 57958ea8abfSGary R Hook /*TODO: Wait for active DMA to complete before returning? */ 58058ea8abfSGary R Hook 58158ea8abfSGary R Hook return 0; 58258ea8abfSGary R Hook } 58358ea8abfSGary R Hook 58458ea8abfSGary R Hook static int ccp_resume(struct dma_chan *dma_chan) 58558ea8abfSGary R Hook { 58658ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 58758ea8abfSGary R Hook dma_chan); 58858ea8abfSGary R Hook struct ccp_dma_desc *desc; 58958ea8abfSGary R Hook unsigned long flags; 59058ea8abfSGary R Hook 59158ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 59258ea8abfSGary R Hook 59358ea8abfSGary R Hook desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc, 59458ea8abfSGary R Hook entry); 59558ea8abfSGary R Hook 59658ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 59758ea8abfSGary R Hook 59858ea8abfSGary R Hook /* Indicate the channel is running again */ 59958ea8abfSGary R Hook chan->status = DMA_IN_PROGRESS; 60058ea8abfSGary R Hook 60158ea8abfSGary R Hook /* If there was something active, re-start */ 60258ea8abfSGary R Hook if (desc) 60358ea8abfSGary R Hook ccp_cmd_callback(desc, 0); 60458ea8abfSGary R Hook 60558ea8abfSGary R Hook return 0; 60658ea8abfSGary R Hook } 60758ea8abfSGary R Hook 60858ea8abfSGary R Hook static int ccp_terminate_all(struct dma_chan *dma_chan) 60958ea8abfSGary R Hook { 61058ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 61158ea8abfSGary R Hook dma_chan); 61258ea8abfSGary R Hook unsigned long flags; 61358ea8abfSGary R Hook 61458ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s\n", __func__); 61558ea8abfSGary R Hook 61658ea8abfSGary R Hook /*TODO: Wait for active DMA to complete before continuing */ 61758ea8abfSGary R Hook 61858ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 61958ea8abfSGary R Hook 62058ea8abfSGary R Hook /*TODO: Purge the complete list? */ 62158ea8abfSGary R Hook ccp_free_desc_resources(chan->ccp, &chan->active); 62258ea8abfSGary R Hook ccp_free_desc_resources(chan->ccp, &chan->pending); 623e5da5c56SGary R Hook ccp_free_desc_resources(chan->ccp, &chan->created); 62458ea8abfSGary R Hook 62558ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 62658ea8abfSGary R Hook 62758ea8abfSGary R Hook return 0; 62858ea8abfSGary R Hook } 62958ea8abfSGary R Hook 63058ea8abfSGary R Hook int ccp_dmaengine_register(struct ccp_device *ccp) 63158ea8abfSGary R Hook { 63258ea8abfSGary R Hook struct ccp_dma_chan *chan; 63358ea8abfSGary R Hook struct dma_device *dma_dev = &ccp->dma_dev; 63458ea8abfSGary R Hook struct dma_chan *dma_chan; 63558ea8abfSGary R Hook char *dma_cmd_cache_name; 63658ea8abfSGary R Hook char *dma_desc_cache_name; 63758ea8abfSGary R Hook unsigned int i; 63858ea8abfSGary R Hook int ret; 63958ea8abfSGary R Hook 64058ea8abfSGary R Hook ccp->ccp_dma_chan = devm_kcalloc(ccp->dev, ccp->cmd_q_count, 64158ea8abfSGary R Hook sizeof(*(ccp->ccp_dma_chan)), 64258ea8abfSGary R Hook GFP_KERNEL); 64358ea8abfSGary R Hook if (!ccp->ccp_dma_chan) 64458ea8abfSGary R Hook return -ENOMEM; 64558ea8abfSGary R Hook 64658ea8abfSGary R Hook dma_cmd_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL, 64758ea8abfSGary R Hook "%s-dmaengine-cmd-cache", 64858ea8abfSGary R Hook ccp->name); 64958ea8abfSGary R Hook if (!dma_cmd_cache_name) 65058ea8abfSGary R Hook return -ENOMEM; 65158ea8abfSGary R Hook 65258ea8abfSGary R Hook ccp->dma_cmd_cache = kmem_cache_create(dma_cmd_cache_name, 65358ea8abfSGary R Hook sizeof(struct ccp_dma_cmd), 65458ea8abfSGary R Hook sizeof(void *), 65558ea8abfSGary R Hook SLAB_HWCACHE_ALIGN, NULL); 65658ea8abfSGary R Hook if (!ccp->dma_cmd_cache) 65758ea8abfSGary R Hook return -ENOMEM; 65858ea8abfSGary R Hook 65958ea8abfSGary R Hook dma_desc_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL, 66058ea8abfSGary R Hook "%s-dmaengine-desc-cache", 66158ea8abfSGary R Hook ccp->name); 6627514e368SWei Yongjun if (!dma_desc_cache_name) { 663ba22a1e2SQuentin Lambert ret = -ENOMEM; 664ba22a1e2SQuentin Lambert goto err_cache; 665ba22a1e2SQuentin Lambert } 666ba22a1e2SQuentin Lambert 66758ea8abfSGary R Hook ccp->dma_desc_cache = kmem_cache_create(dma_desc_cache_name, 66858ea8abfSGary R Hook sizeof(struct ccp_dma_desc), 66958ea8abfSGary R Hook sizeof(void *), 67058ea8abfSGary R Hook SLAB_HWCACHE_ALIGN, NULL); 67158ea8abfSGary R Hook if (!ccp->dma_desc_cache) { 67258ea8abfSGary R Hook ret = -ENOMEM; 67358ea8abfSGary R Hook goto err_cache; 67458ea8abfSGary R Hook } 67558ea8abfSGary R Hook 67658ea8abfSGary R Hook dma_dev->dev = ccp->dev; 67758ea8abfSGary R Hook dma_dev->src_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev)); 67858ea8abfSGary R Hook dma_dev->dst_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev)); 67958ea8abfSGary R Hook dma_dev->directions = DMA_MEM_TO_MEM; 68058ea8abfSGary R Hook dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 68158ea8abfSGary R Hook dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); 68258ea8abfSGary R Hook dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); 68358ea8abfSGary R Hook 684efc989fcSGary R Hook /* The DMA channels for this device can be set to public or private, 685efc989fcSGary R Hook * and overridden by the module parameter dma_chan_attr. 686efc989fcSGary R Hook * Default: according to the value in vdata (dma_chan_attr=0) 687efc989fcSGary R Hook * dma_chan_attr=0x1: all channels private (override vdata) 688efc989fcSGary R Hook * dma_chan_attr=0x2: all channels public (override vdata) 689efc989fcSGary R Hook */ 690efc989fcSGary R Hook if (ccp_get_dma_chan_attr(ccp) == DMA_PRIVATE) 691efc989fcSGary R Hook dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); 692efc989fcSGary R Hook 69358ea8abfSGary R Hook INIT_LIST_HEAD(&dma_dev->channels); 69458ea8abfSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) { 69558ea8abfSGary R Hook chan = ccp->ccp_dma_chan + i; 69658ea8abfSGary R Hook dma_chan = &chan->dma_chan; 69758ea8abfSGary R Hook 69858ea8abfSGary R Hook chan->ccp = ccp; 69958ea8abfSGary R Hook 70058ea8abfSGary R Hook spin_lock_init(&chan->lock); 701e5da5c56SGary R Hook INIT_LIST_HEAD(&chan->created); 70258ea8abfSGary R Hook INIT_LIST_HEAD(&chan->pending); 70358ea8abfSGary R Hook INIT_LIST_HEAD(&chan->active); 70458ea8abfSGary R Hook INIT_LIST_HEAD(&chan->complete); 70558ea8abfSGary R Hook 70658ea8abfSGary R Hook tasklet_init(&chan->cleanup_tasklet, ccp_do_cleanup, 70758ea8abfSGary R Hook (unsigned long)chan); 70858ea8abfSGary R Hook 70958ea8abfSGary R Hook dma_chan->device = dma_dev; 71058ea8abfSGary R Hook dma_cookie_init(dma_chan); 71158ea8abfSGary R Hook 71258ea8abfSGary R Hook list_add_tail(&dma_chan->device_node, &dma_dev->channels); 71358ea8abfSGary R Hook } 71458ea8abfSGary R Hook 71558ea8abfSGary R Hook dma_dev->device_free_chan_resources = ccp_free_chan_resources; 71658ea8abfSGary R Hook dma_dev->device_prep_dma_memcpy = ccp_prep_dma_memcpy; 71758ea8abfSGary R Hook dma_dev->device_prep_dma_interrupt = ccp_prep_dma_interrupt; 71858ea8abfSGary R Hook dma_dev->device_issue_pending = ccp_issue_pending; 71958ea8abfSGary R Hook dma_dev->device_tx_status = ccp_tx_status; 72058ea8abfSGary R Hook dma_dev->device_pause = ccp_pause; 72158ea8abfSGary R Hook dma_dev->device_resume = ccp_resume; 72258ea8abfSGary R Hook dma_dev->device_terminate_all = ccp_terminate_all; 72358ea8abfSGary R Hook 72458ea8abfSGary R Hook ret = dma_async_device_register(dma_dev); 72558ea8abfSGary R Hook if (ret) 72658ea8abfSGary R Hook goto err_reg; 72758ea8abfSGary R Hook 72858ea8abfSGary R Hook return 0; 72958ea8abfSGary R Hook 73058ea8abfSGary R Hook err_reg: 73158ea8abfSGary R Hook kmem_cache_destroy(ccp->dma_desc_cache); 73258ea8abfSGary R Hook 73358ea8abfSGary R Hook err_cache: 73458ea8abfSGary R Hook kmem_cache_destroy(ccp->dma_cmd_cache); 73558ea8abfSGary R Hook 73658ea8abfSGary R Hook return ret; 73758ea8abfSGary R Hook } 73858ea8abfSGary R Hook 73958ea8abfSGary R Hook void ccp_dmaengine_unregister(struct ccp_device *ccp) 74058ea8abfSGary R Hook { 74158ea8abfSGary R Hook struct dma_device *dma_dev = &ccp->dma_dev; 74258ea8abfSGary R Hook 74358ea8abfSGary R Hook dma_async_device_unregister(dma_dev); 74458ea8abfSGary R Hook 74558ea8abfSGary R Hook kmem_cache_destroy(ccp->dma_desc_cache); 74658ea8abfSGary R Hook kmem_cache_destroy(ccp->dma_cmd_cache); 74758ea8abfSGary R Hook } 748