1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 258ea8abfSGary R Hook /* 358ea8abfSGary R Hook * AMD Cryptographic Coprocessor (CCP) driver 458ea8abfSGary R Hook * 5a7c26470SHook, Gary * Copyright (C) 2016,2019 Advanced Micro Devices, Inc. 658ea8abfSGary R Hook * 758ea8abfSGary R Hook * Author: Gary R Hook <gary.hook@amd.com> 858ea8abfSGary R Hook */ 958ea8abfSGary R Hook 10efc989fcSGary R Hook #include <linux/module.h> 1158ea8abfSGary R Hook #include <linux/kernel.h> 12f6b0b78aSBjorn Helgaas #include <linux/dma-mapping.h> 1358ea8abfSGary R Hook #include <linux/dmaengine.h> 1458ea8abfSGary R Hook #include <linux/spinlock.h> 1558ea8abfSGary R Hook #include <linux/mutex.h> 1658ea8abfSGary R Hook #include <linux/ccp.h> 1758ea8abfSGary R Hook 1858ea8abfSGary R Hook #include "ccp-dev.h" 1958ea8abfSGary R Hook #include "../../dma/dmaengine.h" 2058ea8abfSGary R Hook 2158ea8abfSGary R Hook #define CCP_DMA_WIDTH(_mask) \ 2258ea8abfSGary R Hook ({ \ 2358ea8abfSGary R Hook u64 mask = _mask + 1; \ 2458ea8abfSGary R Hook (mask == 0) ? 64 : fls64(mask); \ 2558ea8abfSGary R Hook }) 2658ea8abfSGary R Hook 27efc989fcSGary R Hook /* The CCP as a DMA provider can be configured for public or private 28efc989fcSGary R Hook * channels. Default is specified in the vdata for the device (PCI ID). 29efc989fcSGary R Hook * This module parameter will override for all channels on all devices: 30efc989fcSGary R Hook * dma_chan_attr = 0x2 to force all channels public 31efc989fcSGary R Hook * = 0x1 to force all channels private 32efc989fcSGary R Hook * = 0x0 to defer to the vdata setting 33efc989fcSGary R Hook * = any other value: warning, revert to 0x0 34efc989fcSGary R Hook */ 35efc989fcSGary R Hook static unsigned int dma_chan_attr = CCP_DMA_DFLT; 36efc989fcSGary R Hook module_param(dma_chan_attr, uint, 0444); 37efc989fcSGary R Hook MODULE_PARM_DESC(dma_chan_attr, "Set DMA channel visibility: 0 (default) = device defaults, 1 = make private, 2 = make public"); 38efc989fcSGary R Hook 39a7c26470SHook, Gary static unsigned int dmaengine = 1; 40a7c26470SHook, Gary module_param(dmaengine, uint, 0444); 41a7c26470SHook, Gary MODULE_PARM_DESC(dmaengine, "Register services with the DMA subsystem (any non-zero value, default: 1)"); 42a7c26470SHook, Gary 43404a36a7SColin Ian King static unsigned int ccp_get_dma_chan_attr(struct ccp_device *ccp) 44efc989fcSGary R Hook { 45efc989fcSGary R Hook switch (dma_chan_attr) { 46efc989fcSGary R Hook case CCP_DMA_DFLT: 47efc989fcSGary R Hook return ccp->vdata->dma_chan_attr; 48efc989fcSGary R Hook 49efc989fcSGary R Hook case CCP_DMA_PRIV: 50efc989fcSGary R Hook return DMA_PRIVATE; 51efc989fcSGary R Hook 52efc989fcSGary R Hook case CCP_DMA_PUB: 53efc989fcSGary R Hook return 0; 54efc989fcSGary R Hook 55efc989fcSGary R Hook default: 56efc989fcSGary R Hook dev_info_once(ccp->dev, "Invalid value for dma_chan_attr: %d\n", 57efc989fcSGary R Hook dma_chan_attr); 58efc989fcSGary R Hook return ccp->vdata->dma_chan_attr; 59efc989fcSGary R Hook } 60efc989fcSGary R Hook } 61efc989fcSGary R Hook 6258ea8abfSGary R Hook static void ccp_free_cmd_resources(struct ccp_device *ccp, 6358ea8abfSGary R Hook struct list_head *list) 6458ea8abfSGary R Hook { 6558ea8abfSGary R Hook struct ccp_dma_cmd *cmd, *ctmp; 6658ea8abfSGary R Hook 6758ea8abfSGary R Hook list_for_each_entry_safe(cmd, ctmp, list, entry) { 6858ea8abfSGary R Hook list_del(&cmd->entry); 6958ea8abfSGary R Hook kmem_cache_free(ccp->dma_cmd_cache, cmd); 7058ea8abfSGary R Hook } 7158ea8abfSGary R Hook } 7258ea8abfSGary R Hook 7358ea8abfSGary R Hook static void ccp_free_desc_resources(struct ccp_device *ccp, 7458ea8abfSGary R Hook struct list_head *list) 7558ea8abfSGary R Hook { 7658ea8abfSGary R Hook struct ccp_dma_desc *desc, *dtmp; 7758ea8abfSGary R Hook 7858ea8abfSGary R Hook list_for_each_entry_safe(desc, dtmp, list, entry) { 7958ea8abfSGary R Hook ccp_free_cmd_resources(ccp, &desc->active); 8058ea8abfSGary R Hook ccp_free_cmd_resources(ccp, &desc->pending); 8158ea8abfSGary R Hook 8258ea8abfSGary R Hook list_del(&desc->entry); 8358ea8abfSGary R Hook kmem_cache_free(ccp->dma_desc_cache, desc); 8458ea8abfSGary R Hook } 8558ea8abfSGary R Hook } 8658ea8abfSGary R Hook 8758ea8abfSGary R Hook static void ccp_free_chan_resources(struct dma_chan *dma_chan) 8858ea8abfSGary R Hook { 8958ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 9058ea8abfSGary R Hook dma_chan); 9158ea8abfSGary R Hook unsigned long flags; 9258ea8abfSGary R Hook 9358ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s - chan=%p\n", __func__, chan); 9458ea8abfSGary R Hook 9558ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 9658ea8abfSGary R Hook 9758ea8abfSGary R Hook ccp_free_desc_resources(chan->ccp, &chan->complete); 9858ea8abfSGary R Hook ccp_free_desc_resources(chan->ccp, &chan->active); 9958ea8abfSGary R Hook ccp_free_desc_resources(chan->ccp, &chan->pending); 100e5da5c56SGary R Hook ccp_free_desc_resources(chan->ccp, &chan->created); 10158ea8abfSGary R Hook 10258ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 10358ea8abfSGary R Hook } 10458ea8abfSGary R Hook 10558ea8abfSGary R Hook static void ccp_cleanup_desc_resources(struct ccp_device *ccp, 10658ea8abfSGary R Hook struct list_head *list) 10758ea8abfSGary R Hook { 10858ea8abfSGary R Hook struct ccp_dma_desc *desc, *dtmp; 10958ea8abfSGary R Hook 11058ea8abfSGary R Hook list_for_each_entry_safe_reverse(desc, dtmp, list, entry) { 11158ea8abfSGary R Hook if (!async_tx_test_ack(&desc->tx_desc)) 11258ea8abfSGary R Hook continue; 11358ea8abfSGary R Hook 11458ea8abfSGary R Hook dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc); 11558ea8abfSGary R Hook 11658ea8abfSGary R Hook ccp_free_cmd_resources(ccp, &desc->active); 11758ea8abfSGary R Hook ccp_free_cmd_resources(ccp, &desc->pending); 11858ea8abfSGary R Hook 11958ea8abfSGary R Hook list_del(&desc->entry); 12058ea8abfSGary R Hook kmem_cache_free(ccp->dma_desc_cache, desc); 12158ea8abfSGary R Hook } 12258ea8abfSGary R Hook } 12358ea8abfSGary R Hook 12458ea8abfSGary R Hook static void ccp_do_cleanup(unsigned long data) 12558ea8abfSGary R Hook { 12658ea8abfSGary R Hook struct ccp_dma_chan *chan = (struct ccp_dma_chan *)data; 12758ea8abfSGary R Hook unsigned long flags; 12858ea8abfSGary R Hook 12958ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s - chan=%s\n", __func__, 13058ea8abfSGary R Hook dma_chan_name(&chan->dma_chan)); 13158ea8abfSGary R Hook 13258ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 13358ea8abfSGary R Hook 13458ea8abfSGary R Hook ccp_cleanup_desc_resources(chan->ccp, &chan->complete); 13558ea8abfSGary R Hook 13658ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 13758ea8abfSGary R Hook } 13858ea8abfSGary R Hook 13958ea8abfSGary R Hook static int ccp_issue_next_cmd(struct ccp_dma_desc *desc) 14058ea8abfSGary R Hook { 14158ea8abfSGary R Hook struct ccp_dma_cmd *cmd; 14258ea8abfSGary R Hook int ret; 14358ea8abfSGary R Hook 14458ea8abfSGary R Hook cmd = list_first_entry(&desc->pending, struct ccp_dma_cmd, entry); 14558ea8abfSGary R Hook list_move(&cmd->entry, &desc->active); 14658ea8abfSGary R Hook 14758ea8abfSGary R Hook dev_dbg(desc->ccp->dev, "%s - tx %d, cmd=%p\n", __func__, 14858ea8abfSGary R Hook desc->tx_desc.cookie, cmd); 14958ea8abfSGary R Hook 15058ea8abfSGary R Hook ret = ccp_enqueue_cmd(&cmd->ccp_cmd); 15158ea8abfSGary R Hook if (!ret || (ret == -EINPROGRESS) || (ret == -EBUSY)) 15258ea8abfSGary R Hook return 0; 15358ea8abfSGary R Hook 15458ea8abfSGary R Hook dev_dbg(desc->ccp->dev, "%s - error: ret=%d, tx %d, cmd=%p\n", __func__, 15558ea8abfSGary R Hook ret, desc->tx_desc.cookie, cmd); 15658ea8abfSGary R Hook 15758ea8abfSGary R Hook return ret; 15858ea8abfSGary R Hook } 15958ea8abfSGary R Hook 16058ea8abfSGary R Hook static void ccp_free_active_cmd(struct ccp_dma_desc *desc) 16158ea8abfSGary R Hook { 16258ea8abfSGary R Hook struct ccp_dma_cmd *cmd; 16358ea8abfSGary R Hook 16458ea8abfSGary R Hook cmd = list_first_entry_or_null(&desc->active, struct ccp_dma_cmd, 16558ea8abfSGary R Hook entry); 16658ea8abfSGary R Hook if (!cmd) 16758ea8abfSGary R Hook return; 16858ea8abfSGary R Hook 16958ea8abfSGary R Hook dev_dbg(desc->ccp->dev, "%s - freeing tx %d cmd=%p\n", 17058ea8abfSGary R Hook __func__, desc->tx_desc.cookie, cmd); 17158ea8abfSGary R Hook 17258ea8abfSGary R Hook list_del(&cmd->entry); 17358ea8abfSGary R Hook kmem_cache_free(desc->ccp->dma_cmd_cache, cmd); 17458ea8abfSGary R Hook } 17558ea8abfSGary R Hook 17658ea8abfSGary R Hook static struct ccp_dma_desc *__ccp_next_dma_desc(struct ccp_dma_chan *chan, 17758ea8abfSGary R Hook struct ccp_dma_desc *desc) 17858ea8abfSGary R Hook { 17958ea8abfSGary R Hook /* Move current DMA descriptor to the complete list */ 18058ea8abfSGary R Hook if (desc) 18158ea8abfSGary R Hook list_move(&desc->entry, &chan->complete); 18258ea8abfSGary R Hook 18358ea8abfSGary R Hook /* Get the next DMA descriptor on the active list */ 18458ea8abfSGary R Hook desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc, 18558ea8abfSGary R Hook entry); 18658ea8abfSGary R Hook 18758ea8abfSGary R Hook return desc; 18858ea8abfSGary R Hook } 18958ea8abfSGary R Hook 19058ea8abfSGary R Hook static struct ccp_dma_desc *ccp_handle_active_desc(struct ccp_dma_chan *chan, 19158ea8abfSGary R Hook struct ccp_dma_desc *desc) 19258ea8abfSGary R Hook { 19358ea8abfSGary R Hook struct dma_async_tx_descriptor *tx_desc; 19458ea8abfSGary R Hook unsigned long flags; 19558ea8abfSGary R Hook 19658ea8abfSGary R Hook /* Loop over descriptors until one is found with commands */ 19758ea8abfSGary R Hook do { 19858ea8abfSGary R Hook if (desc) { 19958ea8abfSGary R Hook /* Remove the DMA command from the list and free it */ 20058ea8abfSGary R Hook ccp_free_active_cmd(desc); 20158ea8abfSGary R Hook 20258ea8abfSGary R Hook if (!list_empty(&desc->pending)) { 20358ea8abfSGary R Hook /* No errors, keep going */ 20458ea8abfSGary R Hook if (desc->status != DMA_ERROR) 20558ea8abfSGary R Hook return desc; 20658ea8abfSGary R Hook 20758ea8abfSGary R Hook /* Error, free remaining commands and move on */ 20858ea8abfSGary R Hook ccp_free_cmd_resources(desc->ccp, 20958ea8abfSGary R Hook &desc->pending); 21058ea8abfSGary R Hook } 21158ea8abfSGary R Hook 21258ea8abfSGary R Hook tx_desc = &desc->tx_desc; 21358ea8abfSGary R Hook } else { 21458ea8abfSGary R Hook tx_desc = NULL; 21558ea8abfSGary R Hook } 21658ea8abfSGary R Hook 21758ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 21858ea8abfSGary R Hook 21958ea8abfSGary R Hook if (desc) { 22058ea8abfSGary R Hook if (desc->status != DMA_ERROR) 22158ea8abfSGary R Hook desc->status = DMA_COMPLETE; 22258ea8abfSGary R Hook 22358ea8abfSGary R Hook dev_dbg(desc->ccp->dev, 22458ea8abfSGary R Hook "%s - tx %d complete, status=%u\n", __func__, 22558ea8abfSGary R Hook desc->tx_desc.cookie, desc->status); 22658ea8abfSGary R Hook 22758ea8abfSGary R Hook dma_cookie_complete(tx_desc); 22801c4c097Samd dma_descriptor_unmap(tx_desc); 22958ea8abfSGary R Hook } 23058ea8abfSGary R Hook 23158ea8abfSGary R Hook desc = __ccp_next_dma_desc(chan, desc); 23258ea8abfSGary R Hook 23358ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 23458ea8abfSGary R Hook 23558ea8abfSGary R Hook if (tx_desc) { 236c07f7c29Samd dmaengine_desc_get_callback_invoke(tx_desc, NULL); 23758ea8abfSGary R Hook 23858ea8abfSGary R Hook dma_run_dependencies(tx_desc); 23958ea8abfSGary R Hook } 24058ea8abfSGary R Hook } while (desc); 24158ea8abfSGary R Hook 24258ea8abfSGary R Hook return NULL; 24358ea8abfSGary R Hook } 24458ea8abfSGary R Hook 24558ea8abfSGary R Hook static struct ccp_dma_desc *__ccp_pending_to_active(struct ccp_dma_chan *chan) 24658ea8abfSGary R Hook { 24758ea8abfSGary R Hook struct ccp_dma_desc *desc; 24858ea8abfSGary R Hook 24958ea8abfSGary R Hook if (list_empty(&chan->pending)) 25058ea8abfSGary R Hook return NULL; 25158ea8abfSGary R Hook 25258ea8abfSGary R Hook desc = list_empty(&chan->active) 25358ea8abfSGary R Hook ? list_first_entry(&chan->pending, struct ccp_dma_desc, entry) 25458ea8abfSGary R Hook : NULL; 25558ea8abfSGary R Hook 25658ea8abfSGary R Hook list_splice_tail_init(&chan->pending, &chan->active); 25758ea8abfSGary R Hook 25858ea8abfSGary R Hook return desc; 25958ea8abfSGary R Hook } 26058ea8abfSGary R Hook 26158ea8abfSGary R Hook static void ccp_cmd_callback(void *data, int err) 26258ea8abfSGary R Hook { 26358ea8abfSGary R Hook struct ccp_dma_desc *desc = data; 26458ea8abfSGary R Hook struct ccp_dma_chan *chan; 26558ea8abfSGary R Hook int ret; 26658ea8abfSGary R Hook 26758ea8abfSGary R Hook if (err == -EINPROGRESS) 26858ea8abfSGary R Hook return; 26958ea8abfSGary R Hook 27058ea8abfSGary R Hook chan = container_of(desc->tx_desc.chan, struct ccp_dma_chan, 27158ea8abfSGary R Hook dma_chan); 27258ea8abfSGary R Hook 27358ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s - tx %d callback, err=%d\n", 27458ea8abfSGary R Hook __func__, desc->tx_desc.cookie, err); 27558ea8abfSGary R Hook 27658ea8abfSGary R Hook if (err) 27758ea8abfSGary R Hook desc->status = DMA_ERROR; 27858ea8abfSGary R Hook 27958ea8abfSGary R Hook while (true) { 28058ea8abfSGary R Hook /* Check for DMA descriptor completion */ 28158ea8abfSGary R Hook desc = ccp_handle_active_desc(chan, desc); 28258ea8abfSGary R Hook 28358ea8abfSGary R Hook /* Don't submit cmd if no descriptor or DMA is paused */ 28458ea8abfSGary R Hook if (!desc || (chan->status == DMA_PAUSED)) 28558ea8abfSGary R Hook break; 28658ea8abfSGary R Hook 28758ea8abfSGary R Hook ret = ccp_issue_next_cmd(desc); 28858ea8abfSGary R Hook if (!ret) 28958ea8abfSGary R Hook break; 29058ea8abfSGary R Hook 29158ea8abfSGary R Hook desc->status = DMA_ERROR; 29258ea8abfSGary R Hook } 29358ea8abfSGary R Hook 29458ea8abfSGary R Hook tasklet_schedule(&chan->cleanup_tasklet); 29558ea8abfSGary R Hook } 29658ea8abfSGary R Hook 29758ea8abfSGary R Hook static dma_cookie_t ccp_tx_submit(struct dma_async_tx_descriptor *tx_desc) 29858ea8abfSGary R Hook { 29958ea8abfSGary R Hook struct ccp_dma_desc *desc = container_of(tx_desc, struct ccp_dma_desc, 30058ea8abfSGary R Hook tx_desc); 30158ea8abfSGary R Hook struct ccp_dma_chan *chan; 30258ea8abfSGary R Hook dma_cookie_t cookie; 30358ea8abfSGary R Hook unsigned long flags; 30458ea8abfSGary R Hook 30558ea8abfSGary R Hook chan = container_of(tx_desc->chan, struct ccp_dma_chan, dma_chan); 30658ea8abfSGary R Hook 30758ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 30858ea8abfSGary R Hook 30958ea8abfSGary R Hook cookie = dma_cookie_assign(tx_desc); 310e5da5c56SGary R Hook list_del(&desc->entry); 31158ea8abfSGary R Hook list_add_tail(&desc->entry, &chan->pending); 31258ea8abfSGary R Hook 31358ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 31458ea8abfSGary R Hook 31558ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s - added tx descriptor %d to pending list\n", 31658ea8abfSGary R Hook __func__, cookie); 31758ea8abfSGary R Hook 31858ea8abfSGary R Hook return cookie; 31958ea8abfSGary R Hook } 32058ea8abfSGary R Hook 32158ea8abfSGary R Hook static struct ccp_dma_cmd *ccp_alloc_dma_cmd(struct ccp_dma_chan *chan) 32258ea8abfSGary R Hook { 32358ea8abfSGary R Hook struct ccp_dma_cmd *cmd; 32458ea8abfSGary R Hook 32558ea8abfSGary R Hook cmd = kmem_cache_alloc(chan->ccp->dma_cmd_cache, GFP_NOWAIT); 32658ea8abfSGary R Hook if (cmd) 32758ea8abfSGary R Hook memset(cmd, 0, sizeof(*cmd)); 32858ea8abfSGary R Hook 32958ea8abfSGary R Hook return cmd; 33058ea8abfSGary R Hook } 33158ea8abfSGary R Hook 33258ea8abfSGary R Hook static struct ccp_dma_desc *ccp_alloc_dma_desc(struct ccp_dma_chan *chan, 33358ea8abfSGary R Hook unsigned long flags) 33458ea8abfSGary R Hook { 33558ea8abfSGary R Hook struct ccp_dma_desc *desc; 33658ea8abfSGary R Hook 337664f570aSWei Yongjun desc = kmem_cache_zalloc(chan->ccp->dma_desc_cache, GFP_NOWAIT); 33858ea8abfSGary R Hook if (!desc) 33958ea8abfSGary R Hook return NULL; 34058ea8abfSGary R Hook 34158ea8abfSGary R Hook dma_async_tx_descriptor_init(&desc->tx_desc, &chan->dma_chan); 34258ea8abfSGary R Hook desc->tx_desc.flags = flags; 34358ea8abfSGary R Hook desc->tx_desc.tx_submit = ccp_tx_submit; 34458ea8abfSGary R Hook desc->ccp = chan->ccp; 345691505a8SMark Salter INIT_LIST_HEAD(&desc->entry); 34658ea8abfSGary R Hook INIT_LIST_HEAD(&desc->pending); 34758ea8abfSGary R Hook INIT_LIST_HEAD(&desc->active); 34858ea8abfSGary R Hook desc->status = DMA_IN_PROGRESS; 34958ea8abfSGary R Hook 35058ea8abfSGary R Hook return desc; 35158ea8abfSGary R Hook } 35258ea8abfSGary R Hook 35358ea8abfSGary R Hook static struct ccp_dma_desc *ccp_create_desc(struct dma_chan *dma_chan, 35458ea8abfSGary R Hook struct scatterlist *dst_sg, 35558ea8abfSGary R Hook unsigned int dst_nents, 35658ea8abfSGary R Hook struct scatterlist *src_sg, 35758ea8abfSGary R Hook unsigned int src_nents, 35858ea8abfSGary R Hook unsigned long flags) 35958ea8abfSGary R Hook { 36058ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 36158ea8abfSGary R Hook dma_chan); 36258ea8abfSGary R Hook struct ccp_device *ccp = chan->ccp; 36358ea8abfSGary R Hook struct ccp_dma_desc *desc; 36458ea8abfSGary R Hook struct ccp_dma_cmd *cmd; 36558ea8abfSGary R Hook struct ccp_cmd *ccp_cmd; 36658ea8abfSGary R Hook struct ccp_passthru_nomap_engine *ccp_pt; 36758ea8abfSGary R Hook unsigned int src_offset, src_len; 36858ea8abfSGary R Hook unsigned int dst_offset, dst_len; 36958ea8abfSGary R Hook unsigned int len; 37058ea8abfSGary R Hook unsigned long sflags; 37158ea8abfSGary R Hook size_t total_len; 37258ea8abfSGary R Hook 37358ea8abfSGary R Hook if (!dst_sg || !src_sg) 37458ea8abfSGary R Hook return NULL; 37558ea8abfSGary R Hook 37658ea8abfSGary R Hook if (!dst_nents || !src_nents) 37758ea8abfSGary R Hook return NULL; 37858ea8abfSGary R Hook 37958ea8abfSGary R Hook desc = ccp_alloc_dma_desc(chan, flags); 38058ea8abfSGary R Hook if (!desc) 38158ea8abfSGary R Hook return NULL; 38258ea8abfSGary R Hook 38358ea8abfSGary R Hook total_len = 0; 38458ea8abfSGary R Hook 38558ea8abfSGary R Hook src_len = sg_dma_len(src_sg); 38658ea8abfSGary R Hook src_offset = 0; 38758ea8abfSGary R Hook 38858ea8abfSGary R Hook dst_len = sg_dma_len(dst_sg); 38958ea8abfSGary R Hook dst_offset = 0; 39058ea8abfSGary R Hook 39158ea8abfSGary R Hook while (true) { 39258ea8abfSGary R Hook if (!src_len) { 39358ea8abfSGary R Hook src_nents--; 39458ea8abfSGary R Hook if (!src_nents) 39558ea8abfSGary R Hook break; 39658ea8abfSGary R Hook 39758ea8abfSGary R Hook src_sg = sg_next(src_sg); 39858ea8abfSGary R Hook if (!src_sg) 39958ea8abfSGary R Hook break; 40058ea8abfSGary R Hook 40158ea8abfSGary R Hook src_len = sg_dma_len(src_sg); 40258ea8abfSGary R Hook src_offset = 0; 40358ea8abfSGary R Hook continue; 40458ea8abfSGary R Hook } 40558ea8abfSGary R Hook 40658ea8abfSGary R Hook if (!dst_len) { 40758ea8abfSGary R Hook dst_nents--; 40858ea8abfSGary R Hook if (!dst_nents) 40958ea8abfSGary R Hook break; 41058ea8abfSGary R Hook 41158ea8abfSGary R Hook dst_sg = sg_next(dst_sg); 41258ea8abfSGary R Hook if (!dst_sg) 41358ea8abfSGary R Hook break; 41458ea8abfSGary R Hook 41558ea8abfSGary R Hook dst_len = sg_dma_len(dst_sg); 41658ea8abfSGary R Hook dst_offset = 0; 41758ea8abfSGary R Hook continue; 41858ea8abfSGary R Hook } 41958ea8abfSGary R Hook 42058ea8abfSGary R Hook len = min(dst_len, src_len); 42158ea8abfSGary R Hook 42258ea8abfSGary R Hook cmd = ccp_alloc_dma_cmd(chan); 42358ea8abfSGary R Hook if (!cmd) 42458ea8abfSGary R Hook goto err; 42558ea8abfSGary R Hook 42658ea8abfSGary R Hook ccp_cmd = &cmd->ccp_cmd; 4277c468447SGary R Hook ccp_cmd->ccp = chan->ccp; 42858ea8abfSGary R Hook ccp_pt = &ccp_cmd->u.passthru_nomap; 42958ea8abfSGary R Hook ccp_cmd->flags = CCP_CMD_MAY_BACKLOG; 43058ea8abfSGary R Hook ccp_cmd->flags |= CCP_CMD_PASSTHRU_NO_DMA_MAP; 43158ea8abfSGary R Hook ccp_cmd->engine = CCP_ENGINE_PASSTHRU; 43258ea8abfSGary R Hook ccp_pt->bit_mod = CCP_PASSTHRU_BITWISE_NOOP; 43358ea8abfSGary R Hook ccp_pt->byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP; 43458ea8abfSGary R Hook ccp_pt->src_dma = sg_dma_address(src_sg) + src_offset; 43558ea8abfSGary R Hook ccp_pt->dst_dma = sg_dma_address(dst_sg) + dst_offset; 43658ea8abfSGary R Hook ccp_pt->src_len = len; 43758ea8abfSGary R Hook ccp_pt->final = 1; 43858ea8abfSGary R Hook ccp_cmd->callback = ccp_cmd_callback; 43958ea8abfSGary R Hook ccp_cmd->data = desc; 44058ea8abfSGary R Hook 44158ea8abfSGary R Hook list_add_tail(&cmd->entry, &desc->pending); 44258ea8abfSGary R Hook 44358ea8abfSGary R Hook dev_dbg(ccp->dev, 44458ea8abfSGary R Hook "%s - cmd=%p, src=%pad, dst=%pad, len=%llu\n", __func__, 44558ea8abfSGary R Hook cmd, &ccp_pt->src_dma, 44658ea8abfSGary R Hook &ccp_pt->dst_dma, ccp_pt->src_len); 44758ea8abfSGary R Hook 44858ea8abfSGary R Hook total_len += len; 44958ea8abfSGary R Hook 45058ea8abfSGary R Hook src_len -= len; 45158ea8abfSGary R Hook src_offset += len; 45258ea8abfSGary R Hook 45358ea8abfSGary R Hook dst_len -= len; 45458ea8abfSGary R Hook dst_offset += len; 45558ea8abfSGary R Hook } 45658ea8abfSGary R Hook 45758ea8abfSGary R Hook desc->len = total_len; 45858ea8abfSGary R Hook 45958ea8abfSGary R Hook if (list_empty(&desc->pending)) 46058ea8abfSGary R Hook goto err; 46158ea8abfSGary R Hook 46258ea8abfSGary R Hook dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc); 46358ea8abfSGary R Hook 46458ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, sflags); 46558ea8abfSGary R Hook 466e5da5c56SGary R Hook list_add_tail(&desc->entry, &chan->created); 46758ea8abfSGary R Hook 46858ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, sflags); 46958ea8abfSGary R Hook 47058ea8abfSGary R Hook return desc; 47158ea8abfSGary R Hook 47258ea8abfSGary R Hook err: 47358ea8abfSGary R Hook ccp_free_cmd_resources(ccp, &desc->pending); 47458ea8abfSGary R Hook kmem_cache_free(ccp->dma_desc_cache, desc); 47558ea8abfSGary R Hook 47658ea8abfSGary R Hook return NULL; 47758ea8abfSGary R Hook } 47858ea8abfSGary R Hook 47958ea8abfSGary R Hook static struct dma_async_tx_descriptor *ccp_prep_dma_memcpy( 48058ea8abfSGary R Hook struct dma_chan *dma_chan, dma_addr_t dst, dma_addr_t src, size_t len, 48158ea8abfSGary R Hook unsigned long flags) 48258ea8abfSGary R Hook { 48358ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 48458ea8abfSGary R Hook dma_chan); 48558ea8abfSGary R Hook struct ccp_dma_desc *desc; 48658ea8abfSGary R Hook struct scatterlist dst_sg, src_sg; 48758ea8abfSGary R Hook 48858ea8abfSGary R Hook dev_dbg(chan->ccp->dev, 48958ea8abfSGary R Hook "%s - src=%pad, dst=%pad, len=%zu, flags=%#lx\n", 49058ea8abfSGary R Hook __func__, &src, &dst, len, flags); 49158ea8abfSGary R Hook 49258ea8abfSGary R Hook sg_init_table(&dst_sg, 1); 49358ea8abfSGary R Hook sg_dma_address(&dst_sg) = dst; 49458ea8abfSGary R Hook sg_dma_len(&dst_sg) = len; 49558ea8abfSGary R Hook 49658ea8abfSGary R Hook sg_init_table(&src_sg, 1); 49758ea8abfSGary R Hook sg_dma_address(&src_sg) = src; 49858ea8abfSGary R Hook sg_dma_len(&src_sg) = len; 49958ea8abfSGary R Hook 50058ea8abfSGary R Hook desc = ccp_create_desc(dma_chan, &dst_sg, 1, &src_sg, 1, flags); 50158ea8abfSGary R Hook if (!desc) 50258ea8abfSGary R Hook return NULL; 50358ea8abfSGary R Hook 50458ea8abfSGary R Hook return &desc->tx_desc; 50558ea8abfSGary R Hook } 50658ea8abfSGary R Hook 50758ea8abfSGary R Hook static struct dma_async_tx_descriptor *ccp_prep_dma_interrupt( 50858ea8abfSGary R Hook struct dma_chan *dma_chan, unsigned long flags) 50958ea8abfSGary R Hook { 51058ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 51158ea8abfSGary R Hook dma_chan); 51258ea8abfSGary R Hook struct ccp_dma_desc *desc; 51358ea8abfSGary R Hook 51458ea8abfSGary R Hook desc = ccp_alloc_dma_desc(chan, flags); 51558ea8abfSGary R Hook if (!desc) 51658ea8abfSGary R Hook return NULL; 51758ea8abfSGary R Hook 51858ea8abfSGary R Hook return &desc->tx_desc; 51958ea8abfSGary R Hook } 52058ea8abfSGary R Hook 52158ea8abfSGary R Hook static void ccp_issue_pending(struct dma_chan *dma_chan) 52258ea8abfSGary R Hook { 52358ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 52458ea8abfSGary R Hook dma_chan); 52558ea8abfSGary R Hook struct ccp_dma_desc *desc; 52658ea8abfSGary R Hook unsigned long flags; 52758ea8abfSGary R Hook 52858ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s\n", __func__); 52958ea8abfSGary R Hook 53058ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 53158ea8abfSGary R Hook 53258ea8abfSGary R Hook desc = __ccp_pending_to_active(chan); 53358ea8abfSGary R Hook 53458ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 53558ea8abfSGary R Hook 53658ea8abfSGary R Hook /* If there was nothing active, start processing */ 53758ea8abfSGary R Hook if (desc) 53858ea8abfSGary R Hook ccp_cmd_callback(desc, 0); 53958ea8abfSGary R Hook } 54058ea8abfSGary R Hook 54158ea8abfSGary R Hook static enum dma_status ccp_tx_status(struct dma_chan *dma_chan, 54258ea8abfSGary R Hook dma_cookie_t cookie, 54358ea8abfSGary R Hook struct dma_tx_state *state) 54458ea8abfSGary R Hook { 54558ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 54658ea8abfSGary R Hook dma_chan); 54758ea8abfSGary R Hook struct ccp_dma_desc *desc; 54858ea8abfSGary R Hook enum dma_status ret; 54958ea8abfSGary R Hook unsigned long flags; 55058ea8abfSGary R Hook 55158ea8abfSGary R Hook if (chan->status == DMA_PAUSED) { 55258ea8abfSGary R Hook ret = DMA_PAUSED; 55358ea8abfSGary R Hook goto out; 55458ea8abfSGary R Hook } 55558ea8abfSGary R Hook 55658ea8abfSGary R Hook ret = dma_cookie_status(dma_chan, cookie, state); 55758ea8abfSGary R Hook if (ret == DMA_COMPLETE) { 55858ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 55958ea8abfSGary R Hook 56058ea8abfSGary R Hook /* Get status from complete chain, if still there */ 56158ea8abfSGary R Hook list_for_each_entry(desc, &chan->complete, entry) { 56258ea8abfSGary R Hook if (desc->tx_desc.cookie != cookie) 56358ea8abfSGary R Hook continue; 56458ea8abfSGary R Hook 56558ea8abfSGary R Hook ret = desc->status; 56658ea8abfSGary R Hook break; 56758ea8abfSGary R Hook } 56858ea8abfSGary R Hook 56958ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 57058ea8abfSGary R Hook } 57158ea8abfSGary R Hook 57258ea8abfSGary R Hook out: 57358ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s - %u\n", __func__, ret); 57458ea8abfSGary R Hook 57558ea8abfSGary R Hook return ret; 57658ea8abfSGary R Hook } 57758ea8abfSGary R Hook 57858ea8abfSGary R Hook static int ccp_pause(struct dma_chan *dma_chan) 57958ea8abfSGary R Hook { 58058ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 58158ea8abfSGary R Hook dma_chan); 58258ea8abfSGary R Hook 58358ea8abfSGary R Hook chan->status = DMA_PAUSED; 58458ea8abfSGary R Hook 58558ea8abfSGary R Hook /*TODO: Wait for active DMA to complete before returning? */ 58658ea8abfSGary R Hook 58758ea8abfSGary R Hook return 0; 58858ea8abfSGary R Hook } 58958ea8abfSGary R Hook 59058ea8abfSGary R Hook static int ccp_resume(struct dma_chan *dma_chan) 59158ea8abfSGary R Hook { 59258ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 59358ea8abfSGary R Hook dma_chan); 59458ea8abfSGary R Hook struct ccp_dma_desc *desc; 59558ea8abfSGary R Hook unsigned long flags; 59658ea8abfSGary R Hook 59758ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 59858ea8abfSGary R Hook 59958ea8abfSGary R Hook desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc, 60058ea8abfSGary R Hook entry); 60158ea8abfSGary R Hook 60258ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 60358ea8abfSGary R Hook 60458ea8abfSGary R Hook /* Indicate the channel is running again */ 60558ea8abfSGary R Hook chan->status = DMA_IN_PROGRESS; 60658ea8abfSGary R Hook 60758ea8abfSGary R Hook /* If there was something active, re-start */ 60858ea8abfSGary R Hook if (desc) 60958ea8abfSGary R Hook ccp_cmd_callback(desc, 0); 61058ea8abfSGary R Hook 61158ea8abfSGary R Hook return 0; 61258ea8abfSGary R Hook } 61358ea8abfSGary R Hook 61458ea8abfSGary R Hook static int ccp_terminate_all(struct dma_chan *dma_chan) 61558ea8abfSGary R Hook { 61658ea8abfSGary R Hook struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 61758ea8abfSGary R Hook dma_chan); 61858ea8abfSGary R Hook unsigned long flags; 61958ea8abfSGary R Hook 62058ea8abfSGary R Hook dev_dbg(chan->ccp->dev, "%s\n", __func__); 62158ea8abfSGary R Hook 62258ea8abfSGary R Hook /*TODO: Wait for active DMA to complete before continuing */ 62358ea8abfSGary R Hook 62458ea8abfSGary R Hook spin_lock_irqsave(&chan->lock, flags); 62558ea8abfSGary R Hook 62658ea8abfSGary R Hook /*TODO: Purge the complete list? */ 62758ea8abfSGary R Hook ccp_free_desc_resources(chan->ccp, &chan->active); 62858ea8abfSGary R Hook ccp_free_desc_resources(chan->ccp, &chan->pending); 629e5da5c56SGary R Hook ccp_free_desc_resources(chan->ccp, &chan->created); 63058ea8abfSGary R Hook 63158ea8abfSGary R Hook spin_unlock_irqrestore(&chan->lock, flags); 63258ea8abfSGary R Hook 63358ea8abfSGary R Hook return 0; 63458ea8abfSGary R Hook } 63558ea8abfSGary R Hook 63658ea8abfSGary R Hook int ccp_dmaengine_register(struct ccp_device *ccp) 63758ea8abfSGary R Hook { 63858ea8abfSGary R Hook struct ccp_dma_chan *chan; 63958ea8abfSGary R Hook struct dma_device *dma_dev = &ccp->dma_dev; 64058ea8abfSGary R Hook struct dma_chan *dma_chan; 64158ea8abfSGary R Hook char *dma_cmd_cache_name; 64258ea8abfSGary R Hook char *dma_desc_cache_name; 64358ea8abfSGary R Hook unsigned int i; 64458ea8abfSGary R Hook int ret; 64558ea8abfSGary R Hook 646a7c26470SHook, Gary if (!dmaengine) 647a7c26470SHook, Gary return 0; 648a7c26470SHook, Gary 64958ea8abfSGary R Hook ccp->ccp_dma_chan = devm_kcalloc(ccp->dev, ccp->cmd_q_count, 65058ea8abfSGary R Hook sizeof(*(ccp->ccp_dma_chan)), 65158ea8abfSGary R Hook GFP_KERNEL); 65258ea8abfSGary R Hook if (!ccp->ccp_dma_chan) 65358ea8abfSGary R Hook return -ENOMEM; 65458ea8abfSGary R Hook 65558ea8abfSGary R Hook dma_cmd_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL, 65658ea8abfSGary R Hook "%s-dmaengine-cmd-cache", 65758ea8abfSGary R Hook ccp->name); 65858ea8abfSGary R Hook if (!dma_cmd_cache_name) 65958ea8abfSGary R Hook return -ENOMEM; 66058ea8abfSGary R Hook 66158ea8abfSGary R Hook ccp->dma_cmd_cache = kmem_cache_create(dma_cmd_cache_name, 66258ea8abfSGary R Hook sizeof(struct ccp_dma_cmd), 66358ea8abfSGary R Hook sizeof(void *), 66458ea8abfSGary R Hook SLAB_HWCACHE_ALIGN, NULL); 66558ea8abfSGary R Hook if (!ccp->dma_cmd_cache) 66658ea8abfSGary R Hook return -ENOMEM; 66758ea8abfSGary R Hook 66858ea8abfSGary R Hook dma_desc_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL, 66958ea8abfSGary R Hook "%s-dmaengine-desc-cache", 67058ea8abfSGary R Hook ccp->name); 6717514e368SWei Yongjun if (!dma_desc_cache_name) { 672ba22a1e2SQuentin Lambert ret = -ENOMEM; 673ba22a1e2SQuentin Lambert goto err_cache; 674ba22a1e2SQuentin Lambert } 675ba22a1e2SQuentin Lambert 67658ea8abfSGary R Hook ccp->dma_desc_cache = kmem_cache_create(dma_desc_cache_name, 67758ea8abfSGary R Hook sizeof(struct ccp_dma_desc), 67858ea8abfSGary R Hook sizeof(void *), 67958ea8abfSGary R Hook SLAB_HWCACHE_ALIGN, NULL); 68058ea8abfSGary R Hook if (!ccp->dma_desc_cache) { 68158ea8abfSGary R Hook ret = -ENOMEM; 68258ea8abfSGary R Hook goto err_cache; 68358ea8abfSGary R Hook } 68458ea8abfSGary R Hook 68558ea8abfSGary R Hook dma_dev->dev = ccp->dev; 68658ea8abfSGary R Hook dma_dev->src_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev)); 68758ea8abfSGary R Hook dma_dev->dst_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev)); 68858ea8abfSGary R Hook dma_dev->directions = DMA_MEM_TO_MEM; 68958ea8abfSGary R Hook dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 69058ea8abfSGary R Hook dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); 69158ea8abfSGary R Hook dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); 69258ea8abfSGary R Hook 693efc989fcSGary R Hook /* The DMA channels for this device can be set to public or private, 694efc989fcSGary R Hook * and overridden by the module parameter dma_chan_attr. 695efc989fcSGary R Hook * Default: according to the value in vdata (dma_chan_attr=0) 696efc989fcSGary R Hook * dma_chan_attr=0x1: all channels private (override vdata) 697efc989fcSGary R Hook * dma_chan_attr=0x2: all channels public (override vdata) 698efc989fcSGary R Hook */ 699efc989fcSGary R Hook if (ccp_get_dma_chan_attr(ccp) == DMA_PRIVATE) 700efc989fcSGary R Hook dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); 701efc989fcSGary R Hook 70258ea8abfSGary R Hook INIT_LIST_HEAD(&dma_dev->channels); 70358ea8abfSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) { 70458ea8abfSGary R Hook chan = ccp->ccp_dma_chan + i; 70558ea8abfSGary R Hook dma_chan = &chan->dma_chan; 70658ea8abfSGary R Hook 70758ea8abfSGary R Hook chan->ccp = ccp; 70858ea8abfSGary R Hook 70958ea8abfSGary R Hook spin_lock_init(&chan->lock); 710e5da5c56SGary R Hook INIT_LIST_HEAD(&chan->created); 71158ea8abfSGary R Hook INIT_LIST_HEAD(&chan->pending); 71258ea8abfSGary R Hook INIT_LIST_HEAD(&chan->active); 71358ea8abfSGary R Hook INIT_LIST_HEAD(&chan->complete); 71458ea8abfSGary R Hook 71558ea8abfSGary R Hook tasklet_init(&chan->cleanup_tasklet, ccp_do_cleanup, 71658ea8abfSGary R Hook (unsigned long)chan); 71758ea8abfSGary R Hook 71858ea8abfSGary R Hook dma_chan->device = dma_dev; 71958ea8abfSGary R Hook dma_cookie_init(dma_chan); 72058ea8abfSGary R Hook 72158ea8abfSGary R Hook list_add_tail(&dma_chan->device_node, &dma_dev->channels); 72258ea8abfSGary R Hook } 72358ea8abfSGary R Hook 72458ea8abfSGary R Hook dma_dev->device_free_chan_resources = ccp_free_chan_resources; 72558ea8abfSGary R Hook dma_dev->device_prep_dma_memcpy = ccp_prep_dma_memcpy; 72658ea8abfSGary R Hook dma_dev->device_prep_dma_interrupt = ccp_prep_dma_interrupt; 72758ea8abfSGary R Hook dma_dev->device_issue_pending = ccp_issue_pending; 72858ea8abfSGary R Hook dma_dev->device_tx_status = ccp_tx_status; 72958ea8abfSGary R Hook dma_dev->device_pause = ccp_pause; 73058ea8abfSGary R Hook dma_dev->device_resume = ccp_resume; 73158ea8abfSGary R Hook dma_dev->device_terminate_all = ccp_terminate_all; 73258ea8abfSGary R Hook 73358ea8abfSGary R Hook ret = dma_async_device_register(dma_dev); 73458ea8abfSGary R Hook if (ret) 73558ea8abfSGary R Hook goto err_reg; 73658ea8abfSGary R Hook 73758ea8abfSGary R Hook return 0; 73858ea8abfSGary R Hook 73958ea8abfSGary R Hook err_reg: 74058ea8abfSGary R Hook kmem_cache_destroy(ccp->dma_desc_cache); 74158ea8abfSGary R Hook 74258ea8abfSGary R Hook err_cache: 74358ea8abfSGary R Hook kmem_cache_destroy(ccp->dma_cmd_cache); 74458ea8abfSGary R Hook 74558ea8abfSGary R Hook return ret; 74658ea8abfSGary R Hook } 74758ea8abfSGary R Hook 74858ea8abfSGary R Hook void ccp_dmaengine_unregister(struct ccp_device *ccp) 74958ea8abfSGary R Hook { 75058ea8abfSGary R Hook struct dma_device *dma_dev = &ccp->dma_dev; 75158ea8abfSGary R Hook 752a7c26470SHook, Gary if (!dmaengine) 753a7c26470SHook, Gary return; 754a7c26470SHook, Gary 75558ea8abfSGary R Hook dma_async_device_unregister(dma_dev); 75658ea8abfSGary R Hook 75758ea8abfSGary R Hook kmem_cache_destroy(ccp->dma_desc_cache); 75858ea8abfSGary R Hook kmem_cache_destroy(ccp->dma_cmd_cache); 75958ea8abfSGary R Hook } 760