1b1f01e48SShun-Chih Yu // SPDX-License-Identifier: GPL-2.0 2b1f01e48SShun-Chih Yu // Copyright (c) 2018-2019 MediaTek Inc. 3b1f01e48SShun-Chih Yu 4b1f01e48SShun-Chih Yu /* 5b1f01e48SShun-Chih Yu * Driver for MediaTek Command-Queue DMA Controller 6b1f01e48SShun-Chih Yu * 7b1f01e48SShun-Chih Yu * Author: Shun-Chih Yu <shun-chih.yu@mediatek.com> 8b1f01e48SShun-Chih Yu * 9b1f01e48SShun-Chih Yu */ 10b1f01e48SShun-Chih Yu 11b1f01e48SShun-Chih Yu #include <linux/bitops.h> 12b1f01e48SShun-Chih Yu #include <linux/clk.h> 13b1f01e48SShun-Chih Yu #include <linux/dmaengine.h> 14b1f01e48SShun-Chih Yu #include <linux/dma-mapping.h> 15b1f01e48SShun-Chih Yu #include <linux/err.h> 16b1f01e48SShun-Chih Yu #include <linux/iopoll.h> 17b1f01e48SShun-Chih Yu #include <linux/interrupt.h> 18b1f01e48SShun-Chih Yu #include <linux/list.h> 19b1f01e48SShun-Chih Yu #include <linux/module.h> 20b1f01e48SShun-Chih Yu #include <linux/of.h> 21b1f01e48SShun-Chih Yu #include <linux/of_device.h> 22b1f01e48SShun-Chih Yu #include <linux/of_dma.h> 23b1f01e48SShun-Chih Yu #include <linux/platform_device.h> 24b1f01e48SShun-Chih Yu #include <linux/pm_runtime.h> 25b1f01e48SShun-Chih Yu #include <linux/refcount.h> 26b1f01e48SShun-Chih Yu #include <linux/slab.h> 27b1f01e48SShun-Chih Yu 28b1f01e48SShun-Chih Yu #include "../virt-dma.h" 29b1f01e48SShun-Chih Yu 30b1f01e48SShun-Chih Yu #define MTK_CQDMA_USEC_POLL 10 31b1f01e48SShun-Chih Yu #define MTK_CQDMA_TIMEOUT_POLL 1000 32b1f01e48SShun-Chih Yu #define MTK_CQDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) 33b1f01e48SShun-Chih Yu #define MTK_CQDMA_ALIGN_SIZE 1 34b1f01e48SShun-Chih Yu 35b1f01e48SShun-Chih Yu /* The default number of virtual channel */ 36b1f01e48SShun-Chih Yu #define MTK_CQDMA_NR_VCHANS 32 37b1f01e48SShun-Chih Yu 38b1f01e48SShun-Chih Yu /* The default number of physical channel */ 39b1f01e48SShun-Chih Yu #define MTK_CQDMA_NR_PCHANS 3 40b1f01e48SShun-Chih Yu 41b1f01e48SShun-Chih Yu /* Registers for underlying dma manipulation */ 42b1f01e48SShun-Chih Yu #define MTK_CQDMA_INT_FLAG 0x0 43b1f01e48SShun-Chih Yu #define MTK_CQDMA_INT_EN 0x4 44b1f01e48SShun-Chih Yu #define MTK_CQDMA_EN 0x8 45b1f01e48SShun-Chih Yu #define MTK_CQDMA_RESET 0xc 46b1f01e48SShun-Chih Yu #define MTK_CQDMA_FLUSH 0x14 47b1f01e48SShun-Chih Yu #define MTK_CQDMA_SRC 0x1c 48b1f01e48SShun-Chih Yu #define MTK_CQDMA_DST 0x20 49b1f01e48SShun-Chih Yu #define MTK_CQDMA_LEN1 0x24 50b1f01e48SShun-Chih Yu #define MTK_CQDMA_LEN2 0x28 51b1f01e48SShun-Chih Yu #define MTK_CQDMA_SRC2 0x60 52b1f01e48SShun-Chih Yu #define MTK_CQDMA_DST2 0x64 53b1f01e48SShun-Chih Yu 54b1f01e48SShun-Chih Yu /* Registers setting */ 55b1f01e48SShun-Chih Yu #define MTK_CQDMA_EN_BIT BIT(0) 56b1f01e48SShun-Chih Yu #define MTK_CQDMA_INT_FLAG_BIT BIT(0) 57b1f01e48SShun-Chih Yu #define MTK_CQDMA_INT_EN_BIT BIT(0) 58b1f01e48SShun-Chih Yu #define MTK_CQDMA_FLUSH_BIT BIT(0) 59b1f01e48SShun-Chih Yu 60b1f01e48SShun-Chih Yu #define MTK_CQDMA_WARM_RST_BIT BIT(0) 61b1f01e48SShun-Chih Yu #define MTK_CQDMA_HARD_RST_BIT BIT(1) 62b1f01e48SShun-Chih Yu 63b1f01e48SShun-Chih Yu #define MTK_CQDMA_MAX_LEN GENMASK(27, 0) 64b1f01e48SShun-Chih Yu #define MTK_CQDMA_ADDR_LIMIT GENMASK(31, 0) 65b1f01e48SShun-Chih Yu #define MTK_CQDMA_ADDR2_SHFIT (32) 66b1f01e48SShun-Chih Yu 67b1f01e48SShun-Chih Yu /** 68b1f01e48SShun-Chih Yu * struct mtk_cqdma_vdesc - The struct holding info describing virtual 69b1f01e48SShun-Chih Yu * descriptor (CVD) 70b1f01e48SShun-Chih Yu * @vd: An instance for struct virt_dma_desc 71b1f01e48SShun-Chih Yu * @len: The total data size device wants to move 72b1f01e48SShun-Chih Yu * @residue: The remaining data size device will move 73b1f01e48SShun-Chih Yu * @dest: The destination address device wants to move to 74b1f01e48SShun-Chih Yu * @src: The source address device wants to move from 75b1f01e48SShun-Chih Yu * @ch: The pointer to the corresponding dma channel 76b1f01e48SShun-Chih Yu * @node: The lise_head struct to build link-list for VDs 77b1f01e48SShun-Chih Yu * @parent: The pointer to the parent CVD 78b1f01e48SShun-Chih Yu */ 79b1f01e48SShun-Chih Yu struct mtk_cqdma_vdesc { 80b1f01e48SShun-Chih Yu struct virt_dma_desc vd; 81b1f01e48SShun-Chih Yu size_t len; 82b1f01e48SShun-Chih Yu size_t residue; 83b1f01e48SShun-Chih Yu dma_addr_t dest; 84b1f01e48SShun-Chih Yu dma_addr_t src; 85b1f01e48SShun-Chih Yu struct dma_chan *ch; 86b1f01e48SShun-Chih Yu 87b1f01e48SShun-Chih Yu struct list_head node; 88b1f01e48SShun-Chih Yu struct mtk_cqdma_vdesc *parent; 89b1f01e48SShun-Chih Yu }; 90b1f01e48SShun-Chih Yu 91b1f01e48SShun-Chih Yu /** 92b1f01e48SShun-Chih Yu * struct mtk_cqdma_pchan - The struct holding info describing physical 93b1f01e48SShun-Chih Yu * channel (PC) 94b1f01e48SShun-Chih Yu * @queue: Queue for the PDs issued to this PC 95b1f01e48SShun-Chih Yu * @base: The mapped register I/O base of this PC 96b1f01e48SShun-Chih Yu * @irq: The IRQ that this PC are using 97b1f01e48SShun-Chih Yu * @refcnt: Track how many VCs are using this PC 98b1f01e48SShun-Chih Yu * @tasklet: Tasklet for this PC 99b1f01e48SShun-Chih Yu * @lock: Lock protect agaisting multiple VCs access PC 100b1f01e48SShun-Chih Yu */ 101b1f01e48SShun-Chih Yu struct mtk_cqdma_pchan { 102b1f01e48SShun-Chih Yu struct list_head queue; 103b1f01e48SShun-Chih Yu void __iomem *base; 104b1f01e48SShun-Chih Yu u32 irq; 105b1f01e48SShun-Chih Yu 106b1f01e48SShun-Chih Yu refcount_t refcnt; 107b1f01e48SShun-Chih Yu 108b1f01e48SShun-Chih Yu struct tasklet_struct tasklet; 109b1f01e48SShun-Chih Yu 110b1f01e48SShun-Chih Yu /* lock to protect PC */ 111b1f01e48SShun-Chih Yu spinlock_t lock; 112b1f01e48SShun-Chih Yu }; 113b1f01e48SShun-Chih Yu 114b1f01e48SShun-Chih Yu /** 115b1f01e48SShun-Chih Yu * struct mtk_cqdma_vchan - The struct holding info describing virtual 116b1f01e48SShun-Chih Yu * channel (VC) 117b1f01e48SShun-Chih Yu * @vc: An instance for struct virt_dma_chan 118b1f01e48SShun-Chih Yu * @pc: The pointer to the underlying PC 119b1f01e48SShun-Chih Yu * @issue_completion: The wait for all issued descriptors completited 120b1f01e48SShun-Chih Yu * @issue_synchronize: Bool indicating channel synchronization starts 121b1f01e48SShun-Chih Yu */ 122b1f01e48SShun-Chih Yu struct mtk_cqdma_vchan { 123b1f01e48SShun-Chih Yu struct virt_dma_chan vc; 124b1f01e48SShun-Chih Yu struct mtk_cqdma_pchan *pc; 125b1f01e48SShun-Chih Yu struct completion issue_completion; 126b1f01e48SShun-Chih Yu bool issue_synchronize; 127b1f01e48SShun-Chih Yu }; 128b1f01e48SShun-Chih Yu 129b1f01e48SShun-Chih Yu /** 130b1f01e48SShun-Chih Yu * struct mtk_cqdma_device - The struct holding info describing CQDMA 131b1f01e48SShun-Chih Yu * device 132b1f01e48SShun-Chih Yu * @ddev: An instance for struct dma_device 133b1f01e48SShun-Chih Yu * @clk: The clock that device internal is using 134b1f01e48SShun-Chih Yu * @dma_requests: The number of VCs the device supports to 135b1f01e48SShun-Chih Yu * @dma_channels: The number of PCs the device supports to 136b1f01e48SShun-Chih Yu * @vc: The pointer to all available VCs 137b1f01e48SShun-Chih Yu * @pc: The pointer to all the underlying PCs 138b1f01e48SShun-Chih Yu */ 139b1f01e48SShun-Chih Yu struct mtk_cqdma_device { 140b1f01e48SShun-Chih Yu struct dma_device ddev; 141b1f01e48SShun-Chih Yu struct clk *clk; 142b1f01e48SShun-Chih Yu 143b1f01e48SShun-Chih Yu u32 dma_requests; 144b1f01e48SShun-Chih Yu u32 dma_channels; 145b1f01e48SShun-Chih Yu struct mtk_cqdma_vchan *vc; 146b1f01e48SShun-Chih Yu struct mtk_cqdma_pchan **pc; 147b1f01e48SShun-Chih Yu }; 148b1f01e48SShun-Chih Yu 149b1f01e48SShun-Chih Yu static struct mtk_cqdma_device *to_cqdma_dev(struct dma_chan *chan) 150b1f01e48SShun-Chih Yu { 151b1f01e48SShun-Chih Yu return container_of(chan->device, struct mtk_cqdma_device, ddev); 152b1f01e48SShun-Chih Yu } 153b1f01e48SShun-Chih Yu 154b1f01e48SShun-Chih Yu static struct mtk_cqdma_vchan *to_cqdma_vchan(struct dma_chan *chan) 155b1f01e48SShun-Chih Yu { 156b1f01e48SShun-Chih Yu return container_of(chan, struct mtk_cqdma_vchan, vc.chan); 157b1f01e48SShun-Chih Yu } 158b1f01e48SShun-Chih Yu 159b1f01e48SShun-Chih Yu static struct mtk_cqdma_vdesc *to_cqdma_vdesc(struct virt_dma_desc *vd) 160b1f01e48SShun-Chih Yu { 161b1f01e48SShun-Chih Yu return container_of(vd, struct mtk_cqdma_vdesc, vd); 162b1f01e48SShun-Chih Yu } 163b1f01e48SShun-Chih Yu 164b1f01e48SShun-Chih Yu static struct device *cqdma2dev(struct mtk_cqdma_device *cqdma) 165b1f01e48SShun-Chih Yu { 166b1f01e48SShun-Chih Yu return cqdma->ddev.dev; 167b1f01e48SShun-Chih Yu } 168b1f01e48SShun-Chih Yu 169b1f01e48SShun-Chih Yu static u32 mtk_dma_read(struct mtk_cqdma_pchan *pc, u32 reg) 170b1f01e48SShun-Chih Yu { 171b1f01e48SShun-Chih Yu return readl(pc->base + reg); 172b1f01e48SShun-Chih Yu } 173b1f01e48SShun-Chih Yu 174b1f01e48SShun-Chih Yu static void mtk_dma_write(struct mtk_cqdma_pchan *pc, u32 reg, u32 val) 175b1f01e48SShun-Chih Yu { 176b1f01e48SShun-Chih Yu writel_relaxed(val, pc->base + reg); 177b1f01e48SShun-Chih Yu } 178b1f01e48SShun-Chih Yu 179b1f01e48SShun-Chih Yu static void mtk_dma_rmw(struct mtk_cqdma_pchan *pc, u32 reg, 180b1f01e48SShun-Chih Yu u32 mask, u32 set) 181b1f01e48SShun-Chih Yu { 182b1f01e48SShun-Chih Yu u32 val; 183b1f01e48SShun-Chih Yu 184b1f01e48SShun-Chih Yu val = mtk_dma_read(pc, reg); 185b1f01e48SShun-Chih Yu val &= ~mask; 186b1f01e48SShun-Chih Yu val |= set; 187b1f01e48SShun-Chih Yu mtk_dma_write(pc, reg, val); 188b1f01e48SShun-Chih Yu } 189b1f01e48SShun-Chih Yu 190b1f01e48SShun-Chih Yu static void mtk_dma_set(struct mtk_cqdma_pchan *pc, u32 reg, u32 val) 191b1f01e48SShun-Chih Yu { 192b1f01e48SShun-Chih Yu mtk_dma_rmw(pc, reg, 0, val); 193b1f01e48SShun-Chih Yu } 194b1f01e48SShun-Chih Yu 195b1f01e48SShun-Chih Yu static void mtk_dma_clr(struct mtk_cqdma_pchan *pc, u32 reg, u32 val) 196b1f01e48SShun-Chih Yu { 197b1f01e48SShun-Chih Yu mtk_dma_rmw(pc, reg, val, 0); 198b1f01e48SShun-Chih Yu } 199b1f01e48SShun-Chih Yu 200b1f01e48SShun-Chih Yu static void mtk_cqdma_vdesc_free(struct virt_dma_desc *vd) 201b1f01e48SShun-Chih Yu { 202b1f01e48SShun-Chih Yu kfree(to_cqdma_vdesc(vd)); 203b1f01e48SShun-Chih Yu } 204b1f01e48SShun-Chih Yu 205b1f01e48SShun-Chih Yu static int mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan *pc, bool atomic) 206b1f01e48SShun-Chih Yu { 207b1f01e48SShun-Chih Yu u32 status = 0; 208b1f01e48SShun-Chih Yu 209b1f01e48SShun-Chih Yu if (!atomic) 210b1f01e48SShun-Chih Yu return readl_poll_timeout(pc->base + MTK_CQDMA_EN, 211b1f01e48SShun-Chih Yu status, 212b1f01e48SShun-Chih Yu !(status & MTK_CQDMA_EN_BIT), 213b1f01e48SShun-Chih Yu MTK_CQDMA_USEC_POLL, 214b1f01e48SShun-Chih Yu MTK_CQDMA_TIMEOUT_POLL); 215b1f01e48SShun-Chih Yu 216b1f01e48SShun-Chih Yu return readl_poll_timeout_atomic(pc->base + MTK_CQDMA_EN, 217b1f01e48SShun-Chih Yu status, 218b1f01e48SShun-Chih Yu !(status & MTK_CQDMA_EN_BIT), 219b1f01e48SShun-Chih Yu MTK_CQDMA_USEC_POLL, 220b1f01e48SShun-Chih Yu MTK_CQDMA_TIMEOUT_POLL); 221b1f01e48SShun-Chih Yu } 222b1f01e48SShun-Chih Yu 223b1f01e48SShun-Chih Yu static int mtk_cqdma_hard_reset(struct mtk_cqdma_pchan *pc) 224b1f01e48SShun-Chih Yu { 225b1f01e48SShun-Chih Yu mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT); 226b1f01e48SShun-Chih Yu mtk_dma_clr(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT); 227b1f01e48SShun-Chih Yu 228069b3c42SDan Carpenter return mtk_cqdma_poll_engine_done(pc, true); 229b1f01e48SShun-Chih Yu } 230b1f01e48SShun-Chih Yu 231b1f01e48SShun-Chih Yu static void mtk_cqdma_start(struct mtk_cqdma_pchan *pc, 232b1f01e48SShun-Chih Yu struct mtk_cqdma_vdesc *cvd) 233b1f01e48SShun-Chih Yu { 234b1f01e48SShun-Chih Yu /* wait for the previous transaction done */ 235b1f01e48SShun-Chih Yu if (mtk_cqdma_poll_engine_done(pc, true) < 0) 236b1f01e48SShun-Chih Yu dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma wait transaction timeout\n"); 237b1f01e48SShun-Chih Yu 238b1f01e48SShun-Chih Yu /* warm reset the dma engine for the new transaction */ 239b1f01e48SShun-Chih Yu mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_WARM_RST_BIT); 240b1f01e48SShun-Chih Yu if (mtk_cqdma_poll_engine_done(pc, true) < 0) 241b1f01e48SShun-Chih Yu dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma warm reset timeout\n"); 242b1f01e48SShun-Chih Yu 243b1f01e48SShun-Chih Yu /* setup the source */ 244b1f01e48SShun-Chih Yu mtk_dma_set(pc, MTK_CQDMA_SRC, cvd->src & MTK_CQDMA_ADDR_LIMIT); 245b1f01e48SShun-Chih Yu #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 246b1f01e48SShun-Chih Yu mtk_dma_set(pc, MTK_CQDMA_SRC2, cvd->src >> MTK_CQDMA_ADDR2_SHFIT); 247b1f01e48SShun-Chih Yu #else 248b1f01e48SShun-Chih Yu mtk_dma_set(pc, MTK_CQDMA_SRC2, 0); 249b1f01e48SShun-Chih Yu #endif 250b1f01e48SShun-Chih Yu 251b1f01e48SShun-Chih Yu /* setup the destination */ 252b1f01e48SShun-Chih Yu mtk_dma_set(pc, MTK_CQDMA_DST, cvd->dest & MTK_CQDMA_ADDR_LIMIT); 253b1f01e48SShun-Chih Yu #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 254b1f01e48SShun-Chih Yu mtk_dma_set(pc, MTK_CQDMA_DST2, cvd->dest >> MTK_CQDMA_ADDR2_SHFIT); 255b1f01e48SShun-Chih Yu #else 2565bb5c3a3SShun-Chih Yu mtk_dma_set(pc, MTK_CQDMA_DST2, 0); 257b1f01e48SShun-Chih Yu #endif 258b1f01e48SShun-Chih Yu 259b1f01e48SShun-Chih Yu /* setup the length */ 260b1f01e48SShun-Chih Yu mtk_dma_set(pc, MTK_CQDMA_LEN1, cvd->len); 261b1f01e48SShun-Chih Yu 262b1f01e48SShun-Chih Yu /* start dma engine */ 263b1f01e48SShun-Chih Yu mtk_dma_set(pc, MTK_CQDMA_EN, MTK_CQDMA_EN_BIT); 264b1f01e48SShun-Chih Yu } 265b1f01e48SShun-Chih Yu 266b1f01e48SShun-Chih Yu static void mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan *cvc) 267b1f01e48SShun-Chih Yu { 268b1f01e48SShun-Chih Yu struct virt_dma_desc *vd, *vd2; 269b1f01e48SShun-Chih Yu struct mtk_cqdma_pchan *pc = cvc->pc; 270b1f01e48SShun-Chih Yu struct mtk_cqdma_vdesc *cvd; 271b1f01e48SShun-Chih Yu bool trigger_engine = false; 272b1f01e48SShun-Chih Yu 273b1f01e48SShun-Chih Yu lockdep_assert_held(&cvc->vc.lock); 274b1f01e48SShun-Chih Yu lockdep_assert_held(&pc->lock); 275b1f01e48SShun-Chih Yu 276b1f01e48SShun-Chih Yu list_for_each_entry_safe(vd, vd2, &cvc->vc.desc_issued, node) { 277b1f01e48SShun-Chih Yu /* need to trigger dma engine if PC's queue is empty */ 278b1f01e48SShun-Chih Yu if (list_empty(&pc->queue)) 279b1f01e48SShun-Chih Yu trigger_engine = true; 280b1f01e48SShun-Chih Yu 281b1f01e48SShun-Chih Yu cvd = to_cqdma_vdesc(vd); 282b1f01e48SShun-Chih Yu 283b1f01e48SShun-Chih Yu /* add VD into PC's queue */ 284b1f01e48SShun-Chih Yu list_add_tail(&cvd->node, &pc->queue); 285b1f01e48SShun-Chih Yu 286b1f01e48SShun-Chih Yu /* start the dma engine */ 287b1f01e48SShun-Chih Yu if (trigger_engine) 288b1f01e48SShun-Chih Yu mtk_cqdma_start(pc, cvd); 289b1f01e48SShun-Chih Yu 290b1f01e48SShun-Chih Yu /* remove VD from list desc_issued */ 291b1f01e48SShun-Chih Yu list_del(&vd->node); 292b1f01e48SShun-Chih Yu } 293b1f01e48SShun-Chih Yu } 294b1f01e48SShun-Chih Yu 295b1f01e48SShun-Chih Yu /* 296b1f01e48SShun-Chih Yu * return true if this VC is active, 297b1f01e48SShun-Chih Yu * meaning that there are VDs under processing by the PC 298b1f01e48SShun-Chih Yu */ 299b1f01e48SShun-Chih Yu static bool mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan *cvc) 300b1f01e48SShun-Chih Yu { 301b1f01e48SShun-Chih Yu struct mtk_cqdma_vdesc *cvd; 302b1f01e48SShun-Chih Yu 303b1f01e48SShun-Chih Yu list_for_each_entry(cvd, &cvc->pc->queue, node) 304b1f01e48SShun-Chih Yu if (cvc == to_cqdma_vchan(cvd->ch)) 305b1f01e48SShun-Chih Yu return true; 306b1f01e48SShun-Chih Yu 307b1f01e48SShun-Chih Yu return false; 308b1f01e48SShun-Chih Yu } 309b1f01e48SShun-Chih Yu 310b1f01e48SShun-Chih Yu /* 311b1f01e48SShun-Chih Yu * return the pointer of the CVD that is just consumed by the PC 312b1f01e48SShun-Chih Yu */ 313b1f01e48SShun-Chih Yu static struct mtk_cqdma_vdesc 314b1f01e48SShun-Chih Yu *mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan *pc) 315b1f01e48SShun-Chih Yu { 316b1f01e48SShun-Chih Yu struct mtk_cqdma_vchan *cvc; 317b1f01e48SShun-Chih Yu struct mtk_cqdma_vdesc *cvd, *ret = NULL; 318b1f01e48SShun-Chih Yu 319b1f01e48SShun-Chih Yu /* consume a CVD from PC's queue */ 320b1f01e48SShun-Chih Yu cvd = list_first_entry_or_null(&pc->queue, 321b1f01e48SShun-Chih Yu struct mtk_cqdma_vdesc, node); 322b1f01e48SShun-Chih Yu if (unlikely(!cvd || !cvd->parent)) 323b1f01e48SShun-Chih Yu return NULL; 324b1f01e48SShun-Chih Yu 325b1f01e48SShun-Chih Yu cvc = to_cqdma_vchan(cvd->ch); 326b1f01e48SShun-Chih Yu ret = cvd; 327b1f01e48SShun-Chih Yu 328b1f01e48SShun-Chih Yu /* update residue of the parent CVD */ 329b1f01e48SShun-Chih Yu cvd->parent->residue -= cvd->len; 330b1f01e48SShun-Chih Yu 331b1f01e48SShun-Chih Yu /* delete CVD from PC's queue */ 332b1f01e48SShun-Chih Yu list_del(&cvd->node); 333b1f01e48SShun-Chih Yu 334b1f01e48SShun-Chih Yu spin_lock(&cvc->vc.lock); 335b1f01e48SShun-Chih Yu 336b1f01e48SShun-Chih Yu /* check whether all the child CVDs completed */ 337b1f01e48SShun-Chih Yu if (!cvd->parent->residue) { 338b1f01e48SShun-Chih Yu /* add the parent VD into list desc_completed */ 339b1f01e48SShun-Chih Yu vchan_cookie_complete(&cvd->parent->vd); 340b1f01e48SShun-Chih Yu 341b1f01e48SShun-Chih Yu /* setup completion if this VC is under synchronization */ 342b1f01e48SShun-Chih Yu if (cvc->issue_synchronize && !mtk_cqdma_is_vchan_active(cvc)) { 343b1f01e48SShun-Chih Yu complete(&cvc->issue_completion); 344b1f01e48SShun-Chih Yu cvc->issue_synchronize = false; 345b1f01e48SShun-Chih Yu } 346b1f01e48SShun-Chih Yu } 347b1f01e48SShun-Chih Yu 348b1f01e48SShun-Chih Yu spin_unlock(&cvc->vc.lock); 349b1f01e48SShun-Chih Yu 350b1f01e48SShun-Chih Yu /* start transaction for next CVD in the queue */ 351b1f01e48SShun-Chih Yu cvd = list_first_entry_or_null(&pc->queue, 352b1f01e48SShun-Chih Yu struct mtk_cqdma_vdesc, node); 353b1f01e48SShun-Chih Yu if (cvd) 354b1f01e48SShun-Chih Yu mtk_cqdma_start(pc, cvd); 355b1f01e48SShun-Chih Yu 356b1f01e48SShun-Chih Yu return ret; 357b1f01e48SShun-Chih Yu } 358b1f01e48SShun-Chih Yu 35980ef8869SAllen Pais static void mtk_cqdma_tasklet_cb(struct tasklet_struct *t) 360b1f01e48SShun-Chih Yu { 36180ef8869SAllen Pais struct mtk_cqdma_pchan *pc = from_tasklet(pc, t, tasklet); 362b1f01e48SShun-Chih Yu struct mtk_cqdma_vdesc *cvd = NULL; 363b1f01e48SShun-Chih Yu unsigned long flags; 364b1f01e48SShun-Chih Yu 365b1f01e48SShun-Chih Yu spin_lock_irqsave(&pc->lock, flags); 366b1f01e48SShun-Chih Yu /* consume the queue */ 367b1f01e48SShun-Chih Yu cvd = mtk_cqdma_consume_work_queue(pc); 368b1f01e48SShun-Chih Yu spin_unlock_irqrestore(&pc->lock, flags); 369b1f01e48SShun-Chih Yu 370b1f01e48SShun-Chih Yu /* submit the next CVD */ 371b1f01e48SShun-Chih Yu if (cvd) { 372b1f01e48SShun-Chih Yu dma_run_dependencies(&cvd->vd.tx); 373b1f01e48SShun-Chih Yu 374b1f01e48SShun-Chih Yu /* 375b1f01e48SShun-Chih Yu * free child CVD after completion. 376b1f01e48SShun-Chih Yu * the parent CVD would be freeed with desc_free by user. 377b1f01e48SShun-Chih Yu */ 378b1f01e48SShun-Chih Yu if (cvd->parent != cvd) 379b1f01e48SShun-Chih Yu kfree(cvd); 380b1f01e48SShun-Chih Yu } 381b1f01e48SShun-Chih Yu 382b1f01e48SShun-Chih Yu /* re-enable interrupt before leaving tasklet */ 383b1f01e48SShun-Chih Yu enable_irq(pc->irq); 384b1f01e48SShun-Chih Yu } 385b1f01e48SShun-Chih Yu 386b1f01e48SShun-Chih Yu static irqreturn_t mtk_cqdma_irq(int irq, void *devid) 387b1f01e48SShun-Chih Yu { 388b1f01e48SShun-Chih Yu struct mtk_cqdma_device *cqdma = devid; 389b1f01e48SShun-Chih Yu irqreturn_t ret = IRQ_NONE; 390b1f01e48SShun-Chih Yu bool schedule_tasklet = false; 391b1f01e48SShun-Chih Yu u32 i; 392b1f01e48SShun-Chih Yu 393b1f01e48SShun-Chih Yu /* clear interrupt flags for each PC */ 394b1f01e48SShun-Chih Yu for (i = 0; i < cqdma->dma_channels; ++i, schedule_tasklet = false) { 395b1f01e48SShun-Chih Yu spin_lock(&cqdma->pc[i]->lock); 396b1f01e48SShun-Chih Yu if (mtk_dma_read(cqdma->pc[i], 397b1f01e48SShun-Chih Yu MTK_CQDMA_INT_FLAG) & MTK_CQDMA_INT_FLAG_BIT) { 398b1f01e48SShun-Chih Yu /* clear interrupt */ 399b1f01e48SShun-Chih Yu mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_FLAG, 400b1f01e48SShun-Chih Yu MTK_CQDMA_INT_FLAG_BIT); 401b1f01e48SShun-Chih Yu 402b1f01e48SShun-Chih Yu schedule_tasklet = true; 403b1f01e48SShun-Chih Yu ret = IRQ_HANDLED; 404b1f01e48SShun-Chih Yu } 405b1f01e48SShun-Chih Yu spin_unlock(&cqdma->pc[i]->lock); 406b1f01e48SShun-Chih Yu 407b1f01e48SShun-Chih Yu if (schedule_tasklet) { 408b1f01e48SShun-Chih Yu /* disable interrupt */ 409b1f01e48SShun-Chih Yu disable_irq_nosync(cqdma->pc[i]->irq); 410b1f01e48SShun-Chih Yu 411b1f01e48SShun-Chih Yu /* schedule the tasklet to handle the transactions */ 412b1f01e48SShun-Chih Yu tasklet_schedule(&cqdma->pc[i]->tasklet); 413b1f01e48SShun-Chih Yu } 414b1f01e48SShun-Chih Yu } 415b1f01e48SShun-Chih Yu 416b1f01e48SShun-Chih Yu return ret; 417b1f01e48SShun-Chih Yu } 418b1f01e48SShun-Chih Yu 419b1f01e48SShun-Chih Yu static struct virt_dma_desc *mtk_cqdma_find_active_desc(struct dma_chan *c, 420b1f01e48SShun-Chih Yu dma_cookie_t cookie) 421b1f01e48SShun-Chih Yu { 422b1f01e48SShun-Chih Yu struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); 423b1f01e48SShun-Chih Yu struct virt_dma_desc *vd; 424b1f01e48SShun-Chih Yu unsigned long flags; 425b1f01e48SShun-Chih Yu 426b1f01e48SShun-Chih Yu spin_lock_irqsave(&cvc->pc->lock, flags); 427b1f01e48SShun-Chih Yu list_for_each_entry(vd, &cvc->pc->queue, node) 428b1f01e48SShun-Chih Yu if (vd->tx.cookie == cookie) { 429b1f01e48SShun-Chih Yu spin_unlock_irqrestore(&cvc->pc->lock, flags); 430b1f01e48SShun-Chih Yu return vd; 431b1f01e48SShun-Chih Yu } 432b1f01e48SShun-Chih Yu spin_unlock_irqrestore(&cvc->pc->lock, flags); 433b1f01e48SShun-Chih Yu 434b1f01e48SShun-Chih Yu list_for_each_entry(vd, &cvc->vc.desc_issued, node) 435b1f01e48SShun-Chih Yu if (vd->tx.cookie == cookie) 436b1f01e48SShun-Chih Yu return vd; 437b1f01e48SShun-Chih Yu 438b1f01e48SShun-Chih Yu return NULL; 439b1f01e48SShun-Chih Yu } 440b1f01e48SShun-Chih Yu 441b1f01e48SShun-Chih Yu static enum dma_status mtk_cqdma_tx_status(struct dma_chan *c, 442b1f01e48SShun-Chih Yu dma_cookie_t cookie, 443b1f01e48SShun-Chih Yu struct dma_tx_state *txstate) 444b1f01e48SShun-Chih Yu { 445b1f01e48SShun-Chih Yu struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); 446b1f01e48SShun-Chih Yu struct mtk_cqdma_vdesc *cvd; 447b1f01e48SShun-Chih Yu struct virt_dma_desc *vd; 448b1f01e48SShun-Chih Yu enum dma_status ret; 449b1f01e48SShun-Chih Yu unsigned long flags; 450b1f01e48SShun-Chih Yu size_t bytes = 0; 451b1f01e48SShun-Chih Yu 452b1f01e48SShun-Chih Yu ret = dma_cookie_status(c, cookie, txstate); 453b1f01e48SShun-Chih Yu if (ret == DMA_COMPLETE || !txstate) 454b1f01e48SShun-Chih Yu return ret; 455b1f01e48SShun-Chih Yu 456b1f01e48SShun-Chih Yu spin_lock_irqsave(&cvc->vc.lock, flags); 457b1f01e48SShun-Chih Yu vd = mtk_cqdma_find_active_desc(c, cookie); 458b1f01e48SShun-Chih Yu spin_unlock_irqrestore(&cvc->vc.lock, flags); 459b1f01e48SShun-Chih Yu 460b1f01e48SShun-Chih Yu if (vd) { 461b1f01e48SShun-Chih Yu cvd = to_cqdma_vdesc(vd); 462b1f01e48SShun-Chih Yu bytes = cvd->residue; 463b1f01e48SShun-Chih Yu } 464b1f01e48SShun-Chih Yu 465b1f01e48SShun-Chih Yu dma_set_residue(txstate, bytes); 466b1f01e48SShun-Chih Yu 467b1f01e48SShun-Chih Yu return ret; 468b1f01e48SShun-Chih Yu } 469b1f01e48SShun-Chih Yu 470b1f01e48SShun-Chih Yu static void mtk_cqdma_issue_pending(struct dma_chan *c) 471b1f01e48SShun-Chih Yu { 472b1f01e48SShun-Chih Yu struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); 473b1f01e48SShun-Chih Yu unsigned long pc_flags; 474b1f01e48SShun-Chih Yu unsigned long vc_flags; 475b1f01e48SShun-Chih Yu 476b1f01e48SShun-Chih Yu /* acquire PC's lock before VS's lock for lock dependency in tasklet */ 477b1f01e48SShun-Chih Yu spin_lock_irqsave(&cvc->pc->lock, pc_flags); 478b1f01e48SShun-Chih Yu spin_lock_irqsave(&cvc->vc.lock, vc_flags); 479b1f01e48SShun-Chih Yu 480b1f01e48SShun-Chih Yu if (vchan_issue_pending(&cvc->vc)) 481b1f01e48SShun-Chih Yu mtk_cqdma_issue_vchan_pending(cvc); 482b1f01e48SShun-Chih Yu 483b1f01e48SShun-Chih Yu spin_unlock_irqrestore(&cvc->vc.lock, vc_flags); 484b1f01e48SShun-Chih Yu spin_unlock_irqrestore(&cvc->pc->lock, pc_flags); 485b1f01e48SShun-Chih Yu } 486b1f01e48SShun-Chih Yu 487b1f01e48SShun-Chih Yu static struct dma_async_tx_descriptor * 488b1f01e48SShun-Chih Yu mtk_cqdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, 489b1f01e48SShun-Chih Yu dma_addr_t src, size_t len, unsigned long flags) 490b1f01e48SShun-Chih Yu { 491b1f01e48SShun-Chih Yu struct mtk_cqdma_vdesc **cvd; 492b1f01e48SShun-Chih Yu struct dma_async_tx_descriptor *tx = NULL, *prev_tx = NULL; 493b1f01e48SShun-Chih Yu size_t i, tlen, nr_vd; 494b1f01e48SShun-Chih Yu 495b1f01e48SShun-Chih Yu /* 496b1f01e48SShun-Chih Yu * In the case that trsanction length is larger than the 497b1f01e48SShun-Chih Yu * DMA engine supports, a single memcpy transaction needs 498b1f01e48SShun-Chih Yu * to be separated into several DMA transactions. 499b1f01e48SShun-Chih Yu * Each DMA transaction would be described by a CVD, 500b1f01e48SShun-Chih Yu * and the first one is referred as the parent CVD, 501b1f01e48SShun-Chih Yu * while the others are child CVDs. 502b1f01e48SShun-Chih Yu * The parent CVD's tx descriptor is the only tx descriptor 503b1f01e48SShun-Chih Yu * returned to the DMA user, and it should not be completed 504b1f01e48SShun-Chih Yu * until all the child CVDs completed. 505b1f01e48SShun-Chih Yu */ 506b1f01e48SShun-Chih Yu nr_vd = DIV_ROUND_UP(len, MTK_CQDMA_MAX_LEN); 507b1f01e48SShun-Chih Yu cvd = kcalloc(nr_vd, sizeof(*cvd), GFP_NOWAIT); 508b1f01e48SShun-Chih Yu if (!cvd) 509b1f01e48SShun-Chih Yu return NULL; 510b1f01e48SShun-Chih Yu 511b1f01e48SShun-Chih Yu for (i = 0; i < nr_vd; ++i) { 512b1f01e48SShun-Chih Yu cvd[i] = kzalloc(sizeof(*cvd[i]), GFP_NOWAIT); 513b1f01e48SShun-Chih Yu if (!cvd[i]) { 514b1f01e48SShun-Chih Yu for (; i > 0; --i) 515b1f01e48SShun-Chih Yu kfree(cvd[i - 1]); 516b1f01e48SShun-Chih Yu return NULL; 517b1f01e48SShun-Chih Yu } 518b1f01e48SShun-Chih Yu 519b1f01e48SShun-Chih Yu /* setup dma channel */ 520b1f01e48SShun-Chih Yu cvd[i]->ch = c; 521b1f01e48SShun-Chih Yu 522b1f01e48SShun-Chih Yu /* setup sourece, destination, and length */ 523b1f01e48SShun-Chih Yu tlen = (len > MTK_CQDMA_MAX_LEN) ? MTK_CQDMA_MAX_LEN : len; 524b1f01e48SShun-Chih Yu cvd[i]->len = tlen; 525b1f01e48SShun-Chih Yu cvd[i]->src = src; 526b1f01e48SShun-Chih Yu cvd[i]->dest = dest; 527b1f01e48SShun-Chih Yu 528b1f01e48SShun-Chih Yu /* setup tx descriptor */ 529b1f01e48SShun-Chih Yu tx = vchan_tx_prep(to_virt_chan(c), &cvd[i]->vd, flags); 530b1f01e48SShun-Chih Yu tx->next = NULL; 531b1f01e48SShun-Chih Yu 532b1f01e48SShun-Chih Yu if (!i) { 533b1f01e48SShun-Chih Yu cvd[0]->residue = len; 534b1f01e48SShun-Chih Yu } else { 535b1f01e48SShun-Chih Yu prev_tx->next = tx; 536b1f01e48SShun-Chih Yu cvd[i]->residue = tlen; 537b1f01e48SShun-Chih Yu } 538b1f01e48SShun-Chih Yu 539b1f01e48SShun-Chih Yu cvd[i]->parent = cvd[0]; 540b1f01e48SShun-Chih Yu 541b1f01e48SShun-Chih Yu /* update the src, dest, len, prev_tx for the next CVD */ 542b1f01e48SShun-Chih Yu src += tlen; 543b1f01e48SShun-Chih Yu dest += tlen; 544b1f01e48SShun-Chih Yu len -= tlen; 545b1f01e48SShun-Chih Yu prev_tx = tx; 546b1f01e48SShun-Chih Yu } 547b1f01e48SShun-Chih Yu 548b1f01e48SShun-Chih Yu return &cvd[0]->vd.tx; 549b1f01e48SShun-Chih Yu } 550b1f01e48SShun-Chih Yu 551b1f01e48SShun-Chih Yu static void mtk_cqdma_free_inactive_desc(struct dma_chan *c) 552b1f01e48SShun-Chih Yu { 553b1f01e48SShun-Chih Yu struct virt_dma_chan *vc = to_virt_chan(c); 554b1f01e48SShun-Chih Yu unsigned long flags; 555b1f01e48SShun-Chih Yu LIST_HEAD(head); 556b1f01e48SShun-Chih Yu 557b1f01e48SShun-Chih Yu /* 558b1f01e48SShun-Chih Yu * set desc_allocated, desc_submitted, 559b1f01e48SShun-Chih Yu * and desc_issued as the candicates to be freed 560b1f01e48SShun-Chih Yu */ 561b1f01e48SShun-Chih Yu spin_lock_irqsave(&vc->lock, flags); 562b1f01e48SShun-Chih Yu list_splice_tail_init(&vc->desc_allocated, &head); 563b1f01e48SShun-Chih Yu list_splice_tail_init(&vc->desc_submitted, &head); 564b1f01e48SShun-Chih Yu list_splice_tail_init(&vc->desc_issued, &head); 565b1f01e48SShun-Chih Yu spin_unlock_irqrestore(&vc->lock, flags); 566b1f01e48SShun-Chih Yu 567b1f01e48SShun-Chih Yu /* free descriptor lists */ 568b1f01e48SShun-Chih Yu vchan_dma_desc_free_list(vc, &head); 569b1f01e48SShun-Chih Yu } 570b1f01e48SShun-Chih Yu 571b1f01e48SShun-Chih Yu static void mtk_cqdma_free_active_desc(struct dma_chan *c) 572b1f01e48SShun-Chih Yu { 573b1f01e48SShun-Chih Yu struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); 574b1f01e48SShun-Chih Yu bool sync_needed = false; 575b1f01e48SShun-Chih Yu unsigned long pc_flags; 576b1f01e48SShun-Chih Yu unsigned long vc_flags; 577b1f01e48SShun-Chih Yu 578b1f01e48SShun-Chih Yu /* acquire PC's lock first due to lock dependency in dma ISR */ 579b1f01e48SShun-Chih Yu spin_lock_irqsave(&cvc->pc->lock, pc_flags); 580b1f01e48SShun-Chih Yu spin_lock_irqsave(&cvc->vc.lock, vc_flags); 581b1f01e48SShun-Chih Yu 582b1f01e48SShun-Chih Yu /* synchronization is required if this VC is active */ 583b1f01e48SShun-Chih Yu if (mtk_cqdma_is_vchan_active(cvc)) { 584b1f01e48SShun-Chih Yu cvc->issue_synchronize = true; 585b1f01e48SShun-Chih Yu sync_needed = true; 586b1f01e48SShun-Chih Yu } 587b1f01e48SShun-Chih Yu 588b1f01e48SShun-Chih Yu spin_unlock_irqrestore(&cvc->vc.lock, vc_flags); 589b1f01e48SShun-Chih Yu spin_unlock_irqrestore(&cvc->pc->lock, pc_flags); 590b1f01e48SShun-Chih Yu 591b1f01e48SShun-Chih Yu /* waiting for the completion of this VC */ 592b1f01e48SShun-Chih Yu if (sync_needed) 593b1f01e48SShun-Chih Yu wait_for_completion(&cvc->issue_completion); 594b1f01e48SShun-Chih Yu 595b1f01e48SShun-Chih Yu /* free all descriptors in list desc_completed */ 596b1f01e48SShun-Chih Yu vchan_synchronize(&cvc->vc); 597b1f01e48SShun-Chih Yu 598b1f01e48SShun-Chih Yu WARN_ONCE(!list_empty(&cvc->vc.desc_completed), 599b1f01e48SShun-Chih Yu "Desc pending still in list desc_completed\n"); 600b1f01e48SShun-Chih Yu } 601b1f01e48SShun-Chih Yu 602b1f01e48SShun-Chih Yu static int mtk_cqdma_terminate_all(struct dma_chan *c) 603b1f01e48SShun-Chih Yu { 604b1f01e48SShun-Chih Yu /* free descriptors not processed yet by hardware */ 605b1f01e48SShun-Chih Yu mtk_cqdma_free_inactive_desc(c); 606b1f01e48SShun-Chih Yu 607b1f01e48SShun-Chih Yu /* free descriptors being processed by hardware */ 608b1f01e48SShun-Chih Yu mtk_cqdma_free_active_desc(c); 609b1f01e48SShun-Chih Yu 610b1f01e48SShun-Chih Yu return 0; 611b1f01e48SShun-Chih Yu } 612b1f01e48SShun-Chih Yu 613b1f01e48SShun-Chih Yu static int mtk_cqdma_alloc_chan_resources(struct dma_chan *c) 614b1f01e48SShun-Chih Yu { 615b1f01e48SShun-Chih Yu struct mtk_cqdma_device *cqdma = to_cqdma_dev(c); 616b1f01e48SShun-Chih Yu struct mtk_cqdma_vchan *vc = to_cqdma_vchan(c); 617b1f01e48SShun-Chih Yu struct mtk_cqdma_pchan *pc = NULL; 618b1f01e48SShun-Chih Yu u32 i, min_refcnt = U32_MAX, refcnt; 619b1f01e48SShun-Chih Yu unsigned long flags; 620b1f01e48SShun-Chih Yu 621b1f01e48SShun-Chih Yu /* allocate PC with the minimun refcount */ 622b1f01e48SShun-Chih Yu for (i = 0; i < cqdma->dma_channels; ++i) { 623b1f01e48SShun-Chih Yu refcnt = refcount_read(&cqdma->pc[i]->refcnt); 624b1f01e48SShun-Chih Yu if (refcnt < min_refcnt) { 625b1f01e48SShun-Chih Yu pc = cqdma->pc[i]; 626b1f01e48SShun-Chih Yu min_refcnt = refcnt; 627b1f01e48SShun-Chih Yu } 628b1f01e48SShun-Chih Yu } 629b1f01e48SShun-Chih Yu 630b1f01e48SShun-Chih Yu if (!pc) 631b1f01e48SShun-Chih Yu return -ENOSPC; 632b1f01e48SShun-Chih Yu 633b1f01e48SShun-Chih Yu spin_lock_irqsave(&pc->lock, flags); 634b1f01e48SShun-Chih Yu 635b1f01e48SShun-Chih Yu if (!refcount_read(&pc->refcnt)) { 636b1f01e48SShun-Chih Yu /* allocate PC when the refcount is zero */ 637b1f01e48SShun-Chih Yu mtk_cqdma_hard_reset(pc); 638b1f01e48SShun-Chih Yu 639b1f01e48SShun-Chih Yu /* enable interrupt for this PC */ 640b1f01e48SShun-Chih Yu mtk_dma_set(pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT); 641b1f01e48SShun-Chih Yu 642b1f01e48SShun-Chih Yu /* 643b1f01e48SShun-Chih Yu * refcount_inc would complain increment on 0; use-after-free. 644b1f01e48SShun-Chih Yu * Thus, we need to explicitly set it as 1 initially. 645b1f01e48SShun-Chih Yu */ 646b1f01e48SShun-Chih Yu refcount_set(&pc->refcnt, 1); 647b1f01e48SShun-Chih Yu } else { 648b1f01e48SShun-Chih Yu refcount_inc(&pc->refcnt); 649b1f01e48SShun-Chih Yu } 650b1f01e48SShun-Chih Yu 651b1f01e48SShun-Chih Yu spin_unlock_irqrestore(&pc->lock, flags); 652b1f01e48SShun-Chih Yu 653b1f01e48SShun-Chih Yu vc->pc = pc; 654b1f01e48SShun-Chih Yu 655b1f01e48SShun-Chih Yu return 0; 656b1f01e48SShun-Chih Yu } 657b1f01e48SShun-Chih Yu 658b1f01e48SShun-Chih Yu static void mtk_cqdma_free_chan_resources(struct dma_chan *c) 659b1f01e48SShun-Chih Yu { 660b1f01e48SShun-Chih Yu struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); 661b1f01e48SShun-Chih Yu unsigned long flags; 662b1f01e48SShun-Chih Yu 663b1f01e48SShun-Chih Yu /* free all descriptors in all lists on the VC */ 664b1f01e48SShun-Chih Yu mtk_cqdma_terminate_all(c); 665b1f01e48SShun-Chih Yu 666b1f01e48SShun-Chih Yu spin_lock_irqsave(&cvc->pc->lock, flags); 667b1f01e48SShun-Chih Yu 668b1f01e48SShun-Chih Yu /* PC is not freed until there is no VC mapped to it */ 669b1f01e48SShun-Chih Yu if (refcount_dec_and_test(&cvc->pc->refcnt)) { 670b1f01e48SShun-Chih Yu /* start the flush operation and stop the engine */ 671b1f01e48SShun-Chih Yu mtk_dma_set(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT); 672b1f01e48SShun-Chih Yu 673b1f01e48SShun-Chih Yu /* wait for the completion of flush operation */ 674069b3c42SDan Carpenter if (mtk_cqdma_poll_engine_done(cvc->pc, true) < 0) 675b1f01e48SShun-Chih Yu dev_err(cqdma2dev(to_cqdma_dev(c)), "cqdma flush timeout\n"); 676b1f01e48SShun-Chih Yu 677b1f01e48SShun-Chih Yu /* clear the flush bit and interrupt flag */ 678b1f01e48SShun-Chih Yu mtk_dma_clr(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT); 679b1f01e48SShun-Chih Yu mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_FLAG, 680b1f01e48SShun-Chih Yu MTK_CQDMA_INT_FLAG_BIT); 681b1f01e48SShun-Chih Yu 682b1f01e48SShun-Chih Yu /* disable interrupt for this PC */ 683b1f01e48SShun-Chih Yu mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT); 684b1f01e48SShun-Chih Yu } 685b1f01e48SShun-Chih Yu 686b1f01e48SShun-Chih Yu spin_unlock_irqrestore(&cvc->pc->lock, flags); 687b1f01e48SShun-Chih Yu } 688b1f01e48SShun-Chih Yu 689b1f01e48SShun-Chih Yu static int mtk_cqdma_hw_init(struct mtk_cqdma_device *cqdma) 690b1f01e48SShun-Chih Yu { 691b1f01e48SShun-Chih Yu unsigned long flags; 692b1f01e48SShun-Chih Yu int err; 693b1f01e48SShun-Chih Yu u32 i; 694b1f01e48SShun-Chih Yu 695b1f01e48SShun-Chih Yu pm_runtime_enable(cqdma2dev(cqdma)); 696b1f01e48SShun-Chih Yu pm_runtime_get_sync(cqdma2dev(cqdma)); 697b1f01e48SShun-Chih Yu 698b1f01e48SShun-Chih Yu err = clk_prepare_enable(cqdma->clk); 699b1f01e48SShun-Chih Yu 700b1f01e48SShun-Chih Yu if (err) { 701b1f01e48SShun-Chih Yu pm_runtime_put_sync(cqdma2dev(cqdma)); 702b1f01e48SShun-Chih Yu pm_runtime_disable(cqdma2dev(cqdma)); 703b1f01e48SShun-Chih Yu return err; 704b1f01e48SShun-Chih Yu } 705b1f01e48SShun-Chih Yu 706b1f01e48SShun-Chih Yu /* reset all PCs */ 707b1f01e48SShun-Chih Yu for (i = 0; i < cqdma->dma_channels; ++i) { 708b1f01e48SShun-Chih Yu spin_lock_irqsave(&cqdma->pc[i]->lock, flags); 709b1f01e48SShun-Chih Yu if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0) { 710b1f01e48SShun-Chih Yu dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n"); 711b1f01e48SShun-Chih Yu spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags); 712b1f01e48SShun-Chih Yu 713b1f01e48SShun-Chih Yu clk_disable_unprepare(cqdma->clk); 714b1f01e48SShun-Chih Yu pm_runtime_put_sync(cqdma2dev(cqdma)); 715b1f01e48SShun-Chih Yu pm_runtime_disable(cqdma2dev(cqdma)); 716b1f01e48SShun-Chih Yu return -EINVAL; 717b1f01e48SShun-Chih Yu } 718b1f01e48SShun-Chih Yu spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags); 719b1f01e48SShun-Chih Yu } 720b1f01e48SShun-Chih Yu 721b1f01e48SShun-Chih Yu return 0; 722b1f01e48SShun-Chih Yu } 723b1f01e48SShun-Chih Yu 724b1f01e48SShun-Chih Yu static void mtk_cqdma_hw_deinit(struct mtk_cqdma_device *cqdma) 725b1f01e48SShun-Chih Yu { 726b1f01e48SShun-Chih Yu unsigned long flags; 727b1f01e48SShun-Chih Yu u32 i; 728b1f01e48SShun-Chih Yu 729b1f01e48SShun-Chih Yu /* reset all PCs */ 730b1f01e48SShun-Chih Yu for (i = 0; i < cqdma->dma_channels; ++i) { 731b1f01e48SShun-Chih Yu spin_lock_irqsave(&cqdma->pc[i]->lock, flags); 732b1f01e48SShun-Chih Yu if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0) 733b1f01e48SShun-Chih Yu dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n"); 734b1f01e48SShun-Chih Yu spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags); 735b1f01e48SShun-Chih Yu } 736b1f01e48SShun-Chih Yu 737b1f01e48SShun-Chih Yu clk_disable_unprepare(cqdma->clk); 738b1f01e48SShun-Chih Yu 739b1f01e48SShun-Chih Yu pm_runtime_put_sync(cqdma2dev(cqdma)); 740b1f01e48SShun-Chih Yu pm_runtime_disable(cqdma2dev(cqdma)); 741b1f01e48SShun-Chih Yu } 742b1f01e48SShun-Chih Yu 743b1f01e48SShun-Chih Yu static const struct of_device_id mtk_cqdma_match[] = { 744b1f01e48SShun-Chih Yu { .compatible = "mediatek,mt6765-cqdma" }, 745b1f01e48SShun-Chih Yu { /* sentinel */ } 746b1f01e48SShun-Chih Yu }; 747b1f01e48SShun-Chih Yu MODULE_DEVICE_TABLE(of, mtk_cqdma_match); 748b1f01e48SShun-Chih Yu 749b1f01e48SShun-Chih Yu static int mtk_cqdma_probe(struct platform_device *pdev) 750b1f01e48SShun-Chih Yu { 751b1f01e48SShun-Chih Yu struct mtk_cqdma_device *cqdma; 752b1f01e48SShun-Chih Yu struct mtk_cqdma_vchan *vc; 753b1f01e48SShun-Chih Yu struct dma_device *dd; 754b1f01e48SShun-Chih Yu struct resource *res; 755b1f01e48SShun-Chih Yu int err; 756b1f01e48SShun-Chih Yu u32 i; 757b1f01e48SShun-Chih Yu 758b1f01e48SShun-Chih Yu cqdma = devm_kzalloc(&pdev->dev, sizeof(*cqdma), GFP_KERNEL); 759b1f01e48SShun-Chih Yu if (!cqdma) 760b1f01e48SShun-Chih Yu return -ENOMEM; 761b1f01e48SShun-Chih Yu 762b1f01e48SShun-Chih Yu dd = &cqdma->ddev; 763b1f01e48SShun-Chih Yu 764b1f01e48SShun-Chih Yu cqdma->clk = devm_clk_get(&pdev->dev, "cqdma"); 765b1f01e48SShun-Chih Yu if (IS_ERR(cqdma->clk)) { 766b1f01e48SShun-Chih Yu dev_err(&pdev->dev, "No clock for %s\n", 767b1f01e48SShun-Chih Yu dev_name(&pdev->dev)); 768b1f01e48SShun-Chih Yu return PTR_ERR(cqdma->clk); 769b1f01e48SShun-Chih Yu } 770b1f01e48SShun-Chih Yu 771b1f01e48SShun-Chih Yu dma_cap_set(DMA_MEMCPY, dd->cap_mask); 772b1f01e48SShun-Chih Yu 773b1f01e48SShun-Chih Yu dd->copy_align = MTK_CQDMA_ALIGN_SIZE; 774b1f01e48SShun-Chih Yu dd->device_alloc_chan_resources = mtk_cqdma_alloc_chan_resources; 775b1f01e48SShun-Chih Yu dd->device_free_chan_resources = mtk_cqdma_free_chan_resources; 776b1f01e48SShun-Chih Yu dd->device_tx_status = mtk_cqdma_tx_status; 777b1f01e48SShun-Chih Yu dd->device_issue_pending = mtk_cqdma_issue_pending; 778b1f01e48SShun-Chih Yu dd->device_prep_dma_memcpy = mtk_cqdma_prep_dma_memcpy; 779b1f01e48SShun-Chih Yu dd->device_terminate_all = mtk_cqdma_terminate_all; 780b1f01e48SShun-Chih Yu dd->src_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS; 781b1f01e48SShun-Chih Yu dd->dst_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS; 782b1f01e48SShun-Chih Yu dd->directions = BIT(DMA_MEM_TO_MEM); 783b1f01e48SShun-Chih Yu dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; 784b1f01e48SShun-Chih Yu dd->dev = &pdev->dev; 785b1f01e48SShun-Chih Yu INIT_LIST_HEAD(&dd->channels); 786b1f01e48SShun-Chih Yu 787b1f01e48SShun-Chih Yu if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node, 788b1f01e48SShun-Chih Yu "dma-requests", 789b1f01e48SShun-Chih Yu &cqdma->dma_requests)) { 790b1f01e48SShun-Chih Yu dev_info(&pdev->dev, 791b1f01e48SShun-Chih Yu "Using %u as missing dma-requests property\n", 792b1f01e48SShun-Chih Yu MTK_CQDMA_NR_VCHANS); 793b1f01e48SShun-Chih Yu 794b1f01e48SShun-Chih Yu cqdma->dma_requests = MTK_CQDMA_NR_VCHANS; 795b1f01e48SShun-Chih Yu } 796b1f01e48SShun-Chih Yu 797b1f01e48SShun-Chih Yu if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node, 798b1f01e48SShun-Chih Yu "dma-channels", 799b1f01e48SShun-Chih Yu &cqdma->dma_channels)) { 800b1f01e48SShun-Chih Yu dev_info(&pdev->dev, 801b1f01e48SShun-Chih Yu "Using %u as missing dma-channels property\n", 802b1f01e48SShun-Chih Yu MTK_CQDMA_NR_PCHANS); 803b1f01e48SShun-Chih Yu 804b1f01e48SShun-Chih Yu cqdma->dma_channels = MTK_CQDMA_NR_PCHANS; 805b1f01e48SShun-Chih Yu } 806b1f01e48SShun-Chih Yu 807b1f01e48SShun-Chih Yu cqdma->pc = devm_kcalloc(&pdev->dev, cqdma->dma_channels, 808b1f01e48SShun-Chih Yu sizeof(*cqdma->pc), GFP_KERNEL); 809b1f01e48SShun-Chih Yu if (!cqdma->pc) 810b1f01e48SShun-Chih Yu return -ENOMEM; 811b1f01e48SShun-Chih Yu 812b1f01e48SShun-Chih Yu /* initialization for PCs */ 813b1f01e48SShun-Chih Yu for (i = 0; i < cqdma->dma_channels; ++i) { 814b1f01e48SShun-Chih Yu cqdma->pc[i] = devm_kcalloc(&pdev->dev, 1, 815b1f01e48SShun-Chih Yu sizeof(**cqdma->pc), GFP_KERNEL); 816b1f01e48SShun-Chih Yu if (!cqdma->pc[i]) 817b1f01e48SShun-Chih Yu return -ENOMEM; 818b1f01e48SShun-Chih Yu 819b1f01e48SShun-Chih Yu INIT_LIST_HEAD(&cqdma->pc[i]->queue); 820b1f01e48SShun-Chih Yu spin_lock_init(&cqdma->pc[i]->lock); 821b1f01e48SShun-Chih Yu refcount_set(&cqdma->pc[i]->refcnt, 0); 8229d68427dSMarkus Elfring cqdma->pc[i]->base = devm_platform_ioremap_resource(pdev, i); 823b1f01e48SShun-Chih Yu if (IS_ERR(cqdma->pc[i]->base)) 824b1f01e48SShun-Chih Yu return PTR_ERR(cqdma->pc[i]->base); 825b1f01e48SShun-Chih Yu 826b1f01e48SShun-Chih Yu /* allocate IRQ resource */ 827b1f01e48SShun-Chih Yu res = platform_get_resource(pdev, IORESOURCE_IRQ, i); 828b1f01e48SShun-Chih Yu if (!res) { 829b1f01e48SShun-Chih Yu dev_err(&pdev->dev, "No irq resource for %s\n", 830b1f01e48SShun-Chih Yu dev_name(&pdev->dev)); 831b1f01e48SShun-Chih Yu return -EINVAL; 832b1f01e48SShun-Chih Yu } 833b1f01e48SShun-Chih Yu cqdma->pc[i]->irq = res->start; 834b1f01e48SShun-Chih Yu 835b1f01e48SShun-Chih Yu err = devm_request_irq(&pdev->dev, cqdma->pc[i]->irq, 836b1f01e48SShun-Chih Yu mtk_cqdma_irq, 0, dev_name(&pdev->dev), 837b1f01e48SShun-Chih Yu cqdma); 838b1f01e48SShun-Chih Yu if (err) { 839b1f01e48SShun-Chih Yu dev_err(&pdev->dev, 840b1f01e48SShun-Chih Yu "request_irq failed with err %d\n", err); 841b1f01e48SShun-Chih Yu return -EINVAL; 842b1f01e48SShun-Chih Yu } 843b1f01e48SShun-Chih Yu } 844b1f01e48SShun-Chih Yu 845b1f01e48SShun-Chih Yu /* allocate resource for VCs */ 846b1f01e48SShun-Chih Yu cqdma->vc = devm_kcalloc(&pdev->dev, cqdma->dma_requests, 847b1f01e48SShun-Chih Yu sizeof(*cqdma->vc), GFP_KERNEL); 848b1f01e48SShun-Chih Yu if (!cqdma->vc) 849b1f01e48SShun-Chih Yu return -ENOMEM; 850b1f01e48SShun-Chih Yu 851b1f01e48SShun-Chih Yu for (i = 0; i < cqdma->dma_requests; i++) { 852b1f01e48SShun-Chih Yu vc = &cqdma->vc[i]; 853b1f01e48SShun-Chih Yu vc->vc.desc_free = mtk_cqdma_vdesc_free; 854b1f01e48SShun-Chih Yu vchan_init(&vc->vc, dd); 855b1f01e48SShun-Chih Yu init_completion(&vc->issue_completion); 856b1f01e48SShun-Chih Yu } 857b1f01e48SShun-Chih Yu 858b1f01e48SShun-Chih Yu err = dma_async_device_register(dd); 859b1f01e48SShun-Chih Yu if (err) 860b1f01e48SShun-Chih Yu return err; 861b1f01e48SShun-Chih Yu 862b1f01e48SShun-Chih Yu err = of_dma_controller_register(pdev->dev.of_node, 863b1f01e48SShun-Chih Yu of_dma_xlate_by_chan_id, cqdma); 864b1f01e48SShun-Chih Yu if (err) { 865b1f01e48SShun-Chih Yu dev_err(&pdev->dev, 866b1f01e48SShun-Chih Yu "MediaTek CQDMA OF registration failed %d\n", err); 867b1f01e48SShun-Chih Yu goto err_unregister; 868b1f01e48SShun-Chih Yu } 869b1f01e48SShun-Chih Yu 870b1f01e48SShun-Chih Yu err = mtk_cqdma_hw_init(cqdma); 871b1f01e48SShun-Chih Yu if (err) { 872b1f01e48SShun-Chih Yu dev_err(&pdev->dev, 873b1f01e48SShun-Chih Yu "MediaTek CQDMA HW initialization failed %d\n", err); 874b1f01e48SShun-Chih Yu goto err_unregister; 875b1f01e48SShun-Chih Yu } 876b1f01e48SShun-Chih Yu 877b1f01e48SShun-Chih Yu platform_set_drvdata(pdev, cqdma); 878b1f01e48SShun-Chih Yu 879b1f01e48SShun-Chih Yu /* initialize tasklet for each PC */ 880b1f01e48SShun-Chih Yu for (i = 0; i < cqdma->dma_channels; ++i) 88180ef8869SAllen Pais tasklet_setup(&cqdma->pc[i]->tasklet, mtk_cqdma_tasklet_cb); 882b1f01e48SShun-Chih Yu 883b1f01e48SShun-Chih Yu dev_info(&pdev->dev, "MediaTek CQDMA driver registered\n"); 884b1f01e48SShun-Chih Yu 885b1f01e48SShun-Chih Yu return 0; 886b1f01e48SShun-Chih Yu 887b1f01e48SShun-Chih Yu err_unregister: 888b1f01e48SShun-Chih Yu dma_async_device_unregister(dd); 889b1f01e48SShun-Chih Yu 890b1f01e48SShun-Chih Yu return err; 891b1f01e48SShun-Chih Yu } 892b1f01e48SShun-Chih Yu 893b1f01e48SShun-Chih Yu static int mtk_cqdma_remove(struct platform_device *pdev) 894b1f01e48SShun-Chih Yu { 895b1f01e48SShun-Chih Yu struct mtk_cqdma_device *cqdma = platform_get_drvdata(pdev); 896b1f01e48SShun-Chih Yu struct mtk_cqdma_vchan *vc; 897b1f01e48SShun-Chih Yu unsigned long flags; 898b1f01e48SShun-Chih Yu int i; 899b1f01e48SShun-Chih Yu 900b1f01e48SShun-Chih Yu /* kill VC task */ 901b1f01e48SShun-Chih Yu for (i = 0; i < cqdma->dma_requests; i++) { 902b1f01e48SShun-Chih Yu vc = &cqdma->vc[i]; 903b1f01e48SShun-Chih Yu 904b1f01e48SShun-Chih Yu list_del(&vc->vc.chan.device_node); 905b1f01e48SShun-Chih Yu tasklet_kill(&vc->vc.task); 906b1f01e48SShun-Chih Yu } 907b1f01e48SShun-Chih Yu 908b1f01e48SShun-Chih Yu /* disable interrupt */ 909b1f01e48SShun-Chih Yu for (i = 0; i < cqdma->dma_channels; i++) { 910b1f01e48SShun-Chih Yu spin_lock_irqsave(&cqdma->pc[i]->lock, flags); 911b1f01e48SShun-Chih Yu mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_EN, 912b1f01e48SShun-Chih Yu MTK_CQDMA_INT_EN_BIT); 913b1f01e48SShun-Chih Yu spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags); 914b1f01e48SShun-Chih Yu 915b1f01e48SShun-Chih Yu /* Waits for any pending IRQ handlers to complete */ 916b1f01e48SShun-Chih Yu synchronize_irq(cqdma->pc[i]->irq); 917b1f01e48SShun-Chih Yu 918b1f01e48SShun-Chih Yu tasklet_kill(&cqdma->pc[i]->tasklet); 919b1f01e48SShun-Chih Yu } 920b1f01e48SShun-Chih Yu 921b1f01e48SShun-Chih Yu /* disable hardware */ 922b1f01e48SShun-Chih Yu mtk_cqdma_hw_deinit(cqdma); 923b1f01e48SShun-Chih Yu 924b1f01e48SShun-Chih Yu dma_async_device_unregister(&cqdma->ddev); 925b1f01e48SShun-Chih Yu of_dma_controller_free(pdev->dev.of_node); 926b1f01e48SShun-Chih Yu 927b1f01e48SShun-Chih Yu return 0; 928b1f01e48SShun-Chih Yu } 929b1f01e48SShun-Chih Yu 930b1f01e48SShun-Chih Yu static struct platform_driver mtk_cqdma_driver = { 931b1f01e48SShun-Chih Yu .probe = mtk_cqdma_probe, 932b1f01e48SShun-Chih Yu .remove = mtk_cqdma_remove, 933b1f01e48SShun-Chih Yu .driver = { 934b1f01e48SShun-Chih Yu .name = KBUILD_MODNAME, 935b1f01e48SShun-Chih Yu .of_match_table = mtk_cqdma_match, 936b1f01e48SShun-Chih Yu }, 937b1f01e48SShun-Chih Yu }; 938b1f01e48SShun-Chih Yu module_platform_driver(mtk_cqdma_driver); 939b1f01e48SShun-Chih Yu 940b1f01e48SShun-Chih Yu MODULE_DESCRIPTION("MediaTek CQDMA Controller Driver"); 941b1f01e48SShun-Chih Yu MODULE_AUTHOR("Shun-Chih Yu <shun-chih.yu@mediatek.com>"); 942b1f01e48SShun-Chih Yu MODULE_LICENSE("GPL v2"); 943