1d894fc60SAlex Smith /* 2d894fc60SAlex Smith * Ingenic JZ4780 DMA controller 3d894fc60SAlex Smith * 4d894fc60SAlex Smith * Copyright (c) 2015 Imagination Technologies 5d894fc60SAlex Smith * Author: Alex Smith <alex@alex-smith.me.uk> 6d894fc60SAlex Smith * 7d894fc60SAlex Smith * This program is free software; you can redistribute it and/or modify it 8d894fc60SAlex Smith * under the terms of the GNU General Public License as published by the 9d894fc60SAlex Smith * Free Software Foundation; either version 2 of the License, or (at your 10d894fc60SAlex Smith * option) any later version. 11d894fc60SAlex Smith */ 12d894fc60SAlex Smith 13d894fc60SAlex Smith #include <linux/clk.h> 14d894fc60SAlex Smith #include <linux/dmapool.h> 15d894fc60SAlex Smith #include <linux/init.h> 16d894fc60SAlex Smith #include <linux/interrupt.h> 17d894fc60SAlex Smith #include <linux/module.h> 18d894fc60SAlex Smith #include <linux/of.h> 196147b032SPaul Cercueil #include <linux/of_device.h> 20d894fc60SAlex Smith #include <linux/of_dma.h> 21d894fc60SAlex Smith #include <linux/platform_device.h> 22d894fc60SAlex Smith #include <linux/slab.h> 23d894fc60SAlex Smith 24d894fc60SAlex Smith #include "dmaengine.h" 25d894fc60SAlex Smith #include "virt-dma.h" 26d894fc60SAlex Smith 27d894fc60SAlex Smith /* Global registers. */ 2833633583SPaul Cercueil #define JZ_DMA_REG_DMAC 0x00 2933633583SPaul Cercueil #define JZ_DMA_REG_DIRQP 0x04 3033633583SPaul Cercueil #define JZ_DMA_REG_DDR 0x08 3133633583SPaul Cercueil #define JZ_DMA_REG_DDRS 0x0c 3229870eb7SPaul Cercueil #define JZ_DMA_REG_DCKE 0x10 3329870eb7SPaul Cercueil #define JZ_DMA_REG_DCKES 0x14 3429870eb7SPaul Cercueil #define JZ_DMA_REG_DCKEC 0x18 3533633583SPaul Cercueil #define JZ_DMA_REG_DMACP 0x1c 3633633583SPaul Cercueil #define JZ_DMA_REG_DSIRQP 0x20 3733633583SPaul Cercueil #define JZ_DMA_REG_DSIRQM 0x24 3833633583SPaul Cercueil #define JZ_DMA_REG_DCIRQP 0x28 3933633583SPaul Cercueil #define JZ_DMA_REG_DCIRQM 0x2c 40d894fc60SAlex Smith 41d894fc60SAlex Smith /* Per-channel registers. */ 42d894fc60SAlex Smith #define JZ_DMA_REG_CHAN(n) (n * 0x20) 4333633583SPaul Cercueil #define JZ_DMA_REG_DSA 0x00 4433633583SPaul Cercueil #define JZ_DMA_REG_DTA 0x04 4533633583SPaul Cercueil #define JZ_DMA_REG_DTC 0x08 4633633583SPaul Cercueil #define JZ_DMA_REG_DRT 0x0c 4733633583SPaul Cercueil #define JZ_DMA_REG_DCS 0x10 4833633583SPaul Cercueil #define JZ_DMA_REG_DCM 0x14 4933633583SPaul Cercueil #define JZ_DMA_REG_DDA 0x18 5033633583SPaul Cercueil #define JZ_DMA_REG_DSD 0x1c 51d894fc60SAlex Smith 52d894fc60SAlex Smith #define JZ_DMA_DMAC_DMAE BIT(0) 53d894fc60SAlex Smith #define JZ_DMA_DMAC_AR BIT(2) 54d894fc60SAlex Smith #define JZ_DMA_DMAC_HLT BIT(3) 5517a8e30eSPaul Cercueil #define JZ_DMA_DMAC_FAIC BIT(27) 56d894fc60SAlex Smith #define JZ_DMA_DMAC_FMSC BIT(31) 57d894fc60SAlex Smith 58d894fc60SAlex Smith #define JZ_DMA_DRT_AUTO 0x8 59d894fc60SAlex Smith 60d894fc60SAlex Smith #define JZ_DMA_DCS_CTE BIT(0) 61d894fc60SAlex Smith #define JZ_DMA_DCS_HLT BIT(2) 62d894fc60SAlex Smith #define JZ_DMA_DCS_TT BIT(3) 63d894fc60SAlex Smith #define JZ_DMA_DCS_AR BIT(4) 64d894fc60SAlex Smith #define JZ_DMA_DCS_DES8 BIT(30) 65d894fc60SAlex Smith 66d894fc60SAlex Smith #define JZ_DMA_DCM_LINK BIT(0) 67d894fc60SAlex Smith #define JZ_DMA_DCM_TIE BIT(1) 68d894fc60SAlex Smith #define JZ_DMA_DCM_STDE BIT(2) 69d894fc60SAlex Smith #define JZ_DMA_DCM_TSZ_SHIFT 8 70d894fc60SAlex Smith #define JZ_DMA_DCM_TSZ_MASK (0x7 << JZ_DMA_DCM_TSZ_SHIFT) 71d894fc60SAlex Smith #define JZ_DMA_DCM_DP_SHIFT 12 72d894fc60SAlex Smith #define JZ_DMA_DCM_SP_SHIFT 14 73d894fc60SAlex Smith #define JZ_DMA_DCM_DAI BIT(22) 74d894fc60SAlex Smith #define JZ_DMA_DCM_SAI BIT(23) 75d894fc60SAlex Smith 76d894fc60SAlex Smith #define JZ_DMA_SIZE_4_BYTE 0x0 77d894fc60SAlex Smith #define JZ_DMA_SIZE_1_BYTE 0x1 78d894fc60SAlex Smith #define JZ_DMA_SIZE_2_BYTE 0x2 79d894fc60SAlex Smith #define JZ_DMA_SIZE_16_BYTE 0x3 80d894fc60SAlex Smith #define JZ_DMA_SIZE_32_BYTE 0x4 81d894fc60SAlex Smith #define JZ_DMA_SIZE_64_BYTE 0x5 82d894fc60SAlex Smith #define JZ_DMA_SIZE_128_BYTE 0x6 83d894fc60SAlex Smith 84d894fc60SAlex Smith #define JZ_DMA_WIDTH_32_BIT 0x0 85d894fc60SAlex Smith #define JZ_DMA_WIDTH_8_BIT 0x1 86d894fc60SAlex Smith #define JZ_DMA_WIDTH_16_BIT 0x2 87d894fc60SAlex Smith 88d894fc60SAlex Smith #define JZ_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 89d894fc60SAlex Smith BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 90d894fc60SAlex Smith BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) 91d894fc60SAlex Smith 9233633583SPaul Cercueil #define JZ4780_DMA_CTRL_OFFSET 0x1000 9333633583SPaul Cercueil 9429870eb7SPaul Cercueil /* macros for use with jz4780_dma_soc_data.flags */ 9529870eb7SPaul Cercueil #define JZ_SOC_DATA_ALLOW_LEGACY_DT BIT(0) 9629870eb7SPaul Cercueil #define JZ_SOC_DATA_PROGRAMMABLE_DMA BIT(1) 9729870eb7SPaul Cercueil #define JZ_SOC_DATA_PER_CHAN_PM BIT(2) 98ae9156b6SPaul Cercueil #define JZ_SOC_DATA_NO_DCKES_DCKEC BIT(3) 9929870eb7SPaul Cercueil 100d894fc60SAlex Smith /** 101d894fc60SAlex Smith * struct jz4780_dma_hwdesc - descriptor structure read by the DMA controller. 102d894fc60SAlex Smith * @dcm: value for the DCM (channel command) register 103d894fc60SAlex Smith * @dsa: source address 104d894fc60SAlex Smith * @dta: target address 105d894fc60SAlex Smith * @dtc: transfer count (number of blocks of the transfer size specified in DCM 106d894fc60SAlex Smith * to transfer) in the low 24 bits, offset of the next descriptor from the 107d894fc60SAlex Smith * descriptor base address in the upper 8 bits. 108d894fc60SAlex Smith */ 109d894fc60SAlex Smith struct jz4780_dma_hwdesc { 110d894fc60SAlex Smith uint32_t dcm; 111d894fc60SAlex Smith uint32_t dsa; 112d894fc60SAlex Smith uint32_t dta; 113d894fc60SAlex Smith uint32_t dtc; 114d894fc60SAlex Smith }; 115d894fc60SAlex Smith 116d894fc60SAlex Smith /* Size of allocations for hardware descriptor blocks. */ 117d894fc60SAlex Smith #define JZ_DMA_DESC_BLOCK_SIZE PAGE_SIZE 118d894fc60SAlex Smith #define JZ_DMA_MAX_DESC \ 119d894fc60SAlex Smith (JZ_DMA_DESC_BLOCK_SIZE / sizeof(struct jz4780_dma_hwdesc)) 120d894fc60SAlex Smith 121d894fc60SAlex Smith struct jz4780_dma_desc { 122d894fc60SAlex Smith struct virt_dma_desc vdesc; 123d894fc60SAlex Smith 124d894fc60SAlex Smith struct jz4780_dma_hwdesc *desc; 125d894fc60SAlex Smith dma_addr_t desc_phys; 126d894fc60SAlex Smith unsigned int count; 127d894fc60SAlex Smith enum dma_transaction_type type; 128d894fc60SAlex Smith uint32_t status; 129d894fc60SAlex Smith }; 130d894fc60SAlex Smith 131d894fc60SAlex Smith struct jz4780_dma_chan { 132d894fc60SAlex Smith struct virt_dma_chan vchan; 133d894fc60SAlex Smith unsigned int id; 134d894fc60SAlex Smith struct dma_pool *desc_pool; 135d894fc60SAlex Smith 136d894fc60SAlex Smith uint32_t transfer_type; 137d894fc60SAlex Smith uint32_t transfer_shift; 138d894fc60SAlex Smith struct dma_slave_config config; 139d894fc60SAlex Smith 140d894fc60SAlex Smith struct jz4780_dma_desc *desc; 141d894fc60SAlex Smith unsigned int curr_hwdesc; 142d894fc60SAlex Smith }; 143d894fc60SAlex Smith 1446147b032SPaul Cercueil struct jz4780_dma_soc_data { 1456147b032SPaul Cercueil unsigned int nb_channels; 14629870eb7SPaul Cercueil unsigned int transfer_ord_max; 14729870eb7SPaul Cercueil unsigned long flags; 1486147b032SPaul Cercueil }; 1496147b032SPaul Cercueil 150d894fc60SAlex Smith struct jz4780_dma_dev { 151d894fc60SAlex Smith struct dma_device dma_device; 15233633583SPaul Cercueil void __iomem *chn_base; 15333633583SPaul Cercueil void __iomem *ctrl_base; 154d894fc60SAlex Smith struct clk *clk; 155d894fc60SAlex Smith unsigned int irq; 1566147b032SPaul Cercueil const struct jz4780_dma_soc_data *soc_data; 157d894fc60SAlex Smith 158d894fc60SAlex Smith uint32_t chan_reserved; 1596147b032SPaul Cercueil struct jz4780_dma_chan chan[]; 160d894fc60SAlex Smith }; 161d894fc60SAlex Smith 162026fd406SAlex Smith struct jz4780_dma_filter_data { 163026fd406SAlex Smith struct device_node *of_node; 164d894fc60SAlex Smith uint32_t transfer_type; 165d894fc60SAlex Smith int channel; 166d894fc60SAlex Smith }; 167d894fc60SAlex Smith 168d894fc60SAlex Smith static inline struct jz4780_dma_chan *to_jz4780_dma_chan(struct dma_chan *chan) 169d894fc60SAlex Smith { 170d894fc60SAlex Smith return container_of(chan, struct jz4780_dma_chan, vchan.chan); 171d894fc60SAlex Smith } 172d894fc60SAlex Smith 173d894fc60SAlex Smith static inline struct jz4780_dma_desc *to_jz4780_dma_desc( 174d894fc60SAlex Smith struct virt_dma_desc *vdesc) 175d894fc60SAlex Smith { 176d894fc60SAlex Smith return container_of(vdesc, struct jz4780_dma_desc, vdesc); 177d894fc60SAlex Smith } 178d894fc60SAlex Smith 179d894fc60SAlex Smith static inline struct jz4780_dma_dev *jz4780_dma_chan_parent( 180d894fc60SAlex Smith struct jz4780_dma_chan *jzchan) 181d894fc60SAlex Smith { 182d894fc60SAlex Smith return container_of(jzchan->vchan.chan.device, struct jz4780_dma_dev, 183d894fc60SAlex Smith dma_device); 184d894fc60SAlex Smith } 185d894fc60SAlex Smith 18633633583SPaul Cercueil static inline uint32_t jz4780_dma_chn_readl(struct jz4780_dma_dev *jzdma, 18733633583SPaul Cercueil unsigned int chn, unsigned int reg) 188d894fc60SAlex Smith { 18933633583SPaul Cercueil return readl(jzdma->chn_base + reg + JZ_DMA_REG_CHAN(chn)); 190d894fc60SAlex Smith } 191d894fc60SAlex Smith 19233633583SPaul Cercueil static inline void jz4780_dma_chn_writel(struct jz4780_dma_dev *jzdma, 19333633583SPaul Cercueil unsigned int chn, unsigned int reg, uint32_t val) 19433633583SPaul Cercueil { 19533633583SPaul Cercueil writel(val, jzdma->chn_base + reg + JZ_DMA_REG_CHAN(chn)); 19633633583SPaul Cercueil } 19733633583SPaul Cercueil 19833633583SPaul Cercueil static inline uint32_t jz4780_dma_ctrl_readl(struct jz4780_dma_dev *jzdma, 19933633583SPaul Cercueil unsigned int reg) 20033633583SPaul Cercueil { 20133633583SPaul Cercueil return readl(jzdma->ctrl_base + reg); 20233633583SPaul Cercueil } 20333633583SPaul Cercueil 20433633583SPaul Cercueil static inline void jz4780_dma_ctrl_writel(struct jz4780_dma_dev *jzdma, 205d894fc60SAlex Smith unsigned int reg, uint32_t val) 206d894fc60SAlex Smith { 20733633583SPaul Cercueil writel(val, jzdma->ctrl_base + reg); 208d894fc60SAlex Smith } 209d894fc60SAlex Smith 21029870eb7SPaul Cercueil static inline void jz4780_dma_chan_enable(struct jz4780_dma_dev *jzdma, 21129870eb7SPaul Cercueil unsigned int chn) 21229870eb7SPaul Cercueil { 213ae9156b6SPaul Cercueil if (jzdma->soc_data->flags & JZ_SOC_DATA_PER_CHAN_PM) { 214ae9156b6SPaul Cercueil unsigned int reg; 215ae9156b6SPaul Cercueil 216ae9156b6SPaul Cercueil if (jzdma->soc_data->flags & JZ_SOC_DATA_NO_DCKES_DCKEC) 217ae9156b6SPaul Cercueil reg = JZ_DMA_REG_DCKE; 218ae9156b6SPaul Cercueil else 219ae9156b6SPaul Cercueil reg = JZ_DMA_REG_DCKES; 220ae9156b6SPaul Cercueil 221ae9156b6SPaul Cercueil jz4780_dma_ctrl_writel(jzdma, reg, BIT(chn)); 222ae9156b6SPaul Cercueil } 22329870eb7SPaul Cercueil } 22429870eb7SPaul Cercueil 22529870eb7SPaul Cercueil static inline void jz4780_dma_chan_disable(struct jz4780_dma_dev *jzdma, 22629870eb7SPaul Cercueil unsigned int chn) 22729870eb7SPaul Cercueil { 228ae9156b6SPaul Cercueil if ((jzdma->soc_data->flags & JZ_SOC_DATA_PER_CHAN_PM) && 229ae9156b6SPaul Cercueil !(jzdma->soc_data->flags & JZ_SOC_DATA_NO_DCKES_DCKEC)) 23029870eb7SPaul Cercueil jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DCKEC, BIT(chn)); 23129870eb7SPaul Cercueil } 23229870eb7SPaul Cercueil 233d894fc60SAlex Smith static struct jz4780_dma_desc *jz4780_dma_desc_alloc( 234d894fc60SAlex Smith struct jz4780_dma_chan *jzchan, unsigned int count, 235d894fc60SAlex Smith enum dma_transaction_type type) 236d894fc60SAlex Smith { 237d894fc60SAlex Smith struct jz4780_dma_desc *desc; 238d894fc60SAlex Smith 239d894fc60SAlex Smith if (count > JZ_DMA_MAX_DESC) 240d894fc60SAlex Smith return NULL; 241d894fc60SAlex Smith 242d894fc60SAlex Smith desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 243d894fc60SAlex Smith if (!desc) 244d894fc60SAlex Smith return NULL; 245d894fc60SAlex Smith 246d894fc60SAlex Smith desc->desc = dma_pool_alloc(jzchan->desc_pool, GFP_NOWAIT, 247d894fc60SAlex Smith &desc->desc_phys); 248d894fc60SAlex Smith if (!desc->desc) { 249d894fc60SAlex Smith kfree(desc); 250d894fc60SAlex Smith return NULL; 251d894fc60SAlex Smith } 252d894fc60SAlex Smith 253d894fc60SAlex Smith desc->count = count; 254d894fc60SAlex Smith desc->type = type; 255d894fc60SAlex Smith return desc; 256d894fc60SAlex Smith } 257d894fc60SAlex Smith 258d894fc60SAlex Smith static void jz4780_dma_desc_free(struct virt_dma_desc *vdesc) 259d894fc60SAlex Smith { 260d894fc60SAlex Smith struct jz4780_dma_desc *desc = to_jz4780_dma_desc(vdesc); 261d894fc60SAlex Smith struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(vdesc->tx.chan); 262d894fc60SAlex Smith 263d894fc60SAlex Smith dma_pool_free(jzchan->desc_pool, desc->desc, desc->desc_phys); 264d894fc60SAlex Smith kfree(desc); 265d894fc60SAlex Smith } 266d894fc60SAlex Smith 26729870eb7SPaul Cercueil static uint32_t jz4780_dma_transfer_size(struct jz4780_dma_chan *jzchan, 26829870eb7SPaul Cercueil unsigned long val, uint32_t *shift) 269d894fc60SAlex Smith { 27029870eb7SPaul Cercueil struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 271dc578f31SAlex Smith int ord = ffs(val) - 1; 272d894fc60SAlex Smith 273dc578f31SAlex Smith /* 274dc578f31SAlex Smith * 8 byte transfer sizes unsupported so fall back on 4. If it's larger 275dc578f31SAlex Smith * than the maximum, just limit it. It is perfectly safe to fall back 276dc578f31SAlex Smith * in this way since we won't exceed the maximum burst size supported 277dc578f31SAlex Smith * by the device, the only effect is reduced efficiency. This is better 278dc578f31SAlex Smith * than refusing to perform the request at all. 279dc578f31SAlex Smith */ 280dc578f31SAlex Smith if (ord == 3) 281dc578f31SAlex Smith ord = 2; 28229870eb7SPaul Cercueil else if (ord > jzdma->soc_data->transfer_ord_max) 28329870eb7SPaul Cercueil ord = jzdma->soc_data->transfer_ord_max; 284dc578f31SAlex Smith 285dc578f31SAlex Smith *shift = ord; 286dc578f31SAlex Smith 287dc578f31SAlex Smith switch (ord) { 288d894fc60SAlex Smith case 0: 289d894fc60SAlex Smith return JZ_DMA_SIZE_1_BYTE; 290d894fc60SAlex Smith case 1: 291d894fc60SAlex Smith return JZ_DMA_SIZE_2_BYTE; 292d894fc60SAlex Smith case 2: 293d894fc60SAlex Smith return JZ_DMA_SIZE_4_BYTE; 294d894fc60SAlex Smith case 4: 295d894fc60SAlex Smith return JZ_DMA_SIZE_16_BYTE; 296d894fc60SAlex Smith case 5: 297d894fc60SAlex Smith return JZ_DMA_SIZE_32_BYTE; 298d894fc60SAlex Smith case 6: 299d894fc60SAlex Smith return JZ_DMA_SIZE_64_BYTE; 300d894fc60SAlex Smith default: 301dc578f31SAlex Smith return JZ_DMA_SIZE_128_BYTE; 302d894fc60SAlex Smith } 303d894fc60SAlex Smith } 304d894fc60SAlex Smith 305839896efSAlex Smith static int jz4780_dma_setup_hwdesc(struct jz4780_dma_chan *jzchan, 306d894fc60SAlex Smith struct jz4780_dma_hwdesc *desc, dma_addr_t addr, size_t len, 307d894fc60SAlex Smith enum dma_transfer_direction direction) 308d894fc60SAlex Smith { 309d894fc60SAlex Smith struct dma_slave_config *config = &jzchan->config; 310d894fc60SAlex Smith uint32_t width, maxburst, tsz; 311d894fc60SAlex Smith 312d894fc60SAlex Smith if (direction == DMA_MEM_TO_DEV) { 313d894fc60SAlex Smith desc->dcm = JZ_DMA_DCM_SAI; 314d894fc60SAlex Smith desc->dsa = addr; 315d894fc60SAlex Smith desc->dta = config->dst_addr; 316d894fc60SAlex Smith 317d894fc60SAlex Smith width = config->dst_addr_width; 318d894fc60SAlex Smith maxburst = config->dst_maxburst; 319d894fc60SAlex Smith } else { 320d894fc60SAlex Smith desc->dcm = JZ_DMA_DCM_DAI; 321d894fc60SAlex Smith desc->dsa = config->src_addr; 322d894fc60SAlex Smith desc->dta = addr; 323d894fc60SAlex Smith 324d894fc60SAlex Smith width = config->src_addr_width; 325d894fc60SAlex Smith maxburst = config->src_maxburst; 326d894fc60SAlex Smith } 327d894fc60SAlex Smith 328d894fc60SAlex Smith /* 329d894fc60SAlex Smith * This calculates the maximum transfer size that can be used with the 330d894fc60SAlex Smith * given address, length, width and maximum burst size. The address 331d894fc60SAlex Smith * must be aligned to the transfer size, the total length must be 332d894fc60SAlex Smith * divisible by the transfer size, and we must not use more than the 333d894fc60SAlex Smith * maximum burst specified by the user. 334d894fc60SAlex Smith */ 33529870eb7SPaul Cercueil tsz = jz4780_dma_transfer_size(jzchan, addr | len | (width * maxburst), 336dc578f31SAlex Smith &jzchan->transfer_shift); 337d894fc60SAlex Smith 338d894fc60SAlex Smith switch (width) { 339d894fc60SAlex Smith case DMA_SLAVE_BUSWIDTH_1_BYTE: 340d894fc60SAlex Smith case DMA_SLAVE_BUSWIDTH_2_BYTES: 341d894fc60SAlex Smith break; 342d894fc60SAlex Smith case DMA_SLAVE_BUSWIDTH_4_BYTES: 343d894fc60SAlex Smith width = JZ_DMA_WIDTH_32_BIT; 344d894fc60SAlex Smith break; 345d894fc60SAlex Smith default: 346d894fc60SAlex Smith return -EINVAL; 347d894fc60SAlex Smith } 348d894fc60SAlex Smith 349d894fc60SAlex Smith desc->dcm |= tsz << JZ_DMA_DCM_TSZ_SHIFT; 350d894fc60SAlex Smith desc->dcm |= width << JZ_DMA_DCM_SP_SHIFT; 351d894fc60SAlex Smith desc->dcm |= width << JZ_DMA_DCM_DP_SHIFT; 352d894fc60SAlex Smith 353dc578f31SAlex Smith desc->dtc = len >> jzchan->transfer_shift; 354839896efSAlex Smith return 0; 355d894fc60SAlex Smith } 356d894fc60SAlex Smith 357d894fc60SAlex Smith static struct dma_async_tx_descriptor *jz4780_dma_prep_slave_sg( 358d894fc60SAlex Smith struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, 35946fa5168SAlex Smith enum dma_transfer_direction direction, unsigned long flags, 36046fa5168SAlex Smith void *context) 361d894fc60SAlex Smith { 362d894fc60SAlex Smith struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 363d894fc60SAlex Smith struct jz4780_dma_desc *desc; 364d894fc60SAlex Smith unsigned int i; 365d894fc60SAlex Smith int err; 366d894fc60SAlex Smith 367d894fc60SAlex Smith desc = jz4780_dma_desc_alloc(jzchan, sg_len, DMA_SLAVE); 368d894fc60SAlex Smith if (!desc) 369d894fc60SAlex Smith return NULL; 370d894fc60SAlex Smith 371d894fc60SAlex Smith for (i = 0; i < sg_len; i++) { 372d894fc60SAlex Smith err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], 373d894fc60SAlex Smith sg_dma_address(&sgl[i]), 374d894fc60SAlex Smith sg_dma_len(&sgl[i]), 375d894fc60SAlex Smith direction); 376fc878efeSColin Ian King if (err < 0) { 377fc878efeSColin Ian King jz4780_dma_desc_free(&jzchan->desc->vdesc); 378839896efSAlex Smith return NULL; 379fc878efeSColin Ian King } 380d894fc60SAlex Smith 381d894fc60SAlex Smith desc->desc[i].dcm |= JZ_DMA_DCM_TIE; 382d894fc60SAlex Smith 383d894fc60SAlex Smith if (i != (sg_len - 1)) { 384d894fc60SAlex Smith /* Automatically proceeed to the next descriptor. */ 385d894fc60SAlex Smith desc->desc[i].dcm |= JZ_DMA_DCM_LINK; 386d894fc60SAlex Smith 387d894fc60SAlex Smith /* 388d894fc60SAlex Smith * The upper 8 bits of the DTC field in the descriptor 389d894fc60SAlex Smith * must be set to (offset from descriptor base of next 390d894fc60SAlex Smith * descriptor >> 4). 391d894fc60SAlex Smith */ 392d894fc60SAlex Smith desc->desc[i].dtc |= 393d894fc60SAlex Smith (((i + 1) * sizeof(*desc->desc)) >> 4) << 24; 394d894fc60SAlex Smith } 395d894fc60SAlex Smith } 396d894fc60SAlex Smith 397d894fc60SAlex Smith return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); 398d894fc60SAlex Smith } 399d894fc60SAlex Smith 400d894fc60SAlex Smith static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_cyclic( 401d894fc60SAlex Smith struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 402d894fc60SAlex Smith size_t period_len, enum dma_transfer_direction direction, 403d894fc60SAlex Smith unsigned long flags) 404d894fc60SAlex Smith { 405d894fc60SAlex Smith struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 406d894fc60SAlex Smith struct jz4780_dma_desc *desc; 407d894fc60SAlex Smith unsigned int periods, i; 408d894fc60SAlex Smith int err; 409d894fc60SAlex Smith 410d894fc60SAlex Smith if (buf_len % period_len) 411d894fc60SAlex Smith return NULL; 412d894fc60SAlex Smith 413d894fc60SAlex Smith periods = buf_len / period_len; 414d894fc60SAlex Smith 415d894fc60SAlex Smith desc = jz4780_dma_desc_alloc(jzchan, periods, DMA_CYCLIC); 416d894fc60SAlex Smith if (!desc) 417d894fc60SAlex Smith return NULL; 418d894fc60SAlex Smith 419d894fc60SAlex Smith for (i = 0; i < periods; i++) { 420d894fc60SAlex Smith err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], buf_addr, 421d894fc60SAlex Smith period_len, direction); 422fc878efeSColin Ian King if (err < 0) { 423fc878efeSColin Ian King jz4780_dma_desc_free(&jzchan->desc->vdesc); 424839896efSAlex Smith return NULL; 425fc878efeSColin Ian King } 426d894fc60SAlex Smith 427d894fc60SAlex Smith buf_addr += period_len; 428d894fc60SAlex Smith 429d894fc60SAlex Smith /* 430d894fc60SAlex Smith * Set the link bit to indicate that the controller should 431d894fc60SAlex Smith * automatically proceed to the next descriptor. In 432d894fc60SAlex Smith * jz4780_dma_begin(), this will be cleared if we need to issue 433d894fc60SAlex Smith * an interrupt after each period. 434d894fc60SAlex Smith */ 435d894fc60SAlex Smith desc->desc[i].dcm |= JZ_DMA_DCM_TIE | JZ_DMA_DCM_LINK; 436d894fc60SAlex Smith 437d894fc60SAlex Smith /* 438d894fc60SAlex Smith * The upper 8 bits of the DTC field in the descriptor must be 439d894fc60SAlex Smith * set to (offset from descriptor base of next descriptor >> 4). 440d894fc60SAlex Smith * If this is the last descriptor, link it back to the first, 441d894fc60SAlex Smith * i.e. leave offset set to 0, otherwise point to the next one. 442d894fc60SAlex Smith */ 443d894fc60SAlex Smith if (i != (periods - 1)) { 444d894fc60SAlex Smith desc->desc[i].dtc |= 445d894fc60SAlex Smith (((i + 1) * sizeof(*desc->desc)) >> 4) << 24; 446d894fc60SAlex Smith } 447d894fc60SAlex Smith } 448d894fc60SAlex Smith 449d894fc60SAlex Smith return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); 450d894fc60SAlex Smith } 451d894fc60SAlex Smith 4524f5db8c8SVinod Koul static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_memcpy( 453d894fc60SAlex Smith struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 454d894fc60SAlex Smith size_t len, unsigned long flags) 455d894fc60SAlex Smith { 456d894fc60SAlex Smith struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 457d894fc60SAlex Smith struct jz4780_dma_desc *desc; 458d894fc60SAlex Smith uint32_t tsz; 459d894fc60SAlex Smith 460d894fc60SAlex Smith desc = jz4780_dma_desc_alloc(jzchan, 1, DMA_MEMCPY); 461d894fc60SAlex Smith if (!desc) 462d894fc60SAlex Smith return NULL; 463d894fc60SAlex Smith 46429870eb7SPaul Cercueil tsz = jz4780_dma_transfer_size(jzchan, dest | src | len, 465dc578f31SAlex Smith &jzchan->transfer_shift); 466d894fc60SAlex Smith 4675eed7d84SPaul Cercueil jzchan->transfer_type = JZ_DMA_DRT_AUTO; 4685eed7d84SPaul Cercueil 469d894fc60SAlex Smith desc->desc[0].dsa = src; 470d894fc60SAlex Smith desc->desc[0].dta = dest; 471d894fc60SAlex Smith desc->desc[0].dcm = JZ_DMA_DCM_TIE | JZ_DMA_DCM_SAI | JZ_DMA_DCM_DAI | 472d894fc60SAlex Smith tsz << JZ_DMA_DCM_TSZ_SHIFT | 473d894fc60SAlex Smith JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_SP_SHIFT | 474d894fc60SAlex Smith JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_DP_SHIFT; 475839896efSAlex Smith desc->desc[0].dtc = len >> jzchan->transfer_shift; 476d894fc60SAlex Smith 477d894fc60SAlex Smith return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); 478d894fc60SAlex Smith } 479d894fc60SAlex Smith 480d894fc60SAlex Smith static void jz4780_dma_begin(struct jz4780_dma_chan *jzchan) 481d894fc60SAlex Smith { 482d894fc60SAlex Smith struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 483d894fc60SAlex Smith struct virt_dma_desc *vdesc; 484d894fc60SAlex Smith unsigned int i; 485d894fc60SAlex Smith dma_addr_t desc_phys; 486d894fc60SAlex Smith 487d894fc60SAlex Smith if (!jzchan->desc) { 488d894fc60SAlex Smith vdesc = vchan_next_desc(&jzchan->vchan); 489d894fc60SAlex Smith if (!vdesc) 490d894fc60SAlex Smith return; 491d894fc60SAlex Smith 492d894fc60SAlex Smith list_del(&vdesc->node); 493d894fc60SAlex Smith 494d894fc60SAlex Smith jzchan->desc = to_jz4780_dma_desc(vdesc); 495d894fc60SAlex Smith jzchan->curr_hwdesc = 0; 496d894fc60SAlex Smith 497d894fc60SAlex Smith if (jzchan->desc->type == DMA_CYCLIC && vdesc->tx.callback) { 498d894fc60SAlex Smith /* 499d894fc60SAlex Smith * The DMA controller doesn't support triggering an 500d894fc60SAlex Smith * interrupt after processing each descriptor, only 501d894fc60SAlex Smith * after processing an entire terminated list of 502d894fc60SAlex Smith * descriptors. For a cyclic DMA setup the list of 503d894fc60SAlex Smith * descriptors is not terminated so we can never get an 504d894fc60SAlex Smith * interrupt. 505d894fc60SAlex Smith * 506d894fc60SAlex Smith * If the user requested a callback for a cyclic DMA 507d894fc60SAlex Smith * setup then we workaround this hardware limitation 508d894fc60SAlex Smith * here by degrading to a set of unlinked descriptors 509d894fc60SAlex Smith * which we will submit in sequence in response to the 510d894fc60SAlex Smith * completion of processing the previous descriptor. 511d894fc60SAlex Smith */ 512d894fc60SAlex Smith for (i = 0; i < jzchan->desc->count; i++) 513d894fc60SAlex Smith jzchan->desc->desc[i].dcm &= ~JZ_DMA_DCM_LINK; 514d894fc60SAlex Smith } 515d894fc60SAlex Smith } else { 516d894fc60SAlex Smith /* 517d894fc60SAlex Smith * There is an existing transfer, therefore this must be one 518d894fc60SAlex Smith * for which we unlinked the descriptors above. Advance to the 519d894fc60SAlex Smith * next one in the list. 520d894fc60SAlex Smith */ 521d894fc60SAlex Smith jzchan->curr_hwdesc = 522d894fc60SAlex Smith (jzchan->curr_hwdesc + 1) % jzchan->desc->count; 523d894fc60SAlex Smith } 524d894fc60SAlex Smith 52529870eb7SPaul Cercueil /* Enable the channel's clock. */ 52629870eb7SPaul Cercueil jz4780_dma_chan_enable(jzdma, jzchan->id); 52729870eb7SPaul Cercueil 5285eed7d84SPaul Cercueil /* Use 4-word descriptors. */ 5295eed7d84SPaul Cercueil jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0); 5305eed7d84SPaul Cercueil 5315eed7d84SPaul Cercueil /* Set transfer type. */ 5325eed7d84SPaul Cercueil jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DRT, 5335eed7d84SPaul Cercueil jzchan->transfer_type); 534d894fc60SAlex Smith 5359e4e3a4cSDaniel Silsby /* 5369e4e3a4cSDaniel Silsby * Set the transfer count. This is redundant for a descriptor-driven 5379e4e3a4cSDaniel Silsby * transfer. However, there can be a delay between the transfer start 5389e4e3a4cSDaniel Silsby * time and when DTCn reg contains the new transfer count. Setting 5399e4e3a4cSDaniel Silsby * it explicitly ensures residue is computed correctly at all times. 5409e4e3a4cSDaniel Silsby */ 5419e4e3a4cSDaniel Silsby jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DTC, 5429e4e3a4cSDaniel Silsby jzchan->desc->desc[jzchan->curr_hwdesc].dtc); 5439e4e3a4cSDaniel Silsby 544d894fc60SAlex Smith /* Write descriptor address and initiate descriptor fetch. */ 545d894fc60SAlex Smith desc_phys = jzchan->desc->desc_phys + 546d894fc60SAlex Smith (jzchan->curr_hwdesc * sizeof(*jzchan->desc->desc)); 54733633583SPaul Cercueil jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DDA, desc_phys); 54833633583SPaul Cercueil jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DDRS, BIT(jzchan->id)); 549d894fc60SAlex Smith 550d894fc60SAlex Smith /* Enable the channel. */ 55133633583SPaul Cercueil jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 5525eed7d84SPaul Cercueil JZ_DMA_DCS_CTE); 553d894fc60SAlex Smith } 554d894fc60SAlex Smith 555d894fc60SAlex Smith static void jz4780_dma_issue_pending(struct dma_chan *chan) 556d894fc60SAlex Smith { 557d894fc60SAlex Smith struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 558d894fc60SAlex Smith unsigned long flags; 559d894fc60SAlex Smith 560d894fc60SAlex Smith spin_lock_irqsave(&jzchan->vchan.lock, flags); 561d894fc60SAlex Smith 562d894fc60SAlex Smith if (vchan_issue_pending(&jzchan->vchan) && !jzchan->desc) 563d894fc60SAlex Smith jz4780_dma_begin(jzchan); 564d894fc60SAlex Smith 565d894fc60SAlex Smith spin_unlock_irqrestore(&jzchan->vchan.lock, flags); 566d894fc60SAlex Smith } 567d894fc60SAlex Smith 56846fa5168SAlex Smith static int jz4780_dma_terminate_all(struct dma_chan *chan) 569d894fc60SAlex Smith { 57046fa5168SAlex Smith struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 571d894fc60SAlex Smith struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 572d894fc60SAlex Smith unsigned long flags; 573d894fc60SAlex Smith LIST_HEAD(head); 574d894fc60SAlex Smith 575d894fc60SAlex Smith spin_lock_irqsave(&jzchan->vchan.lock, flags); 576d894fc60SAlex Smith 577d894fc60SAlex Smith /* Clear the DMA status and stop the transfer. */ 57833633583SPaul Cercueil jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0); 579d894fc60SAlex Smith if (jzchan->desc) { 580f0dd52c8SPeter Ujfalusi vchan_terminate_vdesc(&jzchan->desc->vdesc); 581d894fc60SAlex Smith jzchan->desc = NULL; 582d894fc60SAlex Smith } 583d894fc60SAlex Smith 58429870eb7SPaul Cercueil jz4780_dma_chan_disable(jzdma, jzchan->id); 58529870eb7SPaul Cercueil 586d894fc60SAlex Smith vchan_get_all_descriptors(&jzchan->vchan, &head); 587d894fc60SAlex Smith 588d894fc60SAlex Smith spin_unlock_irqrestore(&jzchan->vchan.lock, flags); 589d894fc60SAlex Smith 590d894fc60SAlex Smith vchan_dma_desc_free_list(&jzchan->vchan, &head); 591d894fc60SAlex Smith return 0; 592d894fc60SAlex Smith } 593d894fc60SAlex Smith 594f0dd52c8SPeter Ujfalusi static void jz4780_dma_synchronize(struct dma_chan *chan) 595f0dd52c8SPeter Ujfalusi { 596f0dd52c8SPeter Ujfalusi struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 59729870eb7SPaul Cercueil struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 598f0dd52c8SPeter Ujfalusi 599f0dd52c8SPeter Ujfalusi vchan_synchronize(&jzchan->vchan); 60029870eb7SPaul Cercueil jz4780_dma_chan_disable(jzdma, jzchan->id); 601f0dd52c8SPeter Ujfalusi } 602f0dd52c8SPeter Ujfalusi 60346fa5168SAlex Smith static int jz4780_dma_config(struct dma_chan *chan, 60446fa5168SAlex Smith struct dma_slave_config *config) 605d894fc60SAlex Smith { 60646fa5168SAlex Smith struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 60746fa5168SAlex Smith 608d894fc60SAlex Smith if ((config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) 609d894fc60SAlex Smith || (config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)) 610d894fc60SAlex Smith return -EINVAL; 611d894fc60SAlex Smith 612d894fc60SAlex Smith /* Copy the reset of the slave configuration, it is used later. */ 613d894fc60SAlex Smith memcpy(&jzchan->config, config, sizeof(jzchan->config)); 614d894fc60SAlex Smith 615d894fc60SAlex Smith return 0; 616d894fc60SAlex Smith } 617d894fc60SAlex Smith 618d894fc60SAlex Smith static size_t jz4780_dma_desc_residue(struct jz4780_dma_chan *jzchan, 619d894fc60SAlex Smith struct jz4780_dma_desc *desc, unsigned int next_sg) 620d894fc60SAlex Smith { 621d894fc60SAlex Smith struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 622f3c045dfSDaniel Silsby unsigned int count = 0; 623d894fc60SAlex Smith unsigned int i; 624d894fc60SAlex Smith 625d894fc60SAlex Smith for (i = next_sg; i < desc->count; i++) 626f3c045dfSDaniel Silsby count += desc->desc[i].dtc & GENMASK(23, 0); 627d894fc60SAlex Smith 628f3c045dfSDaniel Silsby if (next_sg != 0) 629f3c045dfSDaniel Silsby count += jz4780_dma_chn_readl(jzdma, jzchan->id, 63033633583SPaul Cercueil JZ_DMA_REG_DTC); 631d894fc60SAlex Smith 632f3c045dfSDaniel Silsby return count << jzchan->transfer_shift; 633d894fc60SAlex Smith } 634d894fc60SAlex Smith 635d894fc60SAlex Smith static enum dma_status jz4780_dma_tx_status(struct dma_chan *chan, 636d894fc60SAlex Smith dma_cookie_t cookie, struct dma_tx_state *txstate) 637d894fc60SAlex Smith { 638d894fc60SAlex Smith struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 639d894fc60SAlex Smith struct virt_dma_desc *vdesc; 640d894fc60SAlex Smith enum dma_status status; 641d894fc60SAlex Smith unsigned long flags; 642d894fc60SAlex Smith 643d894fc60SAlex Smith status = dma_cookie_status(chan, cookie, txstate); 644d894fc60SAlex Smith if ((status == DMA_COMPLETE) || (txstate == NULL)) 645d894fc60SAlex Smith return status; 646d894fc60SAlex Smith 647d894fc60SAlex Smith spin_lock_irqsave(&jzchan->vchan.lock, flags); 648d894fc60SAlex Smith 649d894fc60SAlex Smith vdesc = vchan_find_desc(&jzchan->vchan, cookie); 650d894fc60SAlex Smith if (vdesc) { 651d894fc60SAlex Smith /* On the issued list, so hasn't been processed yet */ 652d894fc60SAlex Smith txstate->residue = jz4780_dma_desc_residue(jzchan, 653d894fc60SAlex Smith to_jz4780_dma_desc(vdesc), 0); 654d894fc60SAlex Smith } else if (cookie == jzchan->desc->vdesc.tx.cookie) { 655d894fc60SAlex Smith txstate->residue = jz4780_dma_desc_residue(jzchan, jzchan->desc, 65683ef4fb7SDaniel Silsby jzchan->curr_hwdesc + 1); 657d894fc60SAlex Smith } else 658d894fc60SAlex Smith txstate->residue = 0; 659d894fc60SAlex Smith 660d894fc60SAlex Smith if (vdesc && jzchan->desc && vdesc == &jzchan->desc->vdesc 661d894fc60SAlex Smith && jzchan->desc->status & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) 662d894fc60SAlex Smith status = DMA_ERROR; 663d894fc60SAlex Smith 664d894fc60SAlex Smith spin_unlock_irqrestore(&jzchan->vchan.lock, flags); 665d894fc60SAlex Smith return status; 666d894fc60SAlex Smith } 667d894fc60SAlex Smith 668d894fc60SAlex Smith static void jz4780_dma_chan_irq(struct jz4780_dma_dev *jzdma, 669d894fc60SAlex Smith struct jz4780_dma_chan *jzchan) 670d894fc60SAlex Smith { 671d894fc60SAlex Smith uint32_t dcs; 672d894fc60SAlex Smith 673d894fc60SAlex Smith spin_lock(&jzchan->vchan.lock); 674d894fc60SAlex Smith 67533633583SPaul Cercueil dcs = jz4780_dma_chn_readl(jzdma, jzchan->id, JZ_DMA_REG_DCS); 67633633583SPaul Cercueil jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0); 677d894fc60SAlex Smith 678d894fc60SAlex Smith if (dcs & JZ_DMA_DCS_AR) { 679d894fc60SAlex Smith dev_warn(&jzchan->vchan.chan.dev->device, 680d894fc60SAlex Smith "address error (DCS=0x%x)\n", dcs); 681d894fc60SAlex Smith } 682d894fc60SAlex Smith 683d894fc60SAlex Smith if (dcs & JZ_DMA_DCS_HLT) { 684d894fc60SAlex Smith dev_warn(&jzchan->vchan.chan.dev->device, 685d894fc60SAlex Smith "channel halt (DCS=0x%x)\n", dcs); 686d894fc60SAlex Smith } 687d894fc60SAlex Smith 688d894fc60SAlex Smith if (jzchan->desc) { 689d894fc60SAlex Smith jzchan->desc->status = dcs; 690d894fc60SAlex Smith 691d894fc60SAlex Smith if ((dcs & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) == 0) { 692d894fc60SAlex Smith if (jzchan->desc->type == DMA_CYCLIC) { 693d894fc60SAlex Smith vchan_cyclic_callback(&jzchan->desc->vdesc); 694d894fc60SAlex Smith } else { 695d894fc60SAlex Smith vchan_cookie_complete(&jzchan->desc->vdesc); 696d894fc60SAlex Smith jzchan->desc = NULL; 697d894fc60SAlex Smith } 698d894fc60SAlex Smith 699d894fc60SAlex Smith jz4780_dma_begin(jzchan); 700d894fc60SAlex Smith } 701d894fc60SAlex Smith } else { 702d894fc60SAlex Smith dev_err(&jzchan->vchan.chan.dev->device, 703d894fc60SAlex Smith "channel IRQ with no active transfer\n"); 704d894fc60SAlex Smith } 705d894fc60SAlex Smith 706d894fc60SAlex Smith spin_unlock(&jzchan->vchan.lock); 707d894fc60SAlex Smith } 708d894fc60SAlex Smith 709d894fc60SAlex Smith static irqreturn_t jz4780_dma_irq_handler(int irq, void *data) 710d894fc60SAlex Smith { 711d894fc60SAlex Smith struct jz4780_dma_dev *jzdma = data; 712d894fc60SAlex Smith uint32_t pending, dmac; 713d894fc60SAlex Smith int i; 714d894fc60SAlex Smith 71533633583SPaul Cercueil pending = jz4780_dma_ctrl_readl(jzdma, JZ_DMA_REG_DIRQP); 716d894fc60SAlex Smith 7176147b032SPaul Cercueil for (i = 0; i < jzdma->soc_data->nb_channels; i++) { 718d894fc60SAlex Smith if (!(pending & (1<<i))) 719d894fc60SAlex Smith continue; 720d894fc60SAlex Smith 721d894fc60SAlex Smith jz4780_dma_chan_irq(jzdma, &jzdma->chan[i]); 722d894fc60SAlex Smith } 723d894fc60SAlex Smith 724d894fc60SAlex Smith /* Clear halt and address error status of all channels. */ 72533633583SPaul Cercueil dmac = jz4780_dma_ctrl_readl(jzdma, JZ_DMA_REG_DMAC); 726d894fc60SAlex Smith dmac &= ~(JZ_DMA_DMAC_HLT | JZ_DMA_DMAC_AR); 72733633583SPaul Cercueil jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DMAC, dmac); 728d894fc60SAlex Smith 729d894fc60SAlex Smith /* Clear interrupt pending status. */ 73033633583SPaul Cercueil jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DIRQP, 0); 731d894fc60SAlex Smith 732d894fc60SAlex Smith return IRQ_HANDLED; 733d894fc60SAlex Smith } 734d894fc60SAlex Smith 735d894fc60SAlex Smith static int jz4780_dma_alloc_chan_resources(struct dma_chan *chan) 736d894fc60SAlex Smith { 737d894fc60SAlex Smith struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 738d894fc60SAlex Smith 739d894fc60SAlex Smith jzchan->desc_pool = dma_pool_create(dev_name(&chan->dev->device), 740d894fc60SAlex Smith chan->device->dev, 741d894fc60SAlex Smith JZ_DMA_DESC_BLOCK_SIZE, 742d894fc60SAlex Smith PAGE_SIZE, 0); 743d894fc60SAlex Smith if (!jzchan->desc_pool) { 744d894fc60SAlex Smith dev_err(&chan->dev->device, 745d894fc60SAlex Smith "failed to allocate descriptor pool\n"); 746d894fc60SAlex Smith return -ENOMEM; 747d894fc60SAlex Smith } 748d894fc60SAlex Smith 749d894fc60SAlex Smith return 0; 750d894fc60SAlex Smith } 751d894fc60SAlex Smith 752d894fc60SAlex Smith static void jz4780_dma_free_chan_resources(struct dma_chan *chan) 753d894fc60SAlex Smith { 754d894fc60SAlex Smith struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 755d894fc60SAlex Smith 756d894fc60SAlex Smith vchan_free_chan_resources(&jzchan->vchan); 757d894fc60SAlex Smith dma_pool_destroy(jzchan->desc_pool); 758d894fc60SAlex Smith jzchan->desc_pool = NULL; 759d894fc60SAlex Smith } 760d894fc60SAlex Smith 761d894fc60SAlex Smith static bool jz4780_dma_filter_fn(struct dma_chan *chan, void *param) 762d894fc60SAlex Smith { 763d894fc60SAlex Smith struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 764d894fc60SAlex Smith struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 765026fd406SAlex Smith struct jz4780_dma_filter_data *data = param; 766026fd406SAlex Smith 767026fd406SAlex Smith if (jzdma->dma_device.dev->of_node != data->of_node) 768026fd406SAlex Smith return false; 769d894fc60SAlex Smith 770d894fc60SAlex Smith if (data->channel > -1) { 771d894fc60SAlex Smith if (data->channel != jzchan->id) 772d894fc60SAlex Smith return false; 773d894fc60SAlex Smith } else if (jzdma->chan_reserved & BIT(jzchan->id)) { 774d894fc60SAlex Smith return false; 775d894fc60SAlex Smith } 776d894fc60SAlex Smith 777d894fc60SAlex Smith jzchan->transfer_type = data->transfer_type; 778d894fc60SAlex Smith 779d894fc60SAlex Smith return true; 780d894fc60SAlex Smith } 781d894fc60SAlex Smith 782d894fc60SAlex Smith static struct dma_chan *jz4780_of_dma_xlate(struct of_phandle_args *dma_spec, 783d894fc60SAlex Smith struct of_dma *ofdma) 784d894fc60SAlex Smith { 785d894fc60SAlex Smith struct jz4780_dma_dev *jzdma = ofdma->of_dma_data; 786d894fc60SAlex Smith dma_cap_mask_t mask = jzdma->dma_device.cap_mask; 787026fd406SAlex Smith struct jz4780_dma_filter_data data; 788d894fc60SAlex Smith 789d894fc60SAlex Smith if (dma_spec->args_count != 2) 790d894fc60SAlex Smith return NULL; 791d894fc60SAlex Smith 792026fd406SAlex Smith data.of_node = ofdma->of_node; 793d894fc60SAlex Smith data.transfer_type = dma_spec->args[0]; 794d894fc60SAlex Smith data.channel = dma_spec->args[1]; 795d894fc60SAlex Smith 796d894fc60SAlex Smith if (data.channel > -1) { 7976147b032SPaul Cercueil if (data.channel >= jzdma->soc_data->nb_channels) { 798d894fc60SAlex Smith dev_err(jzdma->dma_device.dev, 799d894fc60SAlex Smith "device requested non-existent channel %u\n", 800d894fc60SAlex Smith data.channel); 801d894fc60SAlex Smith return NULL; 802d894fc60SAlex Smith } 803d894fc60SAlex Smith 804d894fc60SAlex Smith /* Can only select a channel marked as reserved. */ 805d894fc60SAlex Smith if (!(jzdma->chan_reserved & BIT(data.channel))) { 806d894fc60SAlex Smith dev_err(jzdma->dma_device.dev, 807d894fc60SAlex Smith "device requested unreserved channel %u\n", 808d894fc60SAlex Smith data.channel); 809d894fc60SAlex Smith return NULL; 810d894fc60SAlex Smith } 811d894fc60SAlex Smith 812d3273e10SAlex Smith jzdma->chan[data.channel].transfer_type = data.transfer_type; 813d3273e10SAlex Smith 814d3273e10SAlex Smith return dma_get_slave_channel( 815d3273e10SAlex Smith &jzdma->chan[data.channel].vchan.chan); 816d3273e10SAlex Smith } else { 817d894fc60SAlex Smith return dma_request_channel(mask, jz4780_dma_filter_fn, &data); 818d894fc60SAlex Smith } 819d3273e10SAlex Smith } 820d894fc60SAlex Smith 821d894fc60SAlex Smith static int jz4780_dma_probe(struct platform_device *pdev) 822d894fc60SAlex Smith { 823d894fc60SAlex Smith struct device *dev = &pdev->dev; 8246147b032SPaul Cercueil const struct jz4780_dma_soc_data *soc_data; 825d894fc60SAlex Smith struct jz4780_dma_dev *jzdma; 826d894fc60SAlex Smith struct jz4780_dma_chan *jzchan; 827d894fc60SAlex Smith struct dma_device *dd; 828d894fc60SAlex Smith struct resource *res; 829d894fc60SAlex Smith int i, ret; 830d894fc60SAlex Smith 83154f919a0SPaul Cercueil if (!dev->of_node) { 83254f919a0SPaul Cercueil dev_err(dev, "This driver must be probed from devicetree\n"); 83354f919a0SPaul Cercueil return -EINVAL; 83454f919a0SPaul Cercueil } 83554f919a0SPaul Cercueil 8366147b032SPaul Cercueil soc_data = device_get_match_data(dev); 8376147b032SPaul Cercueil if (!soc_data) 8386147b032SPaul Cercueil return -EINVAL; 8396147b032SPaul Cercueil 8406147b032SPaul Cercueil jzdma = devm_kzalloc(dev, sizeof(*jzdma) 8416147b032SPaul Cercueil + sizeof(*jzdma->chan) * soc_data->nb_channels, 8426147b032SPaul Cercueil GFP_KERNEL); 843d894fc60SAlex Smith if (!jzdma) 844d894fc60SAlex Smith return -ENOMEM; 845d894fc60SAlex Smith 8466147b032SPaul Cercueil jzdma->soc_data = soc_data; 847d894fc60SAlex Smith platform_set_drvdata(pdev, jzdma); 848d894fc60SAlex Smith 849d894fc60SAlex Smith res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 850d894fc60SAlex Smith if (!res) { 851d894fc60SAlex Smith dev_err(dev, "failed to get I/O memory\n"); 852d894fc60SAlex Smith return -EINVAL; 853d894fc60SAlex Smith } 854d894fc60SAlex Smith 85533633583SPaul Cercueil jzdma->chn_base = devm_ioremap_resource(dev, res); 85633633583SPaul Cercueil if (IS_ERR(jzdma->chn_base)) 85733633583SPaul Cercueil return PTR_ERR(jzdma->chn_base); 85833633583SPaul Cercueil 85933633583SPaul Cercueil res = platform_get_resource(pdev, IORESOURCE_MEM, 1); 86033633583SPaul Cercueil if (res) { 86133633583SPaul Cercueil jzdma->ctrl_base = devm_ioremap_resource(dev, res); 86233633583SPaul Cercueil if (IS_ERR(jzdma->ctrl_base)) 86333633583SPaul Cercueil return PTR_ERR(jzdma->ctrl_base); 86429870eb7SPaul Cercueil } else if (soc_data->flags & JZ_SOC_DATA_ALLOW_LEGACY_DT) { 86533633583SPaul Cercueil /* 86633633583SPaul Cercueil * On JZ4780, if the second memory resource was not supplied, 86733633583SPaul Cercueil * assume we're using an old devicetree, and calculate the 86833633583SPaul Cercueil * offset to the control registers. 86933633583SPaul Cercueil */ 87033633583SPaul Cercueil jzdma->ctrl_base = jzdma->chn_base + JZ4780_DMA_CTRL_OFFSET; 87129870eb7SPaul Cercueil } else { 87229870eb7SPaul Cercueil dev_err(dev, "failed to get I/O memory\n"); 87329870eb7SPaul Cercueil return -EINVAL; 87433633583SPaul Cercueil } 875d894fc60SAlex Smith 876839896efSAlex Smith ret = platform_get_irq(pdev, 0); 877839896efSAlex Smith if (ret < 0) { 878d894fc60SAlex Smith dev_err(dev, "failed to get IRQ: %d\n", ret); 879839896efSAlex Smith return ret; 880d894fc60SAlex Smith } 881d894fc60SAlex Smith 882839896efSAlex Smith jzdma->irq = ret; 883839896efSAlex Smith 884d509a83cSAlex Smith ret = request_irq(jzdma->irq, jz4780_dma_irq_handler, 0, dev_name(dev), 885d509a83cSAlex Smith jzdma); 886d894fc60SAlex Smith if (ret) { 887d894fc60SAlex Smith dev_err(dev, "failed to request IRQ %u!\n", jzdma->irq); 888839896efSAlex Smith return ret; 889d894fc60SAlex Smith } 890d894fc60SAlex Smith 891d894fc60SAlex Smith jzdma->clk = devm_clk_get(dev, NULL); 892d894fc60SAlex Smith if (IS_ERR(jzdma->clk)) { 893d894fc60SAlex Smith dev_err(dev, "failed to get clock\n"); 894d509a83cSAlex Smith ret = PTR_ERR(jzdma->clk); 895d509a83cSAlex Smith goto err_free_irq; 896d894fc60SAlex Smith } 897d894fc60SAlex Smith 898d894fc60SAlex Smith clk_prepare_enable(jzdma->clk); 899d894fc60SAlex Smith 900d894fc60SAlex Smith /* Property is optional, if it doesn't exist the value will remain 0. */ 901d894fc60SAlex Smith of_property_read_u32_index(dev->of_node, "ingenic,reserved-channels", 902d894fc60SAlex Smith 0, &jzdma->chan_reserved); 903d894fc60SAlex Smith 904d894fc60SAlex Smith dd = &jzdma->dma_device; 905d894fc60SAlex Smith 906d894fc60SAlex Smith dma_cap_set(DMA_MEMCPY, dd->cap_mask); 907d894fc60SAlex Smith dma_cap_set(DMA_SLAVE, dd->cap_mask); 908d894fc60SAlex Smith dma_cap_set(DMA_CYCLIC, dd->cap_mask); 909d894fc60SAlex Smith 910d894fc60SAlex Smith dd->dev = dev; 91177a68e56SMaxime Ripard dd->copy_align = DMAENGINE_ALIGN_4_BYTES; 912d894fc60SAlex Smith dd->device_alloc_chan_resources = jz4780_dma_alloc_chan_resources; 913d894fc60SAlex Smith dd->device_free_chan_resources = jz4780_dma_free_chan_resources; 914d894fc60SAlex Smith dd->device_prep_slave_sg = jz4780_dma_prep_slave_sg; 915d894fc60SAlex Smith dd->device_prep_dma_cyclic = jz4780_dma_prep_dma_cyclic; 916d894fc60SAlex Smith dd->device_prep_dma_memcpy = jz4780_dma_prep_dma_memcpy; 91746fa5168SAlex Smith dd->device_config = jz4780_dma_config; 918d894fc60SAlex Smith dd->device_terminate_all = jz4780_dma_terminate_all; 919f0dd52c8SPeter Ujfalusi dd->device_synchronize = jz4780_dma_synchronize; 920d894fc60SAlex Smith dd->device_tx_status = jz4780_dma_tx_status; 921d894fc60SAlex Smith dd->device_issue_pending = jz4780_dma_issue_pending; 922d894fc60SAlex Smith dd->src_addr_widths = JZ_DMA_BUSWIDTHS; 923d894fc60SAlex Smith dd->dst_addr_widths = JZ_DMA_BUSWIDTHS; 924d894fc60SAlex Smith dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 925d894fc60SAlex Smith dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 926d894fc60SAlex Smith 927d894fc60SAlex Smith /* 928d894fc60SAlex Smith * Enable DMA controller, mark all channels as not programmable. 929d894fc60SAlex Smith * Also set the FMSC bit - it increases MSC performance, so it makes 930d894fc60SAlex Smith * little sense not to enable it. 931d894fc60SAlex Smith */ 93217a8e30eSPaul Cercueil jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DMAC, JZ_DMA_DMAC_DMAE | 93317a8e30eSPaul Cercueil JZ_DMA_DMAC_FAIC | JZ_DMA_DMAC_FMSC); 93429870eb7SPaul Cercueil 93529870eb7SPaul Cercueil if (soc_data->flags & JZ_SOC_DATA_PROGRAMMABLE_DMA) 93633633583SPaul Cercueil jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DMACP, 0); 937d894fc60SAlex Smith 938d894fc60SAlex Smith INIT_LIST_HEAD(&dd->channels); 939d894fc60SAlex Smith 9406147b032SPaul Cercueil for (i = 0; i < soc_data->nb_channels; i++) { 941d894fc60SAlex Smith jzchan = &jzdma->chan[i]; 942d894fc60SAlex Smith jzchan->id = i; 943d894fc60SAlex Smith 944d894fc60SAlex Smith vchan_init(&jzchan->vchan, dd); 945d894fc60SAlex Smith jzchan->vchan.desc_free = jz4780_dma_desc_free; 946d894fc60SAlex Smith } 947d894fc60SAlex Smith 948d894fc60SAlex Smith ret = dma_async_device_register(dd); 949d894fc60SAlex Smith if (ret) { 950d894fc60SAlex Smith dev_err(dev, "failed to register device\n"); 951d894fc60SAlex Smith goto err_disable_clk; 952d894fc60SAlex Smith } 953d894fc60SAlex Smith 954d894fc60SAlex Smith /* Register with OF DMA helpers. */ 955d894fc60SAlex Smith ret = of_dma_controller_register(dev->of_node, jz4780_of_dma_xlate, 956d894fc60SAlex Smith jzdma); 957d894fc60SAlex Smith if (ret) { 958d894fc60SAlex Smith dev_err(dev, "failed to register OF DMA controller\n"); 959d894fc60SAlex Smith goto err_unregister_dev; 960d894fc60SAlex Smith } 961d894fc60SAlex Smith 962d894fc60SAlex Smith dev_info(dev, "JZ4780 DMA controller initialised\n"); 963d894fc60SAlex Smith return 0; 964d894fc60SAlex Smith 965d894fc60SAlex Smith err_unregister_dev: 966d894fc60SAlex Smith dma_async_device_unregister(dd); 967d894fc60SAlex Smith 968d894fc60SAlex Smith err_disable_clk: 969d894fc60SAlex Smith clk_disable_unprepare(jzdma->clk); 970d509a83cSAlex Smith 971d509a83cSAlex Smith err_free_irq: 972d509a83cSAlex Smith free_irq(jzdma->irq, jzdma); 973d894fc60SAlex Smith return ret; 974d894fc60SAlex Smith } 975d894fc60SAlex Smith 976d894fc60SAlex Smith static int jz4780_dma_remove(struct platform_device *pdev) 977d894fc60SAlex Smith { 978d894fc60SAlex Smith struct jz4780_dma_dev *jzdma = platform_get_drvdata(pdev); 979ae9c02b4SAlex Smith int i; 980d894fc60SAlex Smith 981d894fc60SAlex Smith of_dma_controller_free(pdev->dev.of_node); 982ae9c02b4SAlex Smith 983d509a83cSAlex Smith free_irq(jzdma->irq, jzdma); 984ae9c02b4SAlex Smith 9856147b032SPaul Cercueil for (i = 0; i < jzdma->soc_data->nb_channels; i++) 986ae9c02b4SAlex Smith tasklet_kill(&jzdma->chan[i].vchan.task); 987ae9c02b4SAlex Smith 988d894fc60SAlex Smith dma_async_device_unregister(&jzdma->dma_device); 989d894fc60SAlex Smith return 0; 990d894fc60SAlex Smith } 991d894fc60SAlex Smith 992ffaaa8ccSPaul Cercueil static const struct jz4780_dma_soc_data jz4740_dma_soc_data = { 993ffaaa8ccSPaul Cercueil .nb_channels = 6, 994ffaaa8ccSPaul Cercueil .transfer_ord_max = 5, 995ffaaa8ccSPaul Cercueil }; 996ffaaa8ccSPaul Cercueil 997ae9156b6SPaul Cercueil static const struct jz4780_dma_soc_data jz4725b_dma_soc_data = { 998ae9156b6SPaul Cercueil .nb_channels = 6, 999ae9156b6SPaul Cercueil .transfer_ord_max = 5, 1000ae9156b6SPaul Cercueil .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC, 1001ae9156b6SPaul Cercueil }; 1002ae9156b6SPaul Cercueil 100329870eb7SPaul Cercueil static const struct jz4780_dma_soc_data jz4770_dma_soc_data = { 100429870eb7SPaul Cercueil .nb_channels = 6, 100529870eb7SPaul Cercueil .transfer_ord_max = 6, 100629870eb7SPaul Cercueil .flags = JZ_SOC_DATA_PER_CHAN_PM, 100729870eb7SPaul Cercueil }; 100829870eb7SPaul Cercueil 10096147b032SPaul Cercueil static const struct jz4780_dma_soc_data jz4780_dma_soc_data = { 10106147b032SPaul Cercueil .nb_channels = 32, 101129870eb7SPaul Cercueil .transfer_ord_max = 7, 101229870eb7SPaul Cercueil .flags = JZ_SOC_DATA_ALLOW_LEGACY_DT | JZ_SOC_DATA_PROGRAMMABLE_DMA, 10136147b032SPaul Cercueil }; 10146147b032SPaul Cercueil 1015d894fc60SAlex Smith static const struct of_device_id jz4780_dma_dt_match[] = { 1016ffaaa8ccSPaul Cercueil { .compatible = "ingenic,jz4740-dma", .data = &jz4740_dma_soc_data }, 1017ae9156b6SPaul Cercueil { .compatible = "ingenic,jz4725b-dma", .data = &jz4725b_dma_soc_data }, 101829870eb7SPaul Cercueil { .compatible = "ingenic,jz4770-dma", .data = &jz4770_dma_soc_data }, 10196147b032SPaul Cercueil { .compatible = "ingenic,jz4780-dma", .data = &jz4780_dma_soc_data }, 1020d894fc60SAlex Smith {}, 1021d894fc60SAlex Smith }; 1022d894fc60SAlex Smith MODULE_DEVICE_TABLE(of, jz4780_dma_dt_match); 1023d894fc60SAlex Smith 1024d894fc60SAlex Smith static struct platform_driver jz4780_dma_driver = { 1025d894fc60SAlex Smith .probe = jz4780_dma_probe, 1026d894fc60SAlex Smith .remove = jz4780_dma_remove, 1027d894fc60SAlex Smith .driver = { 1028d894fc60SAlex Smith .name = "jz4780-dma", 1029d894fc60SAlex Smith .of_match_table = of_match_ptr(jz4780_dma_dt_match), 1030d894fc60SAlex Smith }, 1031d894fc60SAlex Smith }; 1032d894fc60SAlex Smith 1033d894fc60SAlex Smith static int __init jz4780_dma_init(void) 1034d894fc60SAlex Smith { 1035d894fc60SAlex Smith return platform_driver_register(&jz4780_dma_driver); 1036d894fc60SAlex Smith } 1037d894fc60SAlex Smith subsys_initcall(jz4780_dma_init); 1038d894fc60SAlex Smith 1039d894fc60SAlex Smith static void __exit jz4780_dma_exit(void) 1040d894fc60SAlex Smith { 1041d894fc60SAlex Smith platform_driver_unregister(&jz4780_dma_driver); 1042d894fc60SAlex Smith } 1043d894fc60SAlex Smith module_exit(jz4780_dma_exit); 1044d894fc60SAlex Smith 1045d894fc60SAlex Smith MODULE_AUTHOR("Alex Smith <alex@alex-smith.me.uk>"); 1046d894fc60SAlex Smith MODULE_DESCRIPTION("Ingenic JZ4780 DMA controller driver"); 1047d894fc60SAlex Smith MODULE_LICENSE("GPL"); 1048