180c4445eSStefan Wahren // SPDX-License-Identifier: GPL-2.0+
296286b57SFlorian Meier /*
396286b57SFlorian Meier * BCM2835 DMA engine support
496286b57SFlorian Meier *
596286b57SFlorian Meier * Author: Florian Meier <florian.meier@koalo.de>
696286b57SFlorian Meier * Copyright 2013
796286b57SFlorian Meier *
896286b57SFlorian Meier * Based on
996286b57SFlorian Meier * OMAP DMAengine support by Russell King
1096286b57SFlorian Meier *
1196286b57SFlorian Meier * BCM2708 DMA Driver
1296286b57SFlorian Meier * Copyright (C) 2010 Broadcom
1396286b57SFlorian Meier *
1496286b57SFlorian Meier * Raspberry Pi PCM I2S ALSA Driver
1596286b57SFlorian Meier * Copyright (c) by Phil Poole 2013
1696286b57SFlorian Meier *
1796286b57SFlorian Meier * MARVELL MMP Peripheral DMA Driver
1896286b57SFlorian Meier * Copyright 2012 Marvell International Ltd.
1996286b57SFlorian Meier */
2096286b57SFlorian Meier #include <linux/dmaengine.h>
2196286b57SFlorian Meier #include <linux/dma-mapping.h>
2227bc944cSPeter Ujfalusi #include <linux/dmapool.h>
2396286b57SFlorian Meier #include <linux/err.h>
2496286b57SFlorian Meier #include <linux/init.h>
2596286b57SFlorian Meier #include <linux/interrupt.h>
2696286b57SFlorian Meier #include <linux/list.h>
2796286b57SFlorian Meier #include <linux/module.h>
2896286b57SFlorian Meier #include <linux/platform_device.h>
2996286b57SFlorian Meier #include <linux/slab.h>
3096286b57SFlorian Meier #include <linux/io.h>
3196286b57SFlorian Meier #include <linux/spinlock.h>
3296286b57SFlorian Meier #include <linux/of.h>
3396286b57SFlorian Meier #include <linux/of_dma.h>
3496286b57SFlorian Meier
3596286b57SFlorian Meier #include "virt-dma.h"
3696286b57SFlorian Meier
37e2eca638SMartin Sperl #define BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED 14
38e2eca638SMartin Sperl #define BCM2835_DMA_CHAN_NAME_SIZE 8
39e2eca638SMartin Sperl
40c3ef8207SLukas Wunner /**
41c3ef8207SLukas Wunner * struct bcm2835_dmadev - BCM2835 DMA controller
42c3ef8207SLukas Wunner * @ddev: DMA device
43c3ef8207SLukas Wunner * @base: base address of register map
44bf75703dSLukas Wunner * @zero_page: bus address of zero page (to detect transactions copying from
45bf75703dSLukas Wunner * zero page and avoid accessing memory if so)
46c3ef8207SLukas Wunner */
4796286b57SFlorian Meier struct bcm2835_dmadev {
4896286b57SFlorian Meier struct dma_device ddev;
4996286b57SFlorian Meier void __iomem *base;
50bf75703dSLukas Wunner dma_addr_t zero_page;
5196286b57SFlorian Meier };
5296286b57SFlorian Meier
5396286b57SFlorian Meier struct bcm2835_dma_cb {
5496286b57SFlorian Meier uint32_t info;
5596286b57SFlorian Meier uint32_t src;
5696286b57SFlorian Meier uint32_t dst;
5796286b57SFlorian Meier uint32_t length;
5896286b57SFlorian Meier uint32_t stride;
5996286b57SFlorian Meier uint32_t next;
6096286b57SFlorian Meier uint32_t pad[2];
6196286b57SFlorian Meier };
6296286b57SFlorian Meier
6327bc944cSPeter Ujfalusi struct bcm2835_cb_entry {
6427bc944cSPeter Ujfalusi struct bcm2835_dma_cb *cb;
6527bc944cSPeter Ujfalusi dma_addr_t paddr;
6627bc944cSPeter Ujfalusi };
6727bc944cSPeter Ujfalusi
6896286b57SFlorian Meier struct bcm2835_chan {
6996286b57SFlorian Meier struct virt_dma_chan vc;
7096286b57SFlorian Meier
7196286b57SFlorian Meier struct dma_slave_config cfg;
7296286b57SFlorian Meier unsigned int dreq;
7396286b57SFlorian Meier
7496286b57SFlorian Meier int ch;
7596286b57SFlorian Meier struct bcm2835_desc *desc;
7627bc944cSPeter Ujfalusi struct dma_pool *cb_pool;
7796286b57SFlorian Meier
7896286b57SFlorian Meier void __iomem *chan_base;
7996286b57SFlorian Meier int irq_number;
80e2eca638SMartin Sperl unsigned int irq_flags;
8140874122SMartin Sperl
8240874122SMartin Sperl bool is_lite_channel;
8396286b57SFlorian Meier };
8496286b57SFlorian Meier
8596286b57SFlorian Meier struct bcm2835_desc {
8627bc944cSPeter Ujfalusi struct bcm2835_chan *c;
8796286b57SFlorian Meier struct virt_dma_desc vd;
8896286b57SFlorian Meier enum dma_transfer_direction dir;
8996286b57SFlorian Meier
9096286b57SFlorian Meier unsigned int frames;
9196286b57SFlorian Meier size_t size;
92a4dcdd84SMartin Sperl
93a4dcdd84SMartin Sperl bool cyclic;
9492153bb5SMartin Sperl
9592153bb5SMartin Sperl struct bcm2835_cb_entry cb_list[];
9696286b57SFlorian Meier };
9796286b57SFlorian Meier
9896286b57SFlorian Meier #define BCM2835_DMA_CS 0x00
9996286b57SFlorian Meier #define BCM2835_DMA_ADDR 0x04
100e42685d7SMartin Sperl #define BCM2835_DMA_TI 0x08
10196286b57SFlorian Meier #define BCM2835_DMA_SOURCE_AD 0x0c
10296286b57SFlorian Meier #define BCM2835_DMA_DEST_AD 0x10
103e42685d7SMartin Sperl #define BCM2835_DMA_LEN 0x14
104e42685d7SMartin Sperl #define BCM2835_DMA_STRIDE 0x18
105e42685d7SMartin Sperl #define BCM2835_DMA_NEXTCB 0x1c
106e42685d7SMartin Sperl #define BCM2835_DMA_DEBUG 0x20
10796286b57SFlorian Meier
10896286b57SFlorian Meier /* DMA CS Control and Status bits */
109e42685d7SMartin Sperl #define BCM2835_DMA_ACTIVE BIT(0) /* activate the DMA */
110e42685d7SMartin Sperl #define BCM2835_DMA_END BIT(1) /* current CB has ended */
111e42685d7SMartin Sperl #define BCM2835_DMA_INT BIT(2) /* interrupt status */
112e42685d7SMartin Sperl #define BCM2835_DMA_DREQ BIT(3) /* DREQ state */
11396286b57SFlorian Meier #define BCM2835_DMA_ISPAUSED BIT(4) /* Pause requested or not active */
11496286b57SFlorian Meier #define BCM2835_DMA_ISHELD BIT(5) /* Is held by DREQ flow control */
115e42685d7SMartin Sperl #define BCM2835_DMA_WAITING_FOR_WRITES BIT(6) /* waiting for last
116e42685d7SMartin Sperl * AXI-write to ack
117e42685d7SMartin Sperl */
11896286b57SFlorian Meier #define BCM2835_DMA_ERR BIT(8)
119e42685d7SMartin Sperl #define BCM2835_DMA_PRIORITY(x) ((x & 15) << 16) /* AXI priority */
120e42685d7SMartin Sperl #define BCM2835_DMA_PANIC_PRIORITY(x) ((x & 15) << 20) /* panic priority */
121e42685d7SMartin Sperl /* current value of TI.BCM2835_DMA_WAIT_RESP */
122e42685d7SMartin Sperl #define BCM2835_DMA_WAIT_FOR_WRITES BIT(28)
123e42685d7SMartin Sperl #define BCM2835_DMA_DIS_DEBUG BIT(29) /* disable debug pause signal */
12496286b57SFlorian Meier #define BCM2835_DMA_ABORT BIT(30) /* Stop current CB, go to next, WO */
12596286b57SFlorian Meier #define BCM2835_DMA_RESET BIT(31) /* WO, self clearing */
12696286b57SFlorian Meier
127e42685d7SMartin Sperl /* Transfer information bits - also bcm2835_cb.info field */
12896286b57SFlorian Meier #define BCM2835_DMA_INT_EN BIT(0)
129e42685d7SMartin Sperl #define BCM2835_DMA_TDMODE BIT(1) /* 2D-Mode */
130e42685d7SMartin Sperl #define BCM2835_DMA_WAIT_RESP BIT(3) /* wait for AXI-write to be acked */
13196286b57SFlorian Meier #define BCM2835_DMA_D_INC BIT(4)
132e42685d7SMartin Sperl #define BCM2835_DMA_D_WIDTH BIT(5) /* 128bit writes if set */
133e42685d7SMartin Sperl #define BCM2835_DMA_D_DREQ BIT(6) /* enable DREQ for destination */
134e42685d7SMartin Sperl #define BCM2835_DMA_D_IGNORE BIT(7) /* ignore destination writes */
13596286b57SFlorian Meier #define BCM2835_DMA_S_INC BIT(8)
136e42685d7SMartin Sperl #define BCM2835_DMA_S_WIDTH BIT(9) /* 128bit writes if set */
137e42685d7SMartin Sperl #define BCM2835_DMA_S_DREQ BIT(10) /* enable SREQ for source */
138e42685d7SMartin Sperl #define BCM2835_DMA_S_IGNORE BIT(11) /* ignore source reads - read 0 */
139e42685d7SMartin Sperl #define BCM2835_DMA_BURST_LENGTH(x) ((x & 15) << 12)
140e42685d7SMartin Sperl #define BCM2835_DMA_PER_MAP(x) ((x & 31) << 16) /* REQ source */
141e42685d7SMartin Sperl #define BCM2835_DMA_WAIT(x) ((x & 31) << 21) /* add DMA-wait cycles */
142e42685d7SMartin Sperl #define BCM2835_DMA_NO_WIDE_BURSTS BIT(26) /* no 2 beat write bursts */
14396286b57SFlorian Meier
144e42685d7SMartin Sperl /* debug register bits */
145e42685d7SMartin Sperl #define BCM2835_DMA_DEBUG_LAST_NOT_SET_ERR BIT(0)
146e42685d7SMartin Sperl #define BCM2835_DMA_DEBUG_FIFO_ERR BIT(1)
147e42685d7SMartin Sperl #define BCM2835_DMA_DEBUG_READ_ERR BIT(2)
148e42685d7SMartin Sperl #define BCM2835_DMA_DEBUG_OUTSTANDING_WRITES_SHIFT 4
149e42685d7SMartin Sperl #define BCM2835_DMA_DEBUG_OUTSTANDING_WRITES_BITS 4
150e42685d7SMartin Sperl #define BCM2835_DMA_DEBUG_ID_SHIFT 16
151e42685d7SMartin Sperl #define BCM2835_DMA_DEBUG_ID_BITS 9
152e42685d7SMartin Sperl #define BCM2835_DMA_DEBUG_STATE_SHIFT 16
153e42685d7SMartin Sperl #define BCM2835_DMA_DEBUG_STATE_BITS 9
154e42685d7SMartin Sperl #define BCM2835_DMA_DEBUG_VERSION_SHIFT 25
155e42685d7SMartin Sperl #define BCM2835_DMA_DEBUG_VERSION_BITS 3
156e42685d7SMartin Sperl #define BCM2835_DMA_DEBUG_LITE BIT(28)
157e42685d7SMartin Sperl
158e42685d7SMartin Sperl /* shared registers for all dma channels */
159e42685d7SMartin Sperl #define BCM2835_DMA_INT_STATUS 0xfe0
160e42685d7SMartin Sperl #define BCM2835_DMA_ENABLE 0xff0
16196286b57SFlorian Meier
16296286b57SFlorian Meier #define BCM2835_DMA_DATA_TYPE_S8 1
16396286b57SFlorian Meier #define BCM2835_DMA_DATA_TYPE_S16 2
16496286b57SFlorian Meier #define BCM2835_DMA_DATA_TYPE_S32 4
16596286b57SFlorian Meier #define BCM2835_DMA_DATA_TYPE_S128 16
16696286b57SFlorian Meier
16796286b57SFlorian Meier /* Valid only for channels 0 - 14, 15 has its own base address */
16896286b57SFlorian Meier #define BCM2835_DMA_CHAN(n) ((n) << 8) /* Base address */
16996286b57SFlorian Meier #define BCM2835_DMA_CHANIO(base, n) ((base) + BCM2835_DMA_CHAN(n))
17096286b57SFlorian Meier
17140874122SMartin Sperl /* the max dma length for different channels */
17240874122SMartin Sperl #define MAX_DMA_LEN SZ_1G
17340874122SMartin Sperl #define MAX_LITE_DMA_LEN (SZ_64K - 4)
17440874122SMartin Sperl
bcm2835_dma_max_frame_length(struct bcm2835_chan * c)17540874122SMartin Sperl static inline size_t bcm2835_dma_max_frame_length(struct bcm2835_chan *c)
17640874122SMartin Sperl {
17740874122SMartin Sperl /* lite and normal channels have different max frame length */
17840874122SMartin Sperl return c->is_lite_channel ? MAX_LITE_DMA_LEN : MAX_DMA_LEN;
17940874122SMartin Sperl }
18040874122SMartin Sperl
18192153bb5SMartin Sperl /* how many frames of max_len size do we need to transfer len bytes */
bcm2835_dma_frames_for_length(size_t len,size_t max_len)18292153bb5SMartin Sperl static inline size_t bcm2835_dma_frames_for_length(size_t len,
18392153bb5SMartin Sperl size_t max_len)
18492153bb5SMartin Sperl {
18592153bb5SMartin Sperl return DIV_ROUND_UP(len, max_len);
18692153bb5SMartin Sperl }
18792153bb5SMartin Sperl
to_bcm2835_dma_dev(struct dma_device * d)18896286b57SFlorian Meier static inline struct bcm2835_dmadev *to_bcm2835_dma_dev(struct dma_device *d)
18996286b57SFlorian Meier {
19096286b57SFlorian Meier return container_of(d, struct bcm2835_dmadev, ddev);
19196286b57SFlorian Meier }
19296286b57SFlorian Meier
to_bcm2835_dma_chan(struct dma_chan * c)19396286b57SFlorian Meier static inline struct bcm2835_chan *to_bcm2835_dma_chan(struct dma_chan *c)
19496286b57SFlorian Meier {
19596286b57SFlorian Meier return container_of(c, struct bcm2835_chan, vc.chan);
19696286b57SFlorian Meier }
19796286b57SFlorian Meier
to_bcm2835_dma_desc(struct dma_async_tx_descriptor * t)19896286b57SFlorian Meier static inline struct bcm2835_desc *to_bcm2835_dma_desc(
19996286b57SFlorian Meier struct dma_async_tx_descriptor *t)
20096286b57SFlorian Meier {
20196286b57SFlorian Meier return container_of(t, struct bcm2835_desc, vd.tx);
20296286b57SFlorian Meier }
20396286b57SFlorian Meier
bcm2835_dma_free_cb_chain(struct bcm2835_desc * desc)20492153bb5SMartin Sperl static void bcm2835_dma_free_cb_chain(struct bcm2835_desc *desc)
20596286b57SFlorian Meier {
20692153bb5SMartin Sperl size_t i;
20727bc944cSPeter Ujfalusi
20827bc944cSPeter Ujfalusi for (i = 0; i < desc->frames; i++)
20927bc944cSPeter Ujfalusi dma_pool_free(desc->c->cb_pool, desc->cb_list[i].cb,
21027bc944cSPeter Ujfalusi desc->cb_list[i].paddr);
21127bc944cSPeter Ujfalusi
21296286b57SFlorian Meier kfree(desc);
21396286b57SFlorian Meier }
21496286b57SFlorian Meier
bcm2835_dma_desc_free(struct virt_dma_desc * vd)21592153bb5SMartin Sperl static void bcm2835_dma_desc_free(struct virt_dma_desc *vd)
21692153bb5SMartin Sperl {
21792153bb5SMartin Sperl bcm2835_dma_free_cb_chain(
21892153bb5SMartin Sperl container_of(vd, struct bcm2835_desc, vd));
21992153bb5SMartin Sperl }
22092153bb5SMartin Sperl
bcm2835_dma_create_cb_set_length(struct bcm2835_chan * chan,struct bcm2835_dma_cb * control_block,size_t len,size_t period_len,size_t * total_len,u32 finalextrainfo)22192153bb5SMartin Sperl static void bcm2835_dma_create_cb_set_length(
22292153bb5SMartin Sperl struct bcm2835_chan *chan,
22392153bb5SMartin Sperl struct bcm2835_dma_cb *control_block,
22492153bb5SMartin Sperl size_t len,
22592153bb5SMartin Sperl size_t period_len,
22692153bb5SMartin Sperl size_t *total_len,
22792153bb5SMartin Sperl u32 finalextrainfo)
22892153bb5SMartin Sperl {
22940874122SMartin Sperl size_t max_len = bcm2835_dma_max_frame_length(chan);
23040874122SMartin Sperl
23140874122SMartin Sperl /* set the length taking lite-channel limitations into account */
23240874122SMartin Sperl control_block->length = min_t(u32, len, max_len);
23392153bb5SMartin Sperl
23492153bb5SMartin Sperl /* finished if we have no period_length */
23592153bb5SMartin Sperl if (!period_len)
23692153bb5SMartin Sperl return;
23792153bb5SMartin Sperl
23892153bb5SMartin Sperl /*
23992153bb5SMartin Sperl * period_len means: that we need to generate
24092153bb5SMartin Sperl * transfers that are terminating at every
24192153bb5SMartin Sperl * multiple of period_len - this is typically
24292153bb5SMartin Sperl * used to set the interrupt flag in info
24392153bb5SMartin Sperl * which is required during cyclic transfers
24492153bb5SMartin Sperl */
24592153bb5SMartin Sperl
24692153bb5SMartin Sperl /* have we filled in period_length yet? */
2472201ac61SMatthias Reichl if (*total_len + control_block->length < period_len) {
2482201ac61SMatthias Reichl /* update number of bytes in this period so far */
2492201ac61SMatthias Reichl *total_len += control_block->length;
25092153bb5SMartin Sperl return;
2512201ac61SMatthias Reichl }
25292153bb5SMartin Sperl
25392153bb5SMartin Sperl /* calculate the length that remains to reach period_length */
25492153bb5SMartin Sperl control_block->length = period_len - *total_len;
25592153bb5SMartin Sperl
25692153bb5SMartin Sperl /* reset total_length for next period */
25792153bb5SMartin Sperl *total_len = 0;
25892153bb5SMartin Sperl
25992153bb5SMartin Sperl /* add extrainfo bits in info */
26092153bb5SMartin Sperl control_block->info |= finalextrainfo;
26192153bb5SMartin Sperl }
26292153bb5SMartin Sperl
bcm2835_dma_count_frames_for_sg(struct bcm2835_chan * c,struct scatterlist * sgl,unsigned int sg_len)263388cc7a2SMartin Sperl static inline size_t bcm2835_dma_count_frames_for_sg(
264388cc7a2SMartin Sperl struct bcm2835_chan *c,
265388cc7a2SMartin Sperl struct scatterlist *sgl,
266388cc7a2SMartin Sperl unsigned int sg_len)
267388cc7a2SMartin Sperl {
268388cc7a2SMartin Sperl size_t frames = 0;
269388cc7a2SMartin Sperl struct scatterlist *sgent;
270388cc7a2SMartin Sperl unsigned int i;
271388cc7a2SMartin Sperl size_t plength = bcm2835_dma_max_frame_length(c);
272388cc7a2SMartin Sperl
273388cc7a2SMartin Sperl for_each_sg(sgl, sgent, sg_len, i)
274388cc7a2SMartin Sperl frames += bcm2835_dma_frames_for_length(
275388cc7a2SMartin Sperl sg_dma_len(sgent), plength);
276388cc7a2SMartin Sperl
277388cc7a2SMartin Sperl return frames;
278388cc7a2SMartin Sperl }
279388cc7a2SMartin Sperl
28092153bb5SMartin Sperl /**
28192153bb5SMartin Sperl * bcm2835_dma_create_cb_chain - create a control block and fills data in
28292153bb5SMartin Sperl *
28392153bb5SMartin Sperl * @chan: the @dma_chan for which we run this
28492153bb5SMartin Sperl * @direction: the direction in which we transfer
28592153bb5SMartin Sperl * @cyclic: it is a cyclic transfer
28692153bb5SMartin Sperl * @info: the default info bits to apply per controlblock
28792153bb5SMartin Sperl * @frames: number of controlblocks to allocate
28892153bb5SMartin Sperl * @src: the src address to assign (if the S_INC bit is set
28992153bb5SMartin Sperl * in @info, then it gets incremented)
29092153bb5SMartin Sperl * @dst: the dst address to assign (if the D_INC bit is set
29192153bb5SMartin Sperl * in @info, then it gets incremented)
29292153bb5SMartin Sperl * @buf_len: the full buffer length (may also be 0)
29392153bb5SMartin Sperl * @period_len: the period length when to apply @finalextrainfo
29492153bb5SMartin Sperl * in addition to the last transfer
29592153bb5SMartin Sperl * this will also break some control-blocks early
29692153bb5SMartin Sperl * @finalextrainfo: additional bits in last controlblock
29792153bb5SMartin Sperl * (or when period_len is reached in case of cyclic)
29892153bb5SMartin Sperl * @gfp: the GFP flag to use for allocation
29992153bb5SMartin Sperl */
bcm2835_dma_create_cb_chain(struct dma_chan * chan,enum dma_transfer_direction direction,bool cyclic,u32 info,u32 finalextrainfo,size_t frames,dma_addr_t src,dma_addr_t dst,size_t buf_len,size_t period_len,gfp_t gfp)30092153bb5SMartin Sperl static struct bcm2835_desc *bcm2835_dma_create_cb_chain(
30192153bb5SMartin Sperl struct dma_chan *chan, enum dma_transfer_direction direction,
30292153bb5SMartin Sperl bool cyclic, u32 info, u32 finalextrainfo, size_t frames,
30392153bb5SMartin Sperl dma_addr_t src, dma_addr_t dst, size_t buf_len,
30492153bb5SMartin Sperl size_t period_len, gfp_t gfp)
30592153bb5SMartin Sperl {
30692153bb5SMartin Sperl struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
30792153bb5SMartin Sperl size_t len = buf_len, total_len;
30892153bb5SMartin Sperl size_t frame;
30992153bb5SMartin Sperl struct bcm2835_desc *d;
31092153bb5SMartin Sperl struct bcm2835_cb_entry *cb_entry;
31192153bb5SMartin Sperl struct bcm2835_dma_cb *control_block;
31292153bb5SMartin Sperl
313d9f094a0SMartin Sperl if (!frames)
314d9f094a0SMartin Sperl return NULL;
315d9f094a0SMartin Sperl
31692153bb5SMartin Sperl /* allocate and setup the descriptor. */
3175fde6005SGustavo A. R. Silva d = kzalloc(struct_size(d, cb_list, frames), gfp);
31892153bb5SMartin Sperl if (!d)
31992153bb5SMartin Sperl return NULL;
32092153bb5SMartin Sperl
32192153bb5SMartin Sperl d->c = c;
32292153bb5SMartin Sperl d->dir = direction;
32392153bb5SMartin Sperl d->cyclic = cyclic;
32492153bb5SMartin Sperl
32592153bb5SMartin Sperl /*
32692153bb5SMartin Sperl * Iterate over all frames, create a control block
32792153bb5SMartin Sperl * for each frame and link them together.
32892153bb5SMartin Sperl */
32992153bb5SMartin Sperl for (frame = 0, total_len = 0; frame < frames; d->frames++, frame++) {
33092153bb5SMartin Sperl cb_entry = &d->cb_list[frame];
33192153bb5SMartin Sperl cb_entry->cb = dma_pool_alloc(c->cb_pool, gfp,
33292153bb5SMartin Sperl &cb_entry->paddr);
33392153bb5SMartin Sperl if (!cb_entry->cb)
33492153bb5SMartin Sperl goto error_cb;
33592153bb5SMartin Sperl
33692153bb5SMartin Sperl /* fill in the control block */
33792153bb5SMartin Sperl control_block = cb_entry->cb;
33892153bb5SMartin Sperl control_block->info = info;
33992153bb5SMartin Sperl control_block->src = src;
34092153bb5SMartin Sperl control_block->dst = dst;
34192153bb5SMartin Sperl control_block->stride = 0;
34292153bb5SMartin Sperl control_block->next = 0;
34392153bb5SMartin Sperl /* set up length in control_block if requested */
34492153bb5SMartin Sperl if (buf_len) {
34592153bb5SMartin Sperl /* calculate length honoring period_length */
34692153bb5SMartin Sperl bcm2835_dma_create_cb_set_length(
34792153bb5SMartin Sperl c, control_block,
34892153bb5SMartin Sperl len, period_len, &total_len,
34992153bb5SMartin Sperl cyclic ? finalextrainfo : 0);
35092153bb5SMartin Sperl
35192153bb5SMartin Sperl /* calculate new remaining length */
35292153bb5SMartin Sperl len -= control_block->length;
35392153bb5SMartin Sperl }
35492153bb5SMartin Sperl
35592153bb5SMartin Sperl /* link this the last controlblock */
35692153bb5SMartin Sperl if (frame)
35792153bb5SMartin Sperl d->cb_list[frame - 1].cb->next = cb_entry->paddr;
35892153bb5SMartin Sperl
35992153bb5SMartin Sperl /* update src and dst and length */
36092153bb5SMartin Sperl if (src && (info & BCM2835_DMA_S_INC))
36192153bb5SMartin Sperl src += control_block->length;
36292153bb5SMartin Sperl if (dst && (info & BCM2835_DMA_D_INC))
36392153bb5SMartin Sperl dst += control_block->length;
36492153bb5SMartin Sperl
36592153bb5SMartin Sperl /* Length of total transfer */
36692153bb5SMartin Sperl d->size += control_block->length;
36792153bb5SMartin Sperl }
36892153bb5SMartin Sperl
36992153bb5SMartin Sperl /* the last frame requires extra flags */
37092153bb5SMartin Sperl d->cb_list[d->frames - 1].cb->info |= finalextrainfo;
37192153bb5SMartin Sperl
37292153bb5SMartin Sperl /* detect a size missmatch */
37392153bb5SMartin Sperl if (buf_len && (d->size != buf_len))
37492153bb5SMartin Sperl goto error_cb;
37592153bb5SMartin Sperl
37692153bb5SMartin Sperl return d;
37792153bb5SMartin Sperl error_cb:
37892153bb5SMartin Sperl bcm2835_dma_free_cb_chain(d);
37992153bb5SMartin Sperl
38092153bb5SMartin Sperl return NULL;
38192153bb5SMartin Sperl }
38292153bb5SMartin Sperl
bcm2835_dma_fill_cb_chain_with_sg(struct dma_chan * chan,enum dma_transfer_direction direction,struct bcm2835_cb_entry * cb,struct scatterlist * sgl,unsigned int sg_len)383388cc7a2SMartin Sperl static void bcm2835_dma_fill_cb_chain_with_sg(
384388cc7a2SMartin Sperl struct dma_chan *chan,
385388cc7a2SMartin Sperl enum dma_transfer_direction direction,
386388cc7a2SMartin Sperl struct bcm2835_cb_entry *cb,
387388cc7a2SMartin Sperl struct scatterlist *sgl,
388388cc7a2SMartin Sperl unsigned int sg_len)
389388cc7a2SMartin Sperl {
390388cc7a2SMartin Sperl struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
3914aa819c7SArnd Bergmann size_t len, max_len;
3924aa819c7SArnd Bergmann unsigned int i;
393388cc7a2SMartin Sperl dma_addr_t addr;
394388cc7a2SMartin Sperl struct scatterlist *sgent;
395388cc7a2SMartin Sperl
3964aa819c7SArnd Bergmann max_len = bcm2835_dma_max_frame_length(c);
397388cc7a2SMartin Sperl for_each_sg(sgl, sgent, sg_len, i) {
398388cc7a2SMartin Sperl for (addr = sg_dma_address(sgent), len = sg_dma_len(sgent);
399388cc7a2SMartin Sperl len > 0;
400388cc7a2SMartin Sperl addr += cb->cb->length, len -= cb->cb->length, cb++) {
401388cc7a2SMartin Sperl if (direction == DMA_DEV_TO_MEM)
402388cc7a2SMartin Sperl cb->cb->dst = addr;
403388cc7a2SMartin Sperl else
404388cc7a2SMartin Sperl cb->cb->src = addr;
405388cc7a2SMartin Sperl cb->cb->length = min(len, max_len);
406388cc7a2SMartin Sperl }
407388cc7a2SMartin Sperl }
408388cc7a2SMartin Sperl }
409388cc7a2SMartin Sperl
bcm2835_dma_abort(struct bcm2835_chan * c)4103e05ada0SLukas Wunner static void bcm2835_dma_abort(struct bcm2835_chan *c)
41196286b57SFlorian Meier {
4129e528c79SLukas Wunner void __iomem *chan_base = c->chan_base;
41396286b57SFlorian Meier long int timeout = 10000;
41496286b57SFlorian Meier
415f7da7782SLukas Wunner /*
416f7da7782SLukas Wunner * A zero control block address means the channel is idle.
417f7da7782SLukas Wunner * (The ACTIVE flag in the CS register is not a reliable indicator.)
418f7da7782SLukas Wunner */
419f7da7782SLukas Wunner if (!readl(chan_base + BCM2835_DMA_ADDR))
4203e05ada0SLukas Wunner return;
42196286b57SFlorian Meier
42296286b57SFlorian Meier /* Write 0 to the active bit - Pause the DMA */
42396286b57SFlorian Meier writel(0, chan_base + BCM2835_DMA_CS);
42496286b57SFlorian Meier
42596286b57SFlorian Meier /* Wait for any current AXI transfer to complete */
4269e528c79SLukas Wunner while ((readl(chan_base + BCM2835_DMA_CS) &
4279e528c79SLukas Wunner BCM2835_DMA_WAITING_FOR_WRITES) && --timeout)
42896286b57SFlorian Meier cpu_relax();
42996286b57SFlorian Meier
4309e528c79SLukas Wunner /* Peripheral might be stuck and fail to signal AXI write responses */
43196286b57SFlorian Meier if (!timeout)
4329e528c79SLukas Wunner dev_err(c->vc.chan.device->dev,
4339e528c79SLukas Wunner "failed to complete outstanding writes\n");
43496286b57SFlorian Meier
4359e528c79SLukas Wunner writel(BCM2835_DMA_RESET, chan_base + BCM2835_DMA_CS);
43696286b57SFlorian Meier }
43796286b57SFlorian Meier
bcm2835_dma_start_desc(struct bcm2835_chan * c)43896286b57SFlorian Meier static void bcm2835_dma_start_desc(struct bcm2835_chan *c)
43996286b57SFlorian Meier {
44096286b57SFlorian Meier struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
44196286b57SFlorian Meier struct bcm2835_desc *d;
44296286b57SFlorian Meier
44396286b57SFlorian Meier if (!vd) {
44496286b57SFlorian Meier c->desc = NULL;
44596286b57SFlorian Meier return;
44696286b57SFlorian Meier }
44796286b57SFlorian Meier
44896286b57SFlorian Meier list_del(&vd->node);
44996286b57SFlorian Meier
45096286b57SFlorian Meier c->desc = d = to_bcm2835_dma_desc(&vd->tx);
45196286b57SFlorian Meier
45227bc944cSPeter Ujfalusi writel(d->cb_list[0].paddr, c->chan_base + BCM2835_DMA_ADDR);
45396286b57SFlorian Meier writel(BCM2835_DMA_ACTIVE, c->chan_base + BCM2835_DMA_CS);
45496286b57SFlorian Meier }
45596286b57SFlorian Meier
bcm2835_dma_callback(int irq,void * data)45696286b57SFlorian Meier static irqreturn_t bcm2835_dma_callback(int irq, void *data)
45796286b57SFlorian Meier {
45896286b57SFlorian Meier struct bcm2835_chan *c = data;
45996286b57SFlorian Meier struct bcm2835_desc *d;
46096286b57SFlorian Meier unsigned long flags;
46196286b57SFlorian Meier
462e2eca638SMartin Sperl /* check the shared interrupt */
463e2eca638SMartin Sperl if (c->irq_flags & IRQF_SHARED) {
464e2eca638SMartin Sperl /* check if the interrupt is enabled */
465e2eca638SMartin Sperl flags = readl(c->chan_base + BCM2835_DMA_CS);
466e2eca638SMartin Sperl /* if not set then we are not the reason for the irq */
467e2eca638SMartin Sperl if (!(flags & BCM2835_DMA_INT))
468e2eca638SMartin Sperl return IRQ_NONE;
469e2eca638SMartin Sperl }
470e2eca638SMartin Sperl
47196286b57SFlorian Meier spin_lock_irqsave(&c->vc.lock, flags);
47296286b57SFlorian Meier
473f7da7782SLukas Wunner /*
474f7da7782SLukas Wunner * Clear the INT flag to receive further interrupts. Keep the channel
475f7da7782SLukas Wunner * active in case the descriptor is cyclic or in case the client has
476f7da7782SLukas Wunner * already terminated the descriptor and issued a new one. (May happen
477f7da7782SLukas Wunner * if this IRQ handler is threaded.) If the channel is finished, it
478f7da7782SLukas Wunner * will remain idle despite the ACTIVE flag being set.
479f7da7782SLukas Wunner */
480f7da7782SLukas Wunner writel(BCM2835_DMA_INT | BCM2835_DMA_ACTIVE,
481f7da7782SLukas Wunner c->chan_base + BCM2835_DMA_CS);
48296286b57SFlorian Meier
48396286b57SFlorian Meier d = c->desc;
48496286b57SFlorian Meier
48596286b57SFlorian Meier if (d) {
486388cc7a2SMartin Sperl if (d->cyclic) {
487388cc7a2SMartin Sperl /* call the cyclic callback */
48896286b57SFlorian Meier vchan_cyclic_callback(&d->vd);
489f7da7782SLukas Wunner } else if (!readl(c->chan_base + BCM2835_DMA_ADDR)) {
490388cc7a2SMartin Sperl vchan_cookie_complete(&c->desc->vd);
491388cc7a2SMartin Sperl bcm2835_dma_start_desc(c);
492388cc7a2SMartin Sperl }
493388cc7a2SMartin Sperl }
49496286b57SFlorian Meier
49596286b57SFlorian Meier spin_unlock_irqrestore(&c->vc.lock, flags);
49696286b57SFlorian Meier
49796286b57SFlorian Meier return IRQ_HANDLED;
49896286b57SFlorian Meier }
49996286b57SFlorian Meier
bcm2835_dma_alloc_chan_resources(struct dma_chan * chan)50096286b57SFlorian Meier static int bcm2835_dma_alloc_chan_resources(struct dma_chan *chan)
50196286b57SFlorian Meier {
50296286b57SFlorian Meier struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
50327bc944cSPeter Ujfalusi struct device *dev = c->vc.chan.device->dev;
50496286b57SFlorian Meier
50527bc944cSPeter Ujfalusi dev_dbg(dev, "Allocating DMA channel %d\n", c->ch);
50627bc944cSPeter Ujfalusi
507603fe86bSLukas Wunner /*
508603fe86bSLukas Wunner * Control blocks are 256 bit in length and must start at a 256 bit
509603fe86bSLukas Wunner * (32 byte) aligned address (BCM2835 ARM Peripherals, sec. 4.2.1.1).
510603fe86bSLukas Wunner */
51127bc944cSPeter Ujfalusi c->cb_pool = dma_pool_create(dev_name(dev), dev,
512603fe86bSLukas Wunner sizeof(struct bcm2835_dma_cb), 32, 0);
51327bc944cSPeter Ujfalusi if (!c->cb_pool) {
51427bc944cSPeter Ujfalusi dev_err(dev, "unable to allocate descriptor pool\n");
51527bc944cSPeter Ujfalusi return -ENOMEM;
51627bc944cSPeter Ujfalusi }
51796286b57SFlorian Meier
518e2eca638SMartin Sperl return request_irq(c->irq_number, bcm2835_dma_callback,
519e2eca638SMartin Sperl c->irq_flags, "DMA IRQ", c);
52096286b57SFlorian Meier }
52196286b57SFlorian Meier
bcm2835_dma_free_chan_resources(struct dma_chan * chan)52296286b57SFlorian Meier static void bcm2835_dma_free_chan_resources(struct dma_chan *chan)
52396286b57SFlorian Meier {
52496286b57SFlorian Meier struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
52596286b57SFlorian Meier
52696286b57SFlorian Meier vchan_free_chan_resources(&c->vc);
52796286b57SFlorian Meier free_irq(c->irq_number, c);
52827bc944cSPeter Ujfalusi dma_pool_destroy(c->cb_pool);
52996286b57SFlorian Meier
53096286b57SFlorian Meier dev_dbg(c->vc.chan.device->dev, "Freeing DMA channel %u\n", c->ch);
53196286b57SFlorian Meier }
53296286b57SFlorian Meier
bcm2835_dma_desc_size(struct bcm2835_desc * d)53396286b57SFlorian Meier static size_t bcm2835_dma_desc_size(struct bcm2835_desc *d)
53496286b57SFlorian Meier {
53596286b57SFlorian Meier return d->size;
53696286b57SFlorian Meier }
53796286b57SFlorian Meier
bcm2835_dma_desc_size_pos(struct bcm2835_desc * d,dma_addr_t addr)53896286b57SFlorian Meier static size_t bcm2835_dma_desc_size_pos(struct bcm2835_desc *d, dma_addr_t addr)
53996286b57SFlorian Meier {
54096286b57SFlorian Meier unsigned int i;
54196286b57SFlorian Meier size_t size;
54296286b57SFlorian Meier
54396286b57SFlorian Meier for (size = i = 0; i < d->frames; i++) {
54427bc944cSPeter Ujfalusi struct bcm2835_dma_cb *control_block = d->cb_list[i].cb;
54596286b57SFlorian Meier size_t this_size = control_block->length;
54696286b57SFlorian Meier dma_addr_t dma;
54796286b57SFlorian Meier
54896286b57SFlorian Meier if (d->dir == DMA_DEV_TO_MEM)
54996286b57SFlorian Meier dma = control_block->dst;
55096286b57SFlorian Meier else
55196286b57SFlorian Meier dma = control_block->src;
55296286b57SFlorian Meier
55396286b57SFlorian Meier if (size)
55496286b57SFlorian Meier size += this_size;
55596286b57SFlorian Meier else if (addr >= dma && addr < dma + this_size)
55696286b57SFlorian Meier size += dma + this_size - addr;
55796286b57SFlorian Meier }
55896286b57SFlorian Meier
55996286b57SFlorian Meier return size;
56096286b57SFlorian Meier }
56196286b57SFlorian Meier
bcm2835_dma_tx_status(struct dma_chan * chan,dma_cookie_t cookie,struct dma_tx_state * txstate)56296286b57SFlorian Meier static enum dma_status bcm2835_dma_tx_status(struct dma_chan *chan,
56396286b57SFlorian Meier dma_cookie_t cookie, struct dma_tx_state *txstate)
56496286b57SFlorian Meier {
56596286b57SFlorian Meier struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
56696286b57SFlorian Meier struct virt_dma_desc *vd;
56796286b57SFlorian Meier enum dma_status ret;
56896286b57SFlorian Meier unsigned long flags;
56996286b57SFlorian Meier
57096286b57SFlorian Meier ret = dma_cookie_status(chan, cookie, txstate);
57196286b57SFlorian Meier if (ret == DMA_COMPLETE || !txstate)
57296286b57SFlorian Meier return ret;
57396286b57SFlorian Meier
57496286b57SFlorian Meier spin_lock_irqsave(&c->vc.lock, flags);
57596286b57SFlorian Meier vd = vchan_find_desc(&c->vc, cookie);
57696286b57SFlorian Meier if (vd) {
57796286b57SFlorian Meier txstate->residue =
57896286b57SFlorian Meier bcm2835_dma_desc_size(to_bcm2835_dma_desc(&vd->tx));
57996286b57SFlorian Meier } else if (c->desc && c->desc->vd.tx.cookie == cookie) {
58096286b57SFlorian Meier struct bcm2835_desc *d = c->desc;
58196286b57SFlorian Meier dma_addr_t pos;
58296286b57SFlorian Meier
58396286b57SFlorian Meier if (d->dir == DMA_MEM_TO_DEV)
58496286b57SFlorian Meier pos = readl(c->chan_base + BCM2835_DMA_SOURCE_AD);
58596286b57SFlorian Meier else if (d->dir == DMA_DEV_TO_MEM)
58696286b57SFlorian Meier pos = readl(c->chan_base + BCM2835_DMA_DEST_AD);
58796286b57SFlorian Meier else
58896286b57SFlorian Meier pos = 0;
58996286b57SFlorian Meier
59096286b57SFlorian Meier txstate->residue = bcm2835_dma_desc_size_pos(d, pos);
59196286b57SFlorian Meier } else {
59296286b57SFlorian Meier txstate->residue = 0;
59396286b57SFlorian Meier }
59496286b57SFlorian Meier
59596286b57SFlorian Meier spin_unlock_irqrestore(&c->vc.lock, flags);
59696286b57SFlorian Meier
59796286b57SFlorian Meier return ret;
59896286b57SFlorian Meier }
59996286b57SFlorian Meier
bcm2835_dma_issue_pending(struct dma_chan * chan)60096286b57SFlorian Meier static void bcm2835_dma_issue_pending(struct dma_chan *chan)
60196286b57SFlorian Meier {
60296286b57SFlorian Meier struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
60396286b57SFlorian Meier unsigned long flags;
60496286b57SFlorian Meier
60596286b57SFlorian Meier spin_lock_irqsave(&c->vc.lock, flags);
60696286b57SFlorian Meier if (vchan_issue_pending(&c->vc) && !c->desc)
60796286b57SFlorian Meier bcm2835_dma_start_desc(c);
60896286b57SFlorian Meier
60996286b57SFlorian Meier spin_unlock_irqrestore(&c->vc.lock, flags);
61096286b57SFlorian Meier }
61196286b57SFlorian Meier
bcm2835_dma_prep_dma_memcpy(struct dma_chan * chan,dma_addr_t dst,dma_addr_t src,size_t len,unsigned long flags)61263637228SBen Dooks static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_memcpy(
613d9f094a0SMartin Sperl struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,
614d9f094a0SMartin Sperl size_t len, unsigned long flags)
615d9f094a0SMartin Sperl {
616d9f094a0SMartin Sperl struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
617d9f094a0SMartin Sperl struct bcm2835_desc *d;
618d9f094a0SMartin Sperl u32 info = BCM2835_DMA_D_INC | BCM2835_DMA_S_INC;
619d9f094a0SMartin Sperl u32 extra = BCM2835_DMA_INT_EN | BCM2835_DMA_WAIT_RESP;
620d9f094a0SMartin Sperl size_t max_len = bcm2835_dma_max_frame_length(c);
621d9f094a0SMartin Sperl size_t frames;
622d9f094a0SMartin Sperl
623d9f094a0SMartin Sperl /* if src, dst or len is not given return with an error */
624d9f094a0SMartin Sperl if (!src || !dst || !len)
625d9f094a0SMartin Sperl return NULL;
626d9f094a0SMartin Sperl
627d9f094a0SMartin Sperl /* calculate number of frames */
628d9f094a0SMartin Sperl frames = bcm2835_dma_frames_for_length(len, max_len);
629d9f094a0SMartin Sperl
630d9f094a0SMartin Sperl /* allocate the CB chain - this also fills in the pointers */
631d9f094a0SMartin Sperl d = bcm2835_dma_create_cb_chain(chan, DMA_MEM_TO_MEM, false,
632d9f094a0SMartin Sperl info, extra, frames,
633d9f094a0SMartin Sperl src, dst, len, 0, GFP_KERNEL);
634d9f094a0SMartin Sperl if (!d)
635d9f094a0SMartin Sperl return NULL;
636d9f094a0SMartin Sperl
637d9f094a0SMartin Sperl return vchan_tx_prep(&c->vc, &d->vd, flags);
638d9f094a0SMartin Sperl }
639d9f094a0SMartin Sperl
bcm2835_dma_prep_slave_sg(struct dma_chan * chan,struct scatterlist * sgl,unsigned int sg_len,enum dma_transfer_direction direction,unsigned long flags,void * context)640388cc7a2SMartin Sperl static struct dma_async_tx_descriptor *bcm2835_dma_prep_slave_sg(
641388cc7a2SMartin Sperl struct dma_chan *chan,
642388cc7a2SMartin Sperl struct scatterlist *sgl, unsigned int sg_len,
643388cc7a2SMartin Sperl enum dma_transfer_direction direction,
644388cc7a2SMartin Sperl unsigned long flags, void *context)
645388cc7a2SMartin Sperl {
646388cc7a2SMartin Sperl struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
647388cc7a2SMartin Sperl struct bcm2835_desc *d;
648388cc7a2SMartin Sperl dma_addr_t src = 0, dst = 0;
649388cc7a2SMartin Sperl u32 info = BCM2835_DMA_WAIT_RESP;
650388cc7a2SMartin Sperl u32 extra = BCM2835_DMA_INT_EN;
651388cc7a2SMartin Sperl size_t frames;
652388cc7a2SMartin Sperl
653388cc7a2SMartin Sperl if (!is_slave_direction(direction)) {
654388cc7a2SMartin Sperl dev_err(chan->device->dev,
655388cc7a2SMartin Sperl "%s: bad direction?\n", __func__);
656388cc7a2SMartin Sperl return NULL;
657388cc7a2SMartin Sperl }
658388cc7a2SMartin Sperl
659388cc7a2SMartin Sperl if (c->dreq != 0)
660388cc7a2SMartin Sperl info |= BCM2835_DMA_PER_MAP(c->dreq);
661388cc7a2SMartin Sperl
662388cc7a2SMartin Sperl if (direction == DMA_DEV_TO_MEM) {
663388cc7a2SMartin Sperl if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)
664388cc7a2SMartin Sperl return NULL;
665388cc7a2SMartin Sperl src = c->cfg.src_addr;
666388cc7a2SMartin Sperl info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC;
667388cc7a2SMartin Sperl } else {
668388cc7a2SMartin Sperl if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)
669388cc7a2SMartin Sperl return NULL;
670388cc7a2SMartin Sperl dst = c->cfg.dst_addr;
671388cc7a2SMartin Sperl info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC;
672388cc7a2SMartin Sperl }
673388cc7a2SMartin Sperl
674388cc7a2SMartin Sperl /* count frames in sg list */
675388cc7a2SMartin Sperl frames = bcm2835_dma_count_frames_for_sg(c, sgl, sg_len);
676388cc7a2SMartin Sperl
677388cc7a2SMartin Sperl /* allocate the CB chain */
678388cc7a2SMartin Sperl d = bcm2835_dma_create_cb_chain(chan, direction, false,
679388cc7a2SMartin Sperl info, extra,
680388cc7a2SMartin Sperl frames, src, dst, 0, 0,
681f1473847SStefan Wahren GFP_NOWAIT);
682388cc7a2SMartin Sperl if (!d)
683388cc7a2SMartin Sperl return NULL;
684388cc7a2SMartin Sperl
685388cc7a2SMartin Sperl /* fill in frames with scatterlist pointers */
686388cc7a2SMartin Sperl bcm2835_dma_fill_cb_chain_with_sg(chan, direction, d->cb_list,
687388cc7a2SMartin Sperl sgl, sg_len);
688388cc7a2SMartin Sperl
689388cc7a2SMartin Sperl return vchan_tx_prep(&c->vc, &d->vd, flags);
690388cc7a2SMartin Sperl }
691388cc7a2SMartin Sperl
bcm2835_dma_prep_dma_cyclic(struct dma_chan * chan,dma_addr_t buf_addr,size_t buf_len,size_t period_len,enum dma_transfer_direction direction,unsigned long flags)69296286b57SFlorian Meier static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_cyclic(
69396286b57SFlorian Meier struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
69496286b57SFlorian Meier size_t period_len, enum dma_transfer_direction direction,
69531c1e5a1SLaurent Pinchart unsigned long flags)
69696286b57SFlorian Meier {
697bf75703dSLukas Wunner struct bcm2835_dmadev *od = to_bcm2835_dma_dev(chan->device);
69896286b57SFlorian Meier struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
69996286b57SFlorian Meier struct bcm2835_desc *d;
70092153bb5SMartin Sperl dma_addr_t src, dst;
70192153bb5SMartin Sperl u32 info = BCM2835_DMA_WAIT_RESP;
7024f2228ccSLukas Wunner u32 extra = 0;
70340874122SMartin Sperl size_t max_len = bcm2835_dma_max_frame_length(c);
70492153bb5SMartin Sperl size_t frames;
70596286b57SFlorian Meier
70696286b57SFlorian Meier /* Grab configuration */
70796286b57SFlorian Meier if (!is_slave_direction(direction)) {
70896286b57SFlorian Meier dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
70996286b57SFlorian Meier return NULL;
71096286b57SFlorian Meier }
71196286b57SFlorian Meier
71292153bb5SMartin Sperl if (!buf_len) {
71392153bb5SMartin Sperl dev_err(chan->device->dev,
71492153bb5SMartin Sperl "%s: bad buffer length (= 0)\n", __func__);
71596286b57SFlorian Meier return NULL;
71696286b57SFlorian Meier }
71796286b57SFlorian Meier
7184f2228ccSLukas Wunner if (flags & DMA_PREP_INTERRUPT)
7194f2228ccSLukas Wunner extra |= BCM2835_DMA_INT_EN;
7204f2228ccSLukas Wunner else
7214f2228ccSLukas Wunner period_len = buf_len;
7224f2228ccSLukas Wunner
72396286b57SFlorian Meier /*
72492153bb5SMartin Sperl * warn if buf_len is not a multiple of period_len - this may leed
72592153bb5SMartin Sperl * to unexpected latencies for interrupts and thus audiable clicks
72696286b57SFlorian Meier */
72792153bb5SMartin Sperl if (buf_len % period_len)
72892153bb5SMartin Sperl dev_warn_once(chan->device->dev,
72992153bb5SMartin Sperl "%s: buffer_length (%zd) is not a multiple of period_len (%zd)\n",
73092153bb5SMartin Sperl __func__, buf_len, period_len);
73196286b57SFlorian Meier
73296286b57SFlorian Meier /* Setup DREQ channel */
73396286b57SFlorian Meier if (c->dreq != 0)
73492153bb5SMartin Sperl info |= BCM2835_DMA_PER_MAP(c->dreq);
73596286b57SFlorian Meier
73692153bb5SMartin Sperl if (direction == DMA_DEV_TO_MEM) {
73792153bb5SMartin Sperl if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)
73892153bb5SMartin Sperl return NULL;
73992153bb5SMartin Sperl src = c->cfg.src_addr;
74092153bb5SMartin Sperl dst = buf_addr;
74192153bb5SMartin Sperl info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC;
74292153bb5SMartin Sperl } else {
74392153bb5SMartin Sperl if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)
74492153bb5SMartin Sperl return NULL;
74592153bb5SMartin Sperl dst = c->cfg.dst_addr;
74692153bb5SMartin Sperl src = buf_addr;
74792153bb5SMartin Sperl info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC;
748bf75703dSLukas Wunner
749bf75703dSLukas Wunner /* non-lite channels can write zeroes w/o accessing memory */
750bf75703dSLukas Wunner if (buf_addr == od->zero_page && !c->is_lite_channel)
751bf75703dSLukas Wunner info |= BCM2835_DMA_S_IGNORE;
75292153bb5SMartin Sperl }
75392153bb5SMartin Sperl
75492153bb5SMartin Sperl /* calculate number of frames */
75540874122SMartin Sperl frames = /* number of periods */
75640874122SMartin Sperl DIV_ROUND_UP(buf_len, period_len) *
75740874122SMartin Sperl /* number of frames per period */
75840874122SMartin Sperl bcm2835_dma_frames_for_length(period_len, max_len);
75996286b57SFlorian Meier
76096286b57SFlorian Meier /*
76192153bb5SMartin Sperl * allocate the CB chain
76292153bb5SMartin Sperl * note that we need to use GFP_NOWAIT, as the ALSA i2s dmaengine
76392153bb5SMartin Sperl * implementation calls prep_dma_cyclic with interrupts disabled.
76496286b57SFlorian Meier */
76592153bb5SMartin Sperl d = bcm2835_dma_create_cb_chain(chan, direction, true,
76692153bb5SMartin Sperl info, extra,
76792153bb5SMartin Sperl frames, src, dst, buf_len,
76892153bb5SMartin Sperl period_len, GFP_NOWAIT);
76992153bb5SMartin Sperl if (!d)
77092153bb5SMartin Sperl return NULL;
77192153bb5SMartin Sperl
77292153bb5SMartin Sperl /* wrap around into a loop */
77392153bb5SMartin Sperl d->cb_list[d->frames - 1].cb->next = d->cb_list[0].paddr;
77496286b57SFlorian Meier
77596286b57SFlorian Meier return vchan_tx_prep(&c->vc, &d->vd, flags);
77696286b57SFlorian Meier }
77796286b57SFlorian Meier
bcm2835_dma_slave_config(struct dma_chan * chan,struct dma_slave_config * cfg)77839159beaSMaxime Ripard static int bcm2835_dma_slave_config(struct dma_chan *chan,
77996286b57SFlorian Meier struct dma_slave_config *cfg)
78096286b57SFlorian Meier {
78139159beaSMaxime Ripard struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
78239159beaSMaxime Ripard
78396286b57SFlorian Meier c->cfg = *cfg;
78496286b57SFlorian Meier
78596286b57SFlorian Meier return 0;
78696286b57SFlorian Meier }
78796286b57SFlorian Meier
bcm2835_dma_terminate_all(struct dma_chan * chan)78839159beaSMaxime Ripard static int bcm2835_dma_terminate_all(struct dma_chan *chan)
78996286b57SFlorian Meier {
79039159beaSMaxime Ripard struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
79196286b57SFlorian Meier unsigned long flags;
79296286b57SFlorian Meier LIST_HEAD(head);
79396286b57SFlorian Meier
79496286b57SFlorian Meier spin_lock_irqsave(&c->vc.lock, flags);
79596286b57SFlorian Meier
796f7da7782SLukas Wunner /* stop DMA activity */
79796286b57SFlorian Meier if (c->desc) {
798de92436aSPeter Ujfalusi vchan_terminate_vdesc(&c->desc->vd);
79996286b57SFlorian Meier c->desc = NULL;
8009e528c79SLukas Wunner bcm2835_dma_abort(c);
80196286b57SFlorian Meier }
80296286b57SFlorian Meier
80396286b57SFlorian Meier vchan_get_all_descriptors(&c->vc, &head);
80496286b57SFlorian Meier spin_unlock_irqrestore(&c->vc.lock, flags);
80596286b57SFlorian Meier vchan_dma_desc_free_list(&c->vc, &head);
80696286b57SFlorian Meier
80796286b57SFlorian Meier return 0;
80896286b57SFlorian Meier }
80996286b57SFlorian Meier
bcm2835_dma_synchronize(struct dma_chan * chan)810de92436aSPeter Ujfalusi static void bcm2835_dma_synchronize(struct dma_chan *chan)
811de92436aSPeter Ujfalusi {
812de92436aSPeter Ujfalusi struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
813de92436aSPeter Ujfalusi
814de92436aSPeter Ujfalusi vchan_synchronize(&c->vc);
815de92436aSPeter Ujfalusi }
816de92436aSPeter Ujfalusi
bcm2835_dma_chan_init(struct bcm2835_dmadev * d,int chan_id,int irq,unsigned int irq_flags)817e2eca638SMartin Sperl static int bcm2835_dma_chan_init(struct bcm2835_dmadev *d, int chan_id,
818e2eca638SMartin Sperl int irq, unsigned int irq_flags)
81996286b57SFlorian Meier {
82096286b57SFlorian Meier struct bcm2835_chan *c;
82196286b57SFlorian Meier
82296286b57SFlorian Meier c = devm_kzalloc(d->ddev.dev, sizeof(*c), GFP_KERNEL);
82396286b57SFlorian Meier if (!c)
82496286b57SFlorian Meier return -ENOMEM;
82596286b57SFlorian Meier
82696286b57SFlorian Meier c->vc.desc_free = bcm2835_dma_desc_free;
82796286b57SFlorian Meier vchan_init(&c->vc, &d->ddev);
82896286b57SFlorian Meier
82996286b57SFlorian Meier c->chan_base = BCM2835_DMA_CHANIO(d->base, chan_id);
83096286b57SFlorian Meier c->ch = chan_id;
83196286b57SFlorian Meier c->irq_number = irq;
832e2eca638SMartin Sperl c->irq_flags = irq_flags;
83396286b57SFlorian Meier
83440874122SMartin Sperl /* check in DEBUG register if this is a LITE channel */
83540874122SMartin Sperl if (readl(c->chan_base + BCM2835_DMA_DEBUG) &
83640874122SMartin Sperl BCM2835_DMA_DEBUG_LITE)
83740874122SMartin Sperl c->is_lite_channel = true;
83840874122SMartin Sperl
83996286b57SFlorian Meier return 0;
84096286b57SFlorian Meier }
84196286b57SFlorian Meier
bcm2835_dma_free(struct bcm2835_dmadev * od)84296286b57SFlorian Meier static void bcm2835_dma_free(struct bcm2835_dmadev *od)
84396286b57SFlorian Meier {
84496286b57SFlorian Meier struct bcm2835_chan *c, *next;
84596286b57SFlorian Meier
84696286b57SFlorian Meier list_for_each_entry_safe(c, next, &od->ddev.channels,
84796286b57SFlorian Meier vc.chan.device_node) {
84896286b57SFlorian Meier list_del(&c->vc.chan.device_node);
84996286b57SFlorian Meier tasklet_kill(&c->vc.task);
85096286b57SFlorian Meier }
851bf75703dSLukas Wunner
852bf75703dSLukas Wunner dma_unmap_page_attrs(od->ddev.dev, od->zero_page, PAGE_SIZE,
853bf75703dSLukas Wunner DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
85496286b57SFlorian Meier }
85596286b57SFlorian Meier
85696286b57SFlorian Meier static const struct of_device_id bcm2835_dma_of_match[] = {
85796286b57SFlorian Meier { .compatible = "brcm,bcm2835-dma", },
85896286b57SFlorian Meier {},
85996286b57SFlorian Meier };
86096286b57SFlorian Meier MODULE_DEVICE_TABLE(of, bcm2835_dma_of_match);
86196286b57SFlorian Meier
bcm2835_dma_xlate(struct of_phandle_args * spec,struct of_dma * ofdma)86296286b57SFlorian Meier static struct dma_chan *bcm2835_dma_xlate(struct of_phandle_args *spec,
86396286b57SFlorian Meier struct of_dma *ofdma)
86496286b57SFlorian Meier {
86596286b57SFlorian Meier struct bcm2835_dmadev *d = ofdma->of_dma_data;
86696286b57SFlorian Meier struct dma_chan *chan;
86796286b57SFlorian Meier
86896286b57SFlorian Meier chan = dma_get_any_slave_channel(&d->ddev);
86996286b57SFlorian Meier if (!chan)
87096286b57SFlorian Meier return NULL;
87196286b57SFlorian Meier
87296286b57SFlorian Meier /* Set DREQ from param */
87396286b57SFlorian Meier to_bcm2835_dma_chan(chan)->dreq = spec->args[0];
87496286b57SFlorian Meier
87596286b57SFlorian Meier return chan;
87696286b57SFlorian Meier }
87796286b57SFlorian Meier
bcm2835_dma_probe(struct platform_device * pdev)87896286b57SFlorian Meier static int bcm2835_dma_probe(struct platform_device *pdev)
87996286b57SFlorian Meier {
88096286b57SFlorian Meier struct bcm2835_dmadev *od;
88196286b57SFlorian Meier void __iomem *base;
88296286b57SFlorian Meier int rc;
883e2eca638SMartin Sperl int i, j;
884e2eca638SMartin Sperl int irq[BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED + 1];
885e2eca638SMartin Sperl int irq_flags;
88696286b57SFlorian Meier uint32_t chans_available;
887e2eca638SMartin Sperl char chan_name[BCM2835_DMA_CHAN_NAME_SIZE];
88896286b57SFlorian Meier
88996286b57SFlorian Meier if (!pdev->dev.dma_mask)
89096286b57SFlorian Meier pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask;
89196286b57SFlorian Meier
89296286b57SFlorian Meier rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
89372503b25SStefan Wahren if (rc) {
89472503b25SStefan Wahren dev_err(&pdev->dev, "Unable to set DMA mask\n");
89596286b57SFlorian Meier return rc;
89672503b25SStefan Wahren }
89796286b57SFlorian Meier
89896286b57SFlorian Meier od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
89996286b57SFlorian Meier if (!od)
90096286b57SFlorian Meier return -ENOMEM;
90196286b57SFlorian Meier
90296286b57SFlorian Meier dma_set_max_seg_size(&pdev->dev, 0x3FFFFFFF);
90396286b57SFlorian Meier
904*4b23603aSTudor Ambarus base = devm_platform_ioremap_resource(pdev, 0);
90596286b57SFlorian Meier if (IS_ERR(base))
90696286b57SFlorian Meier return PTR_ERR(base);
90796286b57SFlorian Meier
90896286b57SFlorian Meier od->base = base;
90996286b57SFlorian Meier
91096286b57SFlorian Meier dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
9117f5ae355SFlorian Meier dma_cap_set(DMA_PRIVATE, od->ddev.cap_mask);
91296286b57SFlorian Meier dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);
913d9f094a0SMartin Sperl dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask);
91496286b57SFlorian Meier od->ddev.device_alloc_chan_resources = bcm2835_dma_alloc_chan_resources;
91596286b57SFlorian Meier od->ddev.device_free_chan_resources = bcm2835_dma_free_chan_resources;
91696286b57SFlorian Meier od->ddev.device_tx_status = bcm2835_dma_tx_status;
91796286b57SFlorian Meier od->ddev.device_issue_pending = bcm2835_dma_issue_pending;
91896286b57SFlorian Meier od->ddev.device_prep_dma_cyclic = bcm2835_dma_prep_dma_cyclic;
919388cc7a2SMartin Sperl od->ddev.device_prep_slave_sg = bcm2835_dma_prep_slave_sg;
920d9f094a0SMartin Sperl od->ddev.device_prep_dma_memcpy = bcm2835_dma_prep_dma_memcpy;
92139159beaSMaxime Ripard od->ddev.device_config = bcm2835_dma_slave_config;
92239159beaSMaxime Ripard od->ddev.device_terminate_all = bcm2835_dma_terminate_all;
923de92436aSPeter Ujfalusi od->ddev.device_synchronize = bcm2835_dma_synchronize;
924b5743680SMaxime Ripard od->ddev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
925b5743680SMaxime Ripard od->ddev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
926d9f094a0SMartin Sperl od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) |
927d9f094a0SMartin Sperl BIT(DMA_MEM_TO_MEM);
9280fa5867eSMartin Sperl od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
9296f6869dcSLukas Wunner od->ddev.descriptor_reuse = true;
93096286b57SFlorian Meier od->ddev.dev = &pdev->dev;
93196286b57SFlorian Meier INIT_LIST_HEAD(&od->ddev.channels);
93296286b57SFlorian Meier
93396286b57SFlorian Meier platform_set_drvdata(pdev, od);
93496286b57SFlorian Meier
935bf75703dSLukas Wunner od->zero_page = dma_map_page_attrs(od->ddev.dev, ZERO_PAGE(0), 0,
936bf75703dSLukas Wunner PAGE_SIZE, DMA_TO_DEVICE,
937bf75703dSLukas Wunner DMA_ATTR_SKIP_CPU_SYNC);
938bf75703dSLukas Wunner if (dma_mapping_error(od->ddev.dev, od->zero_page)) {
939bf75703dSLukas Wunner dev_err(&pdev->dev, "Failed to map zero page\n");
940bf75703dSLukas Wunner return -ENOMEM;
941bf75703dSLukas Wunner }
942bf75703dSLukas Wunner
94396286b57SFlorian Meier /* Request DMA channel mask from device tree */
94496286b57SFlorian Meier if (of_property_read_u32(pdev->dev.of_node,
94596286b57SFlorian Meier "brcm,dma-channel-mask",
94696286b57SFlorian Meier &chans_available)) {
94796286b57SFlorian Meier dev_err(&pdev->dev, "Failed to get channel mask\n");
94896286b57SFlorian Meier rc = -EINVAL;
94996286b57SFlorian Meier goto err_no_dma;
95096286b57SFlorian Meier }
95196286b57SFlorian Meier
952e2eca638SMartin Sperl /* get irqs for each channel that we support */
953e2eca638SMartin Sperl for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) {
954e2eca638SMartin Sperl /* skip masked out channels */
955e2eca638SMartin Sperl if (!(chans_available & (1 << i))) {
956e2eca638SMartin Sperl irq[i] = -1;
957e2eca638SMartin Sperl continue;
958e2eca638SMartin Sperl }
95996286b57SFlorian Meier
960e2eca638SMartin Sperl /* get the named irq */
961e2eca638SMartin Sperl snprintf(chan_name, sizeof(chan_name), "dma%i", i);
962e2eca638SMartin Sperl irq[i] = platform_get_irq_byname(pdev, chan_name);
963e2eca638SMartin Sperl if (irq[i] >= 0)
964e2eca638SMartin Sperl continue;
965e2eca638SMartin Sperl
966e2eca638SMartin Sperl /* legacy device tree case handling */
967e2eca638SMartin Sperl dev_warn_once(&pdev->dev,
9680eef727aSMartin Sperl "missing interrupt-names property in device tree - legacy interpretation is used\n");
969e2eca638SMartin Sperl /*
970e2eca638SMartin Sperl * in case of channel >= 11
971e2eca638SMartin Sperl * use the 11th interrupt and that is shared
972e2eca638SMartin Sperl */
973e2eca638SMartin Sperl irq[i] = platform_get_irq(pdev, i < 11 ? i : 11);
974e2eca638SMartin Sperl }
975e2eca638SMartin Sperl
976e2eca638SMartin Sperl /* get irqs for each channel */
977e2eca638SMartin Sperl for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) {
978e2eca638SMartin Sperl /* skip channels without irq */
979e2eca638SMartin Sperl if (irq[i] < 0)
980e2eca638SMartin Sperl continue;
981e2eca638SMartin Sperl
982e2eca638SMartin Sperl /* check if there are other channels that also use this irq */
983e2eca638SMartin Sperl irq_flags = 0;
984e2eca638SMartin Sperl for (j = 0; j <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; j++)
985e2eca638SMartin Sperl if ((i != j) && (irq[j] == irq[i])) {
986e2eca638SMartin Sperl irq_flags = IRQF_SHARED;
987e2eca638SMartin Sperl break;
988e2eca638SMartin Sperl }
989e2eca638SMartin Sperl
990e2eca638SMartin Sperl /* initialize the channel */
991e2eca638SMartin Sperl rc = bcm2835_dma_chan_init(od, i, irq[i], irq_flags);
99296286b57SFlorian Meier if (rc)
99396286b57SFlorian Meier goto err_no_dma;
99496286b57SFlorian Meier }
99596286b57SFlorian Meier
99696286b57SFlorian Meier dev_dbg(&pdev->dev, "Initialized %i DMA channels\n", i);
99796286b57SFlorian Meier
99896286b57SFlorian Meier /* Device-tree DMA controller registration */
99996286b57SFlorian Meier rc = of_dma_controller_register(pdev->dev.of_node,
100096286b57SFlorian Meier bcm2835_dma_xlate, od);
100196286b57SFlorian Meier if (rc) {
100296286b57SFlorian Meier dev_err(&pdev->dev, "Failed to register DMA controller\n");
100396286b57SFlorian Meier goto err_no_dma;
100496286b57SFlorian Meier }
100596286b57SFlorian Meier
100696286b57SFlorian Meier rc = dma_async_device_register(&od->ddev);
100796286b57SFlorian Meier if (rc) {
100896286b57SFlorian Meier dev_err(&pdev->dev,
100996286b57SFlorian Meier "Failed to register slave DMA engine device: %d\n", rc);
101096286b57SFlorian Meier goto err_no_dma;
101196286b57SFlorian Meier }
101296286b57SFlorian Meier
101396286b57SFlorian Meier dev_dbg(&pdev->dev, "Load BCM2835 DMA engine driver\n");
101496286b57SFlorian Meier
101596286b57SFlorian Meier return 0;
101696286b57SFlorian Meier
101796286b57SFlorian Meier err_no_dma:
101896286b57SFlorian Meier bcm2835_dma_free(od);
101996286b57SFlorian Meier return rc;
102096286b57SFlorian Meier }
102196286b57SFlorian Meier
bcm2835_dma_remove(struct platform_device * pdev)102296286b57SFlorian Meier static int bcm2835_dma_remove(struct platform_device *pdev)
102396286b57SFlorian Meier {
102496286b57SFlorian Meier struct bcm2835_dmadev *od = platform_get_drvdata(pdev);
102596286b57SFlorian Meier
102696286b57SFlorian Meier dma_async_device_unregister(&od->ddev);
102796286b57SFlorian Meier bcm2835_dma_free(od);
102896286b57SFlorian Meier
102996286b57SFlorian Meier return 0;
103096286b57SFlorian Meier }
103196286b57SFlorian Meier
103296286b57SFlorian Meier static struct platform_driver bcm2835_dma_driver = {
103396286b57SFlorian Meier .probe = bcm2835_dma_probe,
103496286b57SFlorian Meier .remove = bcm2835_dma_remove,
103596286b57SFlorian Meier .driver = {
103696286b57SFlorian Meier .name = "bcm2835-dma",
103796286b57SFlorian Meier .of_match_table = of_match_ptr(bcm2835_dma_of_match),
103896286b57SFlorian Meier },
103996286b57SFlorian Meier };
104096286b57SFlorian Meier
104196286b57SFlorian Meier module_platform_driver(bcm2835_dma_driver);
104296286b57SFlorian Meier
104396286b57SFlorian Meier MODULE_ALIAS("platform:bcm2835-dma");
104496286b57SFlorian Meier MODULE_DESCRIPTION("BCM2835 DMA engine driver");
104596286b57SFlorian Meier MODULE_AUTHOR("Florian Meier <florian.meier@koalo.de>");
1046ab39e147SStefan Wahren MODULE_LICENSE("GPL");
1047