xref: /openbmc/linux/drivers/dma/sun4i-dma.c (revision adb57164)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright (C) 2014 Emilio López
4  * Emilio López <emilio@elopez.com.ar>
5  */
6 
7 #include <linux/bitmap.h>
8 #include <linux/bitops.h>
9 #include <linux/clk.h>
10 #include <linux/dmaengine.h>
11 #include <linux/dmapool.h>
12 #include <linux/interrupt.h>
13 #include <linux/module.h>
14 #include <linux/of_dma.h>
15 #include <linux/platform_device.h>
16 #include <linux/slab.h>
17 #include <linux/spinlock.h>
18 
19 #include "virt-dma.h"
20 
21 /** Common macros to normal and dedicated DMA registers **/
22 
23 #define SUN4I_DMA_CFG_LOADING			BIT(31)
24 #define SUN4I_DMA_CFG_DST_DATA_WIDTH(width)	((width) << 25)
25 #define SUN4I_DMA_CFG_DST_BURST_LENGTH(len)	((len) << 23)
26 #define SUN4I_DMA_CFG_DST_ADDR_MODE(mode)	((mode) << 21)
27 #define SUN4I_DMA_CFG_DST_DRQ_TYPE(type)	((type) << 16)
28 #define SUN4I_DMA_CFG_SRC_DATA_WIDTH(width)	((width) << 9)
29 #define SUN4I_DMA_CFG_SRC_BURST_LENGTH(len)	((len) << 7)
30 #define SUN4I_DMA_CFG_SRC_ADDR_MODE(mode)	((mode) << 5)
31 #define SUN4I_DMA_CFG_SRC_DRQ_TYPE(type)	(type)
32 
33 /** Normal DMA register values **/
34 
35 /* Normal DMA source/destination data request type values */
36 #define SUN4I_NDMA_DRQ_TYPE_SDRAM		0x16
37 #define SUN4I_NDMA_DRQ_TYPE_LIMIT		(0x1F + 1)
38 
39 /** Normal DMA register layout **/
40 
41 /* Dedicated DMA source/destination address mode values */
42 #define SUN4I_NDMA_ADDR_MODE_LINEAR		0
43 #define SUN4I_NDMA_ADDR_MODE_IO			1
44 
45 /* Normal DMA configuration register layout */
46 #define SUN4I_NDMA_CFG_CONT_MODE		BIT(30)
47 #define SUN4I_NDMA_CFG_WAIT_STATE(n)		((n) << 27)
48 #define SUN4I_NDMA_CFG_DST_NON_SECURE		BIT(22)
49 #define SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN	BIT(15)
50 #define SUN4I_NDMA_CFG_SRC_NON_SECURE		BIT(6)
51 
52 /** Dedicated DMA register values **/
53 
54 /* Dedicated DMA source/destination address mode values */
55 #define SUN4I_DDMA_ADDR_MODE_LINEAR		0
56 #define SUN4I_DDMA_ADDR_MODE_IO			1
57 #define SUN4I_DDMA_ADDR_MODE_HORIZONTAL_PAGE	2
58 #define SUN4I_DDMA_ADDR_MODE_VERTICAL_PAGE	3
59 
60 /* Dedicated DMA source/destination data request type values */
61 #define SUN4I_DDMA_DRQ_TYPE_SDRAM		0x1
62 #define SUN4I_DDMA_DRQ_TYPE_LIMIT		(0x1F + 1)
63 
64 /** Dedicated DMA register layout **/
65 
66 /* Dedicated DMA configuration register layout */
67 #define SUN4I_DDMA_CFG_BUSY			BIT(30)
68 #define SUN4I_DDMA_CFG_CONT_MODE		BIT(29)
69 #define SUN4I_DDMA_CFG_DST_NON_SECURE		BIT(28)
70 #define SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN	BIT(15)
71 #define SUN4I_DDMA_CFG_SRC_NON_SECURE		BIT(12)
72 
73 /* Dedicated DMA parameter register layout */
74 #define SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(n)	(((n) - 1) << 24)
75 #define SUN4I_DDMA_PARA_DST_WAIT_CYCLES(n)	(((n) - 1) << 16)
76 #define SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(n)	(((n) - 1) << 8)
77 #define SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(n)	(((n) - 1) << 0)
78 
79 /** DMA register offsets **/
80 
81 /* General register offsets */
82 #define SUN4I_DMA_IRQ_ENABLE_REG		0x0
83 #define SUN4I_DMA_IRQ_PENDING_STATUS_REG	0x4
84 
85 /* Normal DMA register offsets */
86 #define SUN4I_NDMA_CHANNEL_REG_BASE(n)		(0x100 + (n) * 0x20)
87 #define SUN4I_NDMA_CFG_REG			0x0
88 #define SUN4I_NDMA_SRC_ADDR_REG			0x4
89 #define SUN4I_NDMA_DST_ADDR_REG		0x8
90 #define SUN4I_NDMA_BYTE_COUNT_REG		0xC
91 
92 /* Dedicated DMA register offsets */
93 #define SUN4I_DDMA_CHANNEL_REG_BASE(n)		(0x300 + (n) * 0x20)
94 #define SUN4I_DDMA_CFG_REG			0x0
95 #define SUN4I_DDMA_SRC_ADDR_REG			0x4
96 #define SUN4I_DDMA_DST_ADDR_REG		0x8
97 #define SUN4I_DDMA_BYTE_COUNT_REG		0xC
98 #define SUN4I_DDMA_PARA_REG			0x18
99 
100 /** DMA Driver **/
101 
102 /*
103  * Normal DMA has 8 channels, and Dedicated DMA has another 8, so
104  * that's 16 channels. As for endpoints, there's 29 and 21
105  * respectively. Given that the Normal DMA endpoints (other than
106  * SDRAM) can be used as tx/rx, we need 78 vchans in total
107  */
108 #define SUN4I_NDMA_NR_MAX_CHANNELS	8
109 #define SUN4I_DDMA_NR_MAX_CHANNELS	8
110 #define SUN4I_DMA_NR_MAX_CHANNELS					\
111 	(SUN4I_NDMA_NR_MAX_CHANNELS + SUN4I_DDMA_NR_MAX_CHANNELS)
112 #define SUN4I_NDMA_NR_MAX_VCHANS	(29 * 2 - 1)
113 #define SUN4I_DDMA_NR_MAX_VCHANS	21
114 #define SUN4I_DMA_NR_MAX_VCHANS						\
115 	(SUN4I_NDMA_NR_MAX_VCHANS + SUN4I_DDMA_NR_MAX_VCHANS)
116 
117 /* This set of SUN4I_DDMA timing parameters were found experimentally while
118  * working with the SPI driver and seem to make it behave correctly */
119 #define SUN4I_DDMA_MAGIC_SPI_PARAMETERS \
120 	(SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(1) |			\
121 	 SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(1) |				\
122 	 SUN4I_DDMA_PARA_DST_WAIT_CYCLES(2) |				\
123 	 SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(2))
124 
125 struct sun4i_dma_pchan {
126 	/* Register base of channel */
127 	void __iomem			*base;
128 	/* vchan currently being serviced */
129 	struct sun4i_dma_vchan		*vchan;
130 	/* Is this a dedicated pchan? */
131 	int				is_dedicated;
132 };
133 
134 struct sun4i_dma_vchan {
135 	struct virt_dma_chan		vc;
136 	struct dma_slave_config		cfg;
137 	struct sun4i_dma_pchan		*pchan;
138 	struct sun4i_dma_promise	*processing;
139 	struct sun4i_dma_contract	*contract;
140 	u8				endpoint;
141 	int				is_dedicated;
142 };
143 
144 struct sun4i_dma_promise {
145 	u32				cfg;
146 	u32				para;
147 	dma_addr_t			src;
148 	dma_addr_t			dst;
149 	size_t				len;
150 	struct list_head		list;
151 };
152 
153 /* A contract is a set of promises */
154 struct sun4i_dma_contract {
155 	struct virt_dma_desc		vd;
156 	struct list_head		demands;
157 	struct list_head		completed_demands;
158 	int				is_cyclic;
159 };
160 
161 struct sun4i_dma_dev {
162 	DECLARE_BITMAP(pchans_used, SUN4I_DMA_NR_MAX_CHANNELS);
163 	struct dma_device		slave;
164 	struct sun4i_dma_pchan		*pchans;
165 	struct sun4i_dma_vchan		*vchans;
166 	void __iomem			*base;
167 	struct clk			*clk;
168 	int				irq;
169 	spinlock_t			lock;
170 };
171 
172 static struct sun4i_dma_dev *to_sun4i_dma_dev(struct dma_device *dev)
173 {
174 	return container_of(dev, struct sun4i_dma_dev, slave);
175 }
176 
177 static struct sun4i_dma_vchan *to_sun4i_dma_vchan(struct dma_chan *chan)
178 {
179 	return container_of(chan, struct sun4i_dma_vchan, vc.chan);
180 }
181 
182 static struct sun4i_dma_contract *to_sun4i_dma_contract(struct virt_dma_desc *vd)
183 {
184 	return container_of(vd, struct sun4i_dma_contract, vd);
185 }
186 
187 static struct device *chan2dev(struct dma_chan *chan)
188 {
189 	return &chan->dev->device;
190 }
191 
192 static int convert_burst(u32 maxburst)
193 {
194 	if (maxburst > 8)
195 		return -EINVAL;
196 
197 	/* 1 -> 0, 4 -> 1, 8 -> 2 */
198 	return (maxburst >> 2);
199 }
200 
201 static int convert_buswidth(enum dma_slave_buswidth addr_width)
202 {
203 	if (addr_width > DMA_SLAVE_BUSWIDTH_4_BYTES)
204 		return -EINVAL;
205 
206 	/* 8 (1 byte) -> 0, 16 (2 bytes) -> 1, 32 (4 bytes) -> 2 */
207 	return (addr_width >> 1);
208 }
209 
210 static void sun4i_dma_free_chan_resources(struct dma_chan *chan)
211 {
212 	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
213 
214 	vchan_free_chan_resources(&vchan->vc);
215 }
216 
217 static struct sun4i_dma_pchan *find_and_use_pchan(struct sun4i_dma_dev *priv,
218 						  struct sun4i_dma_vchan *vchan)
219 {
220 	struct sun4i_dma_pchan *pchan = NULL, *pchans = priv->pchans;
221 	unsigned long flags;
222 	int i, max;
223 
224 	/*
225 	 * pchans 0-SUN4I_NDMA_NR_MAX_CHANNELS are normal, and
226 	 * SUN4I_NDMA_NR_MAX_CHANNELS+ are dedicated ones
227 	 */
228 	if (vchan->is_dedicated) {
229 		i = SUN4I_NDMA_NR_MAX_CHANNELS;
230 		max = SUN4I_DMA_NR_MAX_CHANNELS;
231 	} else {
232 		i = 0;
233 		max = SUN4I_NDMA_NR_MAX_CHANNELS;
234 	}
235 
236 	spin_lock_irqsave(&priv->lock, flags);
237 	for_each_clear_bit_from(i, priv->pchans_used, max) {
238 		pchan = &pchans[i];
239 		pchan->vchan = vchan;
240 		set_bit(i, priv->pchans_used);
241 		break;
242 	}
243 	spin_unlock_irqrestore(&priv->lock, flags);
244 
245 	return pchan;
246 }
247 
248 static void release_pchan(struct sun4i_dma_dev *priv,
249 			  struct sun4i_dma_pchan *pchan)
250 {
251 	unsigned long flags;
252 	int nr = pchan - priv->pchans;
253 
254 	spin_lock_irqsave(&priv->lock, flags);
255 
256 	pchan->vchan = NULL;
257 	clear_bit(nr, priv->pchans_used);
258 
259 	spin_unlock_irqrestore(&priv->lock, flags);
260 }
261 
262 static void configure_pchan(struct sun4i_dma_pchan *pchan,
263 			    struct sun4i_dma_promise *d)
264 {
265 	/*
266 	 * Configure addresses and misc parameters depending on type
267 	 * SUN4I_DDMA has an extra field with timing parameters
268 	 */
269 	if (pchan->is_dedicated) {
270 		writel_relaxed(d->src, pchan->base + SUN4I_DDMA_SRC_ADDR_REG);
271 		writel_relaxed(d->dst, pchan->base + SUN4I_DDMA_DST_ADDR_REG);
272 		writel_relaxed(d->len, pchan->base + SUN4I_DDMA_BYTE_COUNT_REG);
273 		writel_relaxed(d->para, pchan->base + SUN4I_DDMA_PARA_REG);
274 		writel_relaxed(d->cfg, pchan->base + SUN4I_DDMA_CFG_REG);
275 	} else {
276 		writel_relaxed(d->src, pchan->base + SUN4I_NDMA_SRC_ADDR_REG);
277 		writel_relaxed(d->dst, pchan->base + SUN4I_NDMA_DST_ADDR_REG);
278 		writel_relaxed(d->len, pchan->base + SUN4I_NDMA_BYTE_COUNT_REG);
279 		writel_relaxed(d->cfg, pchan->base + SUN4I_NDMA_CFG_REG);
280 	}
281 }
282 
283 static void set_pchan_interrupt(struct sun4i_dma_dev *priv,
284 				struct sun4i_dma_pchan *pchan,
285 				int half, int end)
286 {
287 	u32 reg;
288 	int pchan_number = pchan - priv->pchans;
289 	unsigned long flags;
290 
291 	spin_lock_irqsave(&priv->lock, flags);
292 
293 	reg = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
294 
295 	if (half)
296 		reg |= BIT(pchan_number * 2);
297 	else
298 		reg &= ~BIT(pchan_number * 2);
299 
300 	if (end)
301 		reg |= BIT(pchan_number * 2 + 1);
302 	else
303 		reg &= ~BIT(pchan_number * 2 + 1);
304 
305 	writel_relaxed(reg, priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
306 
307 	spin_unlock_irqrestore(&priv->lock, flags);
308 }
309 
310 /**
311  * Execute pending operations on a vchan
312  *
313  * When given a vchan, this function will try to acquire a suitable
314  * pchan and, if successful, will configure it to fulfill a promise
315  * from the next pending contract.
316  *
317  * This function must be called with &vchan->vc.lock held.
318  */
319 static int __execute_vchan_pending(struct sun4i_dma_dev *priv,
320 				   struct sun4i_dma_vchan *vchan)
321 {
322 	struct sun4i_dma_promise *promise = NULL;
323 	struct sun4i_dma_contract *contract = NULL;
324 	struct sun4i_dma_pchan *pchan;
325 	struct virt_dma_desc *vd;
326 	int ret;
327 
328 	lockdep_assert_held(&vchan->vc.lock);
329 
330 	/* We need a pchan to do anything, so secure one if available */
331 	pchan = find_and_use_pchan(priv, vchan);
332 	if (!pchan)
333 		return -EBUSY;
334 
335 	/*
336 	 * Channel endpoints must not be repeated, so if this vchan
337 	 * has already submitted some work, we can't do anything else
338 	 */
339 	if (vchan->processing) {
340 		dev_dbg(chan2dev(&vchan->vc.chan),
341 			"processing something to this endpoint already\n");
342 		ret = -EBUSY;
343 		goto release_pchan;
344 	}
345 
346 	do {
347 		/* Figure out which contract we're working with today */
348 		vd = vchan_next_desc(&vchan->vc);
349 		if (!vd) {
350 			dev_dbg(chan2dev(&vchan->vc.chan),
351 				"No pending contract found");
352 			ret = 0;
353 			goto release_pchan;
354 		}
355 
356 		contract = to_sun4i_dma_contract(vd);
357 		if (list_empty(&contract->demands)) {
358 			/* The contract has been completed so mark it as such */
359 			list_del(&contract->vd.node);
360 			vchan_cookie_complete(&contract->vd);
361 			dev_dbg(chan2dev(&vchan->vc.chan),
362 				"Empty contract found and marked complete");
363 		}
364 	} while (list_empty(&contract->demands));
365 
366 	/* Now find out what we need to do */
367 	promise = list_first_entry(&contract->demands,
368 				   struct sun4i_dma_promise, list);
369 	vchan->processing = promise;
370 
371 	/* ... and make it reality */
372 	if (promise) {
373 		vchan->contract = contract;
374 		vchan->pchan = pchan;
375 		set_pchan_interrupt(priv, pchan, contract->is_cyclic, 1);
376 		configure_pchan(pchan, promise);
377 	}
378 
379 	return 0;
380 
381 release_pchan:
382 	release_pchan(priv, pchan);
383 	return ret;
384 }
385 
386 static int sanitize_config(struct dma_slave_config *sconfig,
387 			   enum dma_transfer_direction direction)
388 {
389 	switch (direction) {
390 	case DMA_MEM_TO_DEV:
391 		if ((sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) ||
392 		    !sconfig->dst_maxburst)
393 			return -EINVAL;
394 
395 		if (sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
396 			sconfig->src_addr_width = sconfig->dst_addr_width;
397 
398 		if (!sconfig->src_maxburst)
399 			sconfig->src_maxburst = sconfig->dst_maxburst;
400 
401 		break;
402 
403 	case DMA_DEV_TO_MEM:
404 		if ((sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) ||
405 		    !sconfig->src_maxburst)
406 			return -EINVAL;
407 
408 		if (sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
409 			sconfig->dst_addr_width = sconfig->src_addr_width;
410 
411 		if (!sconfig->dst_maxburst)
412 			sconfig->dst_maxburst = sconfig->src_maxburst;
413 
414 		break;
415 	default:
416 		return 0;
417 	}
418 
419 	return 0;
420 }
421 
422 /**
423  * Generate a promise, to be used in a normal DMA contract.
424  *
425  * A NDMA promise contains all the information required to program the
426  * normal part of the DMA Engine and get data copied. A non-executed
427  * promise will live in the demands list on a contract. Once it has been
428  * completed, it will be moved to the completed demands list for later freeing.
429  * All linked promises will be freed when the corresponding contract is freed
430  */
431 static struct sun4i_dma_promise *
432 generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest,
433 		      size_t len, struct dma_slave_config *sconfig,
434 		      enum dma_transfer_direction direction)
435 {
436 	struct sun4i_dma_promise *promise;
437 	int ret;
438 
439 	ret = sanitize_config(sconfig, direction);
440 	if (ret)
441 		return NULL;
442 
443 	promise = kzalloc(sizeof(*promise), GFP_NOWAIT);
444 	if (!promise)
445 		return NULL;
446 
447 	promise->src = src;
448 	promise->dst = dest;
449 	promise->len = len;
450 	promise->cfg = SUN4I_DMA_CFG_LOADING |
451 		SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN;
452 
453 	dev_dbg(chan2dev(chan),
454 		"src burst %d, dst burst %d, src buswidth %d, dst buswidth %d",
455 		sconfig->src_maxburst, sconfig->dst_maxburst,
456 		sconfig->src_addr_width, sconfig->dst_addr_width);
457 
458 	/* Source burst */
459 	ret = convert_burst(sconfig->src_maxburst);
460 	if (ret < 0)
461 		goto fail;
462 	promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret);
463 
464 	/* Destination burst */
465 	ret = convert_burst(sconfig->dst_maxburst);
466 	if (ret < 0)
467 		goto fail;
468 	promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret);
469 
470 	/* Source bus width */
471 	ret = convert_buswidth(sconfig->src_addr_width);
472 	if (ret < 0)
473 		goto fail;
474 	promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret);
475 
476 	/* Destination bus width */
477 	ret = convert_buswidth(sconfig->dst_addr_width);
478 	if (ret < 0)
479 		goto fail;
480 	promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret);
481 
482 	return promise;
483 
484 fail:
485 	kfree(promise);
486 	return NULL;
487 }
488 
489 /**
490  * Generate a promise, to be used in a dedicated DMA contract.
491  *
492  * A DDMA promise contains all the information required to program the
493  * Dedicated part of the DMA Engine and get data copied. A non-executed
494  * promise will live in the demands list on a contract. Once it has been
495  * completed, it will be moved to the completed demands list for later freeing.
496  * All linked promises will be freed when the corresponding contract is freed
497  */
498 static struct sun4i_dma_promise *
499 generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest,
500 		      size_t len, struct dma_slave_config *sconfig)
501 {
502 	struct sun4i_dma_promise *promise;
503 	int ret;
504 
505 	promise = kzalloc(sizeof(*promise), GFP_NOWAIT);
506 	if (!promise)
507 		return NULL;
508 
509 	promise->src = src;
510 	promise->dst = dest;
511 	promise->len = len;
512 	promise->cfg = SUN4I_DMA_CFG_LOADING |
513 		SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN;
514 
515 	/* Source burst */
516 	ret = convert_burst(sconfig->src_maxburst);
517 	if (ret < 0)
518 		goto fail;
519 	promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret);
520 
521 	/* Destination burst */
522 	ret = convert_burst(sconfig->dst_maxburst);
523 	if (ret < 0)
524 		goto fail;
525 	promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret);
526 
527 	/* Source bus width */
528 	ret = convert_buswidth(sconfig->src_addr_width);
529 	if (ret < 0)
530 		goto fail;
531 	promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret);
532 
533 	/* Destination bus width */
534 	ret = convert_buswidth(sconfig->dst_addr_width);
535 	if (ret < 0)
536 		goto fail;
537 	promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret);
538 
539 	return promise;
540 
541 fail:
542 	kfree(promise);
543 	return NULL;
544 }
545 
546 /**
547  * Generate a contract
548  *
549  * Contracts function as DMA descriptors. As our hardware does not support
550  * linked lists, we need to implement SG via software. We use a contract
551  * to hold all the pieces of the request and process them serially one
552  * after another. Each piece is represented as a promise.
553  */
554 static struct sun4i_dma_contract *generate_dma_contract(void)
555 {
556 	struct sun4i_dma_contract *contract;
557 
558 	contract = kzalloc(sizeof(*contract), GFP_NOWAIT);
559 	if (!contract)
560 		return NULL;
561 
562 	INIT_LIST_HEAD(&contract->demands);
563 	INIT_LIST_HEAD(&contract->completed_demands);
564 
565 	return contract;
566 }
567 
568 /**
569  * Get next promise on a cyclic transfer
570  *
571  * Cyclic contracts contain a series of promises which are executed on a
572  * loop. This function returns the next promise from a cyclic contract,
573  * so it can be programmed into the hardware.
574  */
575 static struct sun4i_dma_promise *
576 get_next_cyclic_promise(struct sun4i_dma_contract *contract)
577 {
578 	struct sun4i_dma_promise *promise;
579 
580 	promise = list_first_entry_or_null(&contract->demands,
581 					   struct sun4i_dma_promise, list);
582 	if (!promise) {
583 		list_splice_init(&contract->completed_demands,
584 				 &contract->demands);
585 		promise = list_first_entry(&contract->demands,
586 					   struct sun4i_dma_promise, list);
587 	}
588 
589 	return promise;
590 }
591 
592 /**
593  * Free a contract and all its associated promises
594  */
595 static void sun4i_dma_free_contract(struct virt_dma_desc *vd)
596 {
597 	struct sun4i_dma_contract *contract = to_sun4i_dma_contract(vd);
598 	struct sun4i_dma_promise *promise, *tmp;
599 
600 	/* Free all the demands and completed demands */
601 	list_for_each_entry_safe(promise, tmp, &contract->demands, list)
602 		kfree(promise);
603 
604 	list_for_each_entry_safe(promise, tmp, &contract->completed_demands, list)
605 		kfree(promise);
606 
607 	kfree(contract);
608 }
609 
610 static struct dma_async_tx_descriptor *
611 sun4i_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest,
612 			  dma_addr_t src, size_t len, unsigned long flags)
613 {
614 	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
615 	struct dma_slave_config *sconfig = &vchan->cfg;
616 	struct sun4i_dma_promise *promise;
617 	struct sun4i_dma_contract *contract;
618 
619 	contract = generate_dma_contract();
620 	if (!contract)
621 		return NULL;
622 
623 	/*
624 	 * We can only do the copy to bus aligned addresses, so
625 	 * choose the best one so we get decent performance. We also
626 	 * maximize the burst size for this same reason.
627 	 */
628 	sconfig->src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
629 	sconfig->dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
630 	sconfig->src_maxburst = 8;
631 	sconfig->dst_maxburst = 8;
632 
633 	if (vchan->is_dedicated)
634 		promise = generate_ddma_promise(chan, src, dest, len, sconfig);
635 	else
636 		promise = generate_ndma_promise(chan, src, dest, len, sconfig,
637 						DMA_MEM_TO_MEM);
638 
639 	if (!promise) {
640 		kfree(contract);
641 		return NULL;
642 	}
643 
644 	/* Configure memcpy mode */
645 	if (vchan->is_dedicated) {
646 		promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM) |
647 				SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM);
648 	} else {
649 		promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM) |
650 				SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM);
651 	}
652 
653 	/* Fill the contract with our only promise */
654 	list_add_tail(&promise->list, &contract->demands);
655 
656 	/* And add it to the vchan */
657 	return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
658 }
659 
660 static struct dma_async_tx_descriptor *
661 sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len,
662 			  size_t period_len, enum dma_transfer_direction dir,
663 			  unsigned long flags)
664 {
665 	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
666 	struct dma_slave_config *sconfig = &vchan->cfg;
667 	struct sun4i_dma_promise *promise;
668 	struct sun4i_dma_contract *contract;
669 	dma_addr_t src, dest;
670 	u32 endpoints;
671 	int nr_periods, offset, plength, i;
672 	u8 ram_type, io_mode, linear_mode;
673 
674 	if (!is_slave_direction(dir)) {
675 		dev_err(chan2dev(chan), "Invalid DMA direction\n");
676 		return NULL;
677 	}
678 
679 	contract = generate_dma_contract();
680 	if (!contract)
681 		return NULL;
682 
683 	contract->is_cyclic = 1;
684 
685 	if (vchan->is_dedicated) {
686 		io_mode = SUN4I_DDMA_ADDR_MODE_IO;
687 		linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR;
688 		ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM;
689 	} else {
690 		io_mode = SUN4I_NDMA_ADDR_MODE_IO;
691 		linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR;
692 		ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM;
693 	}
694 
695 	if (dir == DMA_MEM_TO_DEV) {
696 		src = buf;
697 		dest = sconfig->dst_addr;
698 		endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) |
699 			    SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) |
700 			    SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) |
701 			    SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode);
702 	} else {
703 		src = sconfig->src_addr;
704 		dest = buf;
705 		endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) |
706 			    SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) |
707 			    SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) |
708 			    SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode);
709 	}
710 
711 	/*
712 	 * We will be using half done interrupts to make two periods
713 	 * out of a promise, so we need to program the DMA engine less
714 	 * often
715 	 */
716 
717 	/*
718 	 * The engine can interrupt on half-transfer, so we can use
719 	 * this feature to program the engine half as often as if we
720 	 * didn't use it (keep in mind the hardware doesn't support
721 	 * linked lists).
722 	 *
723 	 * Say you have a set of periods (| marks the start/end, I for
724 	 * interrupt, P for programming the engine to do a new
725 	 * transfer), the easy but slow way would be to do
726 	 *
727 	 *  |---|---|---|---| (periods / promises)
728 	 *  P  I,P I,P I,P  I
729 	 *
730 	 * Using half transfer interrupts you can do
731 	 *
732 	 *  |-------|-------| (promises as configured on hw)
733 	 *  |---|---|---|---| (periods)
734 	 *  P   I  I,P  I   I
735 	 *
736 	 * Which requires half the engine programming for the same
737 	 * functionality.
738 	 */
739 	nr_periods = DIV_ROUND_UP(len / period_len, 2);
740 	for (i = 0; i < nr_periods; i++) {
741 		/* Calculate the offset in the buffer and the length needed */
742 		offset = i * period_len * 2;
743 		plength = min((len - offset), (period_len * 2));
744 		if (dir == DMA_MEM_TO_DEV)
745 			src = buf + offset;
746 		else
747 			dest = buf + offset;
748 
749 		/* Make the promise */
750 		if (vchan->is_dedicated)
751 			promise = generate_ddma_promise(chan, src, dest,
752 							plength, sconfig);
753 		else
754 			promise = generate_ndma_promise(chan, src, dest,
755 							plength, sconfig, dir);
756 
757 		if (!promise) {
758 			/* TODO: should we free everything? */
759 			return NULL;
760 		}
761 		promise->cfg |= endpoints;
762 
763 		/* Then add it to the contract */
764 		list_add_tail(&promise->list, &contract->demands);
765 	}
766 
767 	/* And add it to the vchan */
768 	return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
769 }
770 
771 static struct dma_async_tx_descriptor *
772 sun4i_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
773 			unsigned int sg_len, enum dma_transfer_direction dir,
774 			unsigned long flags, void *context)
775 {
776 	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
777 	struct dma_slave_config *sconfig = &vchan->cfg;
778 	struct sun4i_dma_promise *promise;
779 	struct sun4i_dma_contract *contract;
780 	u8 ram_type, io_mode, linear_mode;
781 	struct scatterlist *sg;
782 	dma_addr_t srcaddr, dstaddr;
783 	u32 endpoints, para;
784 	int i;
785 
786 	if (!sgl)
787 		return NULL;
788 
789 	if (!is_slave_direction(dir)) {
790 		dev_err(chan2dev(chan), "Invalid DMA direction\n");
791 		return NULL;
792 	}
793 
794 	contract = generate_dma_contract();
795 	if (!contract)
796 		return NULL;
797 
798 	if (vchan->is_dedicated) {
799 		io_mode = SUN4I_DDMA_ADDR_MODE_IO;
800 		linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR;
801 		ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM;
802 	} else {
803 		io_mode = SUN4I_NDMA_ADDR_MODE_IO;
804 		linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR;
805 		ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM;
806 	}
807 
808 	if (dir == DMA_MEM_TO_DEV)
809 		endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) |
810 			    SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) |
811 			    SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) |
812 			    SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode);
813 	else
814 		endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) |
815 			    SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) |
816 			    SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) |
817 			    SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode);
818 
819 	for_each_sg(sgl, sg, sg_len, i) {
820 		/* Figure out addresses */
821 		if (dir == DMA_MEM_TO_DEV) {
822 			srcaddr = sg_dma_address(sg);
823 			dstaddr = sconfig->dst_addr;
824 		} else {
825 			srcaddr = sconfig->src_addr;
826 			dstaddr = sg_dma_address(sg);
827 		}
828 
829 		/*
830 		 * These are the magic DMA engine timings that keep SPI going.
831 		 * I haven't seen any interface on DMAEngine to configure
832 		 * timings, and so far they seem to work for everything we
833 		 * support, so I've kept them here. I don't know if other
834 		 * devices need different timings because, as usual, we only
835 		 * have the "para" bitfield meanings, but no comment on what
836 		 * the values should be when doing a certain operation :|
837 		 */
838 		para = SUN4I_DDMA_MAGIC_SPI_PARAMETERS;
839 
840 		/* And make a suitable promise */
841 		if (vchan->is_dedicated)
842 			promise = generate_ddma_promise(chan, srcaddr, dstaddr,
843 							sg_dma_len(sg),
844 							sconfig);
845 		else
846 			promise = generate_ndma_promise(chan, srcaddr, dstaddr,
847 							sg_dma_len(sg),
848 							sconfig, dir);
849 
850 		if (!promise)
851 			return NULL; /* TODO: should we free everything? */
852 
853 		promise->cfg |= endpoints;
854 		promise->para = para;
855 
856 		/* Then add it to the contract */
857 		list_add_tail(&promise->list, &contract->demands);
858 	}
859 
860 	/*
861 	 * Once we've got all the promises ready, add the contract
862 	 * to the pending list on the vchan
863 	 */
864 	return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
865 }
866 
867 static int sun4i_dma_terminate_all(struct dma_chan *chan)
868 {
869 	struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
870 	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
871 	struct sun4i_dma_pchan *pchan = vchan->pchan;
872 	LIST_HEAD(head);
873 	unsigned long flags;
874 
875 	spin_lock_irqsave(&vchan->vc.lock, flags);
876 	vchan_get_all_descriptors(&vchan->vc, &head);
877 	spin_unlock_irqrestore(&vchan->vc.lock, flags);
878 
879 	/*
880 	 * Clearing the configuration register will halt the pchan. Interrupts
881 	 * may still trigger, so don't forget to disable them.
882 	 */
883 	if (pchan) {
884 		if (pchan->is_dedicated)
885 			writel(0, pchan->base + SUN4I_DDMA_CFG_REG);
886 		else
887 			writel(0, pchan->base + SUN4I_NDMA_CFG_REG);
888 		set_pchan_interrupt(priv, pchan, 0, 0);
889 		release_pchan(priv, pchan);
890 	}
891 
892 	spin_lock_irqsave(&vchan->vc.lock, flags);
893 	/* Clear these so the vchan is usable again */
894 	vchan->processing = NULL;
895 	vchan->pchan = NULL;
896 	spin_unlock_irqrestore(&vchan->vc.lock, flags);
897 
898 	vchan_dma_desc_free_list(&vchan->vc, &head);
899 
900 	return 0;
901 }
902 
903 static int sun4i_dma_config(struct dma_chan *chan,
904 			    struct dma_slave_config *config)
905 {
906 	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
907 
908 	memcpy(&vchan->cfg, config, sizeof(*config));
909 
910 	return 0;
911 }
912 
913 static struct dma_chan *sun4i_dma_of_xlate(struct of_phandle_args *dma_spec,
914 					   struct of_dma *ofdma)
915 {
916 	struct sun4i_dma_dev *priv = ofdma->of_dma_data;
917 	struct sun4i_dma_vchan *vchan;
918 	struct dma_chan *chan;
919 	u8 is_dedicated = dma_spec->args[0];
920 	u8 endpoint = dma_spec->args[1];
921 
922 	/* Check if type is Normal or Dedicated */
923 	if (is_dedicated != 0 && is_dedicated != 1)
924 		return NULL;
925 
926 	/* Make sure the endpoint looks sane */
927 	if ((is_dedicated && endpoint >= SUN4I_DDMA_DRQ_TYPE_LIMIT) ||
928 	    (!is_dedicated && endpoint >= SUN4I_NDMA_DRQ_TYPE_LIMIT))
929 		return NULL;
930 
931 	chan = dma_get_any_slave_channel(&priv->slave);
932 	if (!chan)
933 		return NULL;
934 
935 	/* Assign the endpoint to the vchan */
936 	vchan = to_sun4i_dma_vchan(chan);
937 	vchan->is_dedicated = is_dedicated;
938 	vchan->endpoint = endpoint;
939 
940 	return chan;
941 }
942 
943 static enum dma_status sun4i_dma_tx_status(struct dma_chan *chan,
944 					   dma_cookie_t cookie,
945 					   struct dma_tx_state *state)
946 {
947 	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
948 	struct sun4i_dma_pchan *pchan = vchan->pchan;
949 	struct sun4i_dma_contract *contract;
950 	struct sun4i_dma_promise *promise;
951 	struct virt_dma_desc *vd;
952 	unsigned long flags;
953 	enum dma_status ret;
954 	size_t bytes = 0;
955 
956 	ret = dma_cookie_status(chan, cookie, state);
957 	if (!state || (ret == DMA_COMPLETE))
958 		return ret;
959 
960 	spin_lock_irqsave(&vchan->vc.lock, flags);
961 	vd = vchan_find_desc(&vchan->vc, cookie);
962 	if (!vd)
963 		goto exit;
964 	contract = to_sun4i_dma_contract(vd);
965 
966 	list_for_each_entry(promise, &contract->demands, list)
967 		bytes += promise->len;
968 
969 	/*
970 	 * The hardware is configured to return the remaining byte
971 	 * quantity. If possible, replace the first listed element's
972 	 * full size with the actual remaining amount
973 	 */
974 	promise = list_first_entry_or_null(&contract->demands,
975 					   struct sun4i_dma_promise, list);
976 	if (promise && pchan) {
977 		bytes -= promise->len;
978 		if (pchan->is_dedicated)
979 			bytes += readl(pchan->base + SUN4I_DDMA_BYTE_COUNT_REG);
980 		else
981 			bytes += readl(pchan->base + SUN4I_NDMA_BYTE_COUNT_REG);
982 	}
983 
984 exit:
985 
986 	dma_set_residue(state, bytes);
987 	spin_unlock_irqrestore(&vchan->vc.lock, flags);
988 
989 	return ret;
990 }
991 
992 static void sun4i_dma_issue_pending(struct dma_chan *chan)
993 {
994 	struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
995 	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
996 	unsigned long flags;
997 
998 	spin_lock_irqsave(&vchan->vc.lock, flags);
999 
1000 	/*
1001 	 * If there are pending transactions for this vchan, push one of
1002 	 * them into the engine to get the ball rolling.
1003 	 */
1004 	if (vchan_issue_pending(&vchan->vc))
1005 		__execute_vchan_pending(priv, vchan);
1006 
1007 	spin_unlock_irqrestore(&vchan->vc.lock, flags);
1008 }
1009 
1010 static irqreturn_t sun4i_dma_interrupt(int irq, void *dev_id)
1011 {
1012 	struct sun4i_dma_dev *priv = dev_id;
1013 	struct sun4i_dma_pchan *pchans = priv->pchans, *pchan;
1014 	struct sun4i_dma_vchan *vchan;
1015 	struct sun4i_dma_contract *contract;
1016 	struct sun4i_dma_promise *promise;
1017 	unsigned long pendirq, irqs, disableirqs;
1018 	int bit, i, free_room, allow_mitigation = 1;
1019 
1020 	pendirq = readl_relaxed(priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1021 
1022 handle_pending:
1023 
1024 	disableirqs = 0;
1025 	free_room = 0;
1026 
1027 	for_each_set_bit(bit, &pendirq, 32) {
1028 		pchan = &pchans[bit >> 1];
1029 		vchan = pchan->vchan;
1030 		if (!vchan) /* a terminated channel may still interrupt */
1031 			continue;
1032 		contract = vchan->contract;
1033 
1034 		/*
1035 		 * Disable the IRQ and free the pchan if it's an end
1036 		 * interrupt (odd bit)
1037 		 */
1038 		if (bit & 1) {
1039 			spin_lock(&vchan->vc.lock);
1040 
1041 			/*
1042 			 * Move the promise into the completed list now that
1043 			 * we're done with it
1044 			 */
1045 			list_del(&vchan->processing->list);
1046 			list_add_tail(&vchan->processing->list,
1047 				      &contract->completed_demands);
1048 
1049 			/*
1050 			 * Cyclic DMA transfers are special:
1051 			 * - There's always something we can dispatch
1052 			 * - We need to run the callback
1053 			 * - Latency is very important, as this is used by audio
1054 			 * We therefore just cycle through the list and dispatch
1055 			 * whatever we have here, reusing the pchan. There's
1056 			 * no need to run the thread after this.
1057 			 *
1058 			 * For non-cyclic transfers we need to look around,
1059 			 * so we can program some more work, or notify the
1060 			 * client that their transfers have been completed.
1061 			 */
1062 			if (contract->is_cyclic) {
1063 				promise = get_next_cyclic_promise(contract);
1064 				vchan->processing = promise;
1065 				configure_pchan(pchan, promise);
1066 				vchan_cyclic_callback(&contract->vd);
1067 			} else {
1068 				vchan->processing = NULL;
1069 				vchan->pchan = NULL;
1070 
1071 				free_room = 1;
1072 				disableirqs |= BIT(bit);
1073 				release_pchan(priv, pchan);
1074 			}
1075 
1076 			spin_unlock(&vchan->vc.lock);
1077 		} else {
1078 			/* Half done interrupt */
1079 			if (contract->is_cyclic)
1080 				vchan_cyclic_callback(&contract->vd);
1081 			else
1082 				disableirqs |= BIT(bit);
1083 		}
1084 	}
1085 
1086 	/* Disable the IRQs for events we handled */
1087 	spin_lock(&priv->lock);
1088 	irqs = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1089 	writel_relaxed(irqs & ~disableirqs,
1090 		       priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1091 	spin_unlock(&priv->lock);
1092 
1093 	/* Writing 1 to the pending field will clear the pending interrupt */
1094 	writel_relaxed(pendirq, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1095 
1096 	/*
1097 	 * If a pchan was freed, we may be able to schedule something else,
1098 	 * so have a look around
1099 	 */
1100 	if (free_room) {
1101 		for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) {
1102 			vchan = &priv->vchans[i];
1103 			spin_lock(&vchan->vc.lock);
1104 			__execute_vchan_pending(priv, vchan);
1105 			spin_unlock(&vchan->vc.lock);
1106 		}
1107 	}
1108 
1109 	/*
1110 	 * Handle newer interrupts if some showed up, but only do it once
1111 	 * to avoid a too long a loop
1112 	 */
1113 	if (allow_mitigation) {
1114 		pendirq = readl_relaxed(priv->base +
1115 					SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1116 		if (pendirq) {
1117 			allow_mitigation = 0;
1118 			goto handle_pending;
1119 		}
1120 	}
1121 
1122 	return IRQ_HANDLED;
1123 }
1124 
1125 static int sun4i_dma_probe(struct platform_device *pdev)
1126 {
1127 	struct sun4i_dma_dev *priv;
1128 	struct resource *res;
1129 	int i, j, ret;
1130 
1131 	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
1132 	if (!priv)
1133 		return -ENOMEM;
1134 
1135 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1136 	priv->base = devm_ioremap_resource(&pdev->dev, res);
1137 	if (IS_ERR(priv->base))
1138 		return PTR_ERR(priv->base);
1139 
1140 	priv->irq = platform_get_irq(pdev, 0);
1141 	if (priv->irq < 0)
1142 		return priv->irq;
1143 
1144 	priv->clk = devm_clk_get(&pdev->dev, NULL);
1145 	if (IS_ERR(priv->clk)) {
1146 		dev_err(&pdev->dev, "No clock specified\n");
1147 		return PTR_ERR(priv->clk);
1148 	}
1149 
1150 	platform_set_drvdata(pdev, priv);
1151 	spin_lock_init(&priv->lock);
1152 
1153 	dma_cap_zero(priv->slave.cap_mask);
1154 	dma_cap_set(DMA_PRIVATE, priv->slave.cap_mask);
1155 	dma_cap_set(DMA_MEMCPY, priv->slave.cap_mask);
1156 	dma_cap_set(DMA_CYCLIC, priv->slave.cap_mask);
1157 	dma_cap_set(DMA_SLAVE, priv->slave.cap_mask);
1158 
1159 	INIT_LIST_HEAD(&priv->slave.channels);
1160 	priv->slave.device_free_chan_resources	= sun4i_dma_free_chan_resources;
1161 	priv->slave.device_tx_status		= sun4i_dma_tx_status;
1162 	priv->slave.device_issue_pending	= sun4i_dma_issue_pending;
1163 	priv->slave.device_prep_slave_sg	= sun4i_dma_prep_slave_sg;
1164 	priv->slave.device_prep_dma_memcpy	= sun4i_dma_prep_dma_memcpy;
1165 	priv->slave.device_prep_dma_cyclic	= sun4i_dma_prep_dma_cyclic;
1166 	priv->slave.device_config		= sun4i_dma_config;
1167 	priv->slave.device_terminate_all	= sun4i_dma_terminate_all;
1168 	priv->slave.copy_align			= 2;
1169 	priv->slave.src_addr_widths		= BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1170 						  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1171 						  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1172 	priv->slave.dst_addr_widths		= BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1173 						  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1174 						  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1175 	priv->slave.directions			= BIT(DMA_DEV_TO_MEM) |
1176 						  BIT(DMA_MEM_TO_DEV);
1177 	priv->slave.residue_granularity		= DMA_RESIDUE_GRANULARITY_BURST;
1178 
1179 	priv->slave.dev = &pdev->dev;
1180 
1181 	priv->pchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_CHANNELS,
1182 				    sizeof(struct sun4i_dma_pchan), GFP_KERNEL);
1183 	priv->vchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_VCHANS,
1184 				    sizeof(struct sun4i_dma_vchan), GFP_KERNEL);
1185 	if (!priv->vchans || !priv->pchans)
1186 		return -ENOMEM;
1187 
1188 	/*
1189 	 * [0..SUN4I_NDMA_NR_MAX_CHANNELS) are normal pchans, and
1190 	 * [SUN4I_NDMA_NR_MAX_CHANNELS..SUN4I_DMA_NR_MAX_CHANNELS) are
1191 	 * dedicated ones
1192 	 */
1193 	for (i = 0; i < SUN4I_NDMA_NR_MAX_CHANNELS; i++)
1194 		priv->pchans[i].base = priv->base +
1195 			SUN4I_NDMA_CHANNEL_REG_BASE(i);
1196 
1197 	for (j = 0; i < SUN4I_DMA_NR_MAX_CHANNELS; i++, j++) {
1198 		priv->pchans[i].base = priv->base +
1199 			SUN4I_DDMA_CHANNEL_REG_BASE(j);
1200 		priv->pchans[i].is_dedicated = 1;
1201 	}
1202 
1203 	for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) {
1204 		struct sun4i_dma_vchan *vchan = &priv->vchans[i];
1205 
1206 		spin_lock_init(&vchan->vc.lock);
1207 		vchan->vc.desc_free = sun4i_dma_free_contract;
1208 		vchan_init(&vchan->vc, &priv->slave);
1209 	}
1210 
1211 	ret = clk_prepare_enable(priv->clk);
1212 	if (ret) {
1213 		dev_err(&pdev->dev, "Couldn't enable the clock\n");
1214 		return ret;
1215 	}
1216 
1217 	/*
1218 	 * Make sure the IRQs are all disabled and accounted for. The bootloader
1219 	 * likes to leave these dirty
1220 	 */
1221 	writel(0, priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1222 	writel(0xFFFFFFFF, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1223 
1224 	ret = devm_request_irq(&pdev->dev, priv->irq, sun4i_dma_interrupt,
1225 			       0, dev_name(&pdev->dev), priv);
1226 	if (ret) {
1227 		dev_err(&pdev->dev, "Cannot request IRQ\n");
1228 		goto err_clk_disable;
1229 	}
1230 
1231 	ret = dma_async_device_register(&priv->slave);
1232 	if (ret) {
1233 		dev_warn(&pdev->dev, "Failed to register DMA engine device\n");
1234 		goto err_clk_disable;
1235 	}
1236 
1237 	ret = of_dma_controller_register(pdev->dev.of_node, sun4i_dma_of_xlate,
1238 					 priv);
1239 	if (ret) {
1240 		dev_err(&pdev->dev, "of_dma_controller_register failed\n");
1241 		goto err_dma_unregister;
1242 	}
1243 
1244 	dev_dbg(&pdev->dev, "Successfully probed SUN4I_DMA\n");
1245 
1246 	return 0;
1247 
1248 err_dma_unregister:
1249 	dma_async_device_unregister(&priv->slave);
1250 err_clk_disable:
1251 	clk_disable_unprepare(priv->clk);
1252 	return ret;
1253 }
1254 
1255 static int sun4i_dma_remove(struct platform_device *pdev)
1256 {
1257 	struct sun4i_dma_dev *priv = platform_get_drvdata(pdev);
1258 
1259 	/* Disable IRQ so no more work is scheduled */
1260 	disable_irq(priv->irq);
1261 
1262 	of_dma_controller_free(pdev->dev.of_node);
1263 	dma_async_device_unregister(&priv->slave);
1264 
1265 	clk_disable_unprepare(priv->clk);
1266 
1267 	return 0;
1268 }
1269 
1270 static const struct of_device_id sun4i_dma_match[] = {
1271 	{ .compatible = "allwinner,sun4i-a10-dma" },
1272 	{ /* sentinel */ },
1273 };
1274 MODULE_DEVICE_TABLE(of, sun4i_dma_match);
1275 
1276 static struct platform_driver sun4i_dma_driver = {
1277 	.probe	= sun4i_dma_probe,
1278 	.remove	= sun4i_dma_remove,
1279 	.driver	= {
1280 		.name		= "sun4i-dma",
1281 		.of_match_table	= sun4i_dma_match,
1282 	},
1283 };
1284 
1285 module_platform_driver(sun4i_dma_driver);
1286 
1287 MODULE_DESCRIPTION("Allwinner A10 Dedicated DMA Controller Driver");
1288 MODULE_AUTHOR("Emilio López <emilio@elopez.com.ar>");
1289 MODULE_LICENSE("GPL");
1290