xref: /openbmc/linux/drivers/dma/sun4i-dma.c (revision 4b23603a)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright (C) 2014 Emilio López
4  * Emilio López <emilio@elopez.com.ar>
5  */
6 
7 #include <linux/bitmap.h>
8 #include <linux/bitops.h>
9 #include <linux/clk.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/dmaengine.h>
12 #include <linux/dmapool.h>
13 #include <linux/interrupt.h>
14 #include <linux/module.h>
15 #include <linux/of_dma.h>
16 #include <linux/platform_device.h>
17 #include <linux/slab.h>
18 #include <linux/spinlock.h>
19 
20 #include "virt-dma.h"
21 
22 /** Common macros to normal and dedicated DMA registers **/
23 
24 #define SUN4I_DMA_CFG_LOADING			BIT(31)
25 #define SUN4I_DMA_CFG_DST_DATA_WIDTH(width)	((width) << 25)
26 #define SUN4I_DMA_CFG_DST_BURST_LENGTH(len)	((len) << 23)
27 #define SUN4I_DMA_CFG_DST_ADDR_MODE(mode)	((mode) << 21)
28 #define SUN4I_DMA_CFG_DST_DRQ_TYPE(type)	((type) << 16)
29 #define SUN4I_DMA_CFG_SRC_DATA_WIDTH(width)	((width) << 9)
30 #define SUN4I_DMA_CFG_SRC_BURST_LENGTH(len)	((len) << 7)
31 #define SUN4I_DMA_CFG_SRC_ADDR_MODE(mode)	((mode) << 5)
32 #define SUN4I_DMA_CFG_SRC_DRQ_TYPE(type)	(type)
33 
34 /** Normal DMA register values **/
35 
36 /* Normal DMA source/destination data request type values */
37 #define SUN4I_NDMA_DRQ_TYPE_SDRAM		0x16
38 #define SUN4I_NDMA_DRQ_TYPE_LIMIT		(0x1F + 1)
39 
40 /** Normal DMA register layout **/
41 
42 /* Dedicated DMA source/destination address mode values */
43 #define SUN4I_NDMA_ADDR_MODE_LINEAR		0
44 #define SUN4I_NDMA_ADDR_MODE_IO			1
45 
46 /* Normal DMA configuration register layout */
47 #define SUN4I_NDMA_CFG_CONT_MODE		BIT(30)
48 #define SUN4I_NDMA_CFG_WAIT_STATE(n)		((n) << 27)
49 #define SUN4I_NDMA_CFG_DST_NON_SECURE		BIT(22)
50 #define SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN	BIT(15)
51 #define SUN4I_NDMA_CFG_SRC_NON_SECURE		BIT(6)
52 
53 /** Dedicated DMA register values **/
54 
55 /* Dedicated DMA source/destination address mode values */
56 #define SUN4I_DDMA_ADDR_MODE_LINEAR		0
57 #define SUN4I_DDMA_ADDR_MODE_IO			1
58 #define SUN4I_DDMA_ADDR_MODE_HORIZONTAL_PAGE	2
59 #define SUN4I_DDMA_ADDR_MODE_VERTICAL_PAGE	3
60 
61 /* Dedicated DMA source/destination data request type values */
62 #define SUN4I_DDMA_DRQ_TYPE_SDRAM		0x1
63 #define SUN4I_DDMA_DRQ_TYPE_LIMIT		(0x1F + 1)
64 
65 /** Dedicated DMA register layout **/
66 
67 /* Dedicated DMA configuration register layout */
68 #define SUN4I_DDMA_CFG_BUSY			BIT(30)
69 #define SUN4I_DDMA_CFG_CONT_MODE		BIT(29)
70 #define SUN4I_DDMA_CFG_DST_NON_SECURE		BIT(28)
71 #define SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN	BIT(15)
72 #define SUN4I_DDMA_CFG_SRC_NON_SECURE		BIT(12)
73 
74 /* Dedicated DMA parameter register layout */
75 #define SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(n)	(((n) - 1) << 24)
76 #define SUN4I_DDMA_PARA_DST_WAIT_CYCLES(n)	(((n) - 1) << 16)
77 #define SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(n)	(((n) - 1) << 8)
78 #define SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(n)	(((n) - 1) << 0)
79 
80 /** DMA register offsets **/
81 
82 /* General register offsets */
83 #define SUN4I_DMA_IRQ_ENABLE_REG		0x0
84 #define SUN4I_DMA_IRQ_PENDING_STATUS_REG	0x4
85 
86 /* Normal DMA register offsets */
87 #define SUN4I_NDMA_CHANNEL_REG_BASE(n)		(0x100 + (n) * 0x20)
88 #define SUN4I_NDMA_CFG_REG			0x0
89 #define SUN4I_NDMA_SRC_ADDR_REG			0x4
90 #define SUN4I_NDMA_DST_ADDR_REG		0x8
91 #define SUN4I_NDMA_BYTE_COUNT_REG		0xC
92 
93 /* Dedicated DMA register offsets */
94 #define SUN4I_DDMA_CHANNEL_REG_BASE(n)		(0x300 + (n) * 0x20)
95 #define SUN4I_DDMA_CFG_REG			0x0
96 #define SUN4I_DDMA_SRC_ADDR_REG			0x4
97 #define SUN4I_DDMA_DST_ADDR_REG		0x8
98 #define SUN4I_DDMA_BYTE_COUNT_REG		0xC
99 #define SUN4I_DDMA_PARA_REG			0x18
100 
101 /** DMA Driver **/
102 
103 /*
104  * Normal DMA has 8 channels, and Dedicated DMA has another 8, so
105  * that's 16 channels. As for endpoints, there's 29 and 21
106  * respectively. Given that the Normal DMA endpoints (other than
107  * SDRAM) can be used as tx/rx, we need 78 vchans in total
108  */
109 #define SUN4I_NDMA_NR_MAX_CHANNELS	8
110 #define SUN4I_DDMA_NR_MAX_CHANNELS	8
111 #define SUN4I_DMA_NR_MAX_CHANNELS					\
112 	(SUN4I_NDMA_NR_MAX_CHANNELS + SUN4I_DDMA_NR_MAX_CHANNELS)
113 #define SUN4I_NDMA_NR_MAX_VCHANS	(29 * 2 - 1)
114 #define SUN4I_DDMA_NR_MAX_VCHANS	21
115 #define SUN4I_DMA_NR_MAX_VCHANS						\
116 	(SUN4I_NDMA_NR_MAX_VCHANS + SUN4I_DDMA_NR_MAX_VCHANS)
117 
118 /* This set of SUN4I_DDMA timing parameters were found experimentally while
119  * working with the SPI driver and seem to make it behave correctly */
120 #define SUN4I_DDMA_MAGIC_SPI_PARAMETERS \
121 	(SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(1) |			\
122 	 SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(1) |				\
123 	 SUN4I_DDMA_PARA_DST_WAIT_CYCLES(2) |				\
124 	 SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(2))
125 
126 /*
127  * Normal DMA supports individual transfers (segments) up to 128k.
128  * Dedicated DMA supports transfers up to 16M. We can only report
129  * one size limit, so we have to use the smaller value.
130  */
131 #define SUN4I_NDMA_MAX_SEG_SIZE		SZ_128K
132 #define SUN4I_DDMA_MAX_SEG_SIZE		SZ_16M
133 #define SUN4I_DMA_MAX_SEG_SIZE		SUN4I_NDMA_MAX_SEG_SIZE
134 
135 struct sun4i_dma_pchan {
136 	/* Register base of channel */
137 	void __iomem			*base;
138 	/* vchan currently being serviced */
139 	struct sun4i_dma_vchan		*vchan;
140 	/* Is this a dedicated pchan? */
141 	int				is_dedicated;
142 };
143 
144 struct sun4i_dma_vchan {
145 	struct virt_dma_chan		vc;
146 	struct dma_slave_config		cfg;
147 	struct sun4i_dma_pchan		*pchan;
148 	struct sun4i_dma_promise	*processing;
149 	struct sun4i_dma_contract	*contract;
150 	u8				endpoint;
151 	int				is_dedicated;
152 };
153 
154 struct sun4i_dma_promise {
155 	u32				cfg;
156 	u32				para;
157 	dma_addr_t			src;
158 	dma_addr_t			dst;
159 	size_t				len;
160 	struct list_head		list;
161 };
162 
163 /* A contract is a set of promises */
164 struct sun4i_dma_contract {
165 	struct virt_dma_desc		vd;
166 	struct list_head		demands;
167 	struct list_head		completed_demands;
168 	bool				is_cyclic : 1;
169 	bool				use_half_int : 1;
170 };
171 
172 struct sun4i_dma_dev {
173 	DECLARE_BITMAP(pchans_used, SUN4I_DMA_NR_MAX_CHANNELS);
174 	struct dma_device		slave;
175 	struct sun4i_dma_pchan		*pchans;
176 	struct sun4i_dma_vchan		*vchans;
177 	void __iomem			*base;
178 	struct clk			*clk;
179 	int				irq;
180 	spinlock_t			lock;
181 };
182 
to_sun4i_dma_dev(struct dma_device * dev)183 static struct sun4i_dma_dev *to_sun4i_dma_dev(struct dma_device *dev)
184 {
185 	return container_of(dev, struct sun4i_dma_dev, slave);
186 }
187 
to_sun4i_dma_vchan(struct dma_chan * chan)188 static struct sun4i_dma_vchan *to_sun4i_dma_vchan(struct dma_chan *chan)
189 {
190 	return container_of(chan, struct sun4i_dma_vchan, vc.chan);
191 }
192 
to_sun4i_dma_contract(struct virt_dma_desc * vd)193 static struct sun4i_dma_contract *to_sun4i_dma_contract(struct virt_dma_desc *vd)
194 {
195 	return container_of(vd, struct sun4i_dma_contract, vd);
196 }
197 
chan2dev(struct dma_chan * chan)198 static struct device *chan2dev(struct dma_chan *chan)
199 {
200 	return &chan->dev->device;
201 }
202 
convert_burst(u32 maxburst)203 static int convert_burst(u32 maxburst)
204 {
205 	if (maxburst > 8)
206 		return -EINVAL;
207 
208 	/* 1 -> 0, 4 -> 1, 8 -> 2 */
209 	return (maxburst >> 2);
210 }
211 
convert_buswidth(enum dma_slave_buswidth addr_width)212 static int convert_buswidth(enum dma_slave_buswidth addr_width)
213 {
214 	if (addr_width > DMA_SLAVE_BUSWIDTH_4_BYTES)
215 		return -EINVAL;
216 
217 	/* 8 (1 byte) -> 0, 16 (2 bytes) -> 1, 32 (4 bytes) -> 2 */
218 	return (addr_width >> 1);
219 }
220 
sun4i_dma_free_chan_resources(struct dma_chan * chan)221 static void sun4i_dma_free_chan_resources(struct dma_chan *chan)
222 {
223 	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
224 
225 	vchan_free_chan_resources(&vchan->vc);
226 }
227 
find_and_use_pchan(struct sun4i_dma_dev * priv,struct sun4i_dma_vchan * vchan)228 static struct sun4i_dma_pchan *find_and_use_pchan(struct sun4i_dma_dev *priv,
229 						  struct sun4i_dma_vchan *vchan)
230 {
231 	struct sun4i_dma_pchan *pchan = NULL, *pchans = priv->pchans;
232 	unsigned long flags;
233 	int i, max;
234 
235 	/*
236 	 * pchans 0-SUN4I_NDMA_NR_MAX_CHANNELS are normal, and
237 	 * SUN4I_NDMA_NR_MAX_CHANNELS+ are dedicated ones
238 	 */
239 	if (vchan->is_dedicated) {
240 		i = SUN4I_NDMA_NR_MAX_CHANNELS;
241 		max = SUN4I_DMA_NR_MAX_CHANNELS;
242 	} else {
243 		i = 0;
244 		max = SUN4I_NDMA_NR_MAX_CHANNELS;
245 	}
246 
247 	spin_lock_irqsave(&priv->lock, flags);
248 	for_each_clear_bit_from(i, priv->pchans_used, max) {
249 		pchan = &pchans[i];
250 		pchan->vchan = vchan;
251 		set_bit(i, priv->pchans_used);
252 		break;
253 	}
254 	spin_unlock_irqrestore(&priv->lock, flags);
255 
256 	return pchan;
257 }
258 
release_pchan(struct sun4i_dma_dev * priv,struct sun4i_dma_pchan * pchan)259 static void release_pchan(struct sun4i_dma_dev *priv,
260 			  struct sun4i_dma_pchan *pchan)
261 {
262 	unsigned long flags;
263 	int nr = pchan - priv->pchans;
264 
265 	spin_lock_irqsave(&priv->lock, flags);
266 
267 	pchan->vchan = NULL;
268 	clear_bit(nr, priv->pchans_used);
269 
270 	spin_unlock_irqrestore(&priv->lock, flags);
271 }
272 
configure_pchan(struct sun4i_dma_pchan * pchan,struct sun4i_dma_promise * d)273 static void configure_pchan(struct sun4i_dma_pchan *pchan,
274 			    struct sun4i_dma_promise *d)
275 {
276 	/*
277 	 * Configure addresses and misc parameters depending on type
278 	 * SUN4I_DDMA has an extra field with timing parameters
279 	 */
280 	if (pchan->is_dedicated) {
281 		writel_relaxed(d->src, pchan->base + SUN4I_DDMA_SRC_ADDR_REG);
282 		writel_relaxed(d->dst, pchan->base + SUN4I_DDMA_DST_ADDR_REG);
283 		writel_relaxed(d->len, pchan->base + SUN4I_DDMA_BYTE_COUNT_REG);
284 		writel_relaxed(d->para, pchan->base + SUN4I_DDMA_PARA_REG);
285 		writel_relaxed(d->cfg, pchan->base + SUN4I_DDMA_CFG_REG);
286 	} else {
287 		writel_relaxed(d->src, pchan->base + SUN4I_NDMA_SRC_ADDR_REG);
288 		writel_relaxed(d->dst, pchan->base + SUN4I_NDMA_DST_ADDR_REG);
289 		writel_relaxed(d->len, pchan->base + SUN4I_NDMA_BYTE_COUNT_REG);
290 		writel_relaxed(d->cfg, pchan->base + SUN4I_NDMA_CFG_REG);
291 	}
292 }
293 
set_pchan_interrupt(struct sun4i_dma_dev * priv,struct sun4i_dma_pchan * pchan,int half,int end)294 static void set_pchan_interrupt(struct sun4i_dma_dev *priv,
295 				struct sun4i_dma_pchan *pchan,
296 				int half, int end)
297 {
298 	u32 reg;
299 	int pchan_number = pchan - priv->pchans;
300 	unsigned long flags;
301 
302 	spin_lock_irqsave(&priv->lock, flags);
303 
304 	reg = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
305 
306 	if (half)
307 		reg |= BIT(pchan_number * 2);
308 	else
309 		reg &= ~BIT(pchan_number * 2);
310 
311 	if (end)
312 		reg |= BIT(pchan_number * 2 + 1);
313 	else
314 		reg &= ~BIT(pchan_number * 2 + 1);
315 
316 	writel_relaxed(reg, priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
317 
318 	spin_unlock_irqrestore(&priv->lock, flags);
319 }
320 
321 /*
322  * Execute pending operations on a vchan
323  *
324  * When given a vchan, this function will try to acquire a suitable
325  * pchan and, if successful, will configure it to fulfill a promise
326  * from the next pending contract.
327  *
328  * This function must be called with &vchan->vc.lock held.
329  */
__execute_vchan_pending(struct sun4i_dma_dev * priv,struct sun4i_dma_vchan * vchan)330 static int __execute_vchan_pending(struct sun4i_dma_dev *priv,
331 				   struct sun4i_dma_vchan *vchan)
332 {
333 	struct sun4i_dma_promise *promise = NULL;
334 	struct sun4i_dma_contract *contract = NULL;
335 	struct sun4i_dma_pchan *pchan;
336 	struct virt_dma_desc *vd;
337 	int ret;
338 
339 	lockdep_assert_held(&vchan->vc.lock);
340 
341 	/* We need a pchan to do anything, so secure one if available */
342 	pchan = find_and_use_pchan(priv, vchan);
343 	if (!pchan)
344 		return -EBUSY;
345 
346 	/*
347 	 * Channel endpoints must not be repeated, so if this vchan
348 	 * has already submitted some work, we can't do anything else
349 	 */
350 	if (vchan->processing) {
351 		dev_dbg(chan2dev(&vchan->vc.chan),
352 			"processing something to this endpoint already\n");
353 		ret = -EBUSY;
354 		goto release_pchan;
355 	}
356 
357 	do {
358 		/* Figure out which contract we're working with today */
359 		vd = vchan_next_desc(&vchan->vc);
360 		if (!vd) {
361 			dev_dbg(chan2dev(&vchan->vc.chan),
362 				"No pending contract found");
363 			ret = 0;
364 			goto release_pchan;
365 		}
366 
367 		contract = to_sun4i_dma_contract(vd);
368 		if (list_empty(&contract->demands)) {
369 			/* The contract has been completed so mark it as such */
370 			list_del(&contract->vd.node);
371 			vchan_cookie_complete(&contract->vd);
372 			dev_dbg(chan2dev(&vchan->vc.chan),
373 				"Empty contract found and marked complete");
374 		}
375 	} while (list_empty(&contract->demands));
376 
377 	/* Now find out what we need to do */
378 	promise = list_first_entry(&contract->demands,
379 				   struct sun4i_dma_promise, list);
380 	vchan->processing = promise;
381 
382 	/* ... and make it reality */
383 	if (promise) {
384 		vchan->contract = contract;
385 		vchan->pchan = pchan;
386 		set_pchan_interrupt(priv, pchan, contract->use_half_int, 1);
387 		configure_pchan(pchan, promise);
388 	}
389 
390 	return 0;
391 
392 release_pchan:
393 	release_pchan(priv, pchan);
394 	return ret;
395 }
396 
sanitize_config(struct dma_slave_config * sconfig,enum dma_transfer_direction direction)397 static int sanitize_config(struct dma_slave_config *sconfig,
398 			   enum dma_transfer_direction direction)
399 {
400 	switch (direction) {
401 	case DMA_MEM_TO_DEV:
402 		if ((sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) ||
403 		    !sconfig->dst_maxburst)
404 			return -EINVAL;
405 
406 		if (sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
407 			sconfig->src_addr_width = sconfig->dst_addr_width;
408 
409 		if (!sconfig->src_maxburst)
410 			sconfig->src_maxburst = sconfig->dst_maxburst;
411 
412 		break;
413 
414 	case DMA_DEV_TO_MEM:
415 		if ((sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) ||
416 		    !sconfig->src_maxburst)
417 			return -EINVAL;
418 
419 		if (sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
420 			sconfig->dst_addr_width = sconfig->src_addr_width;
421 
422 		if (!sconfig->dst_maxburst)
423 			sconfig->dst_maxburst = sconfig->src_maxburst;
424 
425 		break;
426 	default:
427 		return 0;
428 	}
429 
430 	return 0;
431 }
432 
433 /*
434  * Generate a promise, to be used in a normal DMA contract.
435  *
436  * A NDMA promise contains all the information required to program the
437  * normal part of the DMA Engine and get data copied. A non-executed
438  * promise will live in the demands list on a contract. Once it has been
439  * completed, it will be moved to the completed demands list for later freeing.
440  * All linked promises will be freed when the corresponding contract is freed
441  */
442 static struct sun4i_dma_promise *
generate_ndma_promise(struct dma_chan * chan,dma_addr_t src,dma_addr_t dest,size_t len,struct dma_slave_config * sconfig,enum dma_transfer_direction direction)443 generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest,
444 		      size_t len, struct dma_slave_config *sconfig,
445 		      enum dma_transfer_direction direction)
446 {
447 	struct sun4i_dma_promise *promise;
448 	int ret;
449 
450 	ret = sanitize_config(sconfig, direction);
451 	if (ret)
452 		return NULL;
453 
454 	promise = kzalloc(sizeof(*promise), GFP_NOWAIT);
455 	if (!promise)
456 		return NULL;
457 
458 	promise->src = src;
459 	promise->dst = dest;
460 	promise->len = len;
461 	promise->cfg = SUN4I_DMA_CFG_LOADING |
462 		SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN;
463 
464 	dev_dbg(chan2dev(chan),
465 		"src burst %d, dst burst %d, src buswidth %d, dst buswidth %d",
466 		sconfig->src_maxburst, sconfig->dst_maxburst,
467 		sconfig->src_addr_width, sconfig->dst_addr_width);
468 
469 	/* Source burst */
470 	ret = convert_burst(sconfig->src_maxburst);
471 	if (ret < 0)
472 		goto fail;
473 	promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret);
474 
475 	/* Destination burst */
476 	ret = convert_burst(sconfig->dst_maxburst);
477 	if (ret < 0)
478 		goto fail;
479 	promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret);
480 
481 	/* Source bus width */
482 	ret = convert_buswidth(sconfig->src_addr_width);
483 	if (ret < 0)
484 		goto fail;
485 	promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret);
486 
487 	/* Destination bus width */
488 	ret = convert_buswidth(sconfig->dst_addr_width);
489 	if (ret < 0)
490 		goto fail;
491 	promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret);
492 
493 	return promise;
494 
495 fail:
496 	kfree(promise);
497 	return NULL;
498 }
499 
500 /*
501  * Generate a promise, to be used in a dedicated DMA contract.
502  *
503  * A DDMA promise contains all the information required to program the
504  * Dedicated part of the DMA Engine and get data copied. A non-executed
505  * promise will live in the demands list on a contract. Once it has been
506  * completed, it will be moved to the completed demands list for later freeing.
507  * All linked promises will be freed when the corresponding contract is freed
508  */
509 static struct sun4i_dma_promise *
generate_ddma_promise(struct dma_chan * chan,dma_addr_t src,dma_addr_t dest,size_t len,struct dma_slave_config * sconfig)510 generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest,
511 		      size_t len, struct dma_slave_config *sconfig)
512 {
513 	struct sun4i_dma_promise *promise;
514 	int ret;
515 
516 	promise = kzalloc(sizeof(*promise), GFP_NOWAIT);
517 	if (!promise)
518 		return NULL;
519 
520 	promise->src = src;
521 	promise->dst = dest;
522 	promise->len = len;
523 	promise->cfg = SUN4I_DMA_CFG_LOADING |
524 		SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN;
525 
526 	/* Source burst */
527 	ret = convert_burst(sconfig->src_maxburst);
528 	if (ret < 0)
529 		goto fail;
530 	promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret);
531 
532 	/* Destination burst */
533 	ret = convert_burst(sconfig->dst_maxburst);
534 	if (ret < 0)
535 		goto fail;
536 	promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret);
537 
538 	/* Source bus width */
539 	ret = convert_buswidth(sconfig->src_addr_width);
540 	if (ret < 0)
541 		goto fail;
542 	promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret);
543 
544 	/* Destination bus width */
545 	ret = convert_buswidth(sconfig->dst_addr_width);
546 	if (ret < 0)
547 		goto fail;
548 	promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret);
549 
550 	return promise;
551 
552 fail:
553 	kfree(promise);
554 	return NULL;
555 }
556 
557 /*
558  * Generate a contract
559  *
560  * Contracts function as DMA descriptors. As our hardware does not support
561  * linked lists, we need to implement SG via software. We use a contract
562  * to hold all the pieces of the request and process them serially one
563  * after another. Each piece is represented as a promise.
564  */
generate_dma_contract(void)565 static struct sun4i_dma_contract *generate_dma_contract(void)
566 {
567 	struct sun4i_dma_contract *contract;
568 
569 	contract = kzalloc(sizeof(*contract), GFP_NOWAIT);
570 	if (!contract)
571 		return NULL;
572 
573 	INIT_LIST_HEAD(&contract->demands);
574 	INIT_LIST_HEAD(&contract->completed_demands);
575 
576 	return contract;
577 }
578 
579 /*
580  * Get next promise on a cyclic transfer
581  *
582  * Cyclic contracts contain a series of promises which are executed on a
583  * loop. This function returns the next promise from a cyclic contract,
584  * so it can be programmed into the hardware.
585  */
586 static struct sun4i_dma_promise *
get_next_cyclic_promise(struct sun4i_dma_contract * contract)587 get_next_cyclic_promise(struct sun4i_dma_contract *contract)
588 {
589 	struct sun4i_dma_promise *promise;
590 
591 	promise = list_first_entry_or_null(&contract->demands,
592 					   struct sun4i_dma_promise, list);
593 	if (!promise) {
594 		list_splice_init(&contract->completed_demands,
595 				 &contract->demands);
596 		promise = list_first_entry(&contract->demands,
597 					   struct sun4i_dma_promise, list);
598 	}
599 
600 	return promise;
601 }
602 
603 /*
604  * Free a contract and all its associated promises
605  */
sun4i_dma_free_contract(struct virt_dma_desc * vd)606 static void sun4i_dma_free_contract(struct virt_dma_desc *vd)
607 {
608 	struct sun4i_dma_contract *contract = to_sun4i_dma_contract(vd);
609 	struct sun4i_dma_promise *promise, *tmp;
610 
611 	/* Free all the demands and completed demands */
612 	list_for_each_entry_safe(promise, tmp, &contract->demands, list)
613 		kfree(promise);
614 
615 	list_for_each_entry_safe(promise, tmp, &contract->completed_demands, list)
616 		kfree(promise);
617 
618 	kfree(contract);
619 }
620 
621 static struct dma_async_tx_descriptor *
sun4i_dma_prep_dma_memcpy(struct dma_chan * chan,dma_addr_t dest,dma_addr_t src,size_t len,unsigned long flags)622 sun4i_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest,
623 			  dma_addr_t src, size_t len, unsigned long flags)
624 {
625 	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
626 	struct dma_slave_config *sconfig = &vchan->cfg;
627 	struct sun4i_dma_promise *promise;
628 	struct sun4i_dma_contract *contract;
629 
630 	contract = generate_dma_contract();
631 	if (!contract)
632 		return NULL;
633 
634 	/*
635 	 * We can only do the copy to bus aligned addresses, so
636 	 * choose the best one so we get decent performance. We also
637 	 * maximize the burst size for this same reason.
638 	 */
639 	sconfig->src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
640 	sconfig->dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
641 	sconfig->src_maxburst = 8;
642 	sconfig->dst_maxburst = 8;
643 
644 	if (vchan->is_dedicated)
645 		promise = generate_ddma_promise(chan, src, dest, len, sconfig);
646 	else
647 		promise = generate_ndma_promise(chan, src, dest, len, sconfig,
648 						DMA_MEM_TO_MEM);
649 
650 	if (!promise) {
651 		kfree(contract);
652 		return NULL;
653 	}
654 
655 	/* Configure memcpy mode */
656 	if (vchan->is_dedicated) {
657 		promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM) |
658 				SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM);
659 	} else {
660 		promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM) |
661 				SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM);
662 	}
663 
664 	/* Fill the contract with our only promise */
665 	list_add_tail(&promise->list, &contract->demands);
666 
667 	/* And add it to the vchan */
668 	return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
669 }
670 
671 static struct dma_async_tx_descriptor *
sun4i_dma_prep_dma_cyclic(struct dma_chan * chan,dma_addr_t buf,size_t len,size_t period_len,enum dma_transfer_direction dir,unsigned long flags)672 sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len,
673 			  size_t period_len, enum dma_transfer_direction dir,
674 			  unsigned long flags)
675 {
676 	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
677 	struct dma_slave_config *sconfig = &vchan->cfg;
678 	struct sun4i_dma_promise *promise;
679 	struct sun4i_dma_contract *contract;
680 	dma_addr_t src, dest;
681 	u32 endpoints;
682 	int nr_periods, offset, plength, i;
683 	u8 ram_type, io_mode, linear_mode;
684 
685 	if (!is_slave_direction(dir)) {
686 		dev_err(chan2dev(chan), "Invalid DMA direction\n");
687 		return NULL;
688 	}
689 
690 	contract = generate_dma_contract();
691 	if (!contract)
692 		return NULL;
693 
694 	contract->is_cyclic = 1;
695 
696 	if (vchan->is_dedicated) {
697 		io_mode = SUN4I_DDMA_ADDR_MODE_IO;
698 		linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR;
699 		ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM;
700 	} else {
701 		io_mode = SUN4I_NDMA_ADDR_MODE_IO;
702 		linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR;
703 		ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM;
704 	}
705 
706 	if (dir == DMA_MEM_TO_DEV) {
707 		src = buf;
708 		dest = sconfig->dst_addr;
709 		endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) |
710 			    SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) |
711 			    SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) |
712 			    SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode);
713 	} else {
714 		src = sconfig->src_addr;
715 		dest = buf;
716 		endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) |
717 			    SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) |
718 			    SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) |
719 			    SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode);
720 	}
721 
722 	/*
723 	 * We will be using half done interrupts to make two periods
724 	 * out of a promise, so we need to program the DMA engine less
725 	 * often
726 	 */
727 
728 	/*
729 	 * The engine can interrupt on half-transfer, so we can use
730 	 * this feature to program the engine half as often as if we
731 	 * didn't use it (keep in mind the hardware doesn't support
732 	 * linked lists).
733 	 *
734 	 * Say you have a set of periods (| marks the start/end, I for
735 	 * interrupt, P for programming the engine to do a new
736 	 * transfer), the easy but slow way would be to do
737 	 *
738 	 *  |---|---|---|---| (periods / promises)
739 	 *  P  I,P I,P I,P  I
740 	 *
741 	 * Using half transfer interrupts you can do
742 	 *
743 	 *  |-------|-------| (promises as configured on hw)
744 	 *  |---|---|---|---| (periods)
745 	 *  P   I  I,P  I   I
746 	 *
747 	 * Which requires half the engine programming for the same
748 	 * functionality.
749 	 *
750 	 * This only works if two periods fit in a single promise. That will
751 	 * always be the case for dedicated DMA, where the hardware has a much
752 	 * larger maximum transfer size than advertised to clients.
753 	 */
754 	if (vchan->is_dedicated || period_len <= SUN4I_NDMA_MAX_SEG_SIZE / 2) {
755 		period_len *= 2;
756 		contract->use_half_int = 1;
757 	}
758 
759 	nr_periods = DIV_ROUND_UP(len, period_len);
760 	for (i = 0; i < nr_periods; i++) {
761 		/* Calculate the offset in the buffer and the length needed */
762 		offset = i * period_len;
763 		plength = min((len - offset), period_len);
764 		if (dir == DMA_MEM_TO_DEV)
765 			src = buf + offset;
766 		else
767 			dest = buf + offset;
768 
769 		/* Make the promise */
770 		if (vchan->is_dedicated)
771 			promise = generate_ddma_promise(chan, src, dest,
772 							plength, sconfig);
773 		else
774 			promise = generate_ndma_promise(chan, src, dest,
775 							plength, sconfig, dir);
776 
777 		if (!promise) {
778 			/* TODO: should we free everything? */
779 			return NULL;
780 		}
781 		promise->cfg |= endpoints;
782 
783 		/* Then add it to the contract */
784 		list_add_tail(&promise->list, &contract->demands);
785 	}
786 
787 	/* And add it to the vchan */
788 	return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
789 }
790 
791 static struct dma_async_tx_descriptor *
sun4i_dma_prep_slave_sg(struct dma_chan * chan,struct scatterlist * sgl,unsigned int sg_len,enum dma_transfer_direction dir,unsigned long flags,void * context)792 sun4i_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
793 			unsigned int sg_len, enum dma_transfer_direction dir,
794 			unsigned long flags, void *context)
795 {
796 	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
797 	struct dma_slave_config *sconfig = &vchan->cfg;
798 	struct sun4i_dma_promise *promise;
799 	struct sun4i_dma_contract *contract;
800 	u8 ram_type, io_mode, linear_mode;
801 	struct scatterlist *sg;
802 	dma_addr_t srcaddr, dstaddr;
803 	u32 endpoints, para;
804 	int i;
805 
806 	if (!sgl)
807 		return NULL;
808 
809 	if (!is_slave_direction(dir)) {
810 		dev_err(chan2dev(chan), "Invalid DMA direction\n");
811 		return NULL;
812 	}
813 
814 	contract = generate_dma_contract();
815 	if (!contract)
816 		return NULL;
817 
818 	if (vchan->is_dedicated) {
819 		io_mode = SUN4I_DDMA_ADDR_MODE_IO;
820 		linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR;
821 		ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM;
822 	} else {
823 		io_mode = SUN4I_NDMA_ADDR_MODE_IO;
824 		linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR;
825 		ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM;
826 	}
827 
828 	if (dir == DMA_MEM_TO_DEV)
829 		endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) |
830 			    SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) |
831 			    SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) |
832 			    SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode);
833 	else
834 		endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) |
835 			    SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) |
836 			    SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) |
837 			    SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode);
838 
839 	for_each_sg(sgl, sg, sg_len, i) {
840 		/* Figure out addresses */
841 		if (dir == DMA_MEM_TO_DEV) {
842 			srcaddr = sg_dma_address(sg);
843 			dstaddr = sconfig->dst_addr;
844 		} else {
845 			srcaddr = sconfig->src_addr;
846 			dstaddr = sg_dma_address(sg);
847 		}
848 
849 		/*
850 		 * These are the magic DMA engine timings that keep SPI going.
851 		 * I haven't seen any interface on DMAEngine to configure
852 		 * timings, and so far they seem to work for everything we
853 		 * support, so I've kept them here. I don't know if other
854 		 * devices need different timings because, as usual, we only
855 		 * have the "para" bitfield meanings, but no comment on what
856 		 * the values should be when doing a certain operation :|
857 		 */
858 		para = SUN4I_DDMA_MAGIC_SPI_PARAMETERS;
859 
860 		/* And make a suitable promise */
861 		if (vchan->is_dedicated)
862 			promise = generate_ddma_promise(chan, srcaddr, dstaddr,
863 							sg_dma_len(sg),
864 							sconfig);
865 		else
866 			promise = generate_ndma_promise(chan, srcaddr, dstaddr,
867 							sg_dma_len(sg),
868 							sconfig, dir);
869 
870 		if (!promise)
871 			return NULL; /* TODO: should we free everything? */
872 
873 		promise->cfg |= endpoints;
874 		promise->para = para;
875 
876 		/* Then add it to the contract */
877 		list_add_tail(&promise->list, &contract->demands);
878 	}
879 
880 	/*
881 	 * Once we've got all the promises ready, add the contract
882 	 * to the pending list on the vchan
883 	 */
884 	return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
885 }
886 
sun4i_dma_terminate_all(struct dma_chan * chan)887 static int sun4i_dma_terminate_all(struct dma_chan *chan)
888 {
889 	struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
890 	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
891 	struct sun4i_dma_pchan *pchan = vchan->pchan;
892 	LIST_HEAD(head);
893 	unsigned long flags;
894 
895 	spin_lock_irqsave(&vchan->vc.lock, flags);
896 	vchan_get_all_descriptors(&vchan->vc, &head);
897 	spin_unlock_irqrestore(&vchan->vc.lock, flags);
898 
899 	/*
900 	 * Clearing the configuration register will halt the pchan. Interrupts
901 	 * may still trigger, so don't forget to disable them.
902 	 */
903 	if (pchan) {
904 		if (pchan->is_dedicated)
905 			writel(0, pchan->base + SUN4I_DDMA_CFG_REG);
906 		else
907 			writel(0, pchan->base + SUN4I_NDMA_CFG_REG);
908 		set_pchan_interrupt(priv, pchan, 0, 0);
909 		release_pchan(priv, pchan);
910 	}
911 
912 	spin_lock_irqsave(&vchan->vc.lock, flags);
913 	/* Clear these so the vchan is usable again */
914 	vchan->processing = NULL;
915 	vchan->pchan = NULL;
916 	spin_unlock_irqrestore(&vchan->vc.lock, flags);
917 
918 	vchan_dma_desc_free_list(&vchan->vc, &head);
919 
920 	return 0;
921 }
922 
sun4i_dma_config(struct dma_chan * chan,struct dma_slave_config * config)923 static int sun4i_dma_config(struct dma_chan *chan,
924 			    struct dma_slave_config *config)
925 {
926 	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
927 
928 	memcpy(&vchan->cfg, config, sizeof(*config));
929 
930 	return 0;
931 }
932 
sun4i_dma_of_xlate(struct of_phandle_args * dma_spec,struct of_dma * ofdma)933 static struct dma_chan *sun4i_dma_of_xlate(struct of_phandle_args *dma_spec,
934 					   struct of_dma *ofdma)
935 {
936 	struct sun4i_dma_dev *priv = ofdma->of_dma_data;
937 	struct sun4i_dma_vchan *vchan;
938 	struct dma_chan *chan;
939 	u8 is_dedicated = dma_spec->args[0];
940 	u8 endpoint = dma_spec->args[1];
941 
942 	/* Check if type is Normal or Dedicated */
943 	if (is_dedicated != 0 && is_dedicated != 1)
944 		return NULL;
945 
946 	/* Make sure the endpoint looks sane */
947 	if ((is_dedicated && endpoint >= SUN4I_DDMA_DRQ_TYPE_LIMIT) ||
948 	    (!is_dedicated && endpoint >= SUN4I_NDMA_DRQ_TYPE_LIMIT))
949 		return NULL;
950 
951 	chan = dma_get_any_slave_channel(&priv->slave);
952 	if (!chan)
953 		return NULL;
954 
955 	/* Assign the endpoint to the vchan */
956 	vchan = to_sun4i_dma_vchan(chan);
957 	vchan->is_dedicated = is_dedicated;
958 	vchan->endpoint = endpoint;
959 
960 	return chan;
961 }
962 
sun4i_dma_tx_status(struct dma_chan * chan,dma_cookie_t cookie,struct dma_tx_state * state)963 static enum dma_status sun4i_dma_tx_status(struct dma_chan *chan,
964 					   dma_cookie_t cookie,
965 					   struct dma_tx_state *state)
966 {
967 	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
968 	struct sun4i_dma_pchan *pchan = vchan->pchan;
969 	struct sun4i_dma_contract *contract;
970 	struct sun4i_dma_promise *promise;
971 	struct virt_dma_desc *vd;
972 	unsigned long flags;
973 	enum dma_status ret;
974 	size_t bytes = 0;
975 
976 	ret = dma_cookie_status(chan, cookie, state);
977 	if (!state || (ret == DMA_COMPLETE))
978 		return ret;
979 
980 	spin_lock_irqsave(&vchan->vc.lock, flags);
981 	vd = vchan_find_desc(&vchan->vc, cookie);
982 	if (!vd)
983 		goto exit;
984 	contract = to_sun4i_dma_contract(vd);
985 
986 	list_for_each_entry(promise, &contract->demands, list)
987 		bytes += promise->len;
988 
989 	/*
990 	 * The hardware is configured to return the remaining byte
991 	 * quantity. If possible, replace the first listed element's
992 	 * full size with the actual remaining amount
993 	 */
994 	promise = list_first_entry_or_null(&contract->demands,
995 					   struct sun4i_dma_promise, list);
996 	if (promise && pchan) {
997 		bytes -= promise->len;
998 		if (pchan->is_dedicated)
999 			bytes += readl(pchan->base + SUN4I_DDMA_BYTE_COUNT_REG);
1000 		else
1001 			bytes += readl(pchan->base + SUN4I_NDMA_BYTE_COUNT_REG);
1002 	}
1003 
1004 exit:
1005 
1006 	dma_set_residue(state, bytes);
1007 	spin_unlock_irqrestore(&vchan->vc.lock, flags);
1008 
1009 	return ret;
1010 }
1011 
sun4i_dma_issue_pending(struct dma_chan * chan)1012 static void sun4i_dma_issue_pending(struct dma_chan *chan)
1013 {
1014 	struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
1015 	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
1016 	unsigned long flags;
1017 
1018 	spin_lock_irqsave(&vchan->vc.lock, flags);
1019 
1020 	/*
1021 	 * If there are pending transactions for this vchan, push one of
1022 	 * them into the engine to get the ball rolling.
1023 	 */
1024 	if (vchan_issue_pending(&vchan->vc))
1025 		__execute_vchan_pending(priv, vchan);
1026 
1027 	spin_unlock_irqrestore(&vchan->vc.lock, flags);
1028 }
1029 
sun4i_dma_interrupt(int irq,void * dev_id)1030 static irqreturn_t sun4i_dma_interrupt(int irq, void *dev_id)
1031 {
1032 	struct sun4i_dma_dev *priv = dev_id;
1033 	struct sun4i_dma_pchan *pchans = priv->pchans, *pchan;
1034 	struct sun4i_dma_vchan *vchan;
1035 	struct sun4i_dma_contract *contract;
1036 	struct sun4i_dma_promise *promise;
1037 	unsigned long pendirq, irqs, disableirqs;
1038 	int bit, i, free_room, allow_mitigation = 1;
1039 
1040 	pendirq = readl_relaxed(priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1041 
1042 handle_pending:
1043 
1044 	disableirqs = 0;
1045 	free_room = 0;
1046 
1047 	for_each_set_bit(bit, &pendirq, 32) {
1048 		pchan = &pchans[bit >> 1];
1049 		vchan = pchan->vchan;
1050 		if (!vchan) /* a terminated channel may still interrupt */
1051 			continue;
1052 		contract = vchan->contract;
1053 
1054 		/*
1055 		 * Disable the IRQ and free the pchan if it's an end
1056 		 * interrupt (odd bit)
1057 		 */
1058 		if (bit & 1) {
1059 			spin_lock(&vchan->vc.lock);
1060 
1061 			/*
1062 			 * Move the promise into the completed list now that
1063 			 * we're done with it
1064 			 */
1065 			list_move_tail(&vchan->processing->list,
1066 				       &contract->completed_demands);
1067 
1068 			/*
1069 			 * Cyclic DMA transfers are special:
1070 			 * - There's always something we can dispatch
1071 			 * - We need to run the callback
1072 			 * - Latency is very important, as this is used by audio
1073 			 * We therefore just cycle through the list and dispatch
1074 			 * whatever we have here, reusing the pchan. There's
1075 			 * no need to run the thread after this.
1076 			 *
1077 			 * For non-cyclic transfers we need to look around,
1078 			 * so we can program some more work, or notify the
1079 			 * client that their transfers have been completed.
1080 			 */
1081 			if (contract->is_cyclic) {
1082 				promise = get_next_cyclic_promise(contract);
1083 				vchan->processing = promise;
1084 				configure_pchan(pchan, promise);
1085 				vchan_cyclic_callback(&contract->vd);
1086 			} else {
1087 				vchan->processing = NULL;
1088 				vchan->pchan = NULL;
1089 
1090 				free_room = 1;
1091 				disableirqs |= BIT(bit);
1092 				release_pchan(priv, pchan);
1093 			}
1094 
1095 			spin_unlock(&vchan->vc.lock);
1096 		} else {
1097 			/* Half done interrupt */
1098 			if (contract->is_cyclic)
1099 				vchan_cyclic_callback(&contract->vd);
1100 			else
1101 				disableirqs |= BIT(bit);
1102 		}
1103 	}
1104 
1105 	/* Disable the IRQs for events we handled */
1106 	spin_lock(&priv->lock);
1107 	irqs = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1108 	writel_relaxed(irqs & ~disableirqs,
1109 		       priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1110 	spin_unlock(&priv->lock);
1111 
1112 	/* Writing 1 to the pending field will clear the pending interrupt */
1113 	writel_relaxed(pendirq, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1114 
1115 	/*
1116 	 * If a pchan was freed, we may be able to schedule something else,
1117 	 * so have a look around
1118 	 */
1119 	if (free_room) {
1120 		for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) {
1121 			vchan = &priv->vchans[i];
1122 			spin_lock(&vchan->vc.lock);
1123 			__execute_vchan_pending(priv, vchan);
1124 			spin_unlock(&vchan->vc.lock);
1125 		}
1126 	}
1127 
1128 	/*
1129 	 * Handle newer interrupts if some showed up, but only do it once
1130 	 * to avoid a too long a loop
1131 	 */
1132 	if (allow_mitigation) {
1133 		pendirq = readl_relaxed(priv->base +
1134 					SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1135 		if (pendirq) {
1136 			allow_mitigation = 0;
1137 			goto handle_pending;
1138 		}
1139 	}
1140 
1141 	return IRQ_HANDLED;
1142 }
1143 
sun4i_dma_probe(struct platform_device * pdev)1144 static int sun4i_dma_probe(struct platform_device *pdev)
1145 {
1146 	struct sun4i_dma_dev *priv;
1147 	int i, j, ret;
1148 
1149 	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
1150 	if (!priv)
1151 		return -ENOMEM;
1152 
1153 	priv->base = devm_platform_ioremap_resource(pdev, 0);
1154 	if (IS_ERR(priv->base))
1155 		return PTR_ERR(priv->base);
1156 
1157 	priv->irq = platform_get_irq(pdev, 0);
1158 	if (priv->irq < 0)
1159 		return priv->irq;
1160 
1161 	priv->clk = devm_clk_get(&pdev->dev, NULL);
1162 	if (IS_ERR(priv->clk)) {
1163 		dev_err(&pdev->dev, "No clock specified\n");
1164 		return PTR_ERR(priv->clk);
1165 	}
1166 
1167 	platform_set_drvdata(pdev, priv);
1168 	spin_lock_init(&priv->lock);
1169 
1170 	dma_set_max_seg_size(&pdev->dev, SUN4I_DMA_MAX_SEG_SIZE);
1171 
1172 	dma_cap_zero(priv->slave.cap_mask);
1173 	dma_cap_set(DMA_PRIVATE, priv->slave.cap_mask);
1174 	dma_cap_set(DMA_MEMCPY, priv->slave.cap_mask);
1175 	dma_cap_set(DMA_CYCLIC, priv->slave.cap_mask);
1176 	dma_cap_set(DMA_SLAVE, priv->slave.cap_mask);
1177 
1178 	INIT_LIST_HEAD(&priv->slave.channels);
1179 	priv->slave.device_free_chan_resources	= sun4i_dma_free_chan_resources;
1180 	priv->slave.device_tx_status		= sun4i_dma_tx_status;
1181 	priv->slave.device_issue_pending	= sun4i_dma_issue_pending;
1182 	priv->slave.device_prep_slave_sg	= sun4i_dma_prep_slave_sg;
1183 	priv->slave.device_prep_dma_memcpy	= sun4i_dma_prep_dma_memcpy;
1184 	priv->slave.device_prep_dma_cyclic	= sun4i_dma_prep_dma_cyclic;
1185 	priv->slave.device_config		= sun4i_dma_config;
1186 	priv->slave.device_terminate_all	= sun4i_dma_terminate_all;
1187 	priv->slave.copy_align			= 2;
1188 	priv->slave.src_addr_widths		= BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1189 						  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1190 						  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1191 	priv->slave.dst_addr_widths		= BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1192 						  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1193 						  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1194 	priv->slave.directions			= BIT(DMA_DEV_TO_MEM) |
1195 						  BIT(DMA_MEM_TO_DEV);
1196 	priv->slave.residue_granularity		= DMA_RESIDUE_GRANULARITY_BURST;
1197 
1198 	priv->slave.dev = &pdev->dev;
1199 
1200 	priv->pchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_CHANNELS,
1201 				    sizeof(struct sun4i_dma_pchan), GFP_KERNEL);
1202 	priv->vchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_VCHANS,
1203 				    sizeof(struct sun4i_dma_vchan), GFP_KERNEL);
1204 	if (!priv->vchans || !priv->pchans)
1205 		return -ENOMEM;
1206 
1207 	/*
1208 	 * [0..SUN4I_NDMA_NR_MAX_CHANNELS) are normal pchans, and
1209 	 * [SUN4I_NDMA_NR_MAX_CHANNELS..SUN4I_DMA_NR_MAX_CHANNELS) are
1210 	 * dedicated ones
1211 	 */
1212 	for (i = 0; i < SUN4I_NDMA_NR_MAX_CHANNELS; i++)
1213 		priv->pchans[i].base = priv->base +
1214 			SUN4I_NDMA_CHANNEL_REG_BASE(i);
1215 
1216 	for (j = 0; i < SUN4I_DMA_NR_MAX_CHANNELS; i++, j++) {
1217 		priv->pchans[i].base = priv->base +
1218 			SUN4I_DDMA_CHANNEL_REG_BASE(j);
1219 		priv->pchans[i].is_dedicated = 1;
1220 	}
1221 
1222 	for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) {
1223 		struct sun4i_dma_vchan *vchan = &priv->vchans[i];
1224 
1225 		spin_lock_init(&vchan->vc.lock);
1226 		vchan->vc.desc_free = sun4i_dma_free_contract;
1227 		vchan_init(&vchan->vc, &priv->slave);
1228 	}
1229 
1230 	ret = clk_prepare_enable(priv->clk);
1231 	if (ret) {
1232 		dev_err(&pdev->dev, "Couldn't enable the clock\n");
1233 		return ret;
1234 	}
1235 
1236 	/*
1237 	 * Make sure the IRQs are all disabled and accounted for. The bootloader
1238 	 * likes to leave these dirty
1239 	 */
1240 	writel(0, priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1241 	writel(0xFFFFFFFF, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1242 
1243 	ret = devm_request_irq(&pdev->dev, priv->irq, sun4i_dma_interrupt,
1244 			       0, dev_name(&pdev->dev), priv);
1245 	if (ret) {
1246 		dev_err(&pdev->dev, "Cannot request IRQ\n");
1247 		goto err_clk_disable;
1248 	}
1249 
1250 	ret = dma_async_device_register(&priv->slave);
1251 	if (ret) {
1252 		dev_warn(&pdev->dev, "Failed to register DMA engine device\n");
1253 		goto err_clk_disable;
1254 	}
1255 
1256 	ret = of_dma_controller_register(pdev->dev.of_node, sun4i_dma_of_xlate,
1257 					 priv);
1258 	if (ret) {
1259 		dev_err(&pdev->dev, "of_dma_controller_register failed\n");
1260 		goto err_dma_unregister;
1261 	}
1262 
1263 	dev_dbg(&pdev->dev, "Successfully probed SUN4I_DMA\n");
1264 
1265 	return 0;
1266 
1267 err_dma_unregister:
1268 	dma_async_device_unregister(&priv->slave);
1269 err_clk_disable:
1270 	clk_disable_unprepare(priv->clk);
1271 	return ret;
1272 }
1273 
sun4i_dma_remove(struct platform_device * pdev)1274 static int sun4i_dma_remove(struct platform_device *pdev)
1275 {
1276 	struct sun4i_dma_dev *priv = platform_get_drvdata(pdev);
1277 
1278 	/* Disable IRQ so no more work is scheduled */
1279 	disable_irq(priv->irq);
1280 
1281 	of_dma_controller_free(pdev->dev.of_node);
1282 	dma_async_device_unregister(&priv->slave);
1283 
1284 	clk_disable_unprepare(priv->clk);
1285 
1286 	return 0;
1287 }
1288 
1289 static const struct of_device_id sun4i_dma_match[] = {
1290 	{ .compatible = "allwinner,sun4i-a10-dma" },
1291 	{ /* sentinel */ },
1292 };
1293 MODULE_DEVICE_TABLE(of, sun4i_dma_match);
1294 
1295 static struct platform_driver sun4i_dma_driver = {
1296 	.probe	= sun4i_dma_probe,
1297 	.remove	= sun4i_dma_remove,
1298 	.driver	= {
1299 		.name		= "sun4i-dma",
1300 		.of_match_table	= sun4i_dma_match,
1301 	},
1302 };
1303 
1304 module_platform_driver(sun4i_dma_driver);
1305 
1306 MODULE_DESCRIPTION("Allwinner A10 Dedicated DMA Controller Driver");
1307 MODULE_AUTHOR("Emilio López <emilio@elopez.com.ar>");
1308 MODULE_LICENSE("GPL");
1309