xref: /openbmc/linux/drivers/dma/imx-dma.c (revision 9d749629)
1 /*
2  * drivers/dma/imx-dma.c
3  *
4  * This file contains a driver for the Freescale i.MX DMA engine
5  * found on i.MX1/21/27
6  *
7  * Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
8  * Copyright 2012 Javier Martin, Vista Silicon <javier.martin@vista-silicon.com>
9  *
10  * The code contained herein is licensed under the GNU General Public
11  * License. You may obtain a copy of the GNU General Public License
12  * Version 2 or later at the following locations:
13  *
14  * http://www.opensource.org/licenses/gpl-license.html
15  * http://www.gnu.org/copyleft/gpl.html
16  */
17 #include <linux/init.h>
18 #include <linux/types.h>
19 #include <linux/mm.h>
20 #include <linux/interrupt.h>
21 #include <linux/spinlock.h>
22 #include <linux/device.h>
23 #include <linux/dma-mapping.h>
24 #include <linux/slab.h>
25 #include <linux/platform_device.h>
26 #include <linux/clk.h>
27 #include <linux/dmaengine.h>
28 #include <linux/module.h>
29 
30 #include <asm/irq.h>
31 #include <linux/platform_data/dma-imx.h>
32 
33 #include "dmaengine.h"
34 #define IMXDMA_MAX_CHAN_DESCRIPTORS	16
35 #define IMX_DMA_CHANNELS  16
36 
37 #define IMX_DMA_2D_SLOTS	2
38 #define IMX_DMA_2D_SLOT_A	0
39 #define IMX_DMA_2D_SLOT_B	1
40 
41 #define IMX_DMA_LENGTH_LOOP	((unsigned int)-1)
42 #define IMX_DMA_MEMSIZE_32	(0 << 4)
43 #define IMX_DMA_MEMSIZE_8	(1 << 4)
44 #define IMX_DMA_MEMSIZE_16	(2 << 4)
45 #define IMX_DMA_TYPE_LINEAR	(0 << 10)
46 #define IMX_DMA_TYPE_2D		(1 << 10)
47 #define IMX_DMA_TYPE_FIFO	(2 << 10)
48 
49 #define IMX_DMA_ERR_BURST     (1 << 0)
50 #define IMX_DMA_ERR_REQUEST   (1 << 1)
51 #define IMX_DMA_ERR_TRANSFER  (1 << 2)
52 #define IMX_DMA_ERR_BUFFER    (1 << 3)
53 #define IMX_DMA_ERR_TIMEOUT   (1 << 4)
54 
55 #define DMA_DCR     0x00		/* Control Register */
56 #define DMA_DISR    0x04		/* Interrupt status Register */
57 #define DMA_DIMR    0x08		/* Interrupt mask Register */
58 #define DMA_DBTOSR  0x0c		/* Burst timeout status Register */
59 #define DMA_DRTOSR  0x10		/* Request timeout Register */
60 #define DMA_DSESR   0x14		/* Transfer Error Status Register */
61 #define DMA_DBOSR   0x18		/* Buffer overflow status Register */
62 #define DMA_DBTOCR  0x1c		/* Burst timeout control Register */
63 #define DMA_WSRA    0x40		/* W-Size Register A */
64 #define DMA_XSRA    0x44		/* X-Size Register A */
65 #define DMA_YSRA    0x48		/* Y-Size Register A */
66 #define DMA_WSRB    0x4c		/* W-Size Register B */
67 #define DMA_XSRB    0x50		/* X-Size Register B */
68 #define DMA_YSRB    0x54		/* Y-Size Register B */
69 #define DMA_SAR(x)  (0x80 + ((x) << 6))	/* Source Address Registers */
70 #define DMA_DAR(x)  (0x84 + ((x) << 6))	/* Destination Address Registers */
71 #define DMA_CNTR(x) (0x88 + ((x) << 6))	/* Count Registers */
72 #define DMA_CCR(x)  (0x8c + ((x) << 6))	/* Control Registers */
73 #define DMA_RSSR(x) (0x90 + ((x) << 6))	/* Request source select Registers */
74 #define DMA_BLR(x)  (0x94 + ((x) << 6))	/* Burst length Registers */
75 #define DMA_RTOR(x) (0x98 + ((x) << 6))	/* Request timeout Registers */
76 #define DMA_BUCR(x) (0x98 + ((x) << 6))	/* Bus Utilization Registers */
77 #define DMA_CCNR(x) (0x9C + ((x) << 6))	/* Channel counter Registers */
78 
79 #define DCR_DRST           (1<<1)
80 #define DCR_DEN            (1<<0)
81 #define DBTOCR_EN          (1<<15)
82 #define DBTOCR_CNT(x)      ((x) & 0x7fff)
83 #define CNTR_CNT(x)        ((x) & 0xffffff)
84 #define CCR_ACRPT          (1<<14)
85 #define CCR_DMOD_LINEAR    (0x0 << 12)
86 #define CCR_DMOD_2D        (0x1 << 12)
87 #define CCR_DMOD_FIFO      (0x2 << 12)
88 #define CCR_DMOD_EOBFIFO   (0x3 << 12)
89 #define CCR_SMOD_LINEAR    (0x0 << 10)
90 #define CCR_SMOD_2D        (0x1 << 10)
91 #define CCR_SMOD_FIFO      (0x2 << 10)
92 #define CCR_SMOD_EOBFIFO   (0x3 << 10)
93 #define CCR_MDIR_DEC       (1<<9)
94 #define CCR_MSEL_B         (1<<8)
95 #define CCR_DSIZ_32        (0x0 << 6)
96 #define CCR_DSIZ_8         (0x1 << 6)
97 #define CCR_DSIZ_16        (0x2 << 6)
98 #define CCR_SSIZ_32        (0x0 << 4)
99 #define CCR_SSIZ_8         (0x1 << 4)
100 #define CCR_SSIZ_16        (0x2 << 4)
101 #define CCR_REN            (1<<3)
102 #define CCR_RPT            (1<<2)
103 #define CCR_FRC            (1<<1)
104 #define CCR_CEN            (1<<0)
105 #define RTOR_EN            (1<<15)
106 #define RTOR_CLK           (1<<14)
107 #define RTOR_PSC           (1<<13)
108 
109 enum  imxdma_prep_type {
110 	IMXDMA_DESC_MEMCPY,
111 	IMXDMA_DESC_INTERLEAVED,
112 	IMXDMA_DESC_SLAVE_SG,
113 	IMXDMA_DESC_CYCLIC,
114 };
115 
116 struct imx_dma_2d_config {
117 	u16		xsr;
118 	u16		ysr;
119 	u16		wsr;
120 	int		count;
121 };
122 
123 struct imxdma_desc {
124 	struct list_head		node;
125 	struct dma_async_tx_descriptor	desc;
126 	enum dma_status			status;
127 	dma_addr_t			src;
128 	dma_addr_t			dest;
129 	size_t				len;
130 	enum dma_transfer_direction	direction;
131 	enum imxdma_prep_type		type;
132 	/* For memcpy and interleaved */
133 	unsigned int			config_port;
134 	unsigned int			config_mem;
135 	/* For interleaved transfers */
136 	unsigned int			x;
137 	unsigned int			y;
138 	unsigned int			w;
139 	/* For slave sg and cyclic */
140 	struct scatterlist		*sg;
141 	unsigned int			sgcount;
142 };
143 
144 struct imxdma_channel {
145 	int				hw_chaining;
146 	struct timer_list		watchdog;
147 	struct imxdma_engine		*imxdma;
148 	unsigned int			channel;
149 
150 	struct tasklet_struct		dma_tasklet;
151 	struct list_head		ld_free;
152 	struct list_head		ld_queue;
153 	struct list_head		ld_active;
154 	int				descs_allocated;
155 	enum dma_slave_buswidth		word_size;
156 	dma_addr_t			per_address;
157 	u32				watermark_level;
158 	struct dma_chan			chan;
159 	struct dma_async_tx_descriptor	desc;
160 	enum dma_status			status;
161 	int				dma_request;
162 	struct scatterlist		*sg_list;
163 	u32				ccr_from_device;
164 	u32				ccr_to_device;
165 	bool				enabled_2d;
166 	int				slot_2d;
167 };
168 
169 enum imx_dma_type {
170 	IMX1_DMA,
171 	IMX21_DMA,
172 	IMX27_DMA,
173 };
174 
175 struct imxdma_engine {
176 	struct device			*dev;
177 	struct device_dma_parameters	dma_parms;
178 	struct dma_device		dma_device;
179 	void __iomem			*base;
180 	struct clk			*dma_ahb;
181 	struct clk			*dma_ipg;
182 	spinlock_t			lock;
183 	struct imx_dma_2d_config	slots_2d[IMX_DMA_2D_SLOTS];
184 	struct imxdma_channel		channel[IMX_DMA_CHANNELS];
185 	enum imx_dma_type		devtype;
186 };
187 
188 static struct platform_device_id imx_dma_devtype[] = {
189 	{
190 		.name = "imx1-dma",
191 		.driver_data = IMX1_DMA,
192 	}, {
193 		.name = "imx21-dma",
194 		.driver_data = IMX21_DMA,
195 	}, {
196 		.name = "imx27-dma",
197 		.driver_data = IMX27_DMA,
198 	}, {
199 		/* sentinel */
200 	}
201 };
202 MODULE_DEVICE_TABLE(platform, imx_dma_devtype);
203 
204 static inline int is_imx1_dma(struct imxdma_engine *imxdma)
205 {
206 	return imxdma->devtype == IMX1_DMA;
207 }
208 
209 static inline int is_imx21_dma(struct imxdma_engine *imxdma)
210 {
211 	return imxdma->devtype == IMX21_DMA;
212 }
213 
214 static inline int is_imx27_dma(struct imxdma_engine *imxdma)
215 {
216 	return imxdma->devtype == IMX27_DMA;
217 }
218 
219 static struct imxdma_channel *to_imxdma_chan(struct dma_chan *chan)
220 {
221 	return container_of(chan, struct imxdma_channel, chan);
222 }
223 
224 static inline bool imxdma_chan_is_doing_cyclic(struct imxdma_channel *imxdmac)
225 {
226 	struct imxdma_desc *desc;
227 
228 	if (!list_empty(&imxdmac->ld_active)) {
229 		desc = list_first_entry(&imxdmac->ld_active, struct imxdma_desc,
230 					node);
231 		if (desc->type == IMXDMA_DESC_CYCLIC)
232 			return true;
233 	}
234 	return false;
235 }
236 
237 
238 
239 static void imx_dmav1_writel(struct imxdma_engine *imxdma, unsigned val,
240 			     unsigned offset)
241 {
242 	__raw_writel(val, imxdma->base + offset);
243 }
244 
245 static unsigned imx_dmav1_readl(struct imxdma_engine *imxdma, unsigned offset)
246 {
247 	return __raw_readl(imxdma->base + offset);
248 }
249 
250 static int imxdma_hw_chain(struct imxdma_channel *imxdmac)
251 {
252 	struct imxdma_engine *imxdma = imxdmac->imxdma;
253 
254 	if (is_imx27_dma(imxdma))
255 		return imxdmac->hw_chaining;
256 	else
257 		return 0;
258 }
259 
260 /*
261  * imxdma_sg_next - prepare next chunk for scatter-gather DMA emulation
262  */
263 static inline int imxdma_sg_next(struct imxdma_desc *d)
264 {
265 	struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan);
266 	struct imxdma_engine *imxdma = imxdmac->imxdma;
267 	struct scatterlist *sg = d->sg;
268 	unsigned long now;
269 
270 	now = min(d->len, sg_dma_len(sg));
271 	if (d->len != IMX_DMA_LENGTH_LOOP)
272 		d->len -= now;
273 
274 	if (d->direction == DMA_DEV_TO_MEM)
275 		imx_dmav1_writel(imxdma, sg->dma_address,
276 				 DMA_DAR(imxdmac->channel));
277 	else
278 		imx_dmav1_writel(imxdma, sg->dma_address,
279 				 DMA_SAR(imxdmac->channel));
280 
281 	imx_dmav1_writel(imxdma, now, DMA_CNTR(imxdmac->channel));
282 
283 	dev_dbg(imxdma->dev, " %s channel: %d dst 0x%08x, src 0x%08x, "
284 		"size 0x%08x\n", __func__, imxdmac->channel,
285 		 imx_dmav1_readl(imxdma, DMA_DAR(imxdmac->channel)),
286 		 imx_dmav1_readl(imxdma, DMA_SAR(imxdmac->channel)),
287 		 imx_dmav1_readl(imxdma, DMA_CNTR(imxdmac->channel)));
288 
289 	return now;
290 }
291 
292 static void imxdma_enable_hw(struct imxdma_desc *d)
293 {
294 	struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan);
295 	struct imxdma_engine *imxdma = imxdmac->imxdma;
296 	int channel = imxdmac->channel;
297 	unsigned long flags;
298 
299 	dev_dbg(imxdma->dev, "%s channel %d\n", __func__, channel);
300 
301 	local_irq_save(flags);
302 
303 	imx_dmav1_writel(imxdma, 1 << channel, DMA_DISR);
304 	imx_dmav1_writel(imxdma, imx_dmav1_readl(imxdma, DMA_DIMR) &
305 			 ~(1 << channel), DMA_DIMR);
306 	imx_dmav1_writel(imxdma, imx_dmav1_readl(imxdma, DMA_CCR(channel)) |
307 			 CCR_CEN | CCR_ACRPT, DMA_CCR(channel));
308 
309 	if (!is_imx1_dma(imxdma) &&
310 			d->sg && imxdma_hw_chain(imxdmac)) {
311 		d->sg = sg_next(d->sg);
312 		if (d->sg) {
313 			u32 tmp;
314 			imxdma_sg_next(d);
315 			tmp = imx_dmav1_readl(imxdma, DMA_CCR(channel));
316 			imx_dmav1_writel(imxdma, tmp | CCR_RPT | CCR_ACRPT,
317 					 DMA_CCR(channel));
318 		}
319 	}
320 
321 	local_irq_restore(flags);
322 }
323 
324 static void imxdma_disable_hw(struct imxdma_channel *imxdmac)
325 {
326 	struct imxdma_engine *imxdma = imxdmac->imxdma;
327 	int channel = imxdmac->channel;
328 	unsigned long flags;
329 
330 	dev_dbg(imxdma->dev, "%s channel %d\n", __func__, channel);
331 
332 	if (imxdma_hw_chain(imxdmac))
333 		del_timer(&imxdmac->watchdog);
334 
335 	local_irq_save(flags);
336 	imx_dmav1_writel(imxdma, imx_dmav1_readl(imxdma, DMA_DIMR) |
337 			 (1 << channel), DMA_DIMR);
338 	imx_dmav1_writel(imxdma, imx_dmav1_readl(imxdma, DMA_CCR(channel)) &
339 			 ~CCR_CEN, DMA_CCR(channel));
340 	imx_dmav1_writel(imxdma, 1 << channel, DMA_DISR);
341 	local_irq_restore(flags);
342 }
343 
344 static void imxdma_watchdog(unsigned long data)
345 {
346 	struct imxdma_channel *imxdmac = (struct imxdma_channel *)data;
347 	struct imxdma_engine *imxdma = imxdmac->imxdma;
348 	int channel = imxdmac->channel;
349 
350 	imx_dmav1_writel(imxdma, 0, DMA_CCR(channel));
351 
352 	/* Tasklet watchdog error handler */
353 	tasklet_schedule(&imxdmac->dma_tasklet);
354 	dev_dbg(imxdma->dev, "channel %d: watchdog timeout!\n",
355 		imxdmac->channel);
356 }
357 
358 static irqreturn_t imxdma_err_handler(int irq, void *dev_id)
359 {
360 	struct imxdma_engine *imxdma = dev_id;
361 	unsigned int err_mask;
362 	int i, disr;
363 	int errcode;
364 
365 	disr = imx_dmav1_readl(imxdma, DMA_DISR);
366 
367 	err_mask = imx_dmav1_readl(imxdma, DMA_DBTOSR) |
368 		   imx_dmav1_readl(imxdma, DMA_DRTOSR) |
369 		   imx_dmav1_readl(imxdma, DMA_DSESR)  |
370 		   imx_dmav1_readl(imxdma, DMA_DBOSR);
371 
372 	if (!err_mask)
373 		return IRQ_HANDLED;
374 
375 	imx_dmav1_writel(imxdma, disr & err_mask, DMA_DISR);
376 
377 	for (i = 0; i < IMX_DMA_CHANNELS; i++) {
378 		if (!(err_mask & (1 << i)))
379 			continue;
380 		errcode = 0;
381 
382 		if (imx_dmav1_readl(imxdma, DMA_DBTOSR) & (1 << i)) {
383 			imx_dmav1_writel(imxdma, 1 << i, DMA_DBTOSR);
384 			errcode |= IMX_DMA_ERR_BURST;
385 		}
386 		if (imx_dmav1_readl(imxdma, DMA_DRTOSR) & (1 << i)) {
387 			imx_dmav1_writel(imxdma, 1 << i, DMA_DRTOSR);
388 			errcode |= IMX_DMA_ERR_REQUEST;
389 		}
390 		if (imx_dmav1_readl(imxdma, DMA_DSESR) & (1 << i)) {
391 			imx_dmav1_writel(imxdma, 1 << i, DMA_DSESR);
392 			errcode |= IMX_DMA_ERR_TRANSFER;
393 		}
394 		if (imx_dmav1_readl(imxdma, DMA_DBOSR) & (1 << i)) {
395 			imx_dmav1_writel(imxdma, 1 << i, DMA_DBOSR);
396 			errcode |= IMX_DMA_ERR_BUFFER;
397 		}
398 		/* Tasklet error handler */
399 		tasklet_schedule(&imxdma->channel[i].dma_tasklet);
400 
401 		printk(KERN_WARNING
402 		       "DMA timeout on channel %d -%s%s%s%s\n", i,
403 		       errcode & IMX_DMA_ERR_BURST ?    " burst" : "",
404 		       errcode & IMX_DMA_ERR_REQUEST ?  " request" : "",
405 		       errcode & IMX_DMA_ERR_TRANSFER ? " transfer" : "",
406 		       errcode & IMX_DMA_ERR_BUFFER ?   " buffer" : "");
407 	}
408 	return IRQ_HANDLED;
409 }
410 
411 static void dma_irq_handle_channel(struct imxdma_channel *imxdmac)
412 {
413 	struct imxdma_engine *imxdma = imxdmac->imxdma;
414 	int chno = imxdmac->channel;
415 	struct imxdma_desc *desc;
416 
417 	spin_lock(&imxdma->lock);
418 	if (list_empty(&imxdmac->ld_active)) {
419 		spin_unlock(&imxdma->lock);
420 		goto out;
421 	}
422 
423 	desc = list_first_entry(&imxdmac->ld_active,
424 				struct imxdma_desc,
425 				node);
426 	spin_unlock(&imxdma->lock);
427 
428 	if (desc->sg) {
429 		u32 tmp;
430 		desc->sg = sg_next(desc->sg);
431 
432 		if (desc->sg) {
433 			imxdma_sg_next(desc);
434 
435 			tmp = imx_dmav1_readl(imxdma, DMA_CCR(chno));
436 
437 			if (imxdma_hw_chain(imxdmac)) {
438 				/* FIXME: The timeout should probably be
439 				 * configurable
440 				 */
441 				mod_timer(&imxdmac->watchdog,
442 					jiffies + msecs_to_jiffies(500));
443 
444 				tmp |= CCR_CEN | CCR_RPT | CCR_ACRPT;
445 				imx_dmav1_writel(imxdma, tmp, DMA_CCR(chno));
446 			} else {
447 				imx_dmav1_writel(imxdma, tmp & ~CCR_CEN,
448 						 DMA_CCR(chno));
449 				tmp |= CCR_CEN;
450 			}
451 
452 			imx_dmav1_writel(imxdma, tmp, DMA_CCR(chno));
453 
454 			if (imxdma_chan_is_doing_cyclic(imxdmac))
455 				/* Tasklet progression */
456 				tasklet_schedule(&imxdmac->dma_tasklet);
457 
458 			return;
459 		}
460 
461 		if (imxdma_hw_chain(imxdmac)) {
462 			del_timer(&imxdmac->watchdog);
463 			return;
464 		}
465 	}
466 
467 out:
468 	imx_dmav1_writel(imxdma, 0, DMA_CCR(chno));
469 	/* Tasklet irq */
470 	tasklet_schedule(&imxdmac->dma_tasklet);
471 }
472 
473 static irqreturn_t dma_irq_handler(int irq, void *dev_id)
474 {
475 	struct imxdma_engine *imxdma = dev_id;
476 	int i, disr;
477 
478 	if (!is_imx1_dma(imxdma))
479 		imxdma_err_handler(irq, dev_id);
480 
481 	disr = imx_dmav1_readl(imxdma, DMA_DISR);
482 
483 	dev_dbg(imxdma->dev, "%s called, disr=0x%08x\n", __func__, disr);
484 
485 	imx_dmav1_writel(imxdma, disr, DMA_DISR);
486 	for (i = 0; i < IMX_DMA_CHANNELS; i++) {
487 		if (disr & (1 << i))
488 			dma_irq_handle_channel(&imxdma->channel[i]);
489 	}
490 
491 	return IRQ_HANDLED;
492 }
493 
494 static int imxdma_xfer_desc(struct imxdma_desc *d)
495 {
496 	struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan);
497 	struct imxdma_engine *imxdma = imxdmac->imxdma;
498 	unsigned long flags;
499 	int slot = -1;
500 	int i;
501 
502 	/* Configure and enable */
503 	switch (d->type) {
504 	case IMXDMA_DESC_INTERLEAVED:
505 		/* Try to get a free 2D slot */
506 		spin_lock_irqsave(&imxdma->lock, flags);
507 		for (i = 0; i < IMX_DMA_2D_SLOTS; i++) {
508 			if ((imxdma->slots_2d[i].count > 0) &&
509 			((imxdma->slots_2d[i].xsr != d->x) ||
510 			(imxdma->slots_2d[i].ysr != d->y) ||
511 			(imxdma->slots_2d[i].wsr != d->w)))
512 				continue;
513 			slot = i;
514 			break;
515 		}
516 		if (slot < 0) {
517 			spin_unlock_irqrestore(&imxdma->lock, flags);
518 			return -EBUSY;
519 		}
520 
521 		imxdma->slots_2d[slot].xsr = d->x;
522 		imxdma->slots_2d[slot].ysr = d->y;
523 		imxdma->slots_2d[slot].wsr = d->w;
524 		imxdma->slots_2d[slot].count++;
525 
526 		imxdmac->slot_2d = slot;
527 		imxdmac->enabled_2d = true;
528 		spin_unlock_irqrestore(&imxdma->lock, flags);
529 
530 		if (slot == IMX_DMA_2D_SLOT_A) {
531 			d->config_mem &= ~CCR_MSEL_B;
532 			d->config_port &= ~CCR_MSEL_B;
533 			imx_dmav1_writel(imxdma, d->x, DMA_XSRA);
534 			imx_dmav1_writel(imxdma, d->y, DMA_YSRA);
535 			imx_dmav1_writel(imxdma, d->w, DMA_WSRA);
536 		} else {
537 			d->config_mem |= CCR_MSEL_B;
538 			d->config_port |= CCR_MSEL_B;
539 			imx_dmav1_writel(imxdma, d->x, DMA_XSRB);
540 			imx_dmav1_writel(imxdma, d->y, DMA_YSRB);
541 			imx_dmav1_writel(imxdma, d->w, DMA_WSRB);
542 		}
543 		/*
544 		 * We fall-through here intentionally, since a 2D transfer is
545 		 * similar to MEMCPY just adding the 2D slot configuration.
546 		 */
547 	case IMXDMA_DESC_MEMCPY:
548 		imx_dmav1_writel(imxdma, d->src, DMA_SAR(imxdmac->channel));
549 		imx_dmav1_writel(imxdma, d->dest, DMA_DAR(imxdmac->channel));
550 		imx_dmav1_writel(imxdma, d->config_mem | (d->config_port << 2),
551 			 DMA_CCR(imxdmac->channel));
552 
553 		imx_dmav1_writel(imxdma, d->len, DMA_CNTR(imxdmac->channel));
554 
555 		dev_dbg(imxdma->dev, "%s channel: %d dest=0x%08x src=0x%08x "
556 			"dma_length=%d\n", __func__, imxdmac->channel,
557 			d->dest, d->src, d->len);
558 
559 		break;
560 	/* Cyclic transfer is the same as slave_sg with special sg configuration. */
561 	case IMXDMA_DESC_CYCLIC:
562 	case IMXDMA_DESC_SLAVE_SG:
563 		if (d->direction == DMA_DEV_TO_MEM) {
564 			imx_dmav1_writel(imxdma, imxdmac->per_address,
565 					 DMA_SAR(imxdmac->channel));
566 			imx_dmav1_writel(imxdma, imxdmac->ccr_from_device,
567 					 DMA_CCR(imxdmac->channel));
568 
569 			dev_dbg(imxdma->dev, "%s channel: %d sg=%p sgcount=%d "
570 				"total length=%d dev_addr=0x%08x (dev2mem)\n",
571 				__func__, imxdmac->channel, d->sg, d->sgcount,
572 				d->len, imxdmac->per_address);
573 		} else if (d->direction == DMA_MEM_TO_DEV) {
574 			imx_dmav1_writel(imxdma, imxdmac->per_address,
575 					 DMA_DAR(imxdmac->channel));
576 			imx_dmav1_writel(imxdma, imxdmac->ccr_to_device,
577 					 DMA_CCR(imxdmac->channel));
578 
579 			dev_dbg(imxdma->dev, "%s channel: %d sg=%p sgcount=%d "
580 				"total length=%d dev_addr=0x%08x (mem2dev)\n",
581 				__func__, imxdmac->channel, d->sg, d->sgcount,
582 				d->len, imxdmac->per_address);
583 		} else {
584 			dev_err(imxdma->dev, "%s channel: %d bad dma mode\n",
585 				__func__, imxdmac->channel);
586 			return -EINVAL;
587 		}
588 
589 		imxdma_sg_next(d);
590 
591 		break;
592 	default:
593 		return -EINVAL;
594 	}
595 	imxdma_enable_hw(d);
596 	return 0;
597 }
598 
599 static void imxdma_tasklet(unsigned long data)
600 {
601 	struct imxdma_channel *imxdmac = (void *)data;
602 	struct imxdma_engine *imxdma = imxdmac->imxdma;
603 	struct imxdma_desc *desc;
604 
605 	spin_lock(&imxdma->lock);
606 
607 	if (list_empty(&imxdmac->ld_active)) {
608 		/* Someone might have called terminate all */
609 		goto out;
610 	}
611 	desc = list_first_entry(&imxdmac->ld_active, struct imxdma_desc, node);
612 
613 	if (desc->desc.callback)
614 		desc->desc.callback(desc->desc.callback_param);
615 
616 	/* If we are dealing with a cyclic descriptor, keep it on ld_active
617 	 * and dont mark the descriptor as complete.
618 	 * Only in non-cyclic cases it would be marked as complete
619 	 */
620 	if (imxdma_chan_is_doing_cyclic(imxdmac))
621 		goto out;
622 	else
623 		dma_cookie_complete(&desc->desc);
624 
625 	/* Free 2D slot if it was an interleaved transfer */
626 	if (imxdmac->enabled_2d) {
627 		imxdma->slots_2d[imxdmac->slot_2d].count--;
628 		imxdmac->enabled_2d = false;
629 	}
630 
631 	list_move_tail(imxdmac->ld_active.next, &imxdmac->ld_free);
632 
633 	if (!list_empty(&imxdmac->ld_queue)) {
634 		desc = list_first_entry(&imxdmac->ld_queue, struct imxdma_desc,
635 					node);
636 		list_move_tail(imxdmac->ld_queue.next, &imxdmac->ld_active);
637 		if (imxdma_xfer_desc(desc) < 0)
638 			dev_warn(imxdma->dev, "%s: channel: %d couldn't xfer desc\n",
639 				 __func__, imxdmac->channel);
640 	}
641 out:
642 	spin_unlock(&imxdma->lock);
643 }
644 
645 static int imxdma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
646 		unsigned long arg)
647 {
648 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);
649 	struct dma_slave_config *dmaengine_cfg = (void *)arg;
650 	struct imxdma_engine *imxdma = imxdmac->imxdma;
651 	unsigned long flags;
652 	unsigned int mode = 0;
653 
654 	switch (cmd) {
655 	case DMA_TERMINATE_ALL:
656 		imxdma_disable_hw(imxdmac);
657 
658 		spin_lock_irqsave(&imxdma->lock, flags);
659 		list_splice_tail_init(&imxdmac->ld_active, &imxdmac->ld_free);
660 		list_splice_tail_init(&imxdmac->ld_queue, &imxdmac->ld_free);
661 		spin_unlock_irqrestore(&imxdma->lock, flags);
662 		return 0;
663 	case DMA_SLAVE_CONFIG:
664 		if (dmaengine_cfg->direction == DMA_DEV_TO_MEM) {
665 			imxdmac->per_address = dmaengine_cfg->src_addr;
666 			imxdmac->watermark_level = dmaengine_cfg->src_maxburst;
667 			imxdmac->word_size = dmaengine_cfg->src_addr_width;
668 		} else {
669 			imxdmac->per_address = dmaengine_cfg->dst_addr;
670 			imxdmac->watermark_level = dmaengine_cfg->dst_maxburst;
671 			imxdmac->word_size = dmaengine_cfg->dst_addr_width;
672 		}
673 
674 		switch (imxdmac->word_size) {
675 		case DMA_SLAVE_BUSWIDTH_1_BYTE:
676 			mode = IMX_DMA_MEMSIZE_8;
677 			break;
678 		case DMA_SLAVE_BUSWIDTH_2_BYTES:
679 			mode = IMX_DMA_MEMSIZE_16;
680 			break;
681 		default:
682 		case DMA_SLAVE_BUSWIDTH_4_BYTES:
683 			mode = IMX_DMA_MEMSIZE_32;
684 			break;
685 		}
686 
687 		imxdmac->hw_chaining = 0;
688 
689 		imxdmac->ccr_from_device = (mode | IMX_DMA_TYPE_FIFO) |
690 			((IMX_DMA_MEMSIZE_32 | IMX_DMA_TYPE_LINEAR) << 2) |
691 			CCR_REN;
692 		imxdmac->ccr_to_device =
693 			(IMX_DMA_MEMSIZE_32 | IMX_DMA_TYPE_LINEAR) |
694 			((mode | IMX_DMA_TYPE_FIFO) << 2) | CCR_REN;
695 		imx_dmav1_writel(imxdma, imxdmac->dma_request,
696 				 DMA_RSSR(imxdmac->channel));
697 
698 		/* Set burst length */
699 		imx_dmav1_writel(imxdma, imxdmac->watermark_level *
700 				imxdmac->word_size, DMA_BLR(imxdmac->channel));
701 
702 		return 0;
703 	default:
704 		return -ENOSYS;
705 	}
706 
707 	return -EINVAL;
708 }
709 
710 static enum dma_status imxdma_tx_status(struct dma_chan *chan,
711 					    dma_cookie_t cookie,
712 					    struct dma_tx_state *txstate)
713 {
714 	return dma_cookie_status(chan, cookie, txstate);
715 }
716 
717 static dma_cookie_t imxdma_tx_submit(struct dma_async_tx_descriptor *tx)
718 {
719 	struct imxdma_channel *imxdmac = to_imxdma_chan(tx->chan);
720 	struct imxdma_engine *imxdma = imxdmac->imxdma;
721 	dma_cookie_t cookie;
722 	unsigned long flags;
723 
724 	spin_lock_irqsave(&imxdma->lock, flags);
725 	list_move_tail(imxdmac->ld_free.next, &imxdmac->ld_queue);
726 	cookie = dma_cookie_assign(tx);
727 	spin_unlock_irqrestore(&imxdma->lock, flags);
728 
729 	return cookie;
730 }
731 
732 static int imxdma_alloc_chan_resources(struct dma_chan *chan)
733 {
734 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);
735 	struct imx_dma_data *data = chan->private;
736 
737 	if (data != NULL)
738 		imxdmac->dma_request = data->dma_request;
739 
740 	while (imxdmac->descs_allocated < IMXDMA_MAX_CHAN_DESCRIPTORS) {
741 		struct imxdma_desc *desc;
742 
743 		desc = kzalloc(sizeof(*desc), GFP_KERNEL);
744 		if (!desc)
745 			break;
746 		__memzero(&desc->desc, sizeof(struct dma_async_tx_descriptor));
747 		dma_async_tx_descriptor_init(&desc->desc, chan);
748 		desc->desc.tx_submit = imxdma_tx_submit;
749 		/* txd.flags will be overwritten in prep funcs */
750 		desc->desc.flags = DMA_CTRL_ACK;
751 		desc->status = DMA_SUCCESS;
752 
753 		list_add_tail(&desc->node, &imxdmac->ld_free);
754 		imxdmac->descs_allocated++;
755 	}
756 
757 	if (!imxdmac->descs_allocated)
758 		return -ENOMEM;
759 
760 	return imxdmac->descs_allocated;
761 }
762 
763 static void imxdma_free_chan_resources(struct dma_chan *chan)
764 {
765 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);
766 	struct imxdma_engine *imxdma = imxdmac->imxdma;
767 	struct imxdma_desc *desc, *_desc;
768 	unsigned long flags;
769 
770 	spin_lock_irqsave(&imxdma->lock, flags);
771 
772 	imxdma_disable_hw(imxdmac);
773 	list_splice_tail_init(&imxdmac->ld_active, &imxdmac->ld_free);
774 	list_splice_tail_init(&imxdmac->ld_queue, &imxdmac->ld_free);
775 
776 	spin_unlock_irqrestore(&imxdma->lock, flags);
777 
778 	list_for_each_entry_safe(desc, _desc, &imxdmac->ld_free, node) {
779 		kfree(desc);
780 		imxdmac->descs_allocated--;
781 	}
782 	INIT_LIST_HEAD(&imxdmac->ld_free);
783 
784 	if (imxdmac->sg_list) {
785 		kfree(imxdmac->sg_list);
786 		imxdmac->sg_list = NULL;
787 	}
788 }
789 
790 static struct dma_async_tx_descriptor *imxdma_prep_slave_sg(
791 		struct dma_chan *chan, struct scatterlist *sgl,
792 		unsigned int sg_len, enum dma_transfer_direction direction,
793 		unsigned long flags, void *context)
794 {
795 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);
796 	struct scatterlist *sg;
797 	int i, dma_length = 0;
798 	struct imxdma_desc *desc;
799 
800 	if (list_empty(&imxdmac->ld_free) ||
801 	    imxdma_chan_is_doing_cyclic(imxdmac))
802 		return NULL;
803 
804 	desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);
805 
806 	for_each_sg(sgl, sg, sg_len, i) {
807 		dma_length += sg_dma_len(sg);
808 	}
809 
810 	switch (imxdmac->word_size) {
811 	case DMA_SLAVE_BUSWIDTH_4_BYTES:
812 		if (sg_dma_len(sgl) & 3 || sgl->dma_address & 3)
813 			return NULL;
814 		break;
815 	case DMA_SLAVE_BUSWIDTH_2_BYTES:
816 		if (sg_dma_len(sgl) & 1 || sgl->dma_address & 1)
817 			return NULL;
818 		break;
819 	case DMA_SLAVE_BUSWIDTH_1_BYTE:
820 		break;
821 	default:
822 		return NULL;
823 	}
824 
825 	desc->type = IMXDMA_DESC_SLAVE_SG;
826 	desc->sg = sgl;
827 	desc->sgcount = sg_len;
828 	desc->len = dma_length;
829 	desc->direction = direction;
830 	if (direction == DMA_DEV_TO_MEM) {
831 		desc->src = imxdmac->per_address;
832 	} else {
833 		desc->dest = imxdmac->per_address;
834 	}
835 	desc->desc.callback = NULL;
836 	desc->desc.callback_param = NULL;
837 
838 	return &desc->desc;
839 }
840 
841 static struct dma_async_tx_descriptor *imxdma_prep_dma_cyclic(
842 		struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,
843 		size_t period_len, enum dma_transfer_direction direction,
844 		unsigned long flags, void *context)
845 {
846 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);
847 	struct imxdma_engine *imxdma = imxdmac->imxdma;
848 	struct imxdma_desc *desc;
849 	int i;
850 	unsigned int periods = buf_len / period_len;
851 
852 	dev_dbg(imxdma->dev, "%s channel: %d buf_len=%d period_len=%d\n",
853 			__func__, imxdmac->channel, buf_len, period_len);
854 
855 	if (list_empty(&imxdmac->ld_free) ||
856 	    imxdma_chan_is_doing_cyclic(imxdmac))
857 		return NULL;
858 
859 	desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);
860 
861 	if (imxdmac->sg_list)
862 		kfree(imxdmac->sg_list);
863 
864 	imxdmac->sg_list = kcalloc(periods + 1,
865 			sizeof(struct scatterlist), GFP_KERNEL);
866 	if (!imxdmac->sg_list)
867 		return NULL;
868 
869 	sg_init_table(imxdmac->sg_list, periods);
870 
871 	for (i = 0; i < periods; i++) {
872 		imxdmac->sg_list[i].page_link = 0;
873 		imxdmac->sg_list[i].offset = 0;
874 		imxdmac->sg_list[i].dma_address = dma_addr;
875 		sg_dma_len(&imxdmac->sg_list[i]) = period_len;
876 		dma_addr += period_len;
877 	}
878 
879 	/* close the loop */
880 	imxdmac->sg_list[periods].offset = 0;
881 	sg_dma_len(&imxdmac->sg_list[periods]) = 0;
882 	imxdmac->sg_list[periods].page_link =
883 		((unsigned long)imxdmac->sg_list | 0x01) & ~0x02;
884 
885 	desc->type = IMXDMA_DESC_CYCLIC;
886 	desc->sg = imxdmac->sg_list;
887 	desc->sgcount = periods;
888 	desc->len = IMX_DMA_LENGTH_LOOP;
889 	desc->direction = direction;
890 	if (direction == DMA_DEV_TO_MEM) {
891 		desc->src = imxdmac->per_address;
892 	} else {
893 		desc->dest = imxdmac->per_address;
894 	}
895 	desc->desc.callback = NULL;
896 	desc->desc.callback_param = NULL;
897 
898 	return &desc->desc;
899 }
900 
901 static struct dma_async_tx_descriptor *imxdma_prep_dma_memcpy(
902 	struct dma_chan *chan, dma_addr_t dest,
903 	dma_addr_t src, size_t len, unsigned long flags)
904 {
905 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);
906 	struct imxdma_engine *imxdma = imxdmac->imxdma;
907 	struct imxdma_desc *desc;
908 
909 	dev_dbg(imxdma->dev, "%s channel: %d src=0x%x dst=0x%x len=%d\n",
910 			__func__, imxdmac->channel, src, dest, len);
911 
912 	if (list_empty(&imxdmac->ld_free) ||
913 	    imxdma_chan_is_doing_cyclic(imxdmac))
914 		return NULL;
915 
916 	desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);
917 
918 	desc->type = IMXDMA_DESC_MEMCPY;
919 	desc->src = src;
920 	desc->dest = dest;
921 	desc->len = len;
922 	desc->direction = DMA_MEM_TO_MEM;
923 	desc->config_port = IMX_DMA_MEMSIZE_32 | IMX_DMA_TYPE_LINEAR;
924 	desc->config_mem = IMX_DMA_MEMSIZE_32 | IMX_DMA_TYPE_LINEAR;
925 	desc->desc.callback = NULL;
926 	desc->desc.callback_param = NULL;
927 
928 	return &desc->desc;
929 }
930 
931 static struct dma_async_tx_descriptor *imxdma_prep_dma_interleaved(
932 	struct dma_chan *chan, struct dma_interleaved_template *xt,
933 	unsigned long flags)
934 {
935 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);
936 	struct imxdma_engine *imxdma = imxdmac->imxdma;
937 	struct imxdma_desc *desc;
938 
939 	dev_dbg(imxdma->dev, "%s channel: %d src_start=0x%x dst_start=0x%x\n"
940 		"   src_sgl=%s dst_sgl=%s numf=%d frame_size=%d\n", __func__,
941 		imxdmac->channel, xt->src_start, xt->dst_start,
942 		xt->src_sgl ? "true" : "false", xt->dst_sgl ? "true" : "false",
943 		xt->numf, xt->frame_size);
944 
945 	if (list_empty(&imxdmac->ld_free) ||
946 	    imxdma_chan_is_doing_cyclic(imxdmac))
947 		return NULL;
948 
949 	if (xt->frame_size != 1 || xt->numf <= 0 || xt->dir != DMA_MEM_TO_MEM)
950 		return NULL;
951 
952 	desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);
953 
954 	desc->type = IMXDMA_DESC_INTERLEAVED;
955 	desc->src = xt->src_start;
956 	desc->dest = xt->dst_start;
957 	desc->x = xt->sgl[0].size;
958 	desc->y = xt->numf;
959 	desc->w = xt->sgl[0].icg + desc->x;
960 	desc->len = desc->x * desc->y;
961 	desc->direction = DMA_MEM_TO_MEM;
962 	desc->config_port = IMX_DMA_MEMSIZE_32;
963 	desc->config_mem = IMX_DMA_MEMSIZE_32;
964 	if (xt->src_sgl)
965 		desc->config_mem |= IMX_DMA_TYPE_2D;
966 	if (xt->dst_sgl)
967 		desc->config_port |= IMX_DMA_TYPE_2D;
968 	desc->desc.callback = NULL;
969 	desc->desc.callback_param = NULL;
970 
971 	return &desc->desc;
972 }
973 
974 static void imxdma_issue_pending(struct dma_chan *chan)
975 {
976 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);
977 	struct imxdma_engine *imxdma = imxdmac->imxdma;
978 	struct imxdma_desc *desc;
979 	unsigned long flags;
980 
981 	spin_lock_irqsave(&imxdma->lock, flags);
982 	if (list_empty(&imxdmac->ld_active) &&
983 	    !list_empty(&imxdmac->ld_queue)) {
984 		desc = list_first_entry(&imxdmac->ld_queue,
985 					struct imxdma_desc, node);
986 
987 		if (imxdma_xfer_desc(desc) < 0) {
988 			dev_warn(imxdma->dev,
989 				 "%s: channel: %d couldn't issue DMA xfer\n",
990 				 __func__, imxdmac->channel);
991 		} else {
992 			list_move_tail(imxdmac->ld_queue.next,
993 				       &imxdmac->ld_active);
994 		}
995 	}
996 	spin_unlock_irqrestore(&imxdma->lock, flags);
997 }
998 
999 static int __init imxdma_probe(struct platform_device *pdev)
1000 	{
1001 	struct imxdma_engine *imxdma;
1002 	struct resource *res;
1003 	int ret, i;
1004 	int irq, irq_err;
1005 
1006 	imxdma = devm_kzalloc(&pdev->dev, sizeof(*imxdma), GFP_KERNEL);
1007 	if (!imxdma)
1008 		return -ENOMEM;
1009 
1010 	imxdma->devtype = pdev->id_entry->driver_data;
1011 
1012 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1013 	imxdma->base = devm_request_and_ioremap(&pdev->dev, res);
1014 	if (!imxdma->base)
1015 		return -EADDRNOTAVAIL;
1016 
1017 	irq = platform_get_irq(pdev, 0);
1018 	if (irq < 0)
1019 		return irq;
1020 
1021 	imxdma->dma_ipg = devm_clk_get(&pdev->dev, "ipg");
1022 	if (IS_ERR(imxdma->dma_ipg))
1023 		return PTR_ERR(imxdma->dma_ipg);
1024 
1025 	imxdma->dma_ahb = devm_clk_get(&pdev->dev, "ahb");
1026 	if (IS_ERR(imxdma->dma_ahb))
1027 		return PTR_ERR(imxdma->dma_ahb);
1028 
1029 	clk_prepare_enable(imxdma->dma_ipg);
1030 	clk_prepare_enable(imxdma->dma_ahb);
1031 
1032 	/* reset DMA module */
1033 	imx_dmav1_writel(imxdma, DCR_DRST, DMA_DCR);
1034 
1035 	if (is_imx1_dma(imxdma)) {
1036 		ret = devm_request_irq(&pdev->dev, irq,
1037 				       dma_irq_handler, 0, "DMA", imxdma);
1038 		if (ret) {
1039 			dev_warn(imxdma->dev, "Can't register IRQ for DMA\n");
1040 			goto err;
1041 		}
1042 
1043 		irq_err = platform_get_irq(pdev, 1);
1044 		if (irq_err < 0) {
1045 			ret = irq_err;
1046 			goto err;
1047 		}
1048 
1049 		ret = devm_request_irq(&pdev->dev, irq_err,
1050 				       imxdma_err_handler, 0, "DMA", imxdma);
1051 		if (ret) {
1052 			dev_warn(imxdma->dev, "Can't register ERRIRQ for DMA\n");
1053 			goto err;
1054 		}
1055 	}
1056 
1057 	/* enable DMA module */
1058 	imx_dmav1_writel(imxdma, DCR_DEN, DMA_DCR);
1059 
1060 	/* clear all interrupts */
1061 	imx_dmav1_writel(imxdma, (1 << IMX_DMA_CHANNELS) - 1, DMA_DISR);
1062 
1063 	/* disable interrupts */
1064 	imx_dmav1_writel(imxdma, (1 << IMX_DMA_CHANNELS) - 1, DMA_DIMR);
1065 
1066 	INIT_LIST_HEAD(&imxdma->dma_device.channels);
1067 
1068 	dma_cap_set(DMA_SLAVE, imxdma->dma_device.cap_mask);
1069 	dma_cap_set(DMA_CYCLIC, imxdma->dma_device.cap_mask);
1070 	dma_cap_set(DMA_MEMCPY, imxdma->dma_device.cap_mask);
1071 	dma_cap_set(DMA_INTERLEAVE, imxdma->dma_device.cap_mask);
1072 
1073 	/* Initialize 2D global parameters */
1074 	for (i = 0; i < IMX_DMA_2D_SLOTS; i++)
1075 		imxdma->slots_2d[i].count = 0;
1076 
1077 	spin_lock_init(&imxdma->lock);
1078 
1079 	/* Initialize channel parameters */
1080 	for (i = 0; i < IMX_DMA_CHANNELS; i++) {
1081 		struct imxdma_channel *imxdmac = &imxdma->channel[i];
1082 
1083 		if (!is_imx1_dma(imxdma)) {
1084 			ret = devm_request_irq(&pdev->dev, irq + i,
1085 					dma_irq_handler, 0, "DMA", imxdma);
1086 			if (ret) {
1087 				dev_warn(imxdma->dev, "Can't register IRQ %d "
1088 					 "for DMA channel %d\n",
1089 					 irq + i, i);
1090 				goto err;
1091 			}
1092 			init_timer(&imxdmac->watchdog);
1093 			imxdmac->watchdog.function = &imxdma_watchdog;
1094 			imxdmac->watchdog.data = (unsigned long)imxdmac;
1095 		}
1096 
1097 		imxdmac->imxdma = imxdma;
1098 
1099 		INIT_LIST_HEAD(&imxdmac->ld_queue);
1100 		INIT_LIST_HEAD(&imxdmac->ld_free);
1101 		INIT_LIST_HEAD(&imxdmac->ld_active);
1102 
1103 		tasklet_init(&imxdmac->dma_tasklet, imxdma_tasklet,
1104 			     (unsigned long)imxdmac);
1105 		imxdmac->chan.device = &imxdma->dma_device;
1106 		dma_cookie_init(&imxdmac->chan);
1107 		imxdmac->channel = i;
1108 
1109 		/* Add the channel to the DMAC list */
1110 		list_add_tail(&imxdmac->chan.device_node,
1111 			      &imxdma->dma_device.channels);
1112 	}
1113 
1114 	imxdma->dev = &pdev->dev;
1115 	imxdma->dma_device.dev = &pdev->dev;
1116 
1117 	imxdma->dma_device.device_alloc_chan_resources = imxdma_alloc_chan_resources;
1118 	imxdma->dma_device.device_free_chan_resources = imxdma_free_chan_resources;
1119 	imxdma->dma_device.device_tx_status = imxdma_tx_status;
1120 	imxdma->dma_device.device_prep_slave_sg = imxdma_prep_slave_sg;
1121 	imxdma->dma_device.device_prep_dma_cyclic = imxdma_prep_dma_cyclic;
1122 	imxdma->dma_device.device_prep_dma_memcpy = imxdma_prep_dma_memcpy;
1123 	imxdma->dma_device.device_prep_interleaved_dma = imxdma_prep_dma_interleaved;
1124 	imxdma->dma_device.device_control = imxdma_control;
1125 	imxdma->dma_device.device_issue_pending = imxdma_issue_pending;
1126 
1127 	platform_set_drvdata(pdev, imxdma);
1128 
1129 	imxdma->dma_device.copy_align = 2; /* 2^2 = 4 bytes alignment */
1130 	imxdma->dma_device.dev->dma_parms = &imxdma->dma_parms;
1131 	dma_set_max_seg_size(imxdma->dma_device.dev, 0xffffff);
1132 
1133 	ret = dma_async_device_register(&imxdma->dma_device);
1134 	if (ret) {
1135 		dev_err(&pdev->dev, "unable to register\n");
1136 		goto err;
1137 	}
1138 
1139 	return 0;
1140 
1141 err:
1142 	clk_disable_unprepare(imxdma->dma_ipg);
1143 	clk_disable_unprepare(imxdma->dma_ahb);
1144 	return ret;
1145 }
1146 
1147 static int __exit imxdma_remove(struct platform_device *pdev)
1148 {
1149 	struct imxdma_engine *imxdma = platform_get_drvdata(pdev);
1150 
1151         dma_async_device_unregister(&imxdma->dma_device);
1152 
1153 	clk_disable_unprepare(imxdma->dma_ipg);
1154 	clk_disable_unprepare(imxdma->dma_ahb);
1155 
1156         return 0;
1157 }
1158 
1159 static struct platform_driver imxdma_driver = {
1160 	.driver		= {
1161 		.name	= "imx-dma",
1162 	},
1163 	.id_table	= imx_dma_devtype,
1164 	.remove		= __exit_p(imxdma_remove),
1165 };
1166 
1167 static int __init imxdma_module_init(void)
1168 {
1169 	return platform_driver_probe(&imxdma_driver, imxdma_probe);
1170 }
1171 subsys_initcall(imxdma_module_init);
1172 
1173 MODULE_AUTHOR("Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>");
1174 MODULE_DESCRIPTION("i.MX dma driver");
1175 MODULE_LICENSE("GPL");
1176