xref: /openbmc/linux/drivers/dma/mpc512x_dma.c (revision d2999e1b)
1 /*
2  * Copyright (C) Freescale Semicondutor, Inc. 2007, 2008.
3  * Copyright (C) Semihalf 2009
4  * Copyright (C) Ilya Yanok, Emcraft Systems 2010
5  * Copyright (C) Alexander Popov, Promcontroller 2014
6  *
7  * Written by Piotr Ziecik <kosmo@semihalf.com>. Hardware description
8  * (defines, structures and comments) was taken from MPC5121 DMA driver
9  * written by Hongjun Chen <hong-jun.chen@freescale.com>.
10  *
11  * Approved as OSADL project by a majority of OSADL members and funded
12  * by OSADL membership fees in 2009;  for details see www.osadl.org.
13  *
14  * This program is free software; you can redistribute it and/or modify it
15  * under the terms of the GNU General Public License as published by the Free
16  * Software Foundation; either version 2 of the License, or (at your option)
17  * any later version.
18  *
19  * This program is distributed in the hope that it will be useful, but WITHOUT
20  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
21  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
22  * more details.
23  *
24  * You should have received a copy of the GNU General Public License along with
25  * this program; if not, write to the Free Software Foundation, Inc., 59
26  * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
27  *
28  * The full GNU General Public License is included in this distribution in the
29  * file called COPYING.
30  */
31 
32 /*
33  * MPC512x and MPC8308 DMA driver. It supports
34  * memory to memory data transfers (tested using dmatest module) and
35  * data transfers between memory and peripheral I/O memory
36  * by means of slave scatter/gather with these limitations:
37  *  - chunked transfers (described by s/g lists with more than one item)
38  *     are refused as long as proper support for scatter/gather is missing;
39  *  - transfers on MPC8308 always start from software as this SoC appears
40  *     not to have external request lines for peripheral flow control;
41  *  - only peripheral devices with 4-byte FIFO access register are supported;
42  *  - minimal memory <-> I/O memory transfer chunk is 4 bytes and consequently
43  *     source and destination addresses must be 4-byte aligned
44  *     and transfer size must be aligned on (4 * maxburst) boundary;
45  */
46 
47 #include <linux/module.h>
48 #include <linux/dmaengine.h>
49 #include <linux/dma-mapping.h>
50 #include <linux/interrupt.h>
51 #include <linux/io.h>
52 #include <linux/slab.h>
53 #include <linux/of_address.h>
54 #include <linux/of_device.h>
55 #include <linux/of_irq.h>
56 #include <linux/of_platform.h>
57 
58 #include <linux/random.h>
59 
60 #include "dmaengine.h"
61 
62 /* Number of DMA Transfer descriptors allocated per channel */
63 #define MPC_DMA_DESCRIPTORS	64
64 
65 /* Macro definitions */
66 #define MPC_DMA_TCD_OFFSET	0x1000
67 
68 /*
69  * Maximum channel counts for individual hardware variants
70  * and the maximum channel count over all supported controllers,
71  * used for data structure size
72  */
73 #define MPC8308_DMACHAN_MAX	16
74 #define MPC512x_DMACHAN_MAX	64
75 #define MPC_DMA_CHANNELS	64
76 
77 /* Arbitration mode of group and channel */
78 #define MPC_DMA_DMACR_EDCG	(1 << 31)
79 #define MPC_DMA_DMACR_ERGA	(1 << 3)
80 #define MPC_DMA_DMACR_ERCA	(1 << 2)
81 
82 /* Error codes */
83 #define MPC_DMA_DMAES_VLD	(1 << 31)
84 #define MPC_DMA_DMAES_GPE	(1 << 15)
85 #define MPC_DMA_DMAES_CPE	(1 << 14)
86 #define MPC_DMA_DMAES_ERRCHN(err) \
87 				(((err) >> 8) & 0x3f)
88 #define MPC_DMA_DMAES_SAE	(1 << 7)
89 #define MPC_DMA_DMAES_SOE	(1 << 6)
90 #define MPC_DMA_DMAES_DAE	(1 << 5)
91 #define MPC_DMA_DMAES_DOE	(1 << 4)
92 #define MPC_DMA_DMAES_NCE	(1 << 3)
93 #define MPC_DMA_DMAES_SGE	(1 << 2)
94 #define MPC_DMA_DMAES_SBE	(1 << 1)
95 #define MPC_DMA_DMAES_DBE	(1 << 0)
96 
97 #define MPC_DMA_DMAGPOR_SNOOP_ENABLE	(1 << 6)
98 
99 #define MPC_DMA_TSIZE_1		0x00
100 #define MPC_DMA_TSIZE_2		0x01
101 #define MPC_DMA_TSIZE_4		0x02
102 #define MPC_DMA_TSIZE_16	0x04
103 #define MPC_DMA_TSIZE_32	0x05
104 
105 /* MPC5121 DMA engine registers */
106 struct __attribute__ ((__packed__)) mpc_dma_regs {
107 	/* 0x00 */
108 	u32 dmacr;		/* DMA control register */
109 	u32 dmaes;		/* DMA error status */
110 	/* 0x08 */
111 	u32 dmaerqh;		/* DMA enable request high(channels 63~32) */
112 	u32 dmaerql;		/* DMA enable request low(channels 31~0) */
113 	u32 dmaeeih;		/* DMA enable error interrupt high(ch63~32) */
114 	u32 dmaeeil;		/* DMA enable error interrupt low(ch31~0) */
115 	/* 0x18 */
116 	u8 dmaserq;		/* DMA set enable request */
117 	u8 dmacerq;		/* DMA clear enable request */
118 	u8 dmaseei;		/* DMA set enable error interrupt */
119 	u8 dmaceei;		/* DMA clear enable error interrupt */
120 	/* 0x1c */
121 	u8 dmacint;		/* DMA clear interrupt request */
122 	u8 dmacerr;		/* DMA clear error */
123 	u8 dmassrt;		/* DMA set start bit */
124 	u8 dmacdne;		/* DMA clear DONE status bit */
125 	/* 0x20 */
126 	u32 dmainth;		/* DMA interrupt request high(ch63~32) */
127 	u32 dmaintl;		/* DMA interrupt request low(ch31~0) */
128 	u32 dmaerrh;		/* DMA error high(ch63~32) */
129 	u32 dmaerrl;		/* DMA error low(ch31~0) */
130 	/* 0x30 */
131 	u32 dmahrsh;		/* DMA hw request status high(ch63~32) */
132 	u32 dmahrsl;		/* DMA hardware request status low(ch31~0) */
133 	union {
134 		u32 dmaihsa;	/* DMA interrupt high select AXE(ch63~32) */
135 		u32 dmagpor;	/* (General purpose register on MPC8308) */
136 	};
137 	u32 dmailsa;		/* DMA interrupt low select AXE(ch31~0) */
138 	/* 0x40 ~ 0xff */
139 	u32 reserve0[48];	/* Reserved */
140 	/* 0x100 */
141 	u8 dchpri[MPC_DMA_CHANNELS];
142 	/* DMA channels(0~63) priority */
143 };
144 
145 struct __attribute__ ((__packed__)) mpc_dma_tcd {
146 	/* 0x00 */
147 	u32 saddr;		/* Source address */
148 
149 	u32 smod:5;		/* Source address modulo */
150 	u32 ssize:3;		/* Source data transfer size */
151 	u32 dmod:5;		/* Destination address modulo */
152 	u32 dsize:3;		/* Destination data transfer size */
153 	u32 soff:16;		/* Signed source address offset */
154 
155 	/* 0x08 */
156 	u32 nbytes;		/* Inner "minor" byte count */
157 	u32 slast;		/* Last source address adjustment */
158 	u32 daddr;		/* Destination address */
159 
160 	/* 0x14 */
161 	u32 citer_elink:1;	/* Enable channel-to-channel linking on
162 				 * minor loop complete
163 				 */
164 	u32 citer_linkch:6;	/* Link channel for minor loop complete */
165 	u32 citer:9;		/* Current "major" iteration count */
166 	u32 doff:16;		/* Signed destination address offset */
167 
168 	/* 0x18 */
169 	u32 dlast_sga;		/* Last Destination address adjustment/scatter
170 				 * gather address
171 				 */
172 
173 	/* 0x1c */
174 	u32 biter_elink:1;	/* Enable channel-to-channel linking on major
175 				 * loop complete
176 				 */
177 	u32 biter_linkch:6;
178 	u32 biter:9;		/* Beginning "major" iteration count */
179 	u32 bwc:2;		/* Bandwidth control */
180 	u32 major_linkch:6;	/* Link channel number */
181 	u32 done:1;		/* Channel done */
182 	u32 active:1;		/* Channel active */
183 	u32 major_elink:1;	/* Enable channel-to-channel linking on major
184 				 * loop complete
185 				 */
186 	u32 e_sg:1;		/* Enable scatter/gather processing */
187 	u32 d_req:1;		/* Disable request */
188 	u32 int_half:1;		/* Enable an interrupt when major counter is
189 				 * half complete
190 				 */
191 	u32 int_maj:1;		/* Enable an interrupt when major iteration
192 				 * count completes
193 				 */
194 	u32 start:1;		/* Channel start */
195 };
196 
197 struct mpc_dma_desc {
198 	struct dma_async_tx_descriptor	desc;
199 	struct mpc_dma_tcd		*tcd;
200 	dma_addr_t			tcd_paddr;
201 	int				error;
202 	struct list_head		node;
203 	int				will_access_peripheral;
204 };
205 
206 struct mpc_dma_chan {
207 	struct dma_chan			chan;
208 	struct list_head		free;
209 	struct list_head		prepared;
210 	struct list_head		queued;
211 	struct list_head		active;
212 	struct list_head		completed;
213 	struct mpc_dma_tcd		*tcd;
214 	dma_addr_t			tcd_paddr;
215 
216 	/* Settings for access to peripheral FIFO */
217 	dma_addr_t			src_per_paddr;
218 	u32				src_tcd_nunits;
219 	dma_addr_t			dst_per_paddr;
220 	u32				dst_tcd_nunits;
221 
222 	/* Lock for this structure */
223 	spinlock_t			lock;
224 };
225 
226 struct mpc_dma {
227 	struct dma_device		dma;
228 	struct tasklet_struct		tasklet;
229 	struct mpc_dma_chan		channels[MPC_DMA_CHANNELS];
230 	struct mpc_dma_regs __iomem	*regs;
231 	struct mpc_dma_tcd __iomem	*tcd;
232 	int				irq;
233 	int				irq2;
234 	uint				error_status;
235 	int				is_mpc8308;
236 
237 	/* Lock for error_status field in this structure */
238 	spinlock_t			error_status_lock;
239 };
240 
241 #define DRV_NAME	"mpc512x_dma"
242 
243 /* Convert struct dma_chan to struct mpc_dma_chan */
244 static inline struct mpc_dma_chan *dma_chan_to_mpc_dma_chan(struct dma_chan *c)
245 {
246 	return container_of(c, struct mpc_dma_chan, chan);
247 }
248 
249 /* Convert struct dma_chan to struct mpc_dma */
250 static inline struct mpc_dma *dma_chan_to_mpc_dma(struct dma_chan *c)
251 {
252 	struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(c);
253 	return container_of(mchan, struct mpc_dma, channels[c->chan_id]);
254 }
255 
256 /*
257  * Execute all queued DMA descriptors.
258  *
259  * Following requirements must be met while calling mpc_dma_execute():
260  * 	a) mchan->lock is acquired,
261  * 	b) mchan->active list is empty,
262  * 	c) mchan->queued list contains at least one entry.
263  */
264 static void mpc_dma_execute(struct mpc_dma_chan *mchan)
265 {
266 	struct mpc_dma *mdma = dma_chan_to_mpc_dma(&mchan->chan);
267 	struct mpc_dma_desc *first = NULL;
268 	struct mpc_dma_desc *prev = NULL;
269 	struct mpc_dma_desc *mdesc;
270 	int cid = mchan->chan.chan_id;
271 
272 	while (!list_empty(&mchan->queued)) {
273 		mdesc = list_first_entry(&mchan->queued,
274 						struct mpc_dma_desc, node);
275 		/*
276 		 * Grab either several mem-to-mem transfer descriptors
277 		 * or one peripheral transfer descriptor,
278 		 * don't mix mem-to-mem and peripheral transfer descriptors
279 		 * within the same 'active' list.
280 		 */
281 		if (mdesc->will_access_peripheral) {
282 			if (list_empty(&mchan->active))
283 				list_move_tail(&mdesc->node, &mchan->active);
284 			break;
285 		} else {
286 			list_move_tail(&mdesc->node, &mchan->active);
287 		}
288 	}
289 
290 	/* Chain descriptors into one transaction */
291 	list_for_each_entry(mdesc, &mchan->active, node) {
292 		if (!first)
293 			first = mdesc;
294 
295 		if (!prev) {
296 			prev = mdesc;
297 			continue;
298 		}
299 
300 		prev->tcd->dlast_sga = mdesc->tcd_paddr;
301 		prev->tcd->e_sg = 1;
302 		mdesc->tcd->start = 1;
303 
304 		prev = mdesc;
305 	}
306 
307 	prev->tcd->int_maj = 1;
308 
309 	/* Send first descriptor in chain into hardware */
310 	memcpy_toio(&mdma->tcd[cid], first->tcd, sizeof(struct mpc_dma_tcd));
311 
312 	if (first != prev)
313 		mdma->tcd[cid].e_sg = 1;
314 
315 	if (mdma->is_mpc8308) {
316 		/* MPC8308, no request lines, software initiated start */
317 		out_8(&mdma->regs->dmassrt, cid);
318 	} else if (first->will_access_peripheral) {
319 		/* Peripherals involved, start by external request signal */
320 		out_8(&mdma->regs->dmaserq, cid);
321 	} else {
322 		/* Memory to memory transfer, software initiated start */
323 		out_8(&mdma->regs->dmassrt, cid);
324 	}
325 }
326 
327 /* Handle interrupt on one half of DMA controller (32 channels) */
328 static void mpc_dma_irq_process(struct mpc_dma *mdma, u32 is, u32 es, int off)
329 {
330 	struct mpc_dma_chan *mchan;
331 	struct mpc_dma_desc *mdesc;
332 	u32 status = is | es;
333 	int ch;
334 
335 	while ((ch = fls(status) - 1) >= 0) {
336 		status &= ~(1 << ch);
337 		mchan = &mdma->channels[ch + off];
338 
339 		spin_lock(&mchan->lock);
340 
341 		out_8(&mdma->regs->dmacint, ch + off);
342 		out_8(&mdma->regs->dmacerr, ch + off);
343 
344 		/* Check error status */
345 		if (es & (1 << ch))
346 			list_for_each_entry(mdesc, &mchan->active, node)
347 				mdesc->error = -EIO;
348 
349 		/* Execute queued descriptors */
350 		list_splice_tail_init(&mchan->active, &mchan->completed);
351 		if (!list_empty(&mchan->queued))
352 			mpc_dma_execute(mchan);
353 
354 		spin_unlock(&mchan->lock);
355 	}
356 }
357 
358 /* Interrupt handler */
359 static irqreturn_t mpc_dma_irq(int irq, void *data)
360 {
361 	struct mpc_dma *mdma = data;
362 	uint es;
363 
364 	/* Save error status register */
365 	es = in_be32(&mdma->regs->dmaes);
366 	spin_lock(&mdma->error_status_lock);
367 	if ((es & MPC_DMA_DMAES_VLD) && mdma->error_status == 0)
368 		mdma->error_status = es;
369 	spin_unlock(&mdma->error_status_lock);
370 
371 	/* Handle interrupt on each channel */
372 	if (mdma->dma.chancnt > 32) {
373 		mpc_dma_irq_process(mdma, in_be32(&mdma->regs->dmainth),
374 					in_be32(&mdma->regs->dmaerrh), 32);
375 	}
376 	mpc_dma_irq_process(mdma, in_be32(&mdma->regs->dmaintl),
377 					in_be32(&mdma->regs->dmaerrl), 0);
378 
379 	/* Schedule tasklet */
380 	tasklet_schedule(&mdma->tasklet);
381 
382 	return IRQ_HANDLED;
383 }
384 
385 /* process completed descriptors */
386 static void mpc_dma_process_completed(struct mpc_dma *mdma)
387 {
388 	dma_cookie_t last_cookie = 0;
389 	struct mpc_dma_chan *mchan;
390 	struct mpc_dma_desc *mdesc;
391 	struct dma_async_tx_descriptor *desc;
392 	unsigned long flags;
393 	LIST_HEAD(list);
394 	int i;
395 
396 	for (i = 0; i < mdma->dma.chancnt; i++) {
397 		mchan = &mdma->channels[i];
398 
399 		/* Get all completed descriptors */
400 		spin_lock_irqsave(&mchan->lock, flags);
401 		if (!list_empty(&mchan->completed))
402 			list_splice_tail_init(&mchan->completed, &list);
403 		spin_unlock_irqrestore(&mchan->lock, flags);
404 
405 		if (list_empty(&list))
406 			continue;
407 
408 		/* Execute callbacks and run dependencies */
409 		list_for_each_entry(mdesc, &list, node) {
410 			desc = &mdesc->desc;
411 
412 			if (desc->callback)
413 				desc->callback(desc->callback_param);
414 
415 			last_cookie = desc->cookie;
416 			dma_run_dependencies(desc);
417 		}
418 
419 		/* Free descriptors */
420 		spin_lock_irqsave(&mchan->lock, flags);
421 		list_splice_tail_init(&list, &mchan->free);
422 		mchan->chan.completed_cookie = last_cookie;
423 		spin_unlock_irqrestore(&mchan->lock, flags);
424 	}
425 }
426 
427 /* DMA Tasklet */
428 static void mpc_dma_tasklet(unsigned long data)
429 {
430 	struct mpc_dma *mdma = (void *)data;
431 	unsigned long flags;
432 	uint es;
433 
434 	spin_lock_irqsave(&mdma->error_status_lock, flags);
435 	es = mdma->error_status;
436 	mdma->error_status = 0;
437 	spin_unlock_irqrestore(&mdma->error_status_lock, flags);
438 
439 	/* Print nice error report */
440 	if (es) {
441 		dev_err(mdma->dma.dev,
442 			"Hardware reported following error(s) on channel %u:\n",
443 						      MPC_DMA_DMAES_ERRCHN(es));
444 
445 		if (es & MPC_DMA_DMAES_GPE)
446 			dev_err(mdma->dma.dev, "- Group Priority Error\n");
447 		if (es & MPC_DMA_DMAES_CPE)
448 			dev_err(mdma->dma.dev, "- Channel Priority Error\n");
449 		if (es & MPC_DMA_DMAES_SAE)
450 			dev_err(mdma->dma.dev, "- Source Address Error\n");
451 		if (es & MPC_DMA_DMAES_SOE)
452 			dev_err(mdma->dma.dev, "- Source Offset"
453 						" Configuration Error\n");
454 		if (es & MPC_DMA_DMAES_DAE)
455 			dev_err(mdma->dma.dev, "- Destination Address"
456 								" Error\n");
457 		if (es & MPC_DMA_DMAES_DOE)
458 			dev_err(mdma->dma.dev, "- Destination Offset"
459 						" Configuration Error\n");
460 		if (es & MPC_DMA_DMAES_NCE)
461 			dev_err(mdma->dma.dev, "- NBytes/Citter"
462 						" Configuration Error\n");
463 		if (es & MPC_DMA_DMAES_SGE)
464 			dev_err(mdma->dma.dev, "- Scatter/Gather"
465 						" Configuration Error\n");
466 		if (es & MPC_DMA_DMAES_SBE)
467 			dev_err(mdma->dma.dev, "- Source Bus Error\n");
468 		if (es & MPC_DMA_DMAES_DBE)
469 			dev_err(mdma->dma.dev, "- Destination Bus Error\n");
470 	}
471 
472 	mpc_dma_process_completed(mdma);
473 }
474 
475 /* Submit descriptor to hardware */
476 static dma_cookie_t mpc_dma_tx_submit(struct dma_async_tx_descriptor *txd)
477 {
478 	struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(txd->chan);
479 	struct mpc_dma_desc *mdesc;
480 	unsigned long flags;
481 	dma_cookie_t cookie;
482 
483 	mdesc = container_of(txd, struct mpc_dma_desc, desc);
484 
485 	spin_lock_irqsave(&mchan->lock, flags);
486 
487 	/* Move descriptor to queue */
488 	list_move_tail(&mdesc->node, &mchan->queued);
489 
490 	/* If channel is idle, execute all queued descriptors */
491 	if (list_empty(&mchan->active))
492 		mpc_dma_execute(mchan);
493 
494 	/* Update cookie */
495 	cookie = dma_cookie_assign(txd);
496 	spin_unlock_irqrestore(&mchan->lock, flags);
497 
498 	return cookie;
499 }
500 
501 /* Alloc channel resources */
502 static int mpc_dma_alloc_chan_resources(struct dma_chan *chan)
503 {
504 	struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);
505 	struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
506 	struct mpc_dma_desc *mdesc;
507 	struct mpc_dma_tcd *tcd;
508 	dma_addr_t tcd_paddr;
509 	unsigned long flags;
510 	LIST_HEAD(descs);
511 	int i;
512 
513 	/* Alloc DMA memory for Transfer Control Descriptors */
514 	tcd = dma_alloc_coherent(mdma->dma.dev,
515 			MPC_DMA_DESCRIPTORS * sizeof(struct mpc_dma_tcd),
516 							&tcd_paddr, GFP_KERNEL);
517 	if (!tcd)
518 		return -ENOMEM;
519 
520 	/* Alloc descriptors for this channel */
521 	for (i = 0; i < MPC_DMA_DESCRIPTORS; i++) {
522 		mdesc = kzalloc(sizeof(struct mpc_dma_desc), GFP_KERNEL);
523 		if (!mdesc) {
524 			dev_notice(mdma->dma.dev, "Memory allocation error. "
525 					"Allocated only %u descriptors\n", i);
526 			break;
527 		}
528 
529 		dma_async_tx_descriptor_init(&mdesc->desc, chan);
530 		mdesc->desc.flags = DMA_CTRL_ACK;
531 		mdesc->desc.tx_submit = mpc_dma_tx_submit;
532 
533 		mdesc->tcd = &tcd[i];
534 		mdesc->tcd_paddr = tcd_paddr + (i * sizeof(struct mpc_dma_tcd));
535 
536 		list_add_tail(&mdesc->node, &descs);
537 	}
538 
539 	/* Return error only if no descriptors were allocated */
540 	if (i == 0) {
541 		dma_free_coherent(mdma->dma.dev,
542 			MPC_DMA_DESCRIPTORS * sizeof(struct mpc_dma_tcd),
543 								tcd, tcd_paddr);
544 		return -ENOMEM;
545 	}
546 
547 	spin_lock_irqsave(&mchan->lock, flags);
548 	mchan->tcd = tcd;
549 	mchan->tcd_paddr = tcd_paddr;
550 	list_splice_tail_init(&descs, &mchan->free);
551 	spin_unlock_irqrestore(&mchan->lock, flags);
552 
553 	/* Enable Error Interrupt */
554 	out_8(&mdma->regs->dmaseei, chan->chan_id);
555 
556 	return 0;
557 }
558 
559 /* Free channel resources */
560 static void mpc_dma_free_chan_resources(struct dma_chan *chan)
561 {
562 	struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);
563 	struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
564 	struct mpc_dma_desc *mdesc, *tmp;
565 	struct mpc_dma_tcd *tcd;
566 	dma_addr_t tcd_paddr;
567 	unsigned long flags;
568 	LIST_HEAD(descs);
569 
570 	spin_lock_irqsave(&mchan->lock, flags);
571 
572 	/* Channel must be idle */
573 	BUG_ON(!list_empty(&mchan->prepared));
574 	BUG_ON(!list_empty(&mchan->queued));
575 	BUG_ON(!list_empty(&mchan->active));
576 	BUG_ON(!list_empty(&mchan->completed));
577 
578 	/* Move data */
579 	list_splice_tail_init(&mchan->free, &descs);
580 	tcd = mchan->tcd;
581 	tcd_paddr = mchan->tcd_paddr;
582 
583 	spin_unlock_irqrestore(&mchan->lock, flags);
584 
585 	/* Free DMA memory used by descriptors */
586 	dma_free_coherent(mdma->dma.dev,
587 			MPC_DMA_DESCRIPTORS * sizeof(struct mpc_dma_tcd),
588 								tcd, tcd_paddr);
589 
590 	/* Free descriptors */
591 	list_for_each_entry_safe(mdesc, tmp, &descs, node)
592 		kfree(mdesc);
593 
594 	/* Disable Error Interrupt */
595 	out_8(&mdma->regs->dmaceei, chan->chan_id);
596 }
597 
598 /* Send all pending descriptor to hardware */
599 static void mpc_dma_issue_pending(struct dma_chan *chan)
600 {
601 	/*
602 	 * We are posting descriptors to the hardware as soon as
603 	 * they are ready, so this function does nothing.
604 	 */
605 }
606 
607 /* Check request completion status */
608 static enum dma_status
609 mpc_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
610 	       struct dma_tx_state *txstate)
611 {
612 	return dma_cookie_status(chan, cookie, txstate);
613 }
614 
615 /* Prepare descriptor for memory to memory copy */
616 static struct dma_async_tx_descriptor *
617 mpc_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,
618 					size_t len, unsigned long flags)
619 {
620 	struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);
621 	struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
622 	struct mpc_dma_desc *mdesc = NULL;
623 	struct mpc_dma_tcd *tcd;
624 	unsigned long iflags;
625 
626 	/* Get free descriptor */
627 	spin_lock_irqsave(&mchan->lock, iflags);
628 	if (!list_empty(&mchan->free)) {
629 		mdesc = list_first_entry(&mchan->free, struct mpc_dma_desc,
630 									node);
631 		list_del(&mdesc->node);
632 	}
633 	spin_unlock_irqrestore(&mchan->lock, iflags);
634 
635 	if (!mdesc) {
636 		/* try to free completed descriptors */
637 		mpc_dma_process_completed(mdma);
638 		return NULL;
639 	}
640 
641 	mdesc->error = 0;
642 	mdesc->will_access_peripheral = 0;
643 	tcd = mdesc->tcd;
644 
645 	/* Prepare Transfer Control Descriptor for this transaction */
646 	memset(tcd, 0, sizeof(struct mpc_dma_tcd));
647 
648 	if (IS_ALIGNED(src | dst | len, 32)) {
649 		tcd->ssize = MPC_DMA_TSIZE_32;
650 		tcd->dsize = MPC_DMA_TSIZE_32;
651 		tcd->soff = 32;
652 		tcd->doff = 32;
653 	} else if (!mdma->is_mpc8308 && IS_ALIGNED(src | dst | len, 16)) {
654 		/* MPC8308 doesn't support 16 byte transfers */
655 		tcd->ssize = MPC_DMA_TSIZE_16;
656 		tcd->dsize = MPC_DMA_TSIZE_16;
657 		tcd->soff = 16;
658 		tcd->doff = 16;
659 	} else if (IS_ALIGNED(src | dst | len, 4)) {
660 		tcd->ssize = MPC_DMA_TSIZE_4;
661 		tcd->dsize = MPC_DMA_TSIZE_4;
662 		tcd->soff = 4;
663 		tcd->doff = 4;
664 	} else if (IS_ALIGNED(src | dst | len, 2)) {
665 		tcd->ssize = MPC_DMA_TSIZE_2;
666 		tcd->dsize = MPC_DMA_TSIZE_2;
667 		tcd->soff = 2;
668 		tcd->doff = 2;
669 	} else {
670 		tcd->ssize = MPC_DMA_TSIZE_1;
671 		tcd->dsize = MPC_DMA_TSIZE_1;
672 		tcd->soff = 1;
673 		tcd->doff = 1;
674 	}
675 
676 	tcd->saddr = src;
677 	tcd->daddr = dst;
678 	tcd->nbytes = len;
679 	tcd->biter = 1;
680 	tcd->citer = 1;
681 
682 	/* Place descriptor in prepared list */
683 	spin_lock_irqsave(&mchan->lock, iflags);
684 	list_add_tail(&mdesc->node, &mchan->prepared);
685 	spin_unlock_irqrestore(&mchan->lock, iflags);
686 
687 	return &mdesc->desc;
688 }
689 
690 static struct dma_async_tx_descriptor *
691 mpc_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
692 		unsigned int sg_len, enum dma_transfer_direction direction,
693 		unsigned long flags, void *context)
694 {
695 	struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);
696 	struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
697 	struct mpc_dma_desc *mdesc = NULL;
698 	dma_addr_t per_paddr;
699 	u32 tcd_nunits;
700 	struct mpc_dma_tcd *tcd;
701 	unsigned long iflags;
702 	struct scatterlist *sg;
703 	size_t len;
704 	int iter, i;
705 
706 	/* Currently there is no proper support for scatter/gather */
707 	if (sg_len != 1)
708 		return NULL;
709 
710 	if (!is_slave_direction(direction))
711 		return NULL;
712 
713 	for_each_sg(sgl, sg, sg_len, i) {
714 		spin_lock_irqsave(&mchan->lock, iflags);
715 
716 		mdesc = list_first_entry(&mchan->free,
717 						struct mpc_dma_desc, node);
718 		if (!mdesc) {
719 			spin_unlock_irqrestore(&mchan->lock, iflags);
720 			/* Try to free completed descriptors */
721 			mpc_dma_process_completed(mdma);
722 			return NULL;
723 		}
724 
725 		list_del(&mdesc->node);
726 
727 		if (direction == DMA_DEV_TO_MEM) {
728 			per_paddr = mchan->src_per_paddr;
729 			tcd_nunits = mchan->src_tcd_nunits;
730 		} else {
731 			per_paddr = mchan->dst_per_paddr;
732 			tcd_nunits = mchan->dst_tcd_nunits;
733 		}
734 
735 		spin_unlock_irqrestore(&mchan->lock, iflags);
736 
737 		if (per_paddr == 0 || tcd_nunits == 0)
738 			goto err_prep;
739 
740 		mdesc->error = 0;
741 		mdesc->will_access_peripheral = 1;
742 
743 		/* Prepare Transfer Control Descriptor for this transaction */
744 		tcd = mdesc->tcd;
745 
746 		memset(tcd, 0, sizeof(struct mpc_dma_tcd));
747 
748 		if (!IS_ALIGNED(sg_dma_address(sg), 4))
749 			goto err_prep;
750 
751 		if (direction == DMA_DEV_TO_MEM) {
752 			tcd->saddr = per_paddr;
753 			tcd->daddr = sg_dma_address(sg);
754 			tcd->soff = 0;
755 			tcd->doff = 4;
756 		} else {
757 			tcd->saddr = sg_dma_address(sg);
758 			tcd->daddr = per_paddr;
759 			tcd->soff = 4;
760 			tcd->doff = 0;
761 		}
762 
763 		tcd->ssize = MPC_DMA_TSIZE_4;
764 		tcd->dsize = MPC_DMA_TSIZE_4;
765 
766 		len = sg_dma_len(sg);
767 		tcd->nbytes = tcd_nunits * 4;
768 		if (!IS_ALIGNED(len, tcd->nbytes))
769 			goto err_prep;
770 
771 		iter = len / tcd->nbytes;
772 		if (iter >= 1 << 15) {
773 			/* len is too big */
774 			goto err_prep;
775 		}
776 		/* citer_linkch contains the high bits of iter */
777 		tcd->biter = iter & 0x1ff;
778 		tcd->biter_linkch = iter >> 9;
779 		tcd->citer = tcd->biter;
780 		tcd->citer_linkch = tcd->biter_linkch;
781 
782 		tcd->e_sg = 0;
783 		tcd->d_req = 1;
784 
785 		/* Place descriptor in prepared list */
786 		spin_lock_irqsave(&mchan->lock, iflags);
787 		list_add_tail(&mdesc->node, &mchan->prepared);
788 		spin_unlock_irqrestore(&mchan->lock, iflags);
789 	}
790 
791 	return &mdesc->desc;
792 
793 err_prep:
794 	/* Put the descriptor back */
795 	spin_lock_irqsave(&mchan->lock, iflags);
796 	list_add_tail(&mdesc->node, &mchan->free);
797 	spin_unlock_irqrestore(&mchan->lock, iflags);
798 
799 	return NULL;
800 }
801 
802 static int mpc_dma_device_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
803 							unsigned long arg)
804 {
805 	struct mpc_dma_chan *mchan;
806 	struct mpc_dma *mdma;
807 	struct dma_slave_config *cfg;
808 	unsigned long flags;
809 
810 	mchan = dma_chan_to_mpc_dma_chan(chan);
811 	switch (cmd) {
812 	case DMA_TERMINATE_ALL:
813 		/* Disable channel requests */
814 		mdma = dma_chan_to_mpc_dma(chan);
815 
816 		spin_lock_irqsave(&mchan->lock, flags);
817 
818 		out_8(&mdma->regs->dmacerq, chan->chan_id);
819 		list_splice_tail_init(&mchan->prepared, &mchan->free);
820 		list_splice_tail_init(&mchan->queued, &mchan->free);
821 		list_splice_tail_init(&mchan->active, &mchan->free);
822 
823 		spin_unlock_irqrestore(&mchan->lock, flags);
824 
825 		return 0;
826 
827 	case DMA_SLAVE_CONFIG:
828 		/*
829 		 * Software constraints:
830 		 *  - only transfers between a peripheral device and
831 		 *     memory are supported;
832 		 *  - only peripheral devices with 4-byte FIFO access register
833 		 *     are supported;
834 		 *  - minimal transfer chunk is 4 bytes and consequently
835 		 *     source and destination addresses must be 4-byte aligned
836 		 *     and transfer size must be aligned on (4 * maxburst)
837 		 *     boundary;
838 		 *  - during the transfer RAM address is being incremented by
839 		 *     the size of minimal transfer chunk;
840 		 *  - peripheral port's address is constant during the transfer.
841 		 */
842 
843 		cfg = (void *)arg;
844 
845 		if (cfg->src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES ||
846 		    cfg->dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES ||
847 		    !IS_ALIGNED(cfg->src_addr, 4) ||
848 		    !IS_ALIGNED(cfg->dst_addr, 4)) {
849 			return -EINVAL;
850 		}
851 
852 		spin_lock_irqsave(&mchan->lock, flags);
853 
854 		mchan->src_per_paddr = cfg->src_addr;
855 		mchan->src_tcd_nunits = cfg->src_maxburst;
856 		mchan->dst_per_paddr = cfg->dst_addr;
857 		mchan->dst_tcd_nunits = cfg->dst_maxburst;
858 
859 		/* Apply defaults */
860 		if (mchan->src_tcd_nunits == 0)
861 			mchan->src_tcd_nunits = 1;
862 		if (mchan->dst_tcd_nunits == 0)
863 			mchan->dst_tcd_nunits = 1;
864 
865 		spin_unlock_irqrestore(&mchan->lock, flags);
866 
867 		return 0;
868 
869 	default:
870 		/* Unknown command */
871 		break;
872 	}
873 
874 	return -ENXIO;
875 }
876 
877 static int mpc_dma_probe(struct platform_device *op)
878 {
879 	struct device_node *dn = op->dev.of_node;
880 	struct device *dev = &op->dev;
881 	struct dma_device *dma;
882 	struct mpc_dma *mdma;
883 	struct mpc_dma_chan *mchan;
884 	struct resource res;
885 	ulong regs_start, regs_size;
886 	int retval, i;
887 
888 	mdma = devm_kzalloc(dev, sizeof(struct mpc_dma), GFP_KERNEL);
889 	if (!mdma) {
890 		dev_err(dev, "Memory exhausted!\n");
891 		retval = -ENOMEM;
892 		goto err;
893 	}
894 
895 	mdma->irq = irq_of_parse_and_map(dn, 0);
896 	if (mdma->irq == NO_IRQ) {
897 		dev_err(dev, "Error mapping IRQ!\n");
898 		retval = -EINVAL;
899 		goto err;
900 	}
901 
902 	if (of_device_is_compatible(dn, "fsl,mpc8308-dma")) {
903 		mdma->is_mpc8308 = 1;
904 		mdma->irq2 = irq_of_parse_and_map(dn, 1);
905 		if (mdma->irq2 == NO_IRQ) {
906 			dev_err(dev, "Error mapping IRQ!\n");
907 			retval = -EINVAL;
908 			goto err_dispose1;
909 		}
910 	}
911 
912 	retval = of_address_to_resource(dn, 0, &res);
913 	if (retval) {
914 		dev_err(dev, "Error parsing memory region!\n");
915 		goto err_dispose2;
916 	}
917 
918 	regs_start = res.start;
919 	regs_size = resource_size(&res);
920 
921 	if (!devm_request_mem_region(dev, regs_start, regs_size, DRV_NAME)) {
922 		dev_err(dev, "Error requesting memory region!\n");
923 		retval = -EBUSY;
924 		goto err_dispose2;
925 	}
926 
927 	mdma->regs = devm_ioremap(dev, regs_start, regs_size);
928 	if (!mdma->regs) {
929 		dev_err(dev, "Error mapping memory region!\n");
930 		retval = -ENOMEM;
931 		goto err_dispose2;
932 	}
933 
934 	mdma->tcd = (struct mpc_dma_tcd *)((u8 *)(mdma->regs)
935 							+ MPC_DMA_TCD_OFFSET);
936 
937 	retval = request_irq(mdma->irq, &mpc_dma_irq, 0, DRV_NAME, mdma);
938 	if (retval) {
939 		dev_err(dev, "Error requesting IRQ!\n");
940 		retval = -EINVAL;
941 		goto err_dispose2;
942 	}
943 
944 	if (mdma->is_mpc8308) {
945 		retval = request_irq(mdma->irq2, &mpc_dma_irq, 0,
946 							DRV_NAME, mdma);
947 		if (retval) {
948 			dev_err(dev, "Error requesting IRQ2!\n");
949 			retval = -EINVAL;
950 			goto err_free1;
951 		}
952 	}
953 
954 	spin_lock_init(&mdma->error_status_lock);
955 
956 	dma = &mdma->dma;
957 	dma->dev = dev;
958 	if (mdma->is_mpc8308)
959 		dma->chancnt = MPC8308_DMACHAN_MAX;
960 	else
961 		dma->chancnt = MPC512x_DMACHAN_MAX;
962 	dma->device_alloc_chan_resources = mpc_dma_alloc_chan_resources;
963 	dma->device_free_chan_resources = mpc_dma_free_chan_resources;
964 	dma->device_issue_pending = mpc_dma_issue_pending;
965 	dma->device_tx_status = mpc_dma_tx_status;
966 	dma->device_prep_dma_memcpy = mpc_dma_prep_memcpy;
967 	dma->device_prep_slave_sg = mpc_dma_prep_slave_sg;
968 	dma->device_control = mpc_dma_device_control;
969 
970 	INIT_LIST_HEAD(&dma->channels);
971 	dma_cap_set(DMA_MEMCPY, dma->cap_mask);
972 	dma_cap_set(DMA_SLAVE, dma->cap_mask);
973 
974 	for (i = 0; i < dma->chancnt; i++) {
975 		mchan = &mdma->channels[i];
976 
977 		mchan->chan.device = dma;
978 		dma_cookie_init(&mchan->chan);
979 
980 		INIT_LIST_HEAD(&mchan->free);
981 		INIT_LIST_HEAD(&mchan->prepared);
982 		INIT_LIST_HEAD(&mchan->queued);
983 		INIT_LIST_HEAD(&mchan->active);
984 		INIT_LIST_HEAD(&mchan->completed);
985 
986 		spin_lock_init(&mchan->lock);
987 		list_add_tail(&mchan->chan.device_node, &dma->channels);
988 	}
989 
990 	tasklet_init(&mdma->tasklet, mpc_dma_tasklet, (unsigned long)mdma);
991 
992 	/*
993 	 * Configure DMA Engine:
994 	 * - Dynamic clock,
995 	 * - Round-robin group arbitration,
996 	 * - Round-robin channel arbitration.
997 	 */
998 	if (mdma->is_mpc8308) {
999 		/* MPC8308 has 16 channels and lacks some registers */
1000 		out_be32(&mdma->regs->dmacr, MPC_DMA_DMACR_ERCA);
1001 
1002 		/* enable snooping */
1003 		out_be32(&mdma->regs->dmagpor, MPC_DMA_DMAGPOR_SNOOP_ENABLE);
1004 		/* Disable error interrupts */
1005 		out_be32(&mdma->regs->dmaeeil, 0);
1006 
1007 		/* Clear interrupts status */
1008 		out_be32(&mdma->regs->dmaintl, 0xFFFF);
1009 		out_be32(&mdma->regs->dmaerrl, 0xFFFF);
1010 	} else {
1011 		out_be32(&mdma->regs->dmacr, MPC_DMA_DMACR_EDCG |
1012 					MPC_DMA_DMACR_ERGA | MPC_DMA_DMACR_ERCA);
1013 
1014 		/* Disable hardware DMA requests */
1015 		out_be32(&mdma->regs->dmaerqh, 0);
1016 		out_be32(&mdma->regs->dmaerql, 0);
1017 
1018 		/* Disable error interrupts */
1019 		out_be32(&mdma->regs->dmaeeih, 0);
1020 		out_be32(&mdma->regs->dmaeeil, 0);
1021 
1022 		/* Clear interrupts status */
1023 		out_be32(&mdma->regs->dmainth, 0xFFFFFFFF);
1024 		out_be32(&mdma->regs->dmaintl, 0xFFFFFFFF);
1025 		out_be32(&mdma->regs->dmaerrh, 0xFFFFFFFF);
1026 		out_be32(&mdma->regs->dmaerrl, 0xFFFFFFFF);
1027 
1028 		/* Route interrupts to IPIC */
1029 		out_be32(&mdma->regs->dmaihsa, 0);
1030 		out_be32(&mdma->regs->dmailsa, 0);
1031 	}
1032 
1033 	/* Register DMA engine */
1034 	dev_set_drvdata(dev, mdma);
1035 	retval = dma_async_device_register(dma);
1036 	if (retval)
1037 		goto err_free2;
1038 
1039 	return retval;
1040 
1041 err_free2:
1042 	if (mdma->is_mpc8308)
1043 		free_irq(mdma->irq2, mdma);
1044 err_free1:
1045 	free_irq(mdma->irq, mdma);
1046 err_dispose2:
1047 	if (mdma->is_mpc8308)
1048 		irq_dispose_mapping(mdma->irq2);
1049 err_dispose1:
1050 	irq_dispose_mapping(mdma->irq);
1051 err:
1052 	return retval;
1053 }
1054 
1055 static int mpc_dma_remove(struct platform_device *op)
1056 {
1057 	struct device *dev = &op->dev;
1058 	struct mpc_dma *mdma = dev_get_drvdata(dev);
1059 
1060 	dma_async_device_unregister(&mdma->dma);
1061 	if (mdma->is_mpc8308) {
1062 		free_irq(mdma->irq2, mdma);
1063 		irq_dispose_mapping(mdma->irq2);
1064 	}
1065 	free_irq(mdma->irq, mdma);
1066 	irq_dispose_mapping(mdma->irq);
1067 
1068 	return 0;
1069 }
1070 
1071 static struct of_device_id mpc_dma_match[] = {
1072 	{ .compatible = "fsl,mpc5121-dma", },
1073 	{ .compatible = "fsl,mpc8308-dma", },
1074 	{},
1075 };
1076 
1077 static struct platform_driver mpc_dma_driver = {
1078 	.probe		= mpc_dma_probe,
1079 	.remove		= mpc_dma_remove,
1080 	.driver = {
1081 		.name = DRV_NAME,
1082 		.owner = THIS_MODULE,
1083 		.of_match_table	= mpc_dma_match,
1084 	},
1085 };
1086 
1087 module_platform_driver(mpc_dma_driver);
1088 
1089 MODULE_LICENSE("GPL");
1090 MODULE_AUTHOR("Piotr Ziecik <kosmo@semihalf.com>");
1091