xref: /openbmc/linux/drivers/dma/sh/rcar-dmac.c (revision 8795a739)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Renesas R-Car Gen2/Gen3 DMA Controller Driver
4  *
5  * Copyright (C) 2014-2019 Renesas Electronics Inc.
6  *
7  * Author: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
8  */
9 
10 #include <linux/delay.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/dmaengine.h>
13 #include <linux/interrupt.h>
14 #include <linux/list.h>
15 #include <linux/module.h>
16 #include <linux/mutex.h>
17 #include <linux/of.h>
18 #include <linux/of_dma.h>
19 #include <linux/of_platform.h>
20 #include <linux/platform_device.h>
21 #include <linux/pm_runtime.h>
22 #include <linux/slab.h>
23 #include <linux/spinlock.h>
24 
25 #include "../dmaengine.h"
26 
27 /*
28  * struct rcar_dmac_xfer_chunk - Descriptor for a hardware transfer
29  * @node: entry in the parent's chunks list
30  * @src_addr: device source address
31  * @dst_addr: device destination address
32  * @size: transfer size in bytes
33  */
34 struct rcar_dmac_xfer_chunk {
35 	struct list_head node;
36 
37 	dma_addr_t src_addr;
38 	dma_addr_t dst_addr;
39 	u32 size;
40 };
41 
42 /*
43  * struct rcar_dmac_hw_desc - Hardware descriptor for a transfer chunk
44  * @sar: value of the SAR register (source address)
45  * @dar: value of the DAR register (destination address)
46  * @tcr: value of the TCR register (transfer count)
47  */
48 struct rcar_dmac_hw_desc {
49 	u32 sar;
50 	u32 dar;
51 	u32 tcr;
52 	u32 reserved;
53 } __attribute__((__packed__));
54 
55 /*
56  * struct rcar_dmac_desc - R-Car Gen2 DMA Transfer Descriptor
57  * @async_tx: base DMA asynchronous transaction descriptor
58  * @direction: direction of the DMA transfer
59  * @xfer_shift: log2 of the transfer size
60  * @chcr: value of the channel configuration register for this transfer
61  * @node: entry in the channel's descriptors lists
62  * @chunks: list of transfer chunks for this transfer
63  * @running: the transfer chunk being currently processed
64  * @nchunks: number of transfer chunks for this transfer
65  * @hwdescs.use: whether the transfer descriptor uses hardware descriptors
66  * @hwdescs.mem: hardware descriptors memory for the transfer
67  * @hwdescs.dma: device address of the hardware descriptors memory
68  * @hwdescs.size: size of the hardware descriptors in bytes
69  * @size: transfer size in bytes
70  * @cyclic: when set indicates that the DMA transfer is cyclic
71  */
72 struct rcar_dmac_desc {
73 	struct dma_async_tx_descriptor async_tx;
74 	enum dma_transfer_direction direction;
75 	unsigned int xfer_shift;
76 	u32 chcr;
77 
78 	struct list_head node;
79 	struct list_head chunks;
80 	struct rcar_dmac_xfer_chunk *running;
81 	unsigned int nchunks;
82 
83 	struct {
84 		bool use;
85 		struct rcar_dmac_hw_desc *mem;
86 		dma_addr_t dma;
87 		size_t size;
88 	} hwdescs;
89 
90 	unsigned int size;
91 	bool cyclic;
92 };
93 
94 #define to_rcar_dmac_desc(d)	container_of(d, struct rcar_dmac_desc, async_tx)
95 
96 /*
97  * struct rcar_dmac_desc_page - One page worth of descriptors
98  * @node: entry in the channel's pages list
99  * @descs: array of DMA descriptors
100  * @chunks: array of transfer chunk descriptors
101  */
102 struct rcar_dmac_desc_page {
103 	struct list_head node;
104 
105 	union {
106 		struct rcar_dmac_desc descs[0];
107 		struct rcar_dmac_xfer_chunk chunks[0];
108 	};
109 };
110 
111 #define RCAR_DMAC_DESCS_PER_PAGE					\
112 	((PAGE_SIZE - offsetof(struct rcar_dmac_desc_page, descs)) /	\
113 	sizeof(struct rcar_dmac_desc))
114 #define RCAR_DMAC_XFER_CHUNKS_PER_PAGE					\
115 	((PAGE_SIZE - offsetof(struct rcar_dmac_desc_page, chunks)) /	\
116 	sizeof(struct rcar_dmac_xfer_chunk))
117 
118 /*
119  * struct rcar_dmac_chan_slave - Slave configuration
120  * @slave_addr: slave memory address
121  * @xfer_size: size (in bytes) of hardware transfers
122  */
123 struct rcar_dmac_chan_slave {
124 	phys_addr_t slave_addr;
125 	unsigned int xfer_size;
126 };
127 
128 /*
129  * struct rcar_dmac_chan_map - Map of slave device phys to dma address
130  * @addr: slave dma address
131  * @dir: direction of mapping
132  * @slave: slave configuration that is mapped
133  */
134 struct rcar_dmac_chan_map {
135 	dma_addr_t addr;
136 	enum dma_data_direction dir;
137 	struct rcar_dmac_chan_slave slave;
138 };
139 
140 /*
141  * struct rcar_dmac_chan - R-Car Gen2 DMA Controller Channel
142  * @chan: base DMA channel object
143  * @iomem: channel I/O memory base
144  * @index: index of this channel in the controller
145  * @irq: channel IRQ
146  * @src: slave memory address and size on the source side
147  * @dst: slave memory address and size on the destination side
148  * @mid_rid: hardware MID/RID for the DMA client using this channel
149  * @lock: protects the channel CHCR register and the desc members
150  * @desc.free: list of free descriptors
151  * @desc.pending: list of pending descriptors (submitted with tx_submit)
152  * @desc.active: list of active descriptors (activated with issue_pending)
153  * @desc.done: list of completed descriptors
154  * @desc.wait: list of descriptors waiting for an ack
155  * @desc.running: the descriptor being processed (a member of the active list)
156  * @desc.chunks_free: list of free transfer chunk descriptors
157  * @desc.pages: list of pages used by allocated descriptors
158  */
159 struct rcar_dmac_chan {
160 	struct dma_chan chan;
161 	void __iomem *iomem;
162 	unsigned int index;
163 	int irq;
164 
165 	struct rcar_dmac_chan_slave src;
166 	struct rcar_dmac_chan_slave dst;
167 	struct rcar_dmac_chan_map map;
168 	int mid_rid;
169 
170 	spinlock_t lock;
171 
172 	struct {
173 		struct list_head free;
174 		struct list_head pending;
175 		struct list_head active;
176 		struct list_head done;
177 		struct list_head wait;
178 		struct rcar_dmac_desc *running;
179 
180 		struct list_head chunks_free;
181 
182 		struct list_head pages;
183 	} desc;
184 };
185 
186 #define to_rcar_dmac_chan(c)	container_of(c, struct rcar_dmac_chan, chan)
187 
188 /*
189  * struct rcar_dmac - R-Car Gen2 DMA Controller
190  * @engine: base DMA engine object
191  * @dev: the hardware device
192  * @iomem: remapped I/O memory base
193  * @n_channels: number of available channels
194  * @channels: array of DMAC channels
195  * @channels_mask: bitfield of which DMA channels are managed by this driver
196  * @modules: bitmask of client modules in use
197  */
198 struct rcar_dmac {
199 	struct dma_device engine;
200 	struct device *dev;
201 	void __iomem *iomem;
202 	struct device_dma_parameters parms;
203 
204 	unsigned int n_channels;
205 	struct rcar_dmac_chan *channels;
206 	unsigned int channels_mask;
207 
208 	DECLARE_BITMAP(modules, 256);
209 };
210 
211 #define to_rcar_dmac(d)		container_of(d, struct rcar_dmac, engine)
212 
213 /* -----------------------------------------------------------------------------
214  * Registers
215  */
216 
217 #define RCAR_DMAC_CHAN_OFFSET(i)	(0x8000 + 0x80 * (i))
218 
219 #define RCAR_DMAISTA			0x0020
220 #define RCAR_DMASEC			0x0030
221 #define RCAR_DMAOR			0x0060
222 #define RCAR_DMAOR_PRI_FIXED		(0 << 8)
223 #define RCAR_DMAOR_PRI_ROUND_ROBIN	(3 << 8)
224 #define RCAR_DMAOR_AE			(1 << 2)
225 #define RCAR_DMAOR_DME			(1 << 0)
226 #define RCAR_DMACHCLR			0x0080
227 #define RCAR_DMADPSEC			0x00a0
228 
229 #define RCAR_DMASAR			0x0000
230 #define RCAR_DMADAR			0x0004
231 #define RCAR_DMATCR			0x0008
232 #define RCAR_DMATCR_MASK		0x00ffffff
233 #define RCAR_DMATSR			0x0028
234 #define RCAR_DMACHCR			0x000c
235 #define RCAR_DMACHCR_CAE		(1 << 31)
236 #define RCAR_DMACHCR_CAIE		(1 << 30)
237 #define RCAR_DMACHCR_DPM_DISABLED	(0 << 28)
238 #define RCAR_DMACHCR_DPM_ENABLED	(1 << 28)
239 #define RCAR_DMACHCR_DPM_REPEAT		(2 << 28)
240 #define RCAR_DMACHCR_DPM_INFINITE	(3 << 28)
241 #define RCAR_DMACHCR_RPT_SAR		(1 << 27)
242 #define RCAR_DMACHCR_RPT_DAR		(1 << 26)
243 #define RCAR_DMACHCR_RPT_TCR		(1 << 25)
244 #define RCAR_DMACHCR_DPB		(1 << 22)
245 #define RCAR_DMACHCR_DSE		(1 << 19)
246 #define RCAR_DMACHCR_DSIE		(1 << 18)
247 #define RCAR_DMACHCR_TS_1B		((0 << 20) | (0 << 3))
248 #define RCAR_DMACHCR_TS_2B		((0 << 20) | (1 << 3))
249 #define RCAR_DMACHCR_TS_4B		((0 << 20) | (2 << 3))
250 #define RCAR_DMACHCR_TS_16B		((0 << 20) | (3 << 3))
251 #define RCAR_DMACHCR_TS_32B		((1 << 20) | (0 << 3))
252 #define RCAR_DMACHCR_TS_64B		((1 << 20) | (1 << 3))
253 #define RCAR_DMACHCR_TS_8B		((1 << 20) | (3 << 3))
254 #define RCAR_DMACHCR_DM_FIXED		(0 << 14)
255 #define RCAR_DMACHCR_DM_INC		(1 << 14)
256 #define RCAR_DMACHCR_DM_DEC		(2 << 14)
257 #define RCAR_DMACHCR_SM_FIXED		(0 << 12)
258 #define RCAR_DMACHCR_SM_INC		(1 << 12)
259 #define RCAR_DMACHCR_SM_DEC		(2 << 12)
260 #define RCAR_DMACHCR_RS_AUTO		(4 << 8)
261 #define RCAR_DMACHCR_RS_DMARS		(8 << 8)
262 #define RCAR_DMACHCR_IE			(1 << 2)
263 #define RCAR_DMACHCR_TE			(1 << 1)
264 #define RCAR_DMACHCR_DE			(1 << 0)
265 #define RCAR_DMATCRB			0x0018
266 #define RCAR_DMATSRB			0x0038
267 #define RCAR_DMACHCRB			0x001c
268 #define RCAR_DMACHCRB_DCNT(n)		((n) << 24)
269 #define RCAR_DMACHCRB_DPTR_MASK		(0xff << 16)
270 #define RCAR_DMACHCRB_DPTR_SHIFT	16
271 #define RCAR_DMACHCRB_DRST		(1 << 15)
272 #define RCAR_DMACHCRB_DTS		(1 << 8)
273 #define RCAR_DMACHCRB_SLM_NORMAL	(0 << 4)
274 #define RCAR_DMACHCRB_SLM_CLK(n)	((8 | (n)) << 4)
275 #define RCAR_DMACHCRB_PRI(n)		((n) << 0)
276 #define RCAR_DMARS			0x0040
277 #define RCAR_DMABUFCR			0x0048
278 #define RCAR_DMABUFCR_MBU(n)		((n) << 16)
279 #define RCAR_DMABUFCR_ULB(n)		((n) << 0)
280 #define RCAR_DMADPBASE			0x0050
281 #define RCAR_DMADPBASE_MASK		0xfffffff0
282 #define RCAR_DMADPBASE_SEL		(1 << 0)
283 #define RCAR_DMADPCR			0x0054
284 #define RCAR_DMADPCR_DIPT(n)		((n) << 24)
285 #define RCAR_DMAFIXSAR			0x0010
286 #define RCAR_DMAFIXDAR			0x0014
287 #define RCAR_DMAFIXDPBASE		0x0060
288 
289 /* Hardcode the MEMCPY transfer size to 4 bytes. */
290 #define RCAR_DMAC_MEMCPY_XFER_SIZE	4
291 
292 /* -----------------------------------------------------------------------------
293  * Device access
294  */
295 
296 static void rcar_dmac_write(struct rcar_dmac *dmac, u32 reg, u32 data)
297 {
298 	if (reg == RCAR_DMAOR)
299 		writew(data, dmac->iomem + reg);
300 	else
301 		writel(data, dmac->iomem + reg);
302 }
303 
304 static u32 rcar_dmac_read(struct rcar_dmac *dmac, u32 reg)
305 {
306 	if (reg == RCAR_DMAOR)
307 		return readw(dmac->iomem + reg);
308 	else
309 		return readl(dmac->iomem + reg);
310 }
311 
312 static u32 rcar_dmac_chan_read(struct rcar_dmac_chan *chan, u32 reg)
313 {
314 	if (reg == RCAR_DMARS)
315 		return readw(chan->iomem + reg);
316 	else
317 		return readl(chan->iomem + reg);
318 }
319 
320 static void rcar_dmac_chan_write(struct rcar_dmac_chan *chan, u32 reg, u32 data)
321 {
322 	if (reg == RCAR_DMARS)
323 		writew(data, chan->iomem + reg);
324 	else
325 		writel(data, chan->iomem + reg);
326 }
327 
328 /* -----------------------------------------------------------------------------
329  * Initialization and configuration
330  */
331 
332 static bool rcar_dmac_chan_is_busy(struct rcar_dmac_chan *chan)
333 {
334 	u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
335 
336 	return !!(chcr & (RCAR_DMACHCR_DE | RCAR_DMACHCR_TE));
337 }
338 
339 static void rcar_dmac_chan_start_xfer(struct rcar_dmac_chan *chan)
340 {
341 	struct rcar_dmac_desc *desc = chan->desc.running;
342 	u32 chcr = desc->chcr;
343 
344 	WARN_ON_ONCE(rcar_dmac_chan_is_busy(chan));
345 
346 	if (chan->mid_rid >= 0)
347 		rcar_dmac_chan_write(chan, RCAR_DMARS, chan->mid_rid);
348 
349 	if (desc->hwdescs.use) {
350 		struct rcar_dmac_xfer_chunk *chunk =
351 			list_first_entry(&desc->chunks,
352 					 struct rcar_dmac_xfer_chunk, node);
353 
354 		dev_dbg(chan->chan.device->dev,
355 			"chan%u: queue desc %p: %u@%pad\n",
356 			chan->index, desc, desc->nchunks, &desc->hwdescs.dma);
357 
358 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
359 		rcar_dmac_chan_write(chan, RCAR_DMAFIXSAR,
360 				     chunk->src_addr >> 32);
361 		rcar_dmac_chan_write(chan, RCAR_DMAFIXDAR,
362 				     chunk->dst_addr >> 32);
363 		rcar_dmac_chan_write(chan, RCAR_DMAFIXDPBASE,
364 				     desc->hwdescs.dma >> 32);
365 #endif
366 		rcar_dmac_chan_write(chan, RCAR_DMADPBASE,
367 				     (desc->hwdescs.dma & 0xfffffff0) |
368 				     RCAR_DMADPBASE_SEL);
369 		rcar_dmac_chan_write(chan, RCAR_DMACHCRB,
370 				     RCAR_DMACHCRB_DCNT(desc->nchunks - 1) |
371 				     RCAR_DMACHCRB_DRST);
372 
373 		/*
374 		 * Errata: When descriptor memory is accessed through an IOMMU
375 		 * the DMADAR register isn't initialized automatically from the
376 		 * first descriptor at beginning of transfer by the DMAC like it
377 		 * should. Initialize it manually with the destination address
378 		 * of the first chunk.
379 		 */
380 		rcar_dmac_chan_write(chan, RCAR_DMADAR,
381 				     chunk->dst_addr & 0xffffffff);
382 
383 		/*
384 		 * Program the descriptor stage interrupt to occur after the end
385 		 * of the first stage.
386 		 */
387 		rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(1));
388 
389 		chcr |= RCAR_DMACHCR_RPT_SAR | RCAR_DMACHCR_RPT_DAR
390 		     |  RCAR_DMACHCR_RPT_TCR | RCAR_DMACHCR_DPB;
391 
392 		/*
393 		 * If the descriptor isn't cyclic enable normal descriptor mode
394 		 * and the transfer completion interrupt.
395 		 */
396 		if (!desc->cyclic)
397 			chcr |= RCAR_DMACHCR_DPM_ENABLED | RCAR_DMACHCR_IE;
398 		/*
399 		 * If the descriptor is cyclic and has a callback enable the
400 		 * descriptor stage interrupt in infinite repeat mode.
401 		 */
402 		else if (desc->async_tx.callback)
403 			chcr |= RCAR_DMACHCR_DPM_INFINITE | RCAR_DMACHCR_DSIE;
404 		/*
405 		 * Otherwise just select infinite repeat mode without any
406 		 * interrupt.
407 		 */
408 		else
409 			chcr |= RCAR_DMACHCR_DPM_INFINITE;
410 	} else {
411 		struct rcar_dmac_xfer_chunk *chunk = desc->running;
412 
413 		dev_dbg(chan->chan.device->dev,
414 			"chan%u: queue chunk %p: %u@%pad -> %pad\n",
415 			chan->index, chunk, chunk->size, &chunk->src_addr,
416 			&chunk->dst_addr);
417 
418 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
419 		rcar_dmac_chan_write(chan, RCAR_DMAFIXSAR,
420 				     chunk->src_addr >> 32);
421 		rcar_dmac_chan_write(chan, RCAR_DMAFIXDAR,
422 				     chunk->dst_addr >> 32);
423 #endif
424 		rcar_dmac_chan_write(chan, RCAR_DMASAR,
425 				     chunk->src_addr & 0xffffffff);
426 		rcar_dmac_chan_write(chan, RCAR_DMADAR,
427 				     chunk->dst_addr & 0xffffffff);
428 		rcar_dmac_chan_write(chan, RCAR_DMATCR,
429 				     chunk->size >> desc->xfer_shift);
430 
431 		chcr |= RCAR_DMACHCR_DPM_DISABLED | RCAR_DMACHCR_IE;
432 	}
433 
434 	rcar_dmac_chan_write(chan, RCAR_DMACHCR,
435 			     chcr | RCAR_DMACHCR_DE | RCAR_DMACHCR_CAIE);
436 }
437 
438 static int rcar_dmac_init(struct rcar_dmac *dmac)
439 {
440 	u16 dmaor;
441 
442 	/* Clear all channels and enable the DMAC globally. */
443 	rcar_dmac_write(dmac, RCAR_DMACHCLR, dmac->channels_mask);
444 	rcar_dmac_write(dmac, RCAR_DMAOR,
445 			RCAR_DMAOR_PRI_FIXED | RCAR_DMAOR_DME);
446 
447 	dmaor = rcar_dmac_read(dmac, RCAR_DMAOR);
448 	if ((dmaor & (RCAR_DMAOR_AE | RCAR_DMAOR_DME)) != RCAR_DMAOR_DME) {
449 		dev_warn(dmac->dev, "DMAOR initialization failed.\n");
450 		return -EIO;
451 	}
452 
453 	return 0;
454 }
455 
456 /* -----------------------------------------------------------------------------
457  * Descriptors submission
458  */
459 
460 static dma_cookie_t rcar_dmac_tx_submit(struct dma_async_tx_descriptor *tx)
461 {
462 	struct rcar_dmac_chan *chan = to_rcar_dmac_chan(tx->chan);
463 	struct rcar_dmac_desc *desc = to_rcar_dmac_desc(tx);
464 	unsigned long flags;
465 	dma_cookie_t cookie;
466 
467 	spin_lock_irqsave(&chan->lock, flags);
468 
469 	cookie = dma_cookie_assign(tx);
470 
471 	dev_dbg(chan->chan.device->dev, "chan%u: submit #%d@%p\n",
472 		chan->index, tx->cookie, desc);
473 
474 	list_add_tail(&desc->node, &chan->desc.pending);
475 	desc->running = list_first_entry(&desc->chunks,
476 					 struct rcar_dmac_xfer_chunk, node);
477 
478 	spin_unlock_irqrestore(&chan->lock, flags);
479 
480 	return cookie;
481 }
482 
483 /* -----------------------------------------------------------------------------
484  * Descriptors allocation and free
485  */
486 
487 /*
488  * rcar_dmac_desc_alloc - Allocate a page worth of DMA descriptors
489  * @chan: the DMA channel
490  * @gfp: allocation flags
491  */
492 static int rcar_dmac_desc_alloc(struct rcar_dmac_chan *chan, gfp_t gfp)
493 {
494 	struct rcar_dmac_desc_page *page;
495 	unsigned long flags;
496 	LIST_HEAD(list);
497 	unsigned int i;
498 
499 	page = (void *)get_zeroed_page(gfp);
500 	if (!page)
501 		return -ENOMEM;
502 
503 	for (i = 0; i < RCAR_DMAC_DESCS_PER_PAGE; ++i) {
504 		struct rcar_dmac_desc *desc = &page->descs[i];
505 
506 		dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan);
507 		desc->async_tx.tx_submit = rcar_dmac_tx_submit;
508 		INIT_LIST_HEAD(&desc->chunks);
509 
510 		list_add_tail(&desc->node, &list);
511 	}
512 
513 	spin_lock_irqsave(&chan->lock, flags);
514 	list_splice_tail(&list, &chan->desc.free);
515 	list_add_tail(&page->node, &chan->desc.pages);
516 	spin_unlock_irqrestore(&chan->lock, flags);
517 
518 	return 0;
519 }
520 
521 /*
522  * rcar_dmac_desc_put - Release a DMA transfer descriptor
523  * @chan: the DMA channel
524  * @desc: the descriptor
525  *
526  * Put the descriptor and its transfer chunk descriptors back in the channel's
527  * free descriptors lists. The descriptor's chunks list will be reinitialized to
528  * an empty list as a result.
529  *
530  * The descriptor must have been removed from the channel's lists before calling
531  * this function.
532  */
533 static void rcar_dmac_desc_put(struct rcar_dmac_chan *chan,
534 			       struct rcar_dmac_desc *desc)
535 {
536 	unsigned long flags;
537 
538 	spin_lock_irqsave(&chan->lock, flags);
539 	list_splice_tail_init(&desc->chunks, &chan->desc.chunks_free);
540 	list_add(&desc->node, &chan->desc.free);
541 	spin_unlock_irqrestore(&chan->lock, flags);
542 }
543 
544 static void rcar_dmac_desc_recycle_acked(struct rcar_dmac_chan *chan)
545 {
546 	struct rcar_dmac_desc *desc, *_desc;
547 	unsigned long flags;
548 	LIST_HEAD(list);
549 
550 	/*
551 	 * We have to temporarily move all descriptors from the wait list to a
552 	 * local list as iterating over the wait list, even with
553 	 * list_for_each_entry_safe, isn't safe if we release the channel lock
554 	 * around the rcar_dmac_desc_put() call.
555 	 */
556 	spin_lock_irqsave(&chan->lock, flags);
557 	list_splice_init(&chan->desc.wait, &list);
558 	spin_unlock_irqrestore(&chan->lock, flags);
559 
560 	list_for_each_entry_safe(desc, _desc, &list, node) {
561 		if (async_tx_test_ack(&desc->async_tx)) {
562 			list_del(&desc->node);
563 			rcar_dmac_desc_put(chan, desc);
564 		}
565 	}
566 
567 	if (list_empty(&list))
568 		return;
569 
570 	/* Put the remaining descriptors back in the wait list. */
571 	spin_lock_irqsave(&chan->lock, flags);
572 	list_splice(&list, &chan->desc.wait);
573 	spin_unlock_irqrestore(&chan->lock, flags);
574 }
575 
576 /*
577  * rcar_dmac_desc_get - Allocate a descriptor for a DMA transfer
578  * @chan: the DMA channel
579  *
580  * Locking: This function must be called in a non-atomic context.
581  *
582  * Return: A pointer to the allocated descriptor or NULL if no descriptor can
583  * be allocated.
584  */
585 static struct rcar_dmac_desc *rcar_dmac_desc_get(struct rcar_dmac_chan *chan)
586 {
587 	struct rcar_dmac_desc *desc;
588 	unsigned long flags;
589 	int ret;
590 
591 	/* Recycle acked descriptors before attempting allocation. */
592 	rcar_dmac_desc_recycle_acked(chan);
593 
594 	spin_lock_irqsave(&chan->lock, flags);
595 
596 	while (list_empty(&chan->desc.free)) {
597 		/*
598 		 * No free descriptors, allocate a page worth of them and try
599 		 * again, as someone else could race us to get the newly
600 		 * allocated descriptors. If the allocation fails return an
601 		 * error.
602 		 */
603 		spin_unlock_irqrestore(&chan->lock, flags);
604 		ret = rcar_dmac_desc_alloc(chan, GFP_NOWAIT);
605 		if (ret < 0)
606 			return NULL;
607 		spin_lock_irqsave(&chan->lock, flags);
608 	}
609 
610 	desc = list_first_entry(&chan->desc.free, struct rcar_dmac_desc, node);
611 	list_del(&desc->node);
612 
613 	spin_unlock_irqrestore(&chan->lock, flags);
614 
615 	return desc;
616 }
617 
618 /*
619  * rcar_dmac_xfer_chunk_alloc - Allocate a page worth of transfer chunks
620  * @chan: the DMA channel
621  * @gfp: allocation flags
622  */
623 static int rcar_dmac_xfer_chunk_alloc(struct rcar_dmac_chan *chan, gfp_t gfp)
624 {
625 	struct rcar_dmac_desc_page *page;
626 	unsigned long flags;
627 	LIST_HEAD(list);
628 	unsigned int i;
629 
630 	page = (void *)get_zeroed_page(gfp);
631 	if (!page)
632 		return -ENOMEM;
633 
634 	for (i = 0; i < RCAR_DMAC_XFER_CHUNKS_PER_PAGE; ++i) {
635 		struct rcar_dmac_xfer_chunk *chunk = &page->chunks[i];
636 
637 		list_add_tail(&chunk->node, &list);
638 	}
639 
640 	spin_lock_irqsave(&chan->lock, flags);
641 	list_splice_tail(&list, &chan->desc.chunks_free);
642 	list_add_tail(&page->node, &chan->desc.pages);
643 	spin_unlock_irqrestore(&chan->lock, flags);
644 
645 	return 0;
646 }
647 
648 /*
649  * rcar_dmac_xfer_chunk_get - Allocate a transfer chunk for a DMA transfer
650  * @chan: the DMA channel
651  *
652  * Locking: This function must be called in a non-atomic context.
653  *
654  * Return: A pointer to the allocated transfer chunk descriptor or NULL if no
655  * descriptor can be allocated.
656  */
657 static struct rcar_dmac_xfer_chunk *
658 rcar_dmac_xfer_chunk_get(struct rcar_dmac_chan *chan)
659 {
660 	struct rcar_dmac_xfer_chunk *chunk;
661 	unsigned long flags;
662 	int ret;
663 
664 	spin_lock_irqsave(&chan->lock, flags);
665 
666 	while (list_empty(&chan->desc.chunks_free)) {
667 		/*
668 		 * No free descriptors, allocate a page worth of them and try
669 		 * again, as someone else could race us to get the newly
670 		 * allocated descriptors. If the allocation fails return an
671 		 * error.
672 		 */
673 		spin_unlock_irqrestore(&chan->lock, flags);
674 		ret = rcar_dmac_xfer_chunk_alloc(chan, GFP_NOWAIT);
675 		if (ret < 0)
676 			return NULL;
677 		spin_lock_irqsave(&chan->lock, flags);
678 	}
679 
680 	chunk = list_first_entry(&chan->desc.chunks_free,
681 				 struct rcar_dmac_xfer_chunk, node);
682 	list_del(&chunk->node);
683 
684 	spin_unlock_irqrestore(&chan->lock, flags);
685 
686 	return chunk;
687 }
688 
689 static void rcar_dmac_realloc_hwdesc(struct rcar_dmac_chan *chan,
690 				     struct rcar_dmac_desc *desc, size_t size)
691 {
692 	/*
693 	 * dma_alloc_coherent() allocates memory in page size increments. To
694 	 * avoid reallocating the hardware descriptors when the allocated size
695 	 * wouldn't change align the requested size to a multiple of the page
696 	 * size.
697 	 */
698 	size = PAGE_ALIGN(size);
699 
700 	if (desc->hwdescs.size == size)
701 		return;
702 
703 	if (desc->hwdescs.mem) {
704 		dma_free_coherent(chan->chan.device->dev, desc->hwdescs.size,
705 				  desc->hwdescs.mem, desc->hwdescs.dma);
706 		desc->hwdescs.mem = NULL;
707 		desc->hwdescs.size = 0;
708 	}
709 
710 	if (!size)
711 		return;
712 
713 	desc->hwdescs.mem = dma_alloc_coherent(chan->chan.device->dev, size,
714 					       &desc->hwdescs.dma, GFP_NOWAIT);
715 	if (!desc->hwdescs.mem)
716 		return;
717 
718 	desc->hwdescs.size = size;
719 }
720 
721 static int rcar_dmac_fill_hwdesc(struct rcar_dmac_chan *chan,
722 				 struct rcar_dmac_desc *desc)
723 {
724 	struct rcar_dmac_xfer_chunk *chunk;
725 	struct rcar_dmac_hw_desc *hwdesc;
726 
727 	rcar_dmac_realloc_hwdesc(chan, desc, desc->nchunks * sizeof(*hwdesc));
728 
729 	hwdesc = desc->hwdescs.mem;
730 	if (!hwdesc)
731 		return -ENOMEM;
732 
733 	list_for_each_entry(chunk, &desc->chunks, node) {
734 		hwdesc->sar = chunk->src_addr;
735 		hwdesc->dar = chunk->dst_addr;
736 		hwdesc->tcr = chunk->size >> desc->xfer_shift;
737 		hwdesc++;
738 	}
739 
740 	return 0;
741 }
742 
743 /* -----------------------------------------------------------------------------
744  * Stop and reset
745  */
746 static void rcar_dmac_chcr_de_barrier(struct rcar_dmac_chan *chan)
747 {
748 	u32 chcr;
749 	unsigned int i;
750 
751 	/*
752 	 * Ensure that the setting of the DE bit is actually 0 after
753 	 * clearing it.
754 	 */
755 	for (i = 0; i < 1024; i++) {
756 		chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
757 		if (!(chcr & RCAR_DMACHCR_DE))
758 			return;
759 		udelay(1);
760 	}
761 
762 	dev_err(chan->chan.device->dev, "CHCR DE check error\n");
763 }
764 
765 static void rcar_dmac_clear_chcr_de(struct rcar_dmac_chan *chan)
766 {
767 	u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
768 
769 	/* set DE=0 and flush remaining data */
770 	rcar_dmac_chan_write(chan, RCAR_DMACHCR, (chcr & ~RCAR_DMACHCR_DE));
771 
772 	/* make sure all remaining data was flushed */
773 	rcar_dmac_chcr_de_barrier(chan);
774 }
775 
776 static void rcar_dmac_chan_halt(struct rcar_dmac_chan *chan)
777 {
778 	u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
779 
780 	chcr &= ~(RCAR_DMACHCR_DSE | RCAR_DMACHCR_DSIE | RCAR_DMACHCR_IE |
781 		  RCAR_DMACHCR_TE | RCAR_DMACHCR_DE |
782 		  RCAR_DMACHCR_CAE | RCAR_DMACHCR_CAIE);
783 	rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr);
784 	rcar_dmac_chcr_de_barrier(chan);
785 }
786 
787 static void rcar_dmac_chan_reinit(struct rcar_dmac_chan *chan)
788 {
789 	struct rcar_dmac_desc *desc, *_desc;
790 	unsigned long flags;
791 	LIST_HEAD(descs);
792 
793 	spin_lock_irqsave(&chan->lock, flags);
794 
795 	/* Move all non-free descriptors to the local lists. */
796 	list_splice_init(&chan->desc.pending, &descs);
797 	list_splice_init(&chan->desc.active, &descs);
798 	list_splice_init(&chan->desc.done, &descs);
799 	list_splice_init(&chan->desc.wait, &descs);
800 
801 	chan->desc.running = NULL;
802 
803 	spin_unlock_irqrestore(&chan->lock, flags);
804 
805 	list_for_each_entry_safe(desc, _desc, &descs, node) {
806 		list_del(&desc->node);
807 		rcar_dmac_desc_put(chan, desc);
808 	}
809 }
810 
811 static void rcar_dmac_stop_all_chan(struct rcar_dmac *dmac)
812 {
813 	unsigned int i;
814 
815 	/* Stop all channels. */
816 	for (i = 0; i < dmac->n_channels; ++i) {
817 		struct rcar_dmac_chan *chan = &dmac->channels[i];
818 
819 		if (!(dmac->channels_mask & BIT(i)))
820 			continue;
821 
822 		/* Stop and reinitialize the channel. */
823 		spin_lock_irq(&chan->lock);
824 		rcar_dmac_chan_halt(chan);
825 		spin_unlock_irq(&chan->lock);
826 	}
827 }
828 
829 static int rcar_dmac_chan_pause(struct dma_chan *chan)
830 {
831 	unsigned long flags;
832 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
833 
834 	spin_lock_irqsave(&rchan->lock, flags);
835 	rcar_dmac_clear_chcr_de(rchan);
836 	spin_unlock_irqrestore(&rchan->lock, flags);
837 
838 	return 0;
839 }
840 
841 /* -----------------------------------------------------------------------------
842  * Descriptors preparation
843  */
844 
845 static void rcar_dmac_chan_configure_desc(struct rcar_dmac_chan *chan,
846 					  struct rcar_dmac_desc *desc)
847 {
848 	static const u32 chcr_ts[] = {
849 		RCAR_DMACHCR_TS_1B, RCAR_DMACHCR_TS_2B,
850 		RCAR_DMACHCR_TS_4B, RCAR_DMACHCR_TS_8B,
851 		RCAR_DMACHCR_TS_16B, RCAR_DMACHCR_TS_32B,
852 		RCAR_DMACHCR_TS_64B,
853 	};
854 
855 	unsigned int xfer_size;
856 	u32 chcr;
857 
858 	switch (desc->direction) {
859 	case DMA_DEV_TO_MEM:
860 		chcr = RCAR_DMACHCR_DM_INC | RCAR_DMACHCR_SM_FIXED
861 		     | RCAR_DMACHCR_RS_DMARS;
862 		xfer_size = chan->src.xfer_size;
863 		break;
864 
865 	case DMA_MEM_TO_DEV:
866 		chcr = RCAR_DMACHCR_DM_FIXED | RCAR_DMACHCR_SM_INC
867 		     | RCAR_DMACHCR_RS_DMARS;
868 		xfer_size = chan->dst.xfer_size;
869 		break;
870 
871 	case DMA_MEM_TO_MEM:
872 	default:
873 		chcr = RCAR_DMACHCR_DM_INC | RCAR_DMACHCR_SM_INC
874 		     | RCAR_DMACHCR_RS_AUTO;
875 		xfer_size = RCAR_DMAC_MEMCPY_XFER_SIZE;
876 		break;
877 	}
878 
879 	desc->xfer_shift = ilog2(xfer_size);
880 	desc->chcr = chcr | chcr_ts[desc->xfer_shift];
881 }
882 
883 /*
884  * rcar_dmac_chan_prep_sg - prepare transfer descriptors from an SG list
885  *
886  * Common routine for public (MEMCPY) and slave DMA. The MEMCPY case is also
887  * converted to scatter-gather to guarantee consistent locking and a correct
888  * list manipulation. For slave DMA direction carries the usual meaning, and,
889  * logically, the SG list is RAM and the addr variable contains slave address,
890  * e.g., the FIFO I/O register. For MEMCPY direction equals DMA_MEM_TO_MEM
891  * and the SG list contains only one element and points at the source buffer.
892  */
893 static struct dma_async_tx_descriptor *
894 rcar_dmac_chan_prep_sg(struct rcar_dmac_chan *chan, struct scatterlist *sgl,
895 		       unsigned int sg_len, dma_addr_t dev_addr,
896 		       enum dma_transfer_direction dir, unsigned long dma_flags,
897 		       bool cyclic)
898 {
899 	struct rcar_dmac_xfer_chunk *chunk;
900 	struct rcar_dmac_desc *desc;
901 	struct scatterlist *sg;
902 	unsigned int nchunks = 0;
903 	unsigned int max_chunk_size;
904 	unsigned int full_size = 0;
905 	bool cross_boundary = false;
906 	unsigned int i;
907 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
908 	u32 high_dev_addr;
909 	u32 high_mem_addr;
910 #endif
911 
912 	desc = rcar_dmac_desc_get(chan);
913 	if (!desc)
914 		return NULL;
915 
916 	desc->async_tx.flags = dma_flags;
917 	desc->async_tx.cookie = -EBUSY;
918 
919 	desc->cyclic = cyclic;
920 	desc->direction = dir;
921 
922 	rcar_dmac_chan_configure_desc(chan, desc);
923 
924 	max_chunk_size = RCAR_DMATCR_MASK << desc->xfer_shift;
925 
926 	/*
927 	 * Allocate and fill the transfer chunk descriptors. We own the only
928 	 * reference to the DMA descriptor, there's no need for locking.
929 	 */
930 	for_each_sg(sgl, sg, sg_len, i) {
931 		dma_addr_t mem_addr = sg_dma_address(sg);
932 		unsigned int len = sg_dma_len(sg);
933 
934 		full_size += len;
935 
936 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
937 		if (i == 0) {
938 			high_dev_addr = dev_addr >> 32;
939 			high_mem_addr = mem_addr >> 32;
940 		}
941 
942 		if ((dev_addr >> 32 != high_dev_addr) ||
943 		    (mem_addr >> 32 != high_mem_addr))
944 			cross_boundary = true;
945 #endif
946 		while (len) {
947 			unsigned int size = min(len, max_chunk_size);
948 
949 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
950 			/*
951 			 * Prevent individual transfers from crossing 4GB
952 			 * boundaries.
953 			 */
954 			if (dev_addr >> 32 != (dev_addr + size - 1) >> 32) {
955 				size = ALIGN(dev_addr, 1ULL << 32) - dev_addr;
956 				cross_boundary = true;
957 			}
958 			if (mem_addr >> 32 != (mem_addr + size - 1) >> 32) {
959 				size = ALIGN(mem_addr, 1ULL << 32) - mem_addr;
960 				cross_boundary = true;
961 			}
962 #endif
963 
964 			chunk = rcar_dmac_xfer_chunk_get(chan);
965 			if (!chunk) {
966 				rcar_dmac_desc_put(chan, desc);
967 				return NULL;
968 			}
969 
970 			if (dir == DMA_DEV_TO_MEM) {
971 				chunk->src_addr = dev_addr;
972 				chunk->dst_addr = mem_addr;
973 			} else {
974 				chunk->src_addr = mem_addr;
975 				chunk->dst_addr = dev_addr;
976 			}
977 
978 			chunk->size = size;
979 
980 			dev_dbg(chan->chan.device->dev,
981 				"chan%u: chunk %p/%p sgl %u@%p, %u/%u %pad -> %pad\n",
982 				chan->index, chunk, desc, i, sg, size, len,
983 				&chunk->src_addr, &chunk->dst_addr);
984 
985 			mem_addr += size;
986 			if (dir == DMA_MEM_TO_MEM)
987 				dev_addr += size;
988 
989 			len -= size;
990 
991 			list_add_tail(&chunk->node, &desc->chunks);
992 			nchunks++;
993 		}
994 	}
995 
996 	desc->nchunks = nchunks;
997 	desc->size = full_size;
998 
999 	/*
1000 	 * Use hardware descriptor lists if possible when more than one chunk
1001 	 * needs to be transferred (otherwise they don't make much sense).
1002 	 *
1003 	 * Source/Destination address should be located in same 4GiB region
1004 	 * in the 40bit address space when it uses Hardware descriptor,
1005 	 * and cross_boundary is checking it.
1006 	 */
1007 	desc->hwdescs.use = !cross_boundary && nchunks > 1;
1008 	if (desc->hwdescs.use) {
1009 		if (rcar_dmac_fill_hwdesc(chan, desc) < 0)
1010 			desc->hwdescs.use = false;
1011 	}
1012 
1013 	return &desc->async_tx;
1014 }
1015 
1016 /* -----------------------------------------------------------------------------
1017  * DMA engine operations
1018  */
1019 
1020 static int rcar_dmac_alloc_chan_resources(struct dma_chan *chan)
1021 {
1022 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1023 	int ret;
1024 
1025 	INIT_LIST_HEAD(&rchan->desc.chunks_free);
1026 	INIT_LIST_HEAD(&rchan->desc.pages);
1027 
1028 	/* Preallocate descriptors. */
1029 	ret = rcar_dmac_xfer_chunk_alloc(rchan, GFP_KERNEL);
1030 	if (ret < 0)
1031 		return -ENOMEM;
1032 
1033 	ret = rcar_dmac_desc_alloc(rchan, GFP_KERNEL);
1034 	if (ret < 0)
1035 		return -ENOMEM;
1036 
1037 	return pm_runtime_get_sync(chan->device->dev);
1038 }
1039 
1040 static void rcar_dmac_free_chan_resources(struct dma_chan *chan)
1041 {
1042 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1043 	struct rcar_dmac *dmac = to_rcar_dmac(chan->device);
1044 	struct rcar_dmac_chan_map *map = &rchan->map;
1045 	struct rcar_dmac_desc_page *page, *_page;
1046 	struct rcar_dmac_desc *desc;
1047 	LIST_HEAD(list);
1048 
1049 	/* Protect against ISR */
1050 	spin_lock_irq(&rchan->lock);
1051 	rcar_dmac_chan_halt(rchan);
1052 	spin_unlock_irq(&rchan->lock);
1053 
1054 	/*
1055 	 * Now no new interrupts will occur, but one might already be
1056 	 * running. Wait for it to finish before freeing resources.
1057 	 */
1058 	synchronize_irq(rchan->irq);
1059 
1060 	if (rchan->mid_rid >= 0) {
1061 		/* The caller is holding dma_list_mutex */
1062 		clear_bit(rchan->mid_rid, dmac->modules);
1063 		rchan->mid_rid = -EINVAL;
1064 	}
1065 
1066 	list_splice_init(&rchan->desc.free, &list);
1067 	list_splice_init(&rchan->desc.pending, &list);
1068 	list_splice_init(&rchan->desc.active, &list);
1069 	list_splice_init(&rchan->desc.done, &list);
1070 	list_splice_init(&rchan->desc.wait, &list);
1071 
1072 	rchan->desc.running = NULL;
1073 
1074 	list_for_each_entry(desc, &list, node)
1075 		rcar_dmac_realloc_hwdesc(rchan, desc, 0);
1076 
1077 	list_for_each_entry_safe(page, _page, &rchan->desc.pages, node) {
1078 		list_del(&page->node);
1079 		free_page((unsigned long)page);
1080 	}
1081 
1082 	/* Remove slave mapping if present. */
1083 	if (map->slave.xfer_size) {
1084 		dma_unmap_resource(chan->device->dev, map->addr,
1085 				   map->slave.xfer_size, map->dir, 0);
1086 		map->slave.xfer_size = 0;
1087 	}
1088 
1089 	pm_runtime_put(chan->device->dev);
1090 }
1091 
1092 static struct dma_async_tx_descriptor *
1093 rcar_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
1094 			  dma_addr_t dma_src, size_t len, unsigned long flags)
1095 {
1096 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1097 	struct scatterlist sgl;
1098 
1099 	if (!len)
1100 		return NULL;
1101 
1102 	sg_init_table(&sgl, 1);
1103 	sg_set_page(&sgl, pfn_to_page(PFN_DOWN(dma_src)), len,
1104 		    offset_in_page(dma_src));
1105 	sg_dma_address(&sgl) = dma_src;
1106 	sg_dma_len(&sgl) = len;
1107 
1108 	return rcar_dmac_chan_prep_sg(rchan, &sgl, 1, dma_dest,
1109 				      DMA_MEM_TO_MEM, flags, false);
1110 }
1111 
1112 static int rcar_dmac_map_slave_addr(struct dma_chan *chan,
1113 				    enum dma_transfer_direction dir)
1114 {
1115 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1116 	struct rcar_dmac_chan_map *map = &rchan->map;
1117 	phys_addr_t dev_addr;
1118 	size_t dev_size;
1119 	enum dma_data_direction dev_dir;
1120 
1121 	if (dir == DMA_DEV_TO_MEM) {
1122 		dev_addr = rchan->src.slave_addr;
1123 		dev_size = rchan->src.xfer_size;
1124 		dev_dir = DMA_TO_DEVICE;
1125 	} else {
1126 		dev_addr = rchan->dst.slave_addr;
1127 		dev_size = rchan->dst.xfer_size;
1128 		dev_dir = DMA_FROM_DEVICE;
1129 	}
1130 
1131 	/* Reuse current map if possible. */
1132 	if (dev_addr == map->slave.slave_addr &&
1133 	    dev_size == map->slave.xfer_size &&
1134 	    dev_dir == map->dir)
1135 		return 0;
1136 
1137 	/* Remove old mapping if present. */
1138 	if (map->slave.xfer_size)
1139 		dma_unmap_resource(chan->device->dev, map->addr,
1140 				   map->slave.xfer_size, map->dir, 0);
1141 	map->slave.xfer_size = 0;
1142 
1143 	/* Create new slave address map. */
1144 	map->addr = dma_map_resource(chan->device->dev, dev_addr, dev_size,
1145 				     dev_dir, 0);
1146 
1147 	if (dma_mapping_error(chan->device->dev, map->addr)) {
1148 		dev_err(chan->device->dev,
1149 			"chan%u: failed to map %zx@%pap", rchan->index,
1150 			dev_size, &dev_addr);
1151 		return -EIO;
1152 	}
1153 
1154 	dev_dbg(chan->device->dev, "chan%u: map %zx@%pap to %pad dir: %s\n",
1155 		rchan->index, dev_size, &dev_addr, &map->addr,
1156 		dev_dir == DMA_TO_DEVICE ? "DMA_TO_DEVICE" : "DMA_FROM_DEVICE");
1157 
1158 	map->slave.slave_addr = dev_addr;
1159 	map->slave.xfer_size = dev_size;
1160 	map->dir = dev_dir;
1161 
1162 	return 0;
1163 }
1164 
1165 static struct dma_async_tx_descriptor *
1166 rcar_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
1167 			unsigned int sg_len, enum dma_transfer_direction dir,
1168 			unsigned long flags, void *context)
1169 {
1170 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1171 
1172 	/* Someone calling slave DMA on a generic channel? */
1173 	if (rchan->mid_rid < 0 || !sg_len || !sg_dma_len(sgl)) {
1174 		dev_warn(chan->device->dev,
1175 			 "%s: bad parameter: len=%d, id=%d\n",
1176 			 __func__, sg_len, rchan->mid_rid);
1177 		return NULL;
1178 	}
1179 
1180 	if (rcar_dmac_map_slave_addr(chan, dir))
1181 		return NULL;
1182 
1183 	return rcar_dmac_chan_prep_sg(rchan, sgl, sg_len, rchan->map.addr,
1184 				      dir, flags, false);
1185 }
1186 
1187 #define RCAR_DMAC_MAX_SG_LEN	32
1188 
1189 static struct dma_async_tx_descriptor *
1190 rcar_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
1191 			  size_t buf_len, size_t period_len,
1192 			  enum dma_transfer_direction dir, unsigned long flags)
1193 {
1194 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1195 	struct dma_async_tx_descriptor *desc;
1196 	struct scatterlist *sgl;
1197 	unsigned int sg_len;
1198 	unsigned int i;
1199 
1200 	/* Someone calling slave DMA on a generic channel? */
1201 	if (rchan->mid_rid < 0 || buf_len < period_len) {
1202 		dev_warn(chan->device->dev,
1203 			"%s: bad parameter: buf_len=%zu, period_len=%zu, id=%d\n",
1204 			__func__, buf_len, period_len, rchan->mid_rid);
1205 		return NULL;
1206 	}
1207 
1208 	if (rcar_dmac_map_slave_addr(chan, dir))
1209 		return NULL;
1210 
1211 	sg_len = buf_len / period_len;
1212 	if (sg_len > RCAR_DMAC_MAX_SG_LEN) {
1213 		dev_err(chan->device->dev,
1214 			"chan%u: sg length %d exceds limit %d",
1215 			rchan->index, sg_len, RCAR_DMAC_MAX_SG_LEN);
1216 		return NULL;
1217 	}
1218 
1219 	/*
1220 	 * Allocate the sg list dynamically as it would consume too much stack
1221 	 * space.
1222 	 */
1223 	sgl = kcalloc(sg_len, sizeof(*sgl), GFP_NOWAIT);
1224 	if (!sgl)
1225 		return NULL;
1226 
1227 	sg_init_table(sgl, sg_len);
1228 
1229 	for (i = 0; i < sg_len; ++i) {
1230 		dma_addr_t src = buf_addr + (period_len * i);
1231 
1232 		sg_set_page(&sgl[i], pfn_to_page(PFN_DOWN(src)), period_len,
1233 			    offset_in_page(src));
1234 		sg_dma_address(&sgl[i]) = src;
1235 		sg_dma_len(&sgl[i]) = period_len;
1236 	}
1237 
1238 	desc = rcar_dmac_chan_prep_sg(rchan, sgl, sg_len, rchan->map.addr,
1239 				      dir, flags, true);
1240 
1241 	kfree(sgl);
1242 	return desc;
1243 }
1244 
1245 static int rcar_dmac_device_config(struct dma_chan *chan,
1246 				   struct dma_slave_config *cfg)
1247 {
1248 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1249 
1250 	/*
1251 	 * We could lock this, but you shouldn't be configuring the
1252 	 * channel, while using it...
1253 	 */
1254 	rchan->src.slave_addr = cfg->src_addr;
1255 	rchan->dst.slave_addr = cfg->dst_addr;
1256 	rchan->src.xfer_size = cfg->src_addr_width;
1257 	rchan->dst.xfer_size = cfg->dst_addr_width;
1258 
1259 	return 0;
1260 }
1261 
1262 static int rcar_dmac_chan_terminate_all(struct dma_chan *chan)
1263 {
1264 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1265 	unsigned long flags;
1266 
1267 	spin_lock_irqsave(&rchan->lock, flags);
1268 	rcar_dmac_chan_halt(rchan);
1269 	spin_unlock_irqrestore(&rchan->lock, flags);
1270 
1271 	/*
1272 	 * FIXME: No new interrupt can occur now, but the IRQ thread might still
1273 	 * be running.
1274 	 */
1275 
1276 	rcar_dmac_chan_reinit(rchan);
1277 
1278 	return 0;
1279 }
1280 
1281 static unsigned int rcar_dmac_chan_get_residue(struct rcar_dmac_chan *chan,
1282 					       dma_cookie_t cookie)
1283 {
1284 	struct rcar_dmac_desc *desc = chan->desc.running;
1285 	struct rcar_dmac_xfer_chunk *running = NULL;
1286 	struct rcar_dmac_xfer_chunk *chunk;
1287 	enum dma_status status;
1288 	unsigned int residue = 0;
1289 	unsigned int dptr = 0;
1290 	unsigned int chcrb;
1291 	unsigned int tcrb;
1292 	unsigned int i;
1293 
1294 	if (!desc)
1295 		return 0;
1296 
1297 	/*
1298 	 * If the cookie corresponds to a descriptor that has been completed
1299 	 * there is no residue. The same check has already been performed by the
1300 	 * caller but without holding the channel lock, so the descriptor could
1301 	 * now be complete.
1302 	 */
1303 	status = dma_cookie_status(&chan->chan, cookie, NULL);
1304 	if (status == DMA_COMPLETE)
1305 		return 0;
1306 
1307 	/*
1308 	 * If the cookie doesn't correspond to the currently running transfer
1309 	 * then the descriptor hasn't been processed yet, and the residue is
1310 	 * equal to the full descriptor size.
1311 	 * Also, a client driver is possible to call this function before
1312 	 * rcar_dmac_isr_channel_thread() runs. In this case, the "desc.running"
1313 	 * will be the next descriptor, and the done list will appear. So, if
1314 	 * the argument cookie matches the done list's cookie, we can assume
1315 	 * the residue is zero.
1316 	 */
1317 	if (cookie != desc->async_tx.cookie) {
1318 		list_for_each_entry(desc, &chan->desc.done, node) {
1319 			if (cookie == desc->async_tx.cookie)
1320 				return 0;
1321 		}
1322 		list_for_each_entry(desc, &chan->desc.pending, node) {
1323 			if (cookie == desc->async_tx.cookie)
1324 				return desc->size;
1325 		}
1326 		list_for_each_entry(desc, &chan->desc.active, node) {
1327 			if (cookie == desc->async_tx.cookie)
1328 				return desc->size;
1329 		}
1330 
1331 		/*
1332 		 * No descriptor found for the cookie, there's thus no residue.
1333 		 * This shouldn't happen if the calling driver passes a correct
1334 		 * cookie value.
1335 		 */
1336 		WARN(1, "No descriptor for cookie!");
1337 		return 0;
1338 	}
1339 
1340 	/*
1341 	 * We need to read two registers.
1342 	 * Make sure the control register does not skip to next chunk
1343 	 * while reading the counter.
1344 	 * Trying it 3 times should be enough: Initial read, retry, retry
1345 	 * for the paranoid.
1346 	 */
1347 	for (i = 0; i < 3; i++) {
1348 		chcrb = rcar_dmac_chan_read(chan, RCAR_DMACHCRB) &
1349 					    RCAR_DMACHCRB_DPTR_MASK;
1350 		tcrb = rcar_dmac_chan_read(chan, RCAR_DMATCRB);
1351 		/* Still the same? */
1352 		if (chcrb == (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) &
1353 			      RCAR_DMACHCRB_DPTR_MASK))
1354 			break;
1355 	}
1356 	WARN_ONCE(i >= 3, "residue might be not continuous!");
1357 
1358 	/*
1359 	 * In descriptor mode the descriptor running pointer is not maintained
1360 	 * by the interrupt handler, find the running descriptor from the
1361 	 * descriptor pointer field in the CHCRB register. In non-descriptor
1362 	 * mode just use the running descriptor pointer.
1363 	 */
1364 	if (desc->hwdescs.use) {
1365 		dptr = chcrb >> RCAR_DMACHCRB_DPTR_SHIFT;
1366 		if (dptr == 0)
1367 			dptr = desc->nchunks;
1368 		dptr--;
1369 		WARN_ON(dptr >= desc->nchunks);
1370 	} else {
1371 		running = desc->running;
1372 	}
1373 
1374 	/* Compute the size of all chunks still to be transferred. */
1375 	list_for_each_entry_reverse(chunk, &desc->chunks, node) {
1376 		if (chunk == running || ++dptr == desc->nchunks)
1377 			break;
1378 
1379 		residue += chunk->size;
1380 	}
1381 
1382 	/* Add the residue for the current chunk. */
1383 	residue += tcrb << desc->xfer_shift;
1384 
1385 	return residue;
1386 }
1387 
1388 static enum dma_status rcar_dmac_tx_status(struct dma_chan *chan,
1389 					   dma_cookie_t cookie,
1390 					   struct dma_tx_state *txstate)
1391 {
1392 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1393 	enum dma_status status;
1394 	unsigned long flags;
1395 	unsigned int residue;
1396 	bool cyclic;
1397 
1398 	status = dma_cookie_status(chan, cookie, txstate);
1399 	if (status == DMA_COMPLETE || !txstate)
1400 		return status;
1401 
1402 	spin_lock_irqsave(&rchan->lock, flags);
1403 	residue = rcar_dmac_chan_get_residue(rchan, cookie);
1404 	cyclic = rchan->desc.running ? rchan->desc.running->cyclic : false;
1405 	spin_unlock_irqrestore(&rchan->lock, flags);
1406 
1407 	/* if there's no residue, the cookie is complete */
1408 	if (!residue && !cyclic)
1409 		return DMA_COMPLETE;
1410 
1411 	dma_set_residue(txstate, residue);
1412 
1413 	return status;
1414 }
1415 
1416 static void rcar_dmac_issue_pending(struct dma_chan *chan)
1417 {
1418 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1419 	unsigned long flags;
1420 
1421 	spin_lock_irqsave(&rchan->lock, flags);
1422 
1423 	if (list_empty(&rchan->desc.pending))
1424 		goto done;
1425 
1426 	/* Append the pending list to the active list. */
1427 	list_splice_tail_init(&rchan->desc.pending, &rchan->desc.active);
1428 
1429 	/*
1430 	 * If no transfer is running pick the first descriptor from the active
1431 	 * list and start the transfer.
1432 	 */
1433 	if (!rchan->desc.running) {
1434 		struct rcar_dmac_desc *desc;
1435 
1436 		desc = list_first_entry(&rchan->desc.active,
1437 					struct rcar_dmac_desc, node);
1438 		rchan->desc.running = desc;
1439 
1440 		rcar_dmac_chan_start_xfer(rchan);
1441 	}
1442 
1443 done:
1444 	spin_unlock_irqrestore(&rchan->lock, flags);
1445 }
1446 
1447 static void rcar_dmac_device_synchronize(struct dma_chan *chan)
1448 {
1449 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1450 
1451 	synchronize_irq(rchan->irq);
1452 }
1453 
1454 /* -----------------------------------------------------------------------------
1455  * IRQ handling
1456  */
1457 
1458 static irqreturn_t rcar_dmac_isr_desc_stage_end(struct rcar_dmac_chan *chan)
1459 {
1460 	struct rcar_dmac_desc *desc = chan->desc.running;
1461 	unsigned int stage;
1462 
1463 	if (WARN_ON(!desc || !desc->cyclic)) {
1464 		/*
1465 		 * This should never happen, there should always be a running
1466 		 * cyclic descriptor when a descriptor stage end interrupt is
1467 		 * triggered. Warn and return.
1468 		 */
1469 		return IRQ_NONE;
1470 	}
1471 
1472 	/* Program the interrupt pointer to the next stage. */
1473 	stage = (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) &
1474 		 RCAR_DMACHCRB_DPTR_MASK) >> RCAR_DMACHCRB_DPTR_SHIFT;
1475 	rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(stage));
1476 
1477 	return IRQ_WAKE_THREAD;
1478 }
1479 
1480 static irqreturn_t rcar_dmac_isr_transfer_end(struct rcar_dmac_chan *chan)
1481 {
1482 	struct rcar_dmac_desc *desc = chan->desc.running;
1483 	irqreturn_t ret = IRQ_WAKE_THREAD;
1484 
1485 	if (WARN_ON_ONCE(!desc)) {
1486 		/*
1487 		 * This should never happen, there should always be a running
1488 		 * descriptor when a transfer end interrupt is triggered. Warn
1489 		 * and return.
1490 		 */
1491 		return IRQ_NONE;
1492 	}
1493 
1494 	/*
1495 	 * The transfer end interrupt isn't generated for each chunk when using
1496 	 * descriptor mode. Only update the running chunk pointer in
1497 	 * non-descriptor mode.
1498 	 */
1499 	if (!desc->hwdescs.use) {
1500 		/*
1501 		 * If we haven't completed the last transfer chunk simply move
1502 		 * to the next one. Only wake the IRQ thread if the transfer is
1503 		 * cyclic.
1504 		 */
1505 		if (!list_is_last(&desc->running->node, &desc->chunks)) {
1506 			desc->running = list_next_entry(desc->running, node);
1507 			if (!desc->cyclic)
1508 				ret = IRQ_HANDLED;
1509 			goto done;
1510 		}
1511 
1512 		/*
1513 		 * We've completed the last transfer chunk. If the transfer is
1514 		 * cyclic, move back to the first one.
1515 		 */
1516 		if (desc->cyclic) {
1517 			desc->running =
1518 				list_first_entry(&desc->chunks,
1519 						 struct rcar_dmac_xfer_chunk,
1520 						 node);
1521 			goto done;
1522 		}
1523 	}
1524 
1525 	/* The descriptor is complete, move it to the done list. */
1526 	list_move_tail(&desc->node, &chan->desc.done);
1527 
1528 	/* Queue the next descriptor, if any. */
1529 	if (!list_empty(&chan->desc.active))
1530 		chan->desc.running = list_first_entry(&chan->desc.active,
1531 						      struct rcar_dmac_desc,
1532 						      node);
1533 	else
1534 		chan->desc.running = NULL;
1535 
1536 done:
1537 	if (chan->desc.running)
1538 		rcar_dmac_chan_start_xfer(chan);
1539 
1540 	return ret;
1541 }
1542 
1543 static irqreturn_t rcar_dmac_isr_channel(int irq, void *dev)
1544 {
1545 	u32 mask = RCAR_DMACHCR_DSE | RCAR_DMACHCR_TE;
1546 	struct rcar_dmac_chan *chan = dev;
1547 	irqreturn_t ret = IRQ_NONE;
1548 	bool reinit = false;
1549 	u32 chcr;
1550 
1551 	spin_lock(&chan->lock);
1552 
1553 	chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
1554 	if (chcr & RCAR_DMACHCR_CAE) {
1555 		struct rcar_dmac *dmac = to_rcar_dmac(chan->chan.device);
1556 
1557 		/*
1558 		 * We don't need to call rcar_dmac_chan_halt()
1559 		 * because channel is already stopped in error case.
1560 		 * We need to clear register and check DE bit as recovery.
1561 		 */
1562 		rcar_dmac_write(dmac, RCAR_DMACHCLR, 1 << chan->index);
1563 		rcar_dmac_chcr_de_barrier(chan);
1564 		reinit = true;
1565 		goto spin_lock_end;
1566 	}
1567 
1568 	if (chcr & RCAR_DMACHCR_TE)
1569 		mask |= RCAR_DMACHCR_DE;
1570 	rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr & ~mask);
1571 	if (mask & RCAR_DMACHCR_DE)
1572 		rcar_dmac_chcr_de_barrier(chan);
1573 
1574 	if (chcr & RCAR_DMACHCR_DSE)
1575 		ret |= rcar_dmac_isr_desc_stage_end(chan);
1576 
1577 	if (chcr & RCAR_DMACHCR_TE)
1578 		ret |= rcar_dmac_isr_transfer_end(chan);
1579 
1580 spin_lock_end:
1581 	spin_unlock(&chan->lock);
1582 
1583 	if (reinit) {
1584 		dev_err(chan->chan.device->dev, "Channel Address Error\n");
1585 
1586 		rcar_dmac_chan_reinit(chan);
1587 		ret = IRQ_HANDLED;
1588 	}
1589 
1590 	return ret;
1591 }
1592 
1593 static irqreturn_t rcar_dmac_isr_channel_thread(int irq, void *dev)
1594 {
1595 	struct rcar_dmac_chan *chan = dev;
1596 	struct rcar_dmac_desc *desc;
1597 	struct dmaengine_desc_callback cb;
1598 
1599 	spin_lock_irq(&chan->lock);
1600 
1601 	/* For cyclic transfers notify the user after every chunk. */
1602 	if (chan->desc.running && chan->desc.running->cyclic) {
1603 		desc = chan->desc.running;
1604 		dmaengine_desc_get_callback(&desc->async_tx, &cb);
1605 
1606 		if (dmaengine_desc_callback_valid(&cb)) {
1607 			spin_unlock_irq(&chan->lock);
1608 			dmaengine_desc_callback_invoke(&cb, NULL);
1609 			spin_lock_irq(&chan->lock);
1610 		}
1611 	}
1612 
1613 	/*
1614 	 * Call the callback function for all descriptors on the done list and
1615 	 * move them to the ack wait list.
1616 	 */
1617 	while (!list_empty(&chan->desc.done)) {
1618 		desc = list_first_entry(&chan->desc.done, struct rcar_dmac_desc,
1619 					node);
1620 		dma_cookie_complete(&desc->async_tx);
1621 		list_del(&desc->node);
1622 
1623 		dmaengine_desc_get_callback(&desc->async_tx, &cb);
1624 		if (dmaengine_desc_callback_valid(&cb)) {
1625 			spin_unlock_irq(&chan->lock);
1626 			/*
1627 			 * We own the only reference to this descriptor, we can
1628 			 * safely dereference it without holding the channel
1629 			 * lock.
1630 			 */
1631 			dmaengine_desc_callback_invoke(&cb, NULL);
1632 			spin_lock_irq(&chan->lock);
1633 		}
1634 
1635 		list_add_tail(&desc->node, &chan->desc.wait);
1636 	}
1637 
1638 	spin_unlock_irq(&chan->lock);
1639 
1640 	/* Recycle all acked descriptors. */
1641 	rcar_dmac_desc_recycle_acked(chan);
1642 
1643 	return IRQ_HANDLED;
1644 }
1645 
1646 /* -----------------------------------------------------------------------------
1647  * OF xlate and channel filter
1648  */
1649 
1650 static bool rcar_dmac_chan_filter(struct dma_chan *chan, void *arg)
1651 {
1652 	struct rcar_dmac *dmac = to_rcar_dmac(chan->device);
1653 	struct of_phandle_args *dma_spec = arg;
1654 
1655 	/*
1656 	 * FIXME: Using a filter on OF platforms is a nonsense. The OF xlate
1657 	 * function knows from which device it wants to allocate a channel from,
1658 	 * and would be perfectly capable of selecting the channel it wants.
1659 	 * Forcing it to call dma_request_channel() and iterate through all
1660 	 * channels from all controllers is just pointless.
1661 	 */
1662 	if (chan->device->device_config != rcar_dmac_device_config)
1663 		return false;
1664 
1665 	return !test_and_set_bit(dma_spec->args[0], dmac->modules);
1666 }
1667 
1668 static struct dma_chan *rcar_dmac_of_xlate(struct of_phandle_args *dma_spec,
1669 					   struct of_dma *ofdma)
1670 {
1671 	struct rcar_dmac_chan *rchan;
1672 	struct dma_chan *chan;
1673 	dma_cap_mask_t mask;
1674 
1675 	if (dma_spec->args_count != 1)
1676 		return NULL;
1677 
1678 	/* Only slave DMA channels can be allocated via DT */
1679 	dma_cap_zero(mask);
1680 	dma_cap_set(DMA_SLAVE, mask);
1681 
1682 	chan = __dma_request_channel(&mask, rcar_dmac_chan_filter, dma_spec,
1683 				     ofdma->of_node);
1684 	if (!chan)
1685 		return NULL;
1686 
1687 	rchan = to_rcar_dmac_chan(chan);
1688 	rchan->mid_rid = dma_spec->args[0];
1689 
1690 	return chan;
1691 }
1692 
1693 /* -----------------------------------------------------------------------------
1694  * Power management
1695  */
1696 
1697 #ifdef CONFIG_PM
1698 static int rcar_dmac_runtime_suspend(struct device *dev)
1699 {
1700 	return 0;
1701 }
1702 
1703 static int rcar_dmac_runtime_resume(struct device *dev)
1704 {
1705 	struct rcar_dmac *dmac = dev_get_drvdata(dev);
1706 
1707 	return rcar_dmac_init(dmac);
1708 }
1709 #endif
1710 
1711 static const struct dev_pm_ops rcar_dmac_pm = {
1712 	/*
1713 	 * TODO for system sleep/resume:
1714 	 *   - Wait for the current transfer to complete and stop the device,
1715 	 *   - Resume transfers, if any.
1716 	 */
1717 	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
1718 				      pm_runtime_force_resume)
1719 	SET_RUNTIME_PM_OPS(rcar_dmac_runtime_suspend, rcar_dmac_runtime_resume,
1720 			   NULL)
1721 };
1722 
1723 /* -----------------------------------------------------------------------------
1724  * Probe and remove
1725  */
1726 
1727 static int rcar_dmac_chan_probe(struct rcar_dmac *dmac,
1728 				struct rcar_dmac_chan *rchan,
1729 				unsigned int index)
1730 {
1731 	struct platform_device *pdev = to_platform_device(dmac->dev);
1732 	struct dma_chan *chan = &rchan->chan;
1733 	char pdev_irqname[5];
1734 	char *irqname;
1735 	int ret;
1736 
1737 	rchan->index = index;
1738 	rchan->iomem = dmac->iomem + RCAR_DMAC_CHAN_OFFSET(index);
1739 	rchan->mid_rid = -EINVAL;
1740 
1741 	spin_lock_init(&rchan->lock);
1742 
1743 	INIT_LIST_HEAD(&rchan->desc.free);
1744 	INIT_LIST_HEAD(&rchan->desc.pending);
1745 	INIT_LIST_HEAD(&rchan->desc.active);
1746 	INIT_LIST_HEAD(&rchan->desc.done);
1747 	INIT_LIST_HEAD(&rchan->desc.wait);
1748 
1749 	/* Request the channel interrupt. */
1750 	sprintf(pdev_irqname, "ch%u", index);
1751 	rchan->irq = platform_get_irq_byname(pdev, pdev_irqname);
1752 	if (rchan->irq < 0)
1753 		return -ENODEV;
1754 
1755 	irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u",
1756 				 dev_name(dmac->dev), index);
1757 	if (!irqname)
1758 		return -ENOMEM;
1759 
1760 	/*
1761 	 * Initialize the DMA engine channel and add it to the DMA engine
1762 	 * channels list.
1763 	 */
1764 	chan->device = &dmac->engine;
1765 	dma_cookie_init(chan);
1766 
1767 	list_add_tail(&chan->device_node, &dmac->engine.channels);
1768 
1769 	ret = devm_request_threaded_irq(dmac->dev, rchan->irq,
1770 					rcar_dmac_isr_channel,
1771 					rcar_dmac_isr_channel_thread, 0,
1772 					irqname, rchan);
1773 	if (ret) {
1774 		dev_err(dmac->dev, "failed to request IRQ %u (%d)\n",
1775 			rchan->irq, ret);
1776 		return ret;
1777 	}
1778 
1779 	return 0;
1780 }
1781 
1782 #define RCAR_DMAC_MAX_CHANNELS	32
1783 
1784 static int rcar_dmac_parse_of(struct device *dev, struct rcar_dmac *dmac)
1785 {
1786 	struct device_node *np = dev->of_node;
1787 	int ret;
1788 
1789 	ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels);
1790 	if (ret < 0) {
1791 		dev_err(dev, "unable to read dma-channels property\n");
1792 		return ret;
1793 	}
1794 
1795 	/* The hardware and driver don't support more than 32 bits in CHCLR */
1796 	if (dmac->n_channels <= 0 ||
1797 	    dmac->n_channels >= RCAR_DMAC_MAX_CHANNELS) {
1798 		dev_err(dev, "invalid number of channels %u\n",
1799 			dmac->n_channels);
1800 		return -EINVAL;
1801 	}
1802 
1803 	dmac->channels_mask = GENMASK(dmac->n_channels - 1, 0);
1804 
1805 	return 0;
1806 }
1807 
1808 static int rcar_dmac_probe(struct platform_device *pdev)
1809 {
1810 	const enum dma_slave_buswidth widths = DMA_SLAVE_BUSWIDTH_1_BYTE |
1811 		DMA_SLAVE_BUSWIDTH_2_BYTES | DMA_SLAVE_BUSWIDTH_4_BYTES |
1812 		DMA_SLAVE_BUSWIDTH_8_BYTES | DMA_SLAVE_BUSWIDTH_16_BYTES |
1813 		DMA_SLAVE_BUSWIDTH_32_BYTES | DMA_SLAVE_BUSWIDTH_64_BYTES;
1814 	struct dma_device *engine;
1815 	struct rcar_dmac *dmac;
1816 	struct resource *mem;
1817 	unsigned int i;
1818 	int ret;
1819 
1820 	dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL);
1821 	if (!dmac)
1822 		return -ENOMEM;
1823 
1824 	dmac->dev = &pdev->dev;
1825 	platform_set_drvdata(pdev, dmac);
1826 	dmac->dev->dma_parms = &dmac->parms;
1827 	dma_set_max_seg_size(dmac->dev, RCAR_DMATCR_MASK);
1828 	dma_set_mask_and_coherent(dmac->dev, DMA_BIT_MASK(40));
1829 
1830 	ret = rcar_dmac_parse_of(&pdev->dev, dmac);
1831 	if (ret < 0)
1832 		return ret;
1833 
1834 	/*
1835 	 * A still unconfirmed hardware bug prevents the IPMMU microTLB 0 to be
1836 	 * flushed correctly, resulting in memory corruption. DMAC 0 channel 0
1837 	 * is connected to microTLB 0 on currently supported platforms, so we
1838 	 * can't use it with the IPMMU. As the IOMMU API operates at the device
1839 	 * level we can't disable it selectively, so ignore channel 0 for now if
1840 	 * the device is part of an IOMMU group.
1841 	 */
1842 	if (device_iommu_mapped(&pdev->dev))
1843 		dmac->channels_mask &= ~BIT(0);
1844 
1845 	dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels,
1846 				      sizeof(*dmac->channels), GFP_KERNEL);
1847 	if (!dmac->channels)
1848 		return -ENOMEM;
1849 
1850 	/* Request resources. */
1851 	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1852 	dmac->iomem = devm_ioremap_resource(&pdev->dev, mem);
1853 	if (IS_ERR(dmac->iomem))
1854 		return PTR_ERR(dmac->iomem);
1855 
1856 	/* Enable runtime PM and initialize the device. */
1857 	pm_runtime_enable(&pdev->dev);
1858 	ret = pm_runtime_get_sync(&pdev->dev);
1859 	if (ret < 0) {
1860 		dev_err(&pdev->dev, "runtime PM get sync failed (%d)\n", ret);
1861 		return ret;
1862 	}
1863 
1864 	ret = rcar_dmac_init(dmac);
1865 	pm_runtime_put(&pdev->dev);
1866 
1867 	if (ret) {
1868 		dev_err(&pdev->dev, "failed to reset device\n");
1869 		goto error;
1870 	}
1871 
1872 	/* Initialize engine */
1873 	engine = &dmac->engine;
1874 
1875 	dma_cap_set(DMA_MEMCPY, engine->cap_mask);
1876 	dma_cap_set(DMA_SLAVE, engine->cap_mask);
1877 
1878 	engine->dev		= &pdev->dev;
1879 	engine->copy_align	= ilog2(RCAR_DMAC_MEMCPY_XFER_SIZE);
1880 
1881 	engine->src_addr_widths	= widths;
1882 	engine->dst_addr_widths	= widths;
1883 	engine->directions	= BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM);
1884 	engine->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1885 
1886 	engine->device_alloc_chan_resources	= rcar_dmac_alloc_chan_resources;
1887 	engine->device_free_chan_resources	= rcar_dmac_free_chan_resources;
1888 	engine->device_prep_dma_memcpy		= rcar_dmac_prep_dma_memcpy;
1889 	engine->device_prep_slave_sg		= rcar_dmac_prep_slave_sg;
1890 	engine->device_prep_dma_cyclic		= rcar_dmac_prep_dma_cyclic;
1891 	engine->device_config			= rcar_dmac_device_config;
1892 	engine->device_pause			= rcar_dmac_chan_pause;
1893 	engine->device_terminate_all		= rcar_dmac_chan_terminate_all;
1894 	engine->device_tx_status		= rcar_dmac_tx_status;
1895 	engine->device_issue_pending		= rcar_dmac_issue_pending;
1896 	engine->device_synchronize		= rcar_dmac_device_synchronize;
1897 
1898 	INIT_LIST_HEAD(&engine->channels);
1899 
1900 	for (i = 0; i < dmac->n_channels; ++i) {
1901 		if (!(dmac->channels_mask & BIT(i)))
1902 			continue;
1903 
1904 		ret = rcar_dmac_chan_probe(dmac, &dmac->channels[i], i);
1905 		if (ret < 0)
1906 			goto error;
1907 	}
1908 
1909 	/* Register the DMAC as a DMA provider for DT. */
1910 	ret = of_dma_controller_register(pdev->dev.of_node, rcar_dmac_of_xlate,
1911 					 NULL);
1912 	if (ret < 0)
1913 		goto error;
1914 
1915 	/*
1916 	 * Register the DMA engine device.
1917 	 *
1918 	 * Default transfer size of 32 bytes requires 32-byte alignment.
1919 	 */
1920 	ret = dma_async_device_register(engine);
1921 	if (ret < 0)
1922 		goto error;
1923 
1924 	return 0;
1925 
1926 error:
1927 	of_dma_controller_free(pdev->dev.of_node);
1928 	pm_runtime_disable(&pdev->dev);
1929 	return ret;
1930 }
1931 
1932 static int rcar_dmac_remove(struct platform_device *pdev)
1933 {
1934 	struct rcar_dmac *dmac = platform_get_drvdata(pdev);
1935 
1936 	of_dma_controller_free(pdev->dev.of_node);
1937 	dma_async_device_unregister(&dmac->engine);
1938 
1939 	pm_runtime_disable(&pdev->dev);
1940 
1941 	return 0;
1942 }
1943 
1944 static void rcar_dmac_shutdown(struct platform_device *pdev)
1945 {
1946 	struct rcar_dmac *dmac = platform_get_drvdata(pdev);
1947 
1948 	rcar_dmac_stop_all_chan(dmac);
1949 }
1950 
1951 static const struct of_device_id rcar_dmac_of_ids[] = {
1952 	{ .compatible = "renesas,rcar-dmac", },
1953 	{ /* Sentinel */ }
1954 };
1955 MODULE_DEVICE_TABLE(of, rcar_dmac_of_ids);
1956 
1957 static struct platform_driver rcar_dmac_driver = {
1958 	.driver		= {
1959 		.pm	= &rcar_dmac_pm,
1960 		.name	= "rcar-dmac",
1961 		.of_match_table = rcar_dmac_of_ids,
1962 	},
1963 	.probe		= rcar_dmac_probe,
1964 	.remove		= rcar_dmac_remove,
1965 	.shutdown	= rcar_dmac_shutdown,
1966 };
1967 
1968 module_platform_driver(rcar_dmac_driver);
1969 
1970 MODULE_DESCRIPTION("R-Car Gen2 DMA Controller Driver");
1971 MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>");
1972 MODULE_LICENSE("GPL v2");
1973