xref: /openbmc/linux/drivers/dma/sh/usb-dmac.c (revision 82003e04)
1 /*
2  * Renesas USB DMA Controller Driver
3  *
4  * Copyright (C) 2015 Renesas Electronics Corporation
5  *
6  * based on rcar-dmac.c
7  * Copyright (C) 2014 Renesas Electronics Inc.
8  * Author: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
9  *
10  * This is free software; you can redistribute it and/or modify
11  * it under the terms of version 2 of the GNU General Public License as
12  * published by the Free Software Foundation.
13  */
14 
15 #include <linux/delay.h>
16 #include <linux/dma-mapping.h>
17 #include <linux/dmaengine.h>
18 #include <linux/interrupt.h>
19 #include <linux/list.h>
20 #include <linux/module.h>
21 #include <linux/of.h>
22 #include <linux/of_dma.h>
23 #include <linux/of_platform.h>
24 #include <linux/platform_device.h>
25 #include <linux/pm_runtime.h>
26 #include <linux/slab.h>
27 #include <linux/spinlock.h>
28 
29 #include "../dmaengine.h"
30 #include "../virt-dma.h"
31 
32 /*
33  * struct usb_dmac_sg - Descriptor for a hardware transfer
34  * @mem_addr: memory address
35  * @size: transfer size in bytes
36  */
37 struct usb_dmac_sg {
38 	dma_addr_t mem_addr;
39 	u32 size;
40 };
41 
42 /*
43  * struct usb_dmac_desc - USB DMA Transfer Descriptor
44  * @vd: base virtual channel DMA transaction descriptor
45  * @direction: direction of the DMA transfer
46  * @sg_allocated_len: length of allocated sg
47  * @sg_len: length of sg
48  * @sg_index: index of sg
49  * @residue: residue after the DMAC completed a transfer
50  * @node: node for desc_got and desc_freed
51  * @done_cookie: cookie after the DMAC completed a transfer
52  * @sg: information for the transfer
53  */
54 struct usb_dmac_desc {
55 	struct virt_dma_desc vd;
56 	enum dma_transfer_direction direction;
57 	unsigned int sg_allocated_len;
58 	unsigned int sg_len;
59 	unsigned int sg_index;
60 	u32 residue;
61 	struct list_head node;
62 	dma_cookie_t done_cookie;
63 	struct usb_dmac_sg sg[0];
64 };
65 
66 #define to_usb_dmac_desc(vd)	container_of(vd, struct usb_dmac_desc, vd)
67 
68 /*
69  * struct usb_dmac_chan - USB DMA Controller Channel
70  * @vc: base virtual DMA channel object
71  * @iomem: channel I/O memory base
72  * @index: index of this channel in the controller
73  * @irq: irq number of this channel
74  * @desc: the current descriptor
75  * @descs_allocated: number of descriptors allocated
76  * @desc_got: got descriptors
77  * @desc_freed: freed descriptors after the DMAC completed a transfer
78  */
79 struct usb_dmac_chan {
80 	struct virt_dma_chan vc;
81 	void __iomem *iomem;
82 	unsigned int index;
83 	int irq;
84 	struct usb_dmac_desc *desc;
85 	int descs_allocated;
86 	struct list_head desc_got;
87 	struct list_head desc_freed;
88 };
89 
90 #define to_usb_dmac_chan(c) container_of(c, struct usb_dmac_chan, vc.chan)
91 
92 /*
93  * struct usb_dmac - USB DMA Controller
94  * @engine: base DMA engine object
95  * @dev: the hardware device
96  * @iomem: remapped I/O memory base
97  * @n_channels: number of available channels
98  * @channels: array of DMAC channels
99  */
100 struct usb_dmac {
101 	struct dma_device engine;
102 	struct device *dev;
103 	void __iomem *iomem;
104 
105 	unsigned int n_channels;
106 	struct usb_dmac_chan *channels;
107 };
108 
109 #define to_usb_dmac(d)		container_of(d, struct usb_dmac, engine)
110 
111 /* -----------------------------------------------------------------------------
112  * Registers
113  */
114 
115 #define USB_DMAC_CHAN_OFFSET(i)		(0x20 + 0x20 * (i))
116 
117 #define USB_DMASWR			0x0008
118 #define USB_DMASWR_SWR			(1 << 0)
119 #define USB_DMAOR			0x0060
120 #define USB_DMAOR_AE			(1 << 2)
121 #define USB_DMAOR_DME			(1 << 0)
122 
123 #define USB_DMASAR			0x0000
124 #define USB_DMADAR			0x0004
125 #define USB_DMATCR			0x0008
126 #define USB_DMATCR_MASK			0x00ffffff
127 #define USB_DMACHCR			0x0014
128 #define USB_DMACHCR_FTE			(1 << 24)
129 #define USB_DMACHCR_NULLE		(1 << 16)
130 #define USB_DMACHCR_NULL		(1 << 12)
131 #define USB_DMACHCR_TS_8B		((0 << 7) | (0 << 6))
132 #define USB_DMACHCR_TS_16B		((0 << 7) | (1 << 6))
133 #define USB_DMACHCR_TS_32B		((1 << 7) | (0 << 6))
134 #define USB_DMACHCR_IE			(1 << 5)
135 #define USB_DMACHCR_SP			(1 << 2)
136 #define USB_DMACHCR_TE			(1 << 1)
137 #define USB_DMACHCR_DE			(1 << 0)
138 #define USB_DMATEND			0x0018
139 
140 /* Hardcode the xfer_shift to 5 (32bytes) */
141 #define USB_DMAC_XFER_SHIFT	5
142 #define USB_DMAC_XFER_SIZE	(1 << USB_DMAC_XFER_SHIFT)
143 #define USB_DMAC_CHCR_TS	USB_DMACHCR_TS_32B
144 #define USB_DMAC_SLAVE_BUSWIDTH	DMA_SLAVE_BUSWIDTH_32_BYTES
145 
146 /* for descriptors */
147 #define USB_DMAC_INITIAL_NR_DESC	16
148 #define USB_DMAC_INITIAL_NR_SG		8
149 
150 /* -----------------------------------------------------------------------------
151  * Device access
152  */
153 
154 static void usb_dmac_write(struct usb_dmac *dmac, u32 reg, u32 data)
155 {
156 	writel(data, dmac->iomem + reg);
157 }
158 
159 static u32 usb_dmac_read(struct usb_dmac *dmac, u32 reg)
160 {
161 	return readl(dmac->iomem + reg);
162 }
163 
164 static u32 usb_dmac_chan_read(struct usb_dmac_chan *chan, u32 reg)
165 {
166 	return readl(chan->iomem + reg);
167 }
168 
169 static void usb_dmac_chan_write(struct usb_dmac_chan *chan, u32 reg, u32 data)
170 {
171 	writel(data, chan->iomem + reg);
172 }
173 
174 /* -----------------------------------------------------------------------------
175  * Initialization and configuration
176  */
177 
178 static bool usb_dmac_chan_is_busy(struct usb_dmac_chan *chan)
179 {
180 	u32 chcr = usb_dmac_chan_read(chan, USB_DMACHCR);
181 
182 	return (chcr & (USB_DMACHCR_DE | USB_DMACHCR_TE)) == USB_DMACHCR_DE;
183 }
184 
185 static u32 usb_dmac_calc_tend(u32 size)
186 {
187 	/*
188 	 * Please refer to the Figure "Example of Final Transaction Valid
189 	 * Data Transfer Enable (EDTEN) Setting" in the data sheet.
190 	 */
191 	return 0xffffffff << (32 - (size % USB_DMAC_XFER_SIZE ?	:
192 						USB_DMAC_XFER_SIZE));
193 }
194 
195 /* This function is already held by vc.lock */
196 static void usb_dmac_chan_start_sg(struct usb_dmac_chan *chan,
197 				   unsigned int index)
198 {
199 	struct usb_dmac_desc *desc = chan->desc;
200 	struct usb_dmac_sg *sg = desc->sg + index;
201 	dma_addr_t src_addr = 0, dst_addr = 0;
202 
203 	WARN_ON_ONCE(usb_dmac_chan_is_busy(chan));
204 
205 	if (desc->direction == DMA_DEV_TO_MEM)
206 		dst_addr = sg->mem_addr;
207 	else
208 		src_addr = sg->mem_addr;
209 
210 	dev_dbg(chan->vc.chan.device->dev,
211 		"chan%u: queue sg %p: %u@%pad -> %pad\n",
212 		chan->index, sg, sg->size, &src_addr, &dst_addr);
213 
214 	usb_dmac_chan_write(chan, USB_DMASAR, src_addr & 0xffffffff);
215 	usb_dmac_chan_write(chan, USB_DMADAR, dst_addr & 0xffffffff);
216 	usb_dmac_chan_write(chan, USB_DMATCR,
217 			    DIV_ROUND_UP(sg->size, USB_DMAC_XFER_SIZE));
218 	usb_dmac_chan_write(chan, USB_DMATEND, usb_dmac_calc_tend(sg->size));
219 
220 	usb_dmac_chan_write(chan, USB_DMACHCR, USB_DMAC_CHCR_TS |
221 			USB_DMACHCR_NULLE | USB_DMACHCR_IE | USB_DMACHCR_DE);
222 }
223 
224 /* This function is already held by vc.lock */
225 static void usb_dmac_chan_start_desc(struct usb_dmac_chan *chan)
226 {
227 	struct virt_dma_desc *vd;
228 
229 	vd = vchan_next_desc(&chan->vc);
230 	if (!vd) {
231 		chan->desc = NULL;
232 		return;
233 	}
234 
235 	/*
236 	 * Remove this request from vc->desc_issued. Otherwise, this driver
237 	 * will get the previous value from vchan_next_desc() after a transfer
238 	 * was completed.
239 	 */
240 	list_del(&vd->node);
241 
242 	chan->desc = to_usb_dmac_desc(vd);
243 	chan->desc->sg_index = 0;
244 	usb_dmac_chan_start_sg(chan, 0);
245 }
246 
247 static int usb_dmac_init(struct usb_dmac *dmac)
248 {
249 	u16 dmaor;
250 
251 	/* Clear all channels and enable the DMAC globally. */
252 	usb_dmac_write(dmac, USB_DMAOR, USB_DMAOR_DME);
253 
254 	dmaor = usb_dmac_read(dmac, USB_DMAOR);
255 	if ((dmaor & (USB_DMAOR_AE | USB_DMAOR_DME)) != USB_DMAOR_DME) {
256 		dev_warn(dmac->dev, "DMAOR initialization failed.\n");
257 		return -EIO;
258 	}
259 
260 	return 0;
261 }
262 
263 /* -----------------------------------------------------------------------------
264  * Descriptors allocation and free
265  */
266 static int usb_dmac_desc_alloc(struct usb_dmac_chan *chan, unsigned int sg_len,
267 			       gfp_t gfp)
268 {
269 	struct usb_dmac_desc *desc;
270 	unsigned long flags;
271 
272 	desc = kzalloc(sizeof(*desc) + sg_len * sizeof(desc->sg[0]), gfp);
273 	if (!desc)
274 		return -ENOMEM;
275 
276 	desc->sg_allocated_len = sg_len;
277 	INIT_LIST_HEAD(&desc->node);
278 
279 	spin_lock_irqsave(&chan->vc.lock, flags);
280 	list_add_tail(&desc->node, &chan->desc_freed);
281 	spin_unlock_irqrestore(&chan->vc.lock, flags);
282 
283 	return 0;
284 }
285 
286 static void usb_dmac_desc_free(struct usb_dmac_chan *chan)
287 {
288 	struct usb_dmac_desc *desc, *_desc;
289 	LIST_HEAD(list);
290 
291 	list_splice_init(&chan->desc_freed, &list);
292 	list_splice_init(&chan->desc_got, &list);
293 
294 	list_for_each_entry_safe(desc, _desc, &list, node) {
295 		list_del(&desc->node);
296 		kfree(desc);
297 	}
298 	chan->descs_allocated = 0;
299 }
300 
301 static struct usb_dmac_desc *usb_dmac_desc_get(struct usb_dmac_chan *chan,
302 					       unsigned int sg_len, gfp_t gfp)
303 {
304 	struct usb_dmac_desc *desc = NULL;
305 	unsigned long flags;
306 
307 	/* Get a freed descritpor */
308 	spin_lock_irqsave(&chan->vc.lock, flags);
309 	list_for_each_entry(desc, &chan->desc_freed, node) {
310 		if (sg_len <= desc->sg_allocated_len) {
311 			list_move_tail(&desc->node, &chan->desc_got);
312 			spin_unlock_irqrestore(&chan->vc.lock, flags);
313 			return desc;
314 		}
315 	}
316 	spin_unlock_irqrestore(&chan->vc.lock, flags);
317 
318 	/* Allocate a new descriptor */
319 	if (!usb_dmac_desc_alloc(chan, sg_len, gfp)) {
320 		/* If allocated the desc, it was added to tail of the list */
321 		spin_lock_irqsave(&chan->vc.lock, flags);
322 		desc = list_last_entry(&chan->desc_freed, struct usb_dmac_desc,
323 				       node);
324 		list_move_tail(&desc->node, &chan->desc_got);
325 		spin_unlock_irqrestore(&chan->vc.lock, flags);
326 		return desc;
327 	}
328 
329 	return NULL;
330 }
331 
332 static void usb_dmac_desc_put(struct usb_dmac_chan *chan,
333 			      struct usb_dmac_desc *desc)
334 {
335 	unsigned long flags;
336 
337 	spin_lock_irqsave(&chan->vc.lock, flags);
338 	list_move_tail(&desc->node, &chan->desc_freed);
339 	spin_unlock_irqrestore(&chan->vc.lock, flags);
340 }
341 
342 /* -----------------------------------------------------------------------------
343  * Stop and reset
344  */
345 
346 static void usb_dmac_soft_reset(struct usb_dmac_chan *uchan)
347 {
348 	struct dma_chan *chan = &uchan->vc.chan;
349 	struct usb_dmac *dmac = to_usb_dmac(chan->device);
350 	int i;
351 
352 	/* Don't issue soft reset if any one of channels is busy */
353 	for (i = 0; i < dmac->n_channels; ++i) {
354 		if (usb_dmac_chan_is_busy(uchan))
355 			return;
356 	}
357 
358 	usb_dmac_write(dmac, USB_DMAOR, 0);
359 	usb_dmac_write(dmac, USB_DMASWR, USB_DMASWR_SWR);
360 	udelay(100);
361 	usb_dmac_write(dmac, USB_DMASWR, 0);
362 	usb_dmac_write(dmac, USB_DMAOR, 1);
363 }
364 
365 static void usb_dmac_chan_halt(struct usb_dmac_chan *chan)
366 {
367 	u32 chcr = usb_dmac_chan_read(chan, USB_DMACHCR);
368 
369 	chcr &= ~(USB_DMACHCR_IE | USB_DMACHCR_TE | USB_DMACHCR_DE);
370 	usb_dmac_chan_write(chan, USB_DMACHCR, chcr);
371 
372 	usb_dmac_soft_reset(chan);
373 }
374 
375 static void usb_dmac_stop(struct usb_dmac *dmac)
376 {
377 	usb_dmac_write(dmac, USB_DMAOR, 0);
378 }
379 
380 /* -----------------------------------------------------------------------------
381  * DMA engine operations
382  */
383 
384 static int usb_dmac_alloc_chan_resources(struct dma_chan *chan)
385 {
386 	struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan);
387 	int ret;
388 
389 	while (uchan->descs_allocated < USB_DMAC_INITIAL_NR_DESC) {
390 		ret = usb_dmac_desc_alloc(uchan, USB_DMAC_INITIAL_NR_SG,
391 					  GFP_KERNEL);
392 		if (ret < 0) {
393 			usb_dmac_desc_free(uchan);
394 			return ret;
395 		}
396 		uchan->descs_allocated++;
397 	}
398 
399 	return pm_runtime_get_sync(chan->device->dev);
400 }
401 
402 static void usb_dmac_free_chan_resources(struct dma_chan *chan)
403 {
404 	struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan);
405 	unsigned long flags;
406 
407 	/* Protect against ISR */
408 	spin_lock_irqsave(&uchan->vc.lock, flags);
409 	usb_dmac_chan_halt(uchan);
410 	spin_unlock_irqrestore(&uchan->vc.lock, flags);
411 
412 	usb_dmac_desc_free(uchan);
413 	vchan_free_chan_resources(&uchan->vc);
414 
415 	pm_runtime_put(chan->device->dev);
416 }
417 
418 static struct dma_async_tx_descriptor *
419 usb_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
420 		       unsigned int sg_len, enum dma_transfer_direction dir,
421 		       unsigned long dma_flags, void *context)
422 {
423 	struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan);
424 	struct usb_dmac_desc *desc;
425 	struct scatterlist *sg;
426 	int i;
427 
428 	if (!sg_len) {
429 		dev_warn(chan->device->dev,
430 			 "%s: bad parameter: len=%d\n", __func__, sg_len);
431 		return NULL;
432 	}
433 
434 	desc = usb_dmac_desc_get(uchan, sg_len, GFP_NOWAIT);
435 	if (!desc)
436 		return NULL;
437 
438 	desc->direction = dir;
439 	desc->sg_len = sg_len;
440 	for_each_sg(sgl, sg, sg_len, i) {
441 		desc->sg[i].mem_addr = sg_dma_address(sg);
442 		desc->sg[i].size = sg_dma_len(sg);
443 	}
444 
445 	return vchan_tx_prep(&uchan->vc, &desc->vd, dma_flags);
446 }
447 
448 static int usb_dmac_chan_terminate_all(struct dma_chan *chan)
449 {
450 	struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan);
451 	struct usb_dmac_desc *desc, *_desc;
452 	unsigned long flags;
453 	LIST_HEAD(head);
454 	LIST_HEAD(list);
455 
456 	spin_lock_irqsave(&uchan->vc.lock, flags);
457 	usb_dmac_chan_halt(uchan);
458 	vchan_get_all_descriptors(&uchan->vc, &head);
459 	if (uchan->desc)
460 		uchan->desc = NULL;
461 	list_splice_init(&uchan->desc_got, &list);
462 	list_for_each_entry_safe(desc, _desc, &list, node)
463 		list_move_tail(&desc->node, &uchan->desc_freed);
464 	spin_unlock_irqrestore(&uchan->vc.lock, flags);
465 	vchan_dma_desc_free_list(&uchan->vc, &head);
466 
467 	return 0;
468 }
469 
470 static unsigned int usb_dmac_get_current_residue(struct usb_dmac_chan *chan,
471 						 struct usb_dmac_desc *desc,
472 						 int sg_index)
473 {
474 	struct usb_dmac_sg *sg = desc->sg + sg_index;
475 	u32 mem_addr = sg->mem_addr & 0xffffffff;
476 	unsigned int residue = sg->size;
477 
478 	/*
479 	 * We cannot use USB_DMATCR to calculate residue because USB_DMATCR
480 	 * has unsuited value to calculate.
481 	 */
482 	if (desc->direction == DMA_DEV_TO_MEM)
483 		residue -= usb_dmac_chan_read(chan, USB_DMADAR) - mem_addr;
484 	else
485 		residue -= usb_dmac_chan_read(chan, USB_DMASAR) - mem_addr;
486 
487 	return residue;
488 }
489 
490 static u32 usb_dmac_chan_get_residue_if_complete(struct usb_dmac_chan *chan,
491 						 dma_cookie_t cookie)
492 {
493 	struct usb_dmac_desc *desc;
494 	u32 residue = 0;
495 
496 	list_for_each_entry_reverse(desc, &chan->desc_freed, node) {
497 		if (desc->done_cookie == cookie) {
498 			residue = desc->residue;
499 			break;
500 		}
501 	}
502 
503 	return residue;
504 }
505 
506 static u32 usb_dmac_chan_get_residue(struct usb_dmac_chan *chan,
507 				     dma_cookie_t cookie)
508 {
509 	u32 residue = 0;
510 	struct virt_dma_desc *vd;
511 	struct usb_dmac_desc *desc = chan->desc;
512 	int i;
513 
514 	if (!desc) {
515 		vd = vchan_find_desc(&chan->vc, cookie);
516 		if (!vd)
517 			return 0;
518 		desc = to_usb_dmac_desc(vd);
519 	}
520 
521 	/* Compute the size of all usb_dmac_sg still to be transferred */
522 	for (i = desc->sg_index + 1; i < desc->sg_len; i++)
523 		residue += desc->sg[i].size;
524 
525 	/* Add the residue for the current sg */
526 	residue += usb_dmac_get_current_residue(chan, desc, desc->sg_index);
527 
528 	return residue;
529 }
530 
531 static enum dma_status usb_dmac_tx_status(struct dma_chan *chan,
532 					  dma_cookie_t cookie,
533 					  struct dma_tx_state *txstate)
534 {
535 	struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan);
536 	enum dma_status status;
537 	unsigned int residue = 0;
538 	unsigned long flags;
539 
540 	status = dma_cookie_status(chan, cookie, txstate);
541 	/* a client driver will get residue after DMA_COMPLETE */
542 	if (!txstate)
543 		return status;
544 
545 	spin_lock_irqsave(&uchan->vc.lock, flags);
546 	if (status == DMA_COMPLETE)
547 		residue = usb_dmac_chan_get_residue_if_complete(uchan, cookie);
548 	else
549 		residue = usb_dmac_chan_get_residue(uchan, cookie);
550 	spin_unlock_irqrestore(&uchan->vc.lock, flags);
551 
552 	dma_set_residue(txstate, residue);
553 
554 	return status;
555 }
556 
557 static void usb_dmac_issue_pending(struct dma_chan *chan)
558 {
559 	struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan);
560 	unsigned long flags;
561 
562 	spin_lock_irqsave(&uchan->vc.lock, flags);
563 	if (vchan_issue_pending(&uchan->vc) && !uchan->desc)
564 		usb_dmac_chan_start_desc(uchan);
565 	spin_unlock_irqrestore(&uchan->vc.lock, flags);
566 }
567 
568 static void usb_dmac_virt_desc_free(struct virt_dma_desc *vd)
569 {
570 	struct usb_dmac_desc *desc = to_usb_dmac_desc(vd);
571 	struct usb_dmac_chan *chan = to_usb_dmac_chan(vd->tx.chan);
572 
573 	usb_dmac_desc_put(chan, desc);
574 }
575 
576 /* -----------------------------------------------------------------------------
577  * IRQ handling
578  */
579 
580 static void usb_dmac_isr_transfer_end(struct usb_dmac_chan *chan)
581 {
582 	struct usb_dmac_desc *desc = chan->desc;
583 
584 	BUG_ON(!desc);
585 
586 	if (++desc->sg_index < desc->sg_len) {
587 		usb_dmac_chan_start_sg(chan, desc->sg_index);
588 	} else {
589 		desc->residue = usb_dmac_get_current_residue(chan, desc,
590 							desc->sg_index - 1);
591 		desc->done_cookie = desc->vd.tx.cookie;
592 		vchan_cookie_complete(&desc->vd);
593 
594 		/* Restart the next transfer if this driver has a next desc */
595 		usb_dmac_chan_start_desc(chan);
596 	}
597 }
598 
599 static irqreturn_t usb_dmac_isr_channel(int irq, void *dev)
600 {
601 	struct usb_dmac_chan *chan = dev;
602 	irqreturn_t ret = IRQ_NONE;
603 	u32 mask = 0;
604 	u32 chcr;
605 	bool xfer_end = false;
606 
607 	spin_lock(&chan->vc.lock);
608 
609 	chcr = usb_dmac_chan_read(chan, USB_DMACHCR);
610 	if (chcr & (USB_DMACHCR_TE | USB_DMACHCR_SP)) {
611 		mask |= USB_DMACHCR_DE | USB_DMACHCR_TE | USB_DMACHCR_SP;
612 		if (chcr & USB_DMACHCR_DE)
613 			xfer_end = true;
614 		ret |= IRQ_HANDLED;
615 	}
616 	if (chcr & USB_DMACHCR_NULL) {
617 		/* An interruption of TE will happen after we set FTE */
618 		mask |= USB_DMACHCR_NULL;
619 		chcr |= USB_DMACHCR_FTE;
620 		ret |= IRQ_HANDLED;
621 	}
622 	if (mask)
623 		usb_dmac_chan_write(chan, USB_DMACHCR, chcr & ~mask);
624 
625 	if (xfer_end)
626 		usb_dmac_isr_transfer_end(chan);
627 
628 	spin_unlock(&chan->vc.lock);
629 
630 	return ret;
631 }
632 
633 /* -----------------------------------------------------------------------------
634  * OF xlate and channel filter
635  */
636 
637 static bool usb_dmac_chan_filter(struct dma_chan *chan, void *arg)
638 {
639 	struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan);
640 	struct of_phandle_args *dma_spec = arg;
641 
642 	if (dma_spec->np != chan->device->dev->of_node)
643 		return false;
644 
645 	/* USB-DMAC should be used with fixed usb controller's FIFO */
646 	if (uchan->index != dma_spec->args[0])
647 		return false;
648 
649 	return true;
650 }
651 
652 static struct dma_chan *usb_dmac_of_xlate(struct of_phandle_args *dma_spec,
653 					  struct of_dma *ofdma)
654 {
655 	struct usb_dmac_chan *uchan;
656 	struct dma_chan *chan;
657 	dma_cap_mask_t mask;
658 
659 	if (dma_spec->args_count != 1)
660 		return NULL;
661 
662 	/* Only slave DMA channels can be allocated via DT */
663 	dma_cap_zero(mask);
664 	dma_cap_set(DMA_SLAVE, mask);
665 
666 	chan = dma_request_channel(mask, usb_dmac_chan_filter, dma_spec);
667 	if (!chan)
668 		return NULL;
669 
670 	uchan = to_usb_dmac_chan(chan);
671 
672 	return chan;
673 }
674 
675 /* -----------------------------------------------------------------------------
676  * Power management
677  */
678 
679 #ifdef CONFIG_PM
680 static int usb_dmac_runtime_suspend(struct device *dev)
681 {
682 	struct usb_dmac *dmac = dev_get_drvdata(dev);
683 	int i;
684 
685 	for (i = 0; i < dmac->n_channels; ++i) {
686 		if (!dmac->channels[i].iomem)
687 			break;
688 		usb_dmac_chan_halt(&dmac->channels[i]);
689 	}
690 
691 	return 0;
692 }
693 
694 static int usb_dmac_runtime_resume(struct device *dev)
695 {
696 	struct usb_dmac *dmac = dev_get_drvdata(dev);
697 
698 	return usb_dmac_init(dmac);
699 }
700 #endif /* CONFIG_PM */
701 
702 static const struct dev_pm_ops usb_dmac_pm = {
703 	SET_RUNTIME_PM_OPS(usb_dmac_runtime_suspend, usb_dmac_runtime_resume,
704 			   NULL)
705 };
706 
707 /* -----------------------------------------------------------------------------
708  * Probe and remove
709  */
710 
711 static int usb_dmac_chan_probe(struct usb_dmac *dmac,
712 			       struct usb_dmac_chan *uchan,
713 			       unsigned int index)
714 {
715 	struct platform_device *pdev = to_platform_device(dmac->dev);
716 	char pdev_irqname[5];
717 	char *irqname;
718 	int ret;
719 
720 	uchan->index = index;
721 	uchan->iomem = dmac->iomem + USB_DMAC_CHAN_OFFSET(index);
722 
723 	/* Request the channel interrupt. */
724 	sprintf(pdev_irqname, "ch%u", index);
725 	uchan->irq = platform_get_irq_byname(pdev, pdev_irqname);
726 	if (uchan->irq < 0) {
727 		dev_err(dmac->dev, "no IRQ specified for channel %u\n", index);
728 		return -ENODEV;
729 	}
730 
731 	irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u",
732 				 dev_name(dmac->dev), index);
733 	if (!irqname)
734 		return -ENOMEM;
735 
736 	ret = devm_request_irq(dmac->dev, uchan->irq, usb_dmac_isr_channel,
737 			       IRQF_SHARED, irqname, uchan);
738 	if (ret) {
739 		dev_err(dmac->dev, "failed to request IRQ %u (%d)\n",
740 			uchan->irq, ret);
741 		return ret;
742 	}
743 
744 	uchan->vc.desc_free = usb_dmac_virt_desc_free;
745 	vchan_init(&uchan->vc, &dmac->engine);
746 	INIT_LIST_HEAD(&uchan->desc_freed);
747 	INIT_LIST_HEAD(&uchan->desc_got);
748 
749 	return 0;
750 }
751 
752 static int usb_dmac_parse_of(struct device *dev, struct usb_dmac *dmac)
753 {
754 	struct device_node *np = dev->of_node;
755 	int ret;
756 
757 	ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels);
758 	if (ret < 0) {
759 		dev_err(dev, "unable to read dma-channels property\n");
760 		return ret;
761 	}
762 
763 	if (dmac->n_channels <= 0 || dmac->n_channels >= 100) {
764 		dev_err(dev, "invalid number of channels %u\n",
765 			dmac->n_channels);
766 		return -EINVAL;
767 	}
768 
769 	return 0;
770 }
771 
772 static int usb_dmac_probe(struct platform_device *pdev)
773 {
774 	const enum dma_slave_buswidth widths = USB_DMAC_SLAVE_BUSWIDTH;
775 	struct dma_device *engine;
776 	struct usb_dmac *dmac;
777 	struct resource *mem;
778 	unsigned int i;
779 	int ret;
780 
781 	dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL);
782 	if (!dmac)
783 		return -ENOMEM;
784 
785 	dmac->dev = &pdev->dev;
786 	platform_set_drvdata(pdev, dmac);
787 
788 	ret = usb_dmac_parse_of(&pdev->dev, dmac);
789 	if (ret < 0)
790 		return ret;
791 
792 	dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels,
793 				      sizeof(*dmac->channels), GFP_KERNEL);
794 	if (!dmac->channels)
795 		return -ENOMEM;
796 
797 	/* Request resources. */
798 	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
799 	dmac->iomem = devm_ioremap_resource(&pdev->dev, mem);
800 	if (IS_ERR(dmac->iomem))
801 		return PTR_ERR(dmac->iomem);
802 
803 	/* Enable runtime PM and initialize the device. */
804 	pm_runtime_enable(&pdev->dev);
805 	ret = pm_runtime_get_sync(&pdev->dev);
806 	if (ret < 0) {
807 		dev_err(&pdev->dev, "runtime PM get sync failed (%d)\n", ret);
808 		goto error_pm;
809 	}
810 
811 	ret = usb_dmac_init(dmac);
812 
813 	if (ret) {
814 		dev_err(&pdev->dev, "failed to reset device\n");
815 		goto error;
816 	}
817 
818 	/* Initialize the channels. */
819 	INIT_LIST_HEAD(&dmac->engine.channels);
820 
821 	for (i = 0; i < dmac->n_channels; ++i) {
822 		ret = usb_dmac_chan_probe(dmac, &dmac->channels[i], i);
823 		if (ret < 0)
824 			goto error;
825 	}
826 
827 	/* Register the DMAC as a DMA provider for DT. */
828 	ret = of_dma_controller_register(pdev->dev.of_node, usb_dmac_of_xlate,
829 					 NULL);
830 	if (ret < 0)
831 		goto error;
832 
833 	/*
834 	 * Register the DMA engine device.
835 	 *
836 	 * Default transfer size of 32 bytes requires 32-byte alignment.
837 	 */
838 	engine = &dmac->engine;
839 	dma_cap_set(DMA_SLAVE, engine->cap_mask);
840 
841 	engine->dev = &pdev->dev;
842 
843 	engine->src_addr_widths = widths;
844 	engine->dst_addr_widths = widths;
845 	engine->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM);
846 	engine->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
847 
848 	engine->device_alloc_chan_resources = usb_dmac_alloc_chan_resources;
849 	engine->device_free_chan_resources = usb_dmac_free_chan_resources;
850 	engine->device_prep_slave_sg = usb_dmac_prep_slave_sg;
851 	engine->device_terminate_all = usb_dmac_chan_terminate_all;
852 	engine->device_tx_status = usb_dmac_tx_status;
853 	engine->device_issue_pending = usb_dmac_issue_pending;
854 
855 	ret = dma_async_device_register(engine);
856 	if (ret < 0)
857 		goto error;
858 
859 	pm_runtime_put(&pdev->dev);
860 	return 0;
861 
862 error:
863 	of_dma_controller_free(pdev->dev.of_node);
864 	pm_runtime_put(&pdev->dev);
865 error_pm:
866 	pm_runtime_disable(&pdev->dev);
867 	return ret;
868 }
869 
870 static void usb_dmac_chan_remove(struct usb_dmac *dmac,
871 				 struct usb_dmac_chan *uchan)
872 {
873 	usb_dmac_chan_halt(uchan);
874 	devm_free_irq(dmac->dev, uchan->irq, uchan);
875 }
876 
877 static int usb_dmac_remove(struct platform_device *pdev)
878 {
879 	struct usb_dmac *dmac = platform_get_drvdata(pdev);
880 	int i;
881 
882 	for (i = 0; i < dmac->n_channels; ++i)
883 		usb_dmac_chan_remove(dmac, &dmac->channels[i]);
884 	of_dma_controller_free(pdev->dev.of_node);
885 	dma_async_device_unregister(&dmac->engine);
886 
887 	pm_runtime_disable(&pdev->dev);
888 
889 	return 0;
890 }
891 
892 static void usb_dmac_shutdown(struct platform_device *pdev)
893 {
894 	struct usb_dmac *dmac = platform_get_drvdata(pdev);
895 
896 	usb_dmac_stop(dmac);
897 }
898 
899 static const struct of_device_id usb_dmac_of_ids[] = {
900 	{ .compatible = "renesas,usb-dmac", },
901 	{ /* Sentinel */ }
902 };
903 MODULE_DEVICE_TABLE(of, usb_dmac_of_ids);
904 
905 static struct platform_driver usb_dmac_driver = {
906 	.driver		= {
907 		.pm	= &usb_dmac_pm,
908 		.name	= "usb-dmac",
909 		.of_match_table = usb_dmac_of_ids,
910 	},
911 	.probe		= usb_dmac_probe,
912 	.remove		= usb_dmac_remove,
913 	.shutdown	= usb_dmac_shutdown,
914 };
915 
916 module_platform_driver(usb_dmac_driver);
917 
918 MODULE_DESCRIPTION("Renesas USB DMA Controller Driver");
919 MODULE_AUTHOR("Yoshihiro Shimoda <yoshihiro.shimoda.uh@renesas.com>");
920 MODULE_LICENSE("GPL v2");
921