1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
4  * Synopsys DesignWare eDMA core driver
5  *
6  * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
7  */
8 
9 #include <linux/module.h>
10 #include <linux/device.h>
11 #include <linux/kernel.h>
12 #include <linux/dmaengine.h>
13 #include <linux/err.h>
14 #include <linux/interrupt.h>
15 #include <linux/irq.h>
16 #include <linux/dma/edma.h>
17 #include <linux/dma-mapping.h>
18 
19 #include "dw-edma-core.h"
20 #include "dw-edma-v0-core.h"
21 #include "dw-hdma-v0-core.h"
22 #include "../dmaengine.h"
23 #include "../virt-dma.h"
24 
25 static inline
26 struct device *dchan2dev(struct dma_chan *dchan)
27 {
28 	return &dchan->dev->device;
29 }
30 
31 static inline
32 struct device *chan2dev(struct dw_edma_chan *chan)
33 {
34 	return &chan->vc.chan.dev->device;
35 }
36 
37 static inline
38 struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd)
39 {
40 	return container_of(vd, struct dw_edma_desc, vd);
41 }
42 
43 static inline
44 u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr)
45 {
46 	struct dw_edma_chip *chip = chan->dw->chip;
47 
48 	if (chip->ops->pci_address)
49 		return chip->ops->pci_address(chip->dev, cpu_addr);
50 
51 	return cpu_addr;
52 }
53 
54 static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk)
55 {
56 	struct dw_edma_burst *burst;
57 
58 	burst = kzalloc(sizeof(*burst), GFP_NOWAIT);
59 	if (unlikely(!burst))
60 		return NULL;
61 
62 	INIT_LIST_HEAD(&burst->list);
63 	if (chunk->burst) {
64 		/* Create and add new element into the linked list */
65 		chunk->bursts_alloc++;
66 		list_add_tail(&burst->list, &chunk->burst->list);
67 	} else {
68 		/* List head */
69 		chunk->bursts_alloc = 0;
70 		chunk->burst = burst;
71 	}
72 
73 	return burst;
74 }
75 
76 static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc)
77 {
78 	struct dw_edma_chip *chip = desc->chan->dw->chip;
79 	struct dw_edma_chan *chan = desc->chan;
80 	struct dw_edma_chunk *chunk;
81 
82 	chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT);
83 	if (unlikely(!chunk))
84 		return NULL;
85 
86 	INIT_LIST_HEAD(&chunk->list);
87 	chunk->chan = chan;
88 	/* Toggling change bit (CB) in each chunk, this is a mechanism to
89 	 * inform the eDMA HW block that this is a new linked list ready
90 	 * to be consumed.
91 	 *  - Odd chunks originate CB equal to 0
92 	 *  - Even chunks originate CB equal to 1
93 	 */
94 	chunk->cb = !(desc->chunks_alloc % 2);
95 	if (chan->dir == EDMA_DIR_WRITE) {
96 		chunk->ll_region.paddr = chip->ll_region_wr[chan->id].paddr;
97 		chunk->ll_region.vaddr = chip->ll_region_wr[chan->id].vaddr;
98 	} else {
99 		chunk->ll_region.paddr = chip->ll_region_rd[chan->id].paddr;
100 		chunk->ll_region.vaddr = chip->ll_region_rd[chan->id].vaddr;
101 	}
102 
103 	if (desc->chunk) {
104 		/* Create and add new element into the linked list */
105 		if (!dw_edma_alloc_burst(chunk)) {
106 			kfree(chunk);
107 			return NULL;
108 		}
109 		desc->chunks_alloc++;
110 		list_add_tail(&chunk->list, &desc->chunk->list);
111 	} else {
112 		/* List head */
113 		chunk->burst = NULL;
114 		desc->chunks_alloc = 0;
115 		desc->chunk = chunk;
116 	}
117 
118 	return chunk;
119 }
120 
121 static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan)
122 {
123 	struct dw_edma_desc *desc;
124 
125 	desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
126 	if (unlikely(!desc))
127 		return NULL;
128 
129 	desc->chan = chan;
130 	if (!dw_edma_alloc_chunk(desc)) {
131 		kfree(desc);
132 		return NULL;
133 	}
134 
135 	return desc;
136 }
137 
138 static void dw_edma_free_burst(struct dw_edma_chunk *chunk)
139 {
140 	struct dw_edma_burst *child, *_next;
141 
142 	/* Remove all the list elements */
143 	list_for_each_entry_safe(child, _next, &chunk->burst->list, list) {
144 		list_del(&child->list);
145 		kfree(child);
146 		chunk->bursts_alloc--;
147 	}
148 
149 	/* Remove the list head */
150 	kfree(child);
151 	chunk->burst = NULL;
152 }
153 
154 static void dw_edma_free_chunk(struct dw_edma_desc *desc)
155 {
156 	struct dw_edma_chunk *child, *_next;
157 
158 	if (!desc->chunk)
159 		return;
160 
161 	/* Remove all the list elements */
162 	list_for_each_entry_safe(child, _next, &desc->chunk->list, list) {
163 		dw_edma_free_burst(child);
164 		list_del(&child->list);
165 		kfree(child);
166 		desc->chunks_alloc--;
167 	}
168 
169 	/* Remove the list head */
170 	kfree(child);
171 	desc->chunk = NULL;
172 }
173 
174 static void dw_edma_free_desc(struct dw_edma_desc *desc)
175 {
176 	dw_edma_free_chunk(desc);
177 	kfree(desc);
178 }
179 
180 static void vchan_free_desc(struct virt_dma_desc *vdesc)
181 {
182 	dw_edma_free_desc(vd2dw_edma_desc(vdesc));
183 }
184 
185 static int dw_edma_start_transfer(struct dw_edma_chan *chan)
186 {
187 	struct dw_edma *dw = chan->dw;
188 	struct dw_edma_chunk *child;
189 	struct dw_edma_desc *desc;
190 	struct virt_dma_desc *vd;
191 
192 	vd = vchan_next_desc(&chan->vc);
193 	if (!vd)
194 		return 0;
195 
196 	desc = vd2dw_edma_desc(vd);
197 	if (!desc)
198 		return 0;
199 
200 	child = list_first_entry_or_null(&desc->chunk->list,
201 					 struct dw_edma_chunk, list);
202 	if (!child)
203 		return 0;
204 
205 	dw_edma_core_start(dw, child, !desc->xfer_sz);
206 	desc->xfer_sz += child->ll_region.sz;
207 	dw_edma_free_burst(child);
208 	list_del(&child->list);
209 	kfree(child);
210 	desc->chunks_alloc--;
211 
212 	return 1;
213 }
214 
215 static void dw_edma_device_caps(struct dma_chan *dchan,
216 				struct dma_slave_caps *caps)
217 {
218 	struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
219 
220 	if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
221 		if (chan->dir == EDMA_DIR_READ)
222 			caps->directions = BIT(DMA_DEV_TO_MEM);
223 		else
224 			caps->directions = BIT(DMA_MEM_TO_DEV);
225 	} else {
226 		if (chan->dir == EDMA_DIR_WRITE)
227 			caps->directions = BIT(DMA_DEV_TO_MEM);
228 		else
229 			caps->directions = BIT(DMA_MEM_TO_DEV);
230 	}
231 }
232 
233 static int dw_edma_device_config(struct dma_chan *dchan,
234 				 struct dma_slave_config *config)
235 {
236 	struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
237 
238 	memcpy(&chan->config, config, sizeof(*config));
239 	chan->configured = true;
240 
241 	return 0;
242 }
243 
244 static int dw_edma_device_pause(struct dma_chan *dchan)
245 {
246 	struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
247 	int err = 0;
248 
249 	if (!chan->configured)
250 		err = -EPERM;
251 	else if (chan->status != EDMA_ST_BUSY)
252 		err = -EPERM;
253 	else if (chan->request != EDMA_REQ_NONE)
254 		err = -EPERM;
255 	else
256 		chan->request = EDMA_REQ_PAUSE;
257 
258 	return err;
259 }
260 
261 static int dw_edma_device_resume(struct dma_chan *dchan)
262 {
263 	struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
264 	int err = 0;
265 
266 	if (!chan->configured) {
267 		err = -EPERM;
268 	} else if (chan->status != EDMA_ST_PAUSE) {
269 		err = -EPERM;
270 	} else if (chan->request != EDMA_REQ_NONE) {
271 		err = -EPERM;
272 	} else {
273 		chan->status = EDMA_ST_BUSY;
274 		dw_edma_start_transfer(chan);
275 	}
276 
277 	return err;
278 }
279 
280 static int dw_edma_device_terminate_all(struct dma_chan *dchan)
281 {
282 	struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
283 	int err = 0;
284 
285 	if (!chan->configured) {
286 		/* Do nothing */
287 	} else if (chan->status == EDMA_ST_PAUSE) {
288 		chan->status = EDMA_ST_IDLE;
289 		chan->configured = false;
290 	} else if (chan->status == EDMA_ST_IDLE) {
291 		chan->configured = false;
292 	} else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) {
293 		/*
294 		 * The channel is in a false BUSY state, probably didn't
295 		 * receive or lost an interrupt
296 		 */
297 		chan->status = EDMA_ST_IDLE;
298 		chan->configured = false;
299 	} else if (chan->request > EDMA_REQ_PAUSE) {
300 		err = -EPERM;
301 	} else {
302 		chan->request = EDMA_REQ_STOP;
303 	}
304 
305 	return err;
306 }
307 
308 static void dw_edma_device_issue_pending(struct dma_chan *dchan)
309 {
310 	struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
311 	unsigned long flags;
312 
313 	if (!chan->configured)
314 		return;
315 
316 	spin_lock_irqsave(&chan->vc.lock, flags);
317 	if (vchan_issue_pending(&chan->vc) && chan->request == EDMA_REQ_NONE &&
318 	    chan->status == EDMA_ST_IDLE) {
319 		chan->status = EDMA_ST_BUSY;
320 		dw_edma_start_transfer(chan);
321 	}
322 	spin_unlock_irqrestore(&chan->vc.lock, flags);
323 }
324 
325 static enum dma_status
326 dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
327 			 struct dma_tx_state *txstate)
328 {
329 	struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
330 	struct dw_edma_desc *desc;
331 	struct virt_dma_desc *vd;
332 	unsigned long flags;
333 	enum dma_status ret;
334 	u32 residue = 0;
335 
336 	ret = dma_cookie_status(dchan, cookie, txstate);
337 	if (ret == DMA_COMPLETE)
338 		return ret;
339 
340 	if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE)
341 		ret = DMA_PAUSED;
342 
343 	if (!txstate)
344 		goto ret_residue;
345 
346 	spin_lock_irqsave(&chan->vc.lock, flags);
347 	vd = vchan_find_desc(&chan->vc, cookie);
348 	if (vd) {
349 		desc = vd2dw_edma_desc(vd);
350 		if (desc)
351 			residue = desc->alloc_sz - desc->xfer_sz;
352 	}
353 	spin_unlock_irqrestore(&chan->vc.lock, flags);
354 
355 ret_residue:
356 	dma_set_residue(txstate, residue);
357 
358 	return ret;
359 }
360 
361 static struct dma_async_tx_descriptor *
362 dw_edma_device_transfer(struct dw_edma_transfer *xfer)
363 {
364 	struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan);
365 	enum dma_transfer_direction dir = xfer->direction;
366 	struct scatterlist *sg = NULL;
367 	struct dw_edma_chunk *chunk;
368 	struct dw_edma_burst *burst;
369 	struct dw_edma_desc *desc;
370 	u64 src_addr, dst_addr;
371 	size_t fsz = 0;
372 	u32 cnt = 0;
373 	int i;
374 
375 	if (!chan->configured)
376 		return NULL;
377 
378 	/*
379 	 * Local Root Port/End-point              Remote End-point
380 	 * +-----------------------+ PCIe bus +----------------------+
381 	 * |                       |    +-+   |                      |
382 	 * |    DEV_TO_MEM   Rx Ch <----+ +---+ Tx Ch  DEV_TO_MEM    |
383 	 * |                       |    | |   |                      |
384 	 * |    MEM_TO_DEV   Tx Ch +----+ +---> Rx Ch  MEM_TO_DEV    |
385 	 * |                       |    +-+   |                      |
386 	 * +-----------------------+          +----------------------+
387 	 *
388 	 * 1. Normal logic:
389 	 * If eDMA is embedded into the DW PCIe RP/EP and controlled from the
390 	 * CPU/Application side, the Rx channel (EDMA_DIR_READ) will be used
391 	 * for the device read operations (DEV_TO_MEM) and the Tx channel
392 	 * (EDMA_DIR_WRITE) - for the write operations (MEM_TO_DEV).
393 	 *
394 	 * 2. Inverted logic:
395 	 * If eDMA is embedded into a Remote PCIe EP and is controlled by the
396 	 * MWr/MRd TLPs sent from the CPU's PCIe host controller, the Tx
397 	 * channel (EDMA_DIR_WRITE) will be used for the device read operations
398 	 * (DEV_TO_MEM) and the Rx channel (EDMA_DIR_READ) - for the write
399 	 * operations (MEM_TO_DEV).
400 	 *
401 	 * It is the client driver responsibility to choose a proper channel
402 	 * for the DMA transfers.
403 	 */
404 	if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
405 		if ((chan->dir == EDMA_DIR_READ && dir != DMA_DEV_TO_MEM) ||
406 		    (chan->dir == EDMA_DIR_WRITE && dir != DMA_MEM_TO_DEV))
407 			return NULL;
408 	} else {
409 		if ((chan->dir == EDMA_DIR_WRITE && dir != DMA_DEV_TO_MEM) ||
410 		    (chan->dir == EDMA_DIR_READ && dir != DMA_MEM_TO_DEV))
411 			return NULL;
412 	}
413 
414 	if (xfer->type == EDMA_XFER_CYCLIC) {
415 		if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt)
416 			return NULL;
417 	} else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
418 		if (xfer->xfer.sg.len < 1)
419 			return NULL;
420 	} else if (xfer->type == EDMA_XFER_INTERLEAVED) {
421 		if (!xfer->xfer.il->numf || xfer->xfer.il->frame_size < 1)
422 			return NULL;
423 		if (!xfer->xfer.il->src_inc || !xfer->xfer.il->dst_inc)
424 			return NULL;
425 	} else {
426 		return NULL;
427 	}
428 
429 	desc = dw_edma_alloc_desc(chan);
430 	if (unlikely(!desc))
431 		goto err_alloc;
432 
433 	chunk = dw_edma_alloc_chunk(desc);
434 	if (unlikely(!chunk))
435 		goto err_alloc;
436 
437 	if (xfer->type == EDMA_XFER_INTERLEAVED) {
438 		src_addr = xfer->xfer.il->src_start;
439 		dst_addr = xfer->xfer.il->dst_start;
440 	} else {
441 		src_addr = chan->config.src_addr;
442 		dst_addr = chan->config.dst_addr;
443 	}
444 
445 	if (dir == DMA_DEV_TO_MEM)
446 		src_addr = dw_edma_get_pci_address(chan, (phys_addr_t)src_addr);
447 	else
448 		dst_addr = dw_edma_get_pci_address(chan, (phys_addr_t)dst_addr);
449 
450 	if (xfer->type == EDMA_XFER_CYCLIC) {
451 		cnt = xfer->xfer.cyclic.cnt;
452 	} else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
453 		cnt = xfer->xfer.sg.len;
454 		sg = xfer->xfer.sg.sgl;
455 	} else if (xfer->type == EDMA_XFER_INTERLEAVED) {
456 		cnt = xfer->xfer.il->numf * xfer->xfer.il->frame_size;
457 		fsz = xfer->xfer.il->frame_size;
458 	}
459 
460 	for (i = 0; i < cnt; i++) {
461 		if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg)
462 			break;
463 
464 		if (chunk->bursts_alloc == chan->ll_max) {
465 			chunk = dw_edma_alloc_chunk(desc);
466 			if (unlikely(!chunk))
467 				goto err_alloc;
468 		}
469 
470 		burst = dw_edma_alloc_burst(chunk);
471 		if (unlikely(!burst))
472 			goto err_alloc;
473 
474 		if (xfer->type == EDMA_XFER_CYCLIC)
475 			burst->sz = xfer->xfer.cyclic.len;
476 		else if (xfer->type == EDMA_XFER_SCATTER_GATHER)
477 			burst->sz = sg_dma_len(sg);
478 		else if (xfer->type == EDMA_XFER_INTERLEAVED)
479 			burst->sz = xfer->xfer.il->sgl[i % fsz].size;
480 
481 		chunk->ll_region.sz += burst->sz;
482 		desc->alloc_sz += burst->sz;
483 
484 		if (dir == DMA_DEV_TO_MEM) {
485 			burst->sar = src_addr;
486 			if (xfer->type == EDMA_XFER_CYCLIC) {
487 				burst->dar = xfer->xfer.cyclic.paddr;
488 			} else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
489 				src_addr += sg_dma_len(sg);
490 				burst->dar = sg_dma_address(sg);
491 				/* Unlike the typical assumption by other
492 				 * drivers/IPs the peripheral memory isn't
493 				 * a FIFO memory, in this case, it's a
494 				 * linear memory and that why the source
495 				 * and destination addresses are increased
496 				 * by the same portion (data length)
497 				 */
498 			} else if (xfer->type == EDMA_XFER_INTERLEAVED) {
499 				burst->dar = dst_addr;
500 			}
501 		} else {
502 			burst->dar = dst_addr;
503 			if (xfer->type == EDMA_XFER_CYCLIC) {
504 				burst->sar = xfer->xfer.cyclic.paddr;
505 			} else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
506 				dst_addr += sg_dma_len(sg);
507 				burst->sar = sg_dma_address(sg);
508 				/* Unlike the typical assumption by other
509 				 * drivers/IPs the peripheral memory isn't
510 				 * a FIFO memory, in this case, it's a
511 				 * linear memory and that why the source
512 				 * and destination addresses are increased
513 				 * by the same portion (data length)
514 				 */
515 			}  else if (xfer->type == EDMA_XFER_INTERLEAVED) {
516 				burst->sar = src_addr;
517 			}
518 		}
519 
520 		if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
521 			sg = sg_next(sg);
522 		} else if (xfer->type == EDMA_XFER_INTERLEAVED) {
523 			struct dma_interleaved_template *il = xfer->xfer.il;
524 			struct data_chunk *dc = &il->sgl[i % fsz];
525 
526 			src_addr += burst->sz;
527 			if (il->src_sgl)
528 				src_addr += dmaengine_get_src_icg(il, dc);
529 
530 			dst_addr += burst->sz;
531 			if (il->dst_sgl)
532 				dst_addr += dmaengine_get_dst_icg(il, dc);
533 		}
534 	}
535 
536 	return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags);
537 
538 err_alloc:
539 	if (desc)
540 		dw_edma_free_desc(desc);
541 
542 	return NULL;
543 }
544 
545 static struct dma_async_tx_descriptor *
546 dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
547 			     unsigned int len,
548 			     enum dma_transfer_direction direction,
549 			     unsigned long flags, void *context)
550 {
551 	struct dw_edma_transfer xfer;
552 
553 	xfer.dchan = dchan;
554 	xfer.direction = direction;
555 	xfer.xfer.sg.sgl = sgl;
556 	xfer.xfer.sg.len = len;
557 	xfer.flags = flags;
558 	xfer.type = EDMA_XFER_SCATTER_GATHER;
559 
560 	return dw_edma_device_transfer(&xfer);
561 }
562 
563 static struct dma_async_tx_descriptor *
564 dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr,
565 			       size_t len, size_t count,
566 			       enum dma_transfer_direction direction,
567 			       unsigned long flags)
568 {
569 	struct dw_edma_transfer xfer;
570 
571 	xfer.dchan = dchan;
572 	xfer.direction = direction;
573 	xfer.xfer.cyclic.paddr = paddr;
574 	xfer.xfer.cyclic.len = len;
575 	xfer.xfer.cyclic.cnt = count;
576 	xfer.flags = flags;
577 	xfer.type = EDMA_XFER_CYCLIC;
578 
579 	return dw_edma_device_transfer(&xfer);
580 }
581 
582 static struct dma_async_tx_descriptor *
583 dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan,
584 				    struct dma_interleaved_template *ilt,
585 				    unsigned long flags)
586 {
587 	struct dw_edma_transfer xfer;
588 
589 	xfer.dchan = dchan;
590 	xfer.direction = ilt->dir;
591 	xfer.xfer.il = ilt;
592 	xfer.flags = flags;
593 	xfer.type = EDMA_XFER_INTERLEAVED;
594 
595 	return dw_edma_device_transfer(&xfer);
596 }
597 
598 static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
599 {
600 	struct dw_edma_desc *desc;
601 	struct virt_dma_desc *vd;
602 	unsigned long flags;
603 
604 	spin_lock_irqsave(&chan->vc.lock, flags);
605 	vd = vchan_next_desc(&chan->vc);
606 	if (vd) {
607 		switch (chan->request) {
608 		case EDMA_REQ_NONE:
609 			desc = vd2dw_edma_desc(vd);
610 			if (!desc->chunks_alloc) {
611 				list_del(&vd->node);
612 				vchan_cookie_complete(vd);
613 			}
614 
615 			/* Continue transferring if there are remaining chunks or issued requests.
616 			 */
617 			chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE;
618 			break;
619 
620 		case EDMA_REQ_STOP:
621 			list_del(&vd->node);
622 			vchan_cookie_complete(vd);
623 			chan->request = EDMA_REQ_NONE;
624 			chan->status = EDMA_ST_IDLE;
625 			break;
626 
627 		case EDMA_REQ_PAUSE:
628 			chan->request = EDMA_REQ_NONE;
629 			chan->status = EDMA_ST_PAUSE;
630 			break;
631 
632 		default:
633 			break;
634 		}
635 	}
636 	spin_unlock_irqrestore(&chan->vc.lock, flags);
637 }
638 
639 static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
640 {
641 	struct virt_dma_desc *vd;
642 	unsigned long flags;
643 
644 	spin_lock_irqsave(&chan->vc.lock, flags);
645 	vd = vchan_next_desc(&chan->vc);
646 	if (vd) {
647 		list_del(&vd->node);
648 		vchan_cookie_complete(vd);
649 	}
650 	spin_unlock_irqrestore(&chan->vc.lock, flags);
651 	chan->request = EDMA_REQ_NONE;
652 	chan->status = EDMA_ST_IDLE;
653 }
654 
655 static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data)
656 {
657 	struct dw_edma_irq *dw_irq = data;
658 
659 	return dw_edma_core_handle_int(dw_irq, EDMA_DIR_WRITE,
660 				       dw_edma_done_interrupt,
661 				       dw_edma_abort_interrupt);
662 }
663 
664 static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data)
665 {
666 	struct dw_edma_irq *dw_irq = data;
667 
668 	return dw_edma_core_handle_int(dw_irq, EDMA_DIR_READ,
669 				       dw_edma_done_interrupt,
670 				       dw_edma_abort_interrupt);
671 }
672 
673 static irqreturn_t dw_edma_interrupt_common(int irq, void *data)
674 {
675 	irqreturn_t ret = IRQ_NONE;
676 
677 	ret |= dw_edma_interrupt_write(irq, data);
678 	ret |= dw_edma_interrupt_read(irq, data);
679 
680 	return ret;
681 }
682 
683 static int dw_edma_alloc_chan_resources(struct dma_chan *dchan)
684 {
685 	struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
686 
687 	if (chan->status != EDMA_ST_IDLE)
688 		return -EBUSY;
689 
690 	return 0;
691 }
692 
693 static void dw_edma_free_chan_resources(struct dma_chan *dchan)
694 {
695 	unsigned long timeout = jiffies + msecs_to_jiffies(5000);
696 	int ret;
697 
698 	while (time_before(jiffies, timeout)) {
699 		ret = dw_edma_device_terminate_all(dchan);
700 		if (!ret)
701 			break;
702 
703 		if (time_after_eq(jiffies, timeout))
704 			return;
705 
706 		cpu_relax();
707 	}
708 }
709 
710 static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)
711 {
712 	struct dw_edma_chip *chip = dw->chip;
713 	struct device *dev = chip->dev;
714 	struct dw_edma_chan *chan;
715 	struct dw_edma_irq *irq;
716 	struct dma_device *dma;
717 	u32 i, ch_cnt;
718 	u32 pos;
719 
720 	ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;
721 	dma = &dw->dma;
722 
723 	INIT_LIST_HEAD(&dma->channels);
724 
725 	for (i = 0; i < ch_cnt; i++) {
726 		chan = &dw->chan[i];
727 
728 		chan->dw = dw;
729 
730 		if (i < dw->wr_ch_cnt) {
731 			chan->id = i;
732 			chan->dir = EDMA_DIR_WRITE;
733 		} else {
734 			chan->id = i - dw->wr_ch_cnt;
735 			chan->dir = EDMA_DIR_READ;
736 		}
737 
738 		chan->configured = false;
739 		chan->request = EDMA_REQ_NONE;
740 		chan->status = EDMA_ST_IDLE;
741 
742 		if (chan->dir == EDMA_DIR_WRITE)
743 			chan->ll_max = (chip->ll_region_wr[chan->id].sz / EDMA_LL_SZ);
744 		else
745 			chan->ll_max = (chip->ll_region_rd[chan->id].sz / EDMA_LL_SZ);
746 		chan->ll_max -= 1;
747 
748 		dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n",
749 			 chan->dir == EDMA_DIR_WRITE ? "write" : "read",
750 			 chan->id, chan->ll_max);
751 
752 		if (dw->nr_irqs == 1)
753 			pos = 0;
754 		else if (chan->dir == EDMA_DIR_WRITE)
755 			pos = chan->id % wr_alloc;
756 		else
757 			pos = wr_alloc + chan->id % rd_alloc;
758 
759 		irq = &dw->irq[pos];
760 
761 		if (chan->dir == EDMA_DIR_WRITE)
762 			irq->wr_mask |= BIT(chan->id);
763 		else
764 			irq->rd_mask |= BIT(chan->id);
765 
766 		irq->dw = dw;
767 		memcpy(&chan->msi, &irq->msi, sizeof(chan->msi));
768 
769 		dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n",
770 			 chan->dir == EDMA_DIR_WRITE  ? "write" : "read", chan->id,
771 			 chan->msi.address_hi, chan->msi.address_lo,
772 			 chan->msi.data);
773 
774 		chan->vc.desc_free = vchan_free_desc;
775 		chan->vc.chan.private = chan->dir == EDMA_DIR_WRITE ?
776 					&dw->chip->dt_region_wr[chan->id] :
777 					&dw->chip->dt_region_rd[chan->id];
778 
779 		vchan_init(&chan->vc, dma);
780 
781 		dw_edma_core_ch_config(chan);
782 	}
783 
784 	/* Set DMA channel capabilities */
785 	dma_cap_zero(dma->cap_mask);
786 	dma_cap_set(DMA_SLAVE, dma->cap_mask);
787 	dma_cap_set(DMA_CYCLIC, dma->cap_mask);
788 	dma_cap_set(DMA_PRIVATE, dma->cap_mask);
789 	dma_cap_set(DMA_INTERLEAVE, dma->cap_mask);
790 	dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
791 	dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
792 	dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
793 	dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
794 
795 	/* Set DMA channel callbacks */
796 	dma->dev = chip->dev;
797 	dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources;
798 	dma->device_free_chan_resources = dw_edma_free_chan_resources;
799 	dma->device_caps = dw_edma_device_caps;
800 	dma->device_config = dw_edma_device_config;
801 	dma->device_pause = dw_edma_device_pause;
802 	dma->device_resume = dw_edma_device_resume;
803 	dma->device_terminate_all = dw_edma_device_terminate_all;
804 	dma->device_issue_pending = dw_edma_device_issue_pending;
805 	dma->device_tx_status = dw_edma_device_tx_status;
806 	dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg;
807 	dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic;
808 	dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma;
809 
810 	dma_set_max_seg_size(dma->dev, U32_MAX);
811 
812 	/* Register DMA device */
813 	return dma_async_device_register(dma);
814 }
815 
816 static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt)
817 {
818 	if (*nr_irqs && *alloc < cnt) {
819 		(*alloc)++;
820 		(*nr_irqs)--;
821 	}
822 }
823 
824 static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt)
825 {
826 	while (*mask * alloc < cnt)
827 		(*mask)++;
828 }
829 
830 static int dw_edma_irq_request(struct dw_edma *dw,
831 			       u32 *wr_alloc, u32 *rd_alloc)
832 {
833 	struct dw_edma_chip *chip = dw->chip;
834 	struct device *dev = dw->chip->dev;
835 	u32 wr_mask = 1;
836 	u32 rd_mask = 1;
837 	int i, err = 0;
838 	u32 ch_cnt;
839 	int irq;
840 
841 	ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;
842 
843 	if (chip->nr_irqs < 1 || !chip->ops->irq_vector)
844 		return -EINVAL;
845 
846 	dw->irq = devm_kcalloc(dev, chip->nr_irqs, sizeof(*dw->irq), GFP_KERNEL);
847 	if (!dw->irq)
848 		return -ENOMEM;
849 
850 	if (chip->nr_irqs == 1) {
851 		/* Common IRQ shared among all channels */
852 		irq = chip->ops->irq_vector(dev, 0);
853 		err = request_irq(irq, dw_edma_interrupt_common,
854 				  IRQF_SHARED, dw->name, &dw->irq[0]);
855 		if (err) {
856 			dw->nr_irqs = 0;
857 			return err;
858 		}
859 
860 		if (irq_get_msi_desc(irq))
861 			get_cached_msi_msg(irq, &dw->irq[0].msi);
862 
863 		dw->nr_irqs = 1;
864 	} else {
865 		/* Distribute IRQs equally among all channels */
866 		int tmp = chip->nr_irqs;
867 
868 		while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) {
869 			dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt);
870 			dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt);
871 		}
872 
873 		dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt);
874 		dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt);
875 
876 		for (i = 0; i < (*wr_alloc + *rd_alloc); i++) {
877 			irq = chip->ops->irq_vector(dev, i);
878 			err = request_irq(irq,
879 					  i < *wr_alloc ?
880 						dw_edma_interrupt_write :
881 						dw_edma_interrupt_read,
882 					  IRQF_SHARED, dw->name,
883 					  &dw->irq[i]);
884 			if (err)
885 				goto err_irq_free;
886 
887 			if (irq_get_msi_desc(irq))
888 				get_cached_msi_msg(irq, &dw->irq[i].msi);
889 		}
890 
891 		dw->nr_irqs = i;
892 	}
893 
894 	return 0;
895 
896 err_irq_free:
897 	for  (i--; i >= 0; i--) {
898 		irq = chip->ops->irq_vector(dev, i);
899 		free_irq(irq, &dw->irq[i]);
900 	}
901 
902 	return err;
903 }
904 
905 int dw_edma_probe(struct dw_edma_chip *chip)
906 {
907 	struct device *dev;
908 	struct dw_edma *dw;
909 	u32 wr_alloc = 0;
910 	u32 rd_alloc = 0;
911 	int i, err;
912 
913 	if (!chip)
914 		return -EINVAL;
915 
916 	dev = chip->dev;
917 	if (!dev || !chip->ops)
918 		return -EINVAL;
919 
920 	dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL);
921 	if (!dw)
922 		return -ENOMEM;
923 
924 	dw->chip = chip;
925 
926 	if (dw->chip->mf == EDMA_MF_HDMA_NATIVE)
927 		dw_hdma_v0_core_register(dw);
928 	else
929 		dw_edma_v0_core_register(dw);
930 
931 	raw_spin_lock_init(&dw->lock);
932 
933 	dw->wr_ch_cnt = min_t(u16, chip->ll_wr_cnt,
934 			      dw_edma_core_ch_count(dw, EDMA_DIR_WRITE));
935 	dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH);
936 
937 	dw->rd_ch_cnt = min_t(u16, chip->ll_rd_cnt,
938 			      dw_edma_core_ch_count(dw, EDMA_DIR_READ));
939 	dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH);
940 
941 	if (!dw->wr_ch_cnt && !dw->rd_ch_cnt)
942 		return -EINVAL;
943 
944 	dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n",
945 		 dw->wr_ch_cnt, dw->rd_ch_cnt);
946 
947 	/* Allocate channels */
948 	dw->chan = devm_kcalloc(dev, dw->wr_ch_cnt + dw->rd_ch_cnt,
949 				sizeof(*dw->chan), GFP_KERNEL);
950 	if (!dw->chan)
951 		return -ENOMEM;
952 
953 	snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%s",
954 		 dev_name(chip->dev));
955 
956 	/* Disable eDMA, only to establish the ideal initial conditions */
957 	dw_edma_core_off(dw);
958 
959 	/* Request IRQs */
960 	err = dw_edma_irq_request(dw, &wr_alloc, &rd_alloc);
961 	if (err)
962 		return err;
963 
964 	/* Setup write/read channels */
965 	err = dw_edma_channel_setup(dw, wr_alloc, rd_alloc);
966 	if (err)
967 		goto err_irq_free;
968 
969 	/* Turn debugfs on */
970 	dw_edma_core_debugfs_on(dw);
971 
972 	chip->dw = dw;
973 
974 	return 0;
975 
976 err_irq_free:
977 	for (i = (dw->nr_irqs - 1); i >= 0; i--)
978 		free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]);
979 
980 	return err;
981 }
982 EXPORT_SYMBOL_GPL(dw_edma_probe);
983 
984 int dw_edma_remove(struct dw_edma_chip *chip)
985 {
986 	struct dw_edma_chan *chan, *_chan;
987 	struct device *dev = chip->dev;
988 	struct dw_edma *dw = chip->dw;
989 	int i;
990 
991 	/* Skip removal if no private data found */
992 	if (!dw)
993 		return -ENODEV;
994 
995 	/* Disable eDMA */
996 	dw_edma_core_off(dw);
997 
998 	/* Free irqs */
999 	for (i = (dw->nr_irqs - 1); i >= 0; i--)
1000 		free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]);
1001 
1002 	/* Deregister eDMA device */
1003 	dma_async_device_unregister(&dw->dma);
1004 	list_for_each_entry_safe(chan, _chan, &dw->dma.channels,
1005 				 vc.chan.device_node) {
1006 		tasklet_kill(&chan->vc.task);
1007 		list_del(&chan->vc.chan.device_node);
1008 	}
1009 
1010 	return 0;
1011 }
1012 EXPORT_SYMBOL_GPL(dw_edma_remove);
1013 
1014 MODULE_LICENSE("GPL v2");
1015 MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver");
1016 MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>");
1017