xref: /openbmc/linux/drivers/dma/amba-pl08x.c (revision 92a2c6b2)
1 /*
2  * Copyright (c) 2006 ARM Ltd.
3  * Copyright (c) 2010 ST-Ericsson SA
4  *
5  * Author: Peter Pearse <peter.pearse@arm.com>
6  * Author: Linus Walleij <linus.walleij@stericsson.com>
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms of the GNU General Public License as published by the Free
10  * Software Foundation; either version 2 of the License, or (at your option)
11  * any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along with
19  * this program; if not, write to the Free Software Foundation, Inc., 59
20  * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
21  *
22  * The full GNU General Public License is in this distribution in the file
23  * called COPYING.
24  *
25  * Documentation: ARM DDI 0196G == PL080
26  * Documentation: ARM DDI 0218E == PL081
27  * Documentation: S3C6410 User's Manual == PL080S
28  *
29  * PL080 & PL081 both have 16 sets of DMA signals that can be routed to any
30  * channel.
31  *
32  * The PL080 has 8 channels available for simultaneous use, and the PL081
33  * has only two channels. So on these DMA controllers the number of channels
34  * and the number of incoming DMA signals are two totally different things.
35  * It is usually not possible to theoretically handle all physical signals,
36  * so a multiplexing scheme with possible denial of use is necessary.
37  *
38  * The PL080 has a dual bus master, PL081 has a single master.
39  *
40  * PL080S is a version modified by Samsung and used in S3C64xx SoCs.
41  * It differs in following aspects:
42  * - CH_CONFIG register at different offset,
43  * - separate CH_CONTROL2 register for transfer size,
44  * - bigger maximum transfer size,
45  * - 8-word aligned LLI, instead of 4-word, due to extra CCTL2 word,
46  * - no support for peripheral flow control.
47  *
48  * Memory to peripheral transfer may be visualized as
49  *	Get data from memory to DMAC
50  *	Until no data left
51  *		On burst request from peripheral
52  *			Destination burst from DMAC to peripheral
53  *			Clear burst request
54  *	Raise terminal count interrupt
55  *
56  * For peripherals with a FIFO:
57  * Source      burst size == half the depth of the peripheral FIFO
58  * Destination burst size == the depth of the peripheral FIFO
59  *
60  * (Bursts are irrelevant for mem to mem transfers - there are no burst
61  * signals, the DMA controller will simply facilitate its AHB master.)
62  *
63  * ASSUMES default (little) endianness for DMA transfers
64  *
65  * The PL08x has two flow control settings:
66  *  - DMAC flow control: the transfer size defines the number of transfers
67  *    which occur for the current LLI entry, and the DMAC raises TC at the
68  *    end of every LLI entry.  Observed behaviour shows the DMAC listening
69  *    to both the BREQ and SREQ signals (contrary to documented),
70  *    transferring data if either is active.  The LBREQ and LSREQ signals
71  *    are ignored.
72  *
73  *  - Peripheral flow control: the transfer size is ignored (and should be
74  *    zero).  The data is transferred from the current LLI entry, until
75  *    after the final transfer signalled by LBREQ or LSREQ.  The DMAC
76  *    will then move to the next LLI entry. Unsupported by PL080S.
77  */
78 #include <linux/amba/bus.h>
79 #include <linux/amba/pl08x.h>
80 #include <linux/debugfs.h>
81 #include <linux/delay.h>
82 #include <linux/device.h>
83 #include <linux/dmaengine.h>
84 #include <linux/dmapool.h>
85 #include <linux/dma-mapping.h>
86 #include <linux/export.h>
87 #include <linux/init.h>
88 #include <linux/interrupt.h>
89 #include <linux/module.h>
90 #include <linux/pm_runtime.h>
91 #include <linux/seq_file.h>
92 #include <linux/slab.h>
93 #include <linux/amba/pl080.h>
94 
95 #include "dmaengine.h"
96 #include "virt-dma.h"
97 
98 #define DRIVER_NAME	"pl08xdmac"
99 
100 static struct amba_driver pl08x_amba_driver;
101 struct pl08x_driver_data;
102 
103 /**
104  * struct vendor_data - vendor-specific config parameters for PL08x derivatives
105  * @channels: the number of channels available in this variant
106  * @dualmaster: whether this version supports dual AHB masters or not.
107  * @nomadik: whether the channels have Nomadik security extension bits
108  *	that need to be checked for permission before use and some registers are
109  *	missing
110  * @pl080s: whether this version is a PL080S, which has separate register and
111  *	LLI word for transfer size.
112  */
113 struct vendor_data {
114 	u8 config_offset;
115 	u8 channels;
116 	bool dualmaster;
117 	bool nomadik;
118 	bool pl080s;
119 	u32 max_transfer_size;
120 };
121 
122 /**
123  * struct pl08x_bus_data - information of source or destination
124  * busses for a transfer
125  * @addr: current address
126  * @maxwidth: the maximum width of a transfer on this bus
127  * @buswidth: the width of this bus in bytes: 1, 2 or 4
128  */
129 struct pl08x_bus_data {
130 	dma_addr_t addr;
131 	u8 maxwidth;
132 	u8 buswidth;
133 };
134 
135 #define IS_BUS_ALIGNED(bus) IS_ALIGNED((bus)->addr, (bus)->buswidth)
136 
137 /**
138  * struct pl08x_phy_chan - holder for the physical channels
139  * @id: physical index to this channel
140  * @lock: a lock to use when altering an instance of this struct
141  * @serving: the virtual channel currently being served by this physical
142  * channel
143  * @locked: channel unavailable for the system, e.g. dedicated to secure
144  * world
145  */
146 struct pl08x_phy_chan {
147 	unsigned int id;
148 	void __iomem *base;
149 	void __iomem *reg_config;
150 	spinlock_t lock;
151 	struct pl08x_dma_chan *serving;
152 	bool locked;
153 };
154 
155 /**
156  * struct pl08x_sg - structure containing data per sg
157  * @src_addr: src address of sg
158  * @dst_addr: dst address of sg
159  * @len: transfer len in bytes
160  * @node: node for txd's dsg_list
161  */
162 struct pl08x_sg {
163 	dma_addr_t src_addr;
164 	dma_addr_t dst_addr;
165 	size_t len;
166 	struct list_head node;
167 };
168 
169 /**
170  * struct pl08x_txd - wrapper for struct dma_async_tx_descriptor
171  * @vd: virtual DMA descriptor
172  * @dsg_list: list of children sg's
173  * @llis_bus: DMA memory address (physical) start for the LLIs
174  * @llis_va: virtual memory address start for the LLIs
175  * @cctl: control reg values for current txd
176  * @ccfg: config reg values for current txd
177  * @done: this marks completed descriptors, which should not have their
178  *   mux released.
179  * @cyclic: indicate cyclic transfers
180  */
181 struct pl08x_txd {
182 	struct virt_dma_desc vd;
183 	struct list_head dsg_list;
184 	dma_addr_t llis_bus;
185 	u32 *llis_va;
186 	/* Default cctl value for LLIs */
187 	u32 cctl;
188 	/*
189 	 * Settings to be put into the physical channel when we
190 	 * trigger this txd.  Other registers are in llis_va[0].
191 	 */
192 	u32 ccfg;
193 	bool done;
194 	bool cyclic;
195 };
196 
197 /**
198  * struct pl08x_dma_chan_state - holds the PL08x specific virtual channel
199  * states
200  * @PL08X_CHAN_IDLE: the channel is idle
201  * @PL08X_CHAN_RUNNING: the channel has allocated a physical transport
202  * channel and is running a transfer on it
203  * @PL08X_CHAN_PAUSED: the channel has allocated a physical transport
204  * channel, but the transfer is currently paused
205  * @PL08X_CHAN_WAITING: the channel is waiting for a physical transport
206  * channel to become available (only pertains to memcpy channels)
207  */
208 enum pl08x_dma_chan_state {
209 	PL08X_CHAN_IDLE,
210 	PL08X_CHAN_RUNNING,
211 	PL08X_CHAN_PAUSED,
212 	PL08X_CHAN_WAITING,
213 };
214 
215 /**
216  * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel
217  * @vc: wrappped virtual channel
218  * @phychan: the physical channel utilized by this channel, if there is one
219  * @name: name of channel
220  * @cd: channel platform data
221  * @runtime_addr: address for RX/TX according to the runtime config
222  * @at: active transaction on this channel
223  * @lock: a lock for this channel data
224  * @host: a pointer to the host (internal use)
225  * @state: whether the channel is idle, paused, running etc
226  * @slave: whether this channel is a device (slave) or for memcpy
227  * @signal: the physical DMA request signal which this channel is using
228  * @mux_use: count of descriptors using this DMA request signal setting
229  */
230 struct pl08x_dma_chan {
231 	struct virt_dma_chan vc;
232 	struct pl08x_phy_chan *phychan;
233 	const char *name;
234 	const struct pl08x_channel_data *cd;
235 	struct dma_slave_config cfg;
236 	struct pl08x_txd *at;
237 	struct pl08x_driver_data *host;
238 	enum pl08x_dma_chan_state state;
239 	bool slave;
240 	int signal;
241 	unsigned mux_use;
242 };
243 
244 /**
245  * struct pl08x_driver_data - the local state holder for the PL08x
246  * @slave: slave engine for this instance
247  * @memcpy: memcpy engine for this instance
248  * @base: virtual memory base (remapped) for the PL08x
249  * @adev: the corresponding AMBA (PrimeCell) bus entry
250  * @vd: vendor data for this PL08x variant
251  * @pd: platform data passed in from the platform/machine
252  * @phy_chans: array of data for the physical channels
253  * @pool: a pool for the LLI descriptors
254  * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI
255  * fetches
256  * @mem_buses: set to indicate memory transfers on AHB2.
257  * @lock: a spinlock for this struct
258  */
259 struct pl08x_driver_data {
260 	struct dma_device slave;
261 	struct dma_device memcpy;
262 	void __iomem *base;
263 	struct amba_device *adev;
264 	const struct vendor_data *vd;
265 	struct pl08x_platform_data *pd;
266 	struct pl08x_phy_chan *phy_chans;
267 	struct dma_pool *pool;
268 	u8 lli_buses;
269 	u8 mem_buses;
270 	u8 lli_words;
271 };
272 
273 /*
274  * PL08X specific defines
275  */
276 
277 /* The order of words in an LLI. */
278 #define PL080_LLI_SRC		0
279 #define PL080_LLI_DST		1
280 #define PL080_LLI_LLI		2
281 #define PL080_LLI_CCTL		3
282 #define PL080S_LLI_CCTL2	4
283 
284 /* Total words in an LLI. */
285 #define PL080_LLI_WORDS		4
286 #define PL080S_LLI_WORDS	8
287 
288 /*
289  * Number of LLIs in each LLI buffer allocated for one transfer
290  * (maximum times we call dma_pool_alloc on this pool without freeing)
291  */
292 #define MAX_NUM_TSFR_LLIS	512
293 #define PL08X_ALIGN		8
294 
295 static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan)
296 {
297 	return container_of(chan, struct pl08x_dma_chan, vc.chan);
298 }
299 
300 static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx)
301 {
302 	return container_of(tx, struct pl08x_txd, vd.tx);
303 }
304 
305 /*
306  * Mux handling.
307  *
308  * This gives us the DMA request input to the PL08x primecell which the
309  * peripheral described by the channel data will be routed to, possibly
310  * via a board/SoC specific external MUX.  One important point to note
311  * here is that this does not depend on the physical channel.
312  */
313 static int pl08x_request_mux(struct pl08x_dma_chan *plchan)
314 {
315 	const struct pl08x_platform_data *pd = plchan->host->pd;
316 	int ret;
317 
318 	if (plchan->mux_use++ == 0 && pd->get_xfer_signal) {
319 		ret = pd->get_xfer_signal(plchan->cd);
320 		if (ret < 0) {
321 			plchan->mux_use = 0;
322 			return ret;
323 		}
324 
325 		plchan->signal = ret;
326 	}
327 	return 0;
328 }
329 
330 static void pl08x_release_mux(struct pl08x_dma_chan *plchan)
331 {
332 	const struct pl08x_platform_data *pd = plchan->host->pd;
333 
334 	if (plchan->signal >= 0) {
335 		WARN_ON(plchan->mux_use == 0);
336 
337 		if (--plchan->mux_use == 0 && pd->put_xfer_signal) {
338 			pd->put_xfer_signal(plchan->cd, plchan->signal);
339 			plchan->signal = -1;
340 		}
341 	}
342 }
343 
344 /*
345  * Physical channel handling
346  */
347 
348 /* Whether a certain channel is busy or not */
349 static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch)
350 {
351 	unsigned int val;
352 
353 	val = readl(ch->reg_config);
354 	return val & PL080_CONFIG_ACTIVE;
355 }
356 
357 static void pl08x_write_lli(struct pl08x_driver_data *pl08x,
358 		struct pl08x_phy_chan *phychan, const u32 *lli, u32 ccfg)
359 {
360 	if (pl08x->vd->pl080s)
361 		dev_vdbg(&pl08x->adev->dev,
362 			"WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
363 			"clli=0x%08x, cctl=0x%08x, cctl2=0x%08x, ccfg=0x%08x\n",
364 			phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST],
365 			lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL],
366 			lli[PL080S_LLI_CCTL2], ccfg);
367 	else
368 		dev_vdbg(&pl08x->adev->dev,
369 			"WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
370 			"clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n",
371 			phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST],
372 			lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], ccfg);
373 
374 	writel_relaxed(lli[PL080_LLI_SRC], phychan->base + PL080_CH_SRC_ADDR);
375 	writel_relaxed(lli[PL080_LLI_DST], phychan->base + PL080_CH_DST_ADDR);
376 	writel_relaxed(lli[PL080_LLI_LLI], phychan->base + PL080_CH_LLI);
377 	writel_relaxed(lli[PL080_LLI_CCTL], phychan->base + PL080_CH_CONTROL);
378 
379 	if (pl08x->vd->pl080s)
380 		writel_relaxed(lli[PL080S_LLI_CCTL2],
381 				phychan->base + PL080S_CH_CONTROL2);
382 
383 	writel(ccfg, phychan->reg_config);
384 }
385 
386 /*
387  * Set the initial DMA register values i.e. those for the first LLI
388  * The next LLI pointer and the configuration interrupt bit have
389  * been set when the LLIs were constructed.  Poke them into the hardware
390  * and start the transfer.
391  */
392 static void pl08x_start_next_txd(struct pl08x_dma_chan *plchan)
393 {
394 	struct pl08x_driver_data *pl08x = plchan->host;
395 	struct pl08x_phy_chan *phychan = plchan->phychan;
396 	struct virt_dma_desc *vd = vchan_next_desc(&plchan->vc);
397 	struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
398 	u32 val;
399 
400 	list_del(&txd->vd.node);
401 
402 	plchan->at = txd;
403 
404 	/* Wait for channel inactive */
405 	while (pl08x_phy_channel_busy(phychan))
406 		cpu_relax();
407 
408 	pl08x_write_lli(pl08x, phychan, &txd->llis_va[0], txd->ccfg);
409 
410 	/* Enable the DMA channel */
411 	/* Do not access config register until channel shows as disabled */
412 	while (readl(pl08x->base + PL080_EN_CHAN) & (1 << phychan->id))
413 		cpu_relax();
414 
415 	/* Do not access config register until channel shows as inactive */
416 	val = readl(phychan->reg_config);
417 	while ((val & PL080_CONFIG_ACTIVE) || (val & PL080_CONFIG_ENABLE))
418 		val = readl(phychan->reg_config);
419 
420 	writel(val | PL080_CONFIG_ENABLE, phychan->reg_config);
421 }
422 
423 /*
424  * Pause the channel by setting the HALT bit.
425  *
426  * For M->P transfers, pause the DMAC first and then stop the peripheral -
427  * the FIFO can only drain if the peripheral is still requesting data.
428  * (note: this can still timeout if the DMAC FIFO never drains of data.)
429  *
430  * For P->M transfers, disable the peripheral first to stop it filling
431  * the DMAC FIFO, and then pause the DMAC.
432  */
433 static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch)
434 {
435 	u32 val;
436 	int timeout;
437 
438 	/* Set the HALT bit and wait for the FIFO to drain */
439 	val = readl(ch->reg_config);
440 	val |= PL080_CONFIG_HALT;
441 	writel(val, ch->reg_config);
442 
443 	/* Wait for channel inactive */
444 	for (timeout = 1000; timeout; timeout--) {
445 		if (!pl08x_phy_channel_busy(ch))
446 			break;
447 		udelay(1);
448 	}
449 	if (pl08x_phy_channel_busy(ch))
450 		pr_err("pl08x: channel%u timeout waiting for pause\n", ch->id);
451 }
452 
453 static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch)
454 {
455 	u32 val;
456 
457 	/* Clear the HALT bit */
458 	val = readl(ch->reg_config);
459 	val &= ~PL080_CONFIG_HALT;
460 	writel(val, ch->reg_config);
461 }
462 
463 /*
464  * pl08x_terminate_phy_chan() stops the channel, clears the FIFO and
465  * clears any pending interrupt status.  This should not be used for
466  * an on-going transfer, but as a method of shutting down a channel
467  * (eg, when it's no longer used) or terminating a transfer.
468  */
469 static void pl08x_terminate_phy_chan(struct pl08x_driver_data *pl08x,
470 	struct pl08x_phy_chan *ch)
471 {
472 	u32 val = readl(ch->reg_config);
473 
474 	val &= ~(PL080_CONFIG_ENABLE | PL080_CONFIG_ERR_IRQ_MASK |
475 	         PL080_CONFIG_TC_IRQ_MASK);
476 
477 	writel(val, ch->reg_config);
478 
479 	writel(1 << ch->id, pl08x->base + PL080_ERR_CLEAR);
480 	writel(1 << ch->id, pl08x->base + PL080_TC_CLEAR);
481 }
482 
483 static inline u32 get_bytes_in_cctl(u32 cctl)
484 {
485 	/* The source width defines the number of bytes */
486 	u32 bytes = cctl & PL080_CONTROL_TRANSFER_SIZE_MASK;
487 
488 	cctl &= PL080_CONTROL_SWIDTH_MASK;
489 
490 	switch (cctl >> PL080_CONTROL_SWIDTH_SHIFT) {
491 	case PL080_WIDTH_8BIT:
492 		break;
493 	case PL080_WIDTH_16BIT:
494 		bytes *= 2;
495 		break;
496 	case PL080_WIDTH_32BIT:
497 		bytes *= 4;
498 		break;
499 	}
500 	return bytes;
501 }
502 
503 static inline u32 get_bytes_in_cctl_pl080s(u32 cctl, u32 cctl1)
504 {
505 	/* The source width defines the number of bytes */
506 	u32 bytes = cctl1 & PL080S_CONTROL_TRANSFER_SIZE_MASK;
507 
508 	cctl &= PL080_CONTROL_SWIDTH_MASK;
509 
510 	switch (cctl >> PL080_CONTROL_SWIDTH_SHIFT) {
511 	case PL080_WIDTH_8BIT:
512 		break;
513 	case PL080_WIDTH_16BIT:
514 		bytes *= 2;
515 		break;
516 	case PL080_WIDTH_32BIT:
517 		bytes *= 4;
518 		break;
519 	}
520 	return bytes;
521 }
522 
523 /* The channel should be paused when calling this */
524 static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan)
525 {
526 	struct pl08x_driver_data *pl08x = plchan->host;
527 	const u32 *llis_va, *llis_va_limit;
528 	struct pl08x_phy_chan *ch;
529 	dma_addr_t llis_bus;
530 	struct pl08x_txd *txd;
531 	u32 llis_max_words;
532 	size_t bytes;
533 	u32 clli;
534 
535 	ch = plchan->phychan;
536 	txd = plchan->at;
537 
538 	if (!ch || !txd)
539 		return 0;
540 
541 	/*
542 	 * Follow the LLIs to get the number of remaining
543 	 * bytes in the currently active transaction.
544 	 */
545 	clli = readl(ch->base + PL080_CH_LLI) & ~PL080_LLI_LM_AHB2;
546 
547 	/* First get the remaining bytes in the active transfer */
548 	if (pl08x->vd->pl080s)
549 		bytes = get_bytes_in_cctl_pl080s(
550 				readl(ch->base + PL080_CH_CONTROL),
551 				readl(ch->base + PL080S_CH_CONTROL2));
552 	else
553 		bytes = get_bytes_in_cctl(readl(ch->base + PL080_CH_CONTROL));
554 
555 	if (!clli)
556 		return bytes;
557 
558 	llis_va = txd->llis_va;
559 	llis_bus = txd->llis_bus;
560 
561 	llis_max_words = pl08x->lli_words * MAX_NUM_TSFR_LLIS;
562 	BUG_ON(clli < llis_bus || clli >= llis_bus +
563 						sizeof(u32) * llis_max_words);
564 
565 	/*
566 	 * Locate the next LLI - as this is an array,
567 	 * it's simple maths to find.
568 	 */
569 	llis_va += (clli - llis_bus) / sizeof(u32);
570 
571 	llis_va_limit = llis_va + llis_max_words;
572 
573 	for (; llis_va < llis_va_limit; llis_va += pl08x->lli_words) {
574 		if (pl08x->vd->pl080s)
575 			bytes += get_bytes_in_cctl_pl080s(
576 						llis_va[PL080_LLI_CCTL],
577 						llis_va[PL080S_LLI_CCTL2]);
578 		else
579 			bytes += get_bytes_in_cctl(llis_va[PL080_LLI_CCTL]);
580 
581 		/*
582 		 * A LLI pointer going backward terminates the LLI list
583 		 */
584 		if (llis_va[PL080_LLI_LLI] <= clli)
585 			break;
586 	}
587 
588 	return bytes;
589 }
590 
591 /*
592  * Allocate a physical channel for a virtual channel
593  *
594  * Try to locate a physical channel to be used for this transfer. If all
595  * are taken return NULL and the requester will have to cope by using
596  * some fallback PIO mode or retrying later.
597  */
598 static struct pl08x_phy_chan *
599 pl08x_get_phy_channel(struct pl08x_driver_data *pl08x,
600 		      struct pl08x_dma_chan *virt_chan)
601 {
602 	struct pl08x_phy_chan *ch = NULL;
603 	unsigned long flags;
604 	int i;
605 
606 	for (i = 0; i < pl08x->vd->channels; i++) {
607 		ch = &pl08x->phy_chans[i];
608 
609 		spin_lock_irqsave(&ch->lock, flags);
610 
611 		if (!ch->locked && !ch->serving) {
612 			ch->serving = virt_chan;
613 			spin_unlock_irqrestore(&ch->lock, flags);
614 			break;
615 		}
616 
617 		spin_unlock_irqrestore(&ch->lock, flags);
618 	}
619 
620 	if (i == pl08x->vd->channels) {
621 		/* No physical channel available, cope with it */
622 		return NULL;
623 	}
624 
625 	return ch;
626 }
627 
628 /* Mark the physical channel as free.  Note, this write is atomic. */
629 static inline void pl08x_put_phy_channel(struct pl08x_driver_data *pl08x,
630 					 struct pl08x_phy_chan *ch)
631 {
632 	ch->serving = NULL;
633 }
634 
635 /*
636  * Try to allocate a physical channel.  When successful, assign it to
637  * this virtual channel, and initiate the next descriptor.  The
638  * virtual channel lock must be held at this point.
639  */
640 static void pl08x_phy_alloc_and_start(struct pl08x_dma_chan *plchan)
641 {
642 	struct pl08x_driver_data *pl08x = plchan->host;
643 	struct pl08x_phy_chan *ch;
644 
645 	ch = pl08x_get_phy_channel(pl08x, plchan);
646 	if (!ch) {
647 		dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name);
648 		plchan->state = PL08X_CHAN_WAITING;
649 		return;
650 	}
651 
652 	dev_dbg(&pl08x->adev->dev, "allocated physical channel %d for xfer on %s\n",
653 		ch->id, plchan->name);
654 
655 	plchan->phychan = ch;
656 	plchan->state = PL08X_CHAN_RUNNING;
657 	pl08x_start_next_txd(plchan);
658 }
659 
660 static void pl08x_phy_reassign_start(struct pl08x_phy_chan *ch,
661 	struct pl08x_dma_chan *plchan)
662 {
663 	struct pl08x_driver_data *pl08x = plchan->host;
664 
665 	dev_dbg(&pl08x->adev->dev, "reassigned physical channel %d for xfer on %s\n",
666 		ch->id, plchan->name);
667 
668 	/*
669 	 * We do this without taking the lock; we're really only concerned
670 	 * about whether this pointer is NULL or not, and we're guaranteed
671 	 * that this will only be called when it _already_ is non-NULL.
672 	 */
673 	ch->serving = plchan;
674 	plchan->phychan = ch;
675 	plchan->state = PL08X_CHAN_RUNNING;
676 	pl08x_start_next_txd(plchan);
677 }
678 
679 /*
680  * Free a physical DMA channel, potentially reallocating it to another
681  * virtual channel if we have any pending.
682  */
683 static void pl08x_phy_free(struct pl08x_dma_chan *plchan)
684 {
685 	struct pl08x_driver_data *pl08x = plchan->host;
686 	struct pl08x_dma_chan *p, *next;
687 
688  retry:
689 	next = NULL;
690 
691 	/* Find a waiting virtual channel for the next transfer. */
692 	list_for_each_entry(p, &pl08x->memcpy.channels, vc.chan.device_node)
693 		if (p->state == PL08X_CHAN_WAITING) {
694 			next = p;
695 			break;
696 		}
697 
698 	if (!next) {
699 		list_for_each_entry(p, &pl08x->slave.channels, vc.chan.device_node)
700 			if (p->state == PL08X_CHAN_WAITING) {
701 				next = p;
702 				break;
703 			}
704 	}
705 
706 	/* Ensure that the physical channel is stopped */
707 	pl08x_terminate_phy_chan(pl08x, plchan->phychan);
708 
709 	if (next) {
710 		bool success;
711 
712 		/*
713 		 * Eww.  We know this isn't going to deadlock
714 		 * but lockdep probably doesn't.
715 		 */
716 		spin_lock(&next->vc.lock);
717 		/* Re-check the state now that we have the lock */
718 		success = next->state == PL08X_CHAN_WAITING;
719 		if (success)
720 			pl08x_phy_reassign_start(plchan->phychan, next);
721 		spin_unlock(&next->vc.lock);
722 
723 		/* If the state changed, try to find another channel */
724 		if (!success)
725 			goto retry;
726 	} else {
727 		/* No more jobs, so free up the physical channel */
728 		pl08x_put_phy_channel(pl08x, plchan->phychan);
729 	}
730 
731 	plchan->phychan = NULL;
732 	plchan->state = PL08X_CHAN_IDLE;
733 }
734 
735 /*
736  * LLI handling
737  */
738 
739 static inline unsigned int pl08x_get_bytes_for_cctl(unsigned int coded)
740 {
741 	switch (coded) {
742 	case PL080_WIDTH_8BIT:
743 		return 1;
744 	case PL080_WIDTH_16BIT:
745 		return 2;
746 	case PL080_WIDTH_32BIT:
747 		return 4;
748 	default:
749 		break;
750 	}
751 	BUG();
752 	return 0;
753 }
754 
755 static inline u32 pl08x_cctl_bits(u32 cctl, u8 srcwidth, u8 dstwidth,
756 				  size_t tsize)
757 {
758 	u32 retbits = cctl;
759 
760 	/* Remove all src, dst and transfer size bits */
761 	retbits &= ~PL080_CONTROL_DWIDTH_MASK;
762 	retbits &= ~PL080_CONTROL_SWIDTH_MASK;
763 	retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK;
764 
765 	/* Then set the bits according to the parameters */
766 	switch (srcwidth) {
767 	case 1:
768 		retbits |= PL080_WIDTH_8BIT << PL080_CONTROL_SWIDTH_SHIFT;
769 		break;
770 	case 2:
771 		retbits |= PL080_WIDTH_16BIT << PL080_CONTROL_SWIDTH_SHIFT;
772 		break;
773 	case 4:
774 		retbits |= PL080_WIDTH_32BIT << PL080_CONTROL_SWIDTH_SHIFT;
775 		break;
776 	default:
777 		BUG();
778 		break;
779 	}
780 
781 	switch (dstwidth) {
782 	case 1:
783 		retbits |= PL080_WIDTH_8BIT << PL080_CONTROL_DWIDTH_SHIFT;
784 		break;
785 	case 2:
786 		retbits |= PL080_WIDTH_16BIT << PL080_CONTROL_DWIDTH_SHIFT;
787 		break;
788 	case 4:
789 		retbits |= PL080_WIDTH_32BIT << PL080_CONTROL_DWIDTH_SHIFT;
790 		break;
791 	default:
792 		BUG();
793 		break;
794 	}
795 
796 	tsize &= PL080_CONTROL_TRANSFER_SIZE_MASK;
797 	retbits |= tsize << PL080_CONTROL_TRANSFER_SIZE_SHIFT;
798 	return retbits;
799 }
800 
801 struct pl08x_lli_build_data {
802 	struct pl08x_txd *txd;
803 	struct pl08x_bus_data srcbus;
804 	struct pl08x_bus_data dstbus;
805 	size_t remainder;
806 	u32 lli_bus;
807 };
808 
809 /*
810  * Autoselect a master bus to use for the transfer. Slave will be the chosen as
811  * victim in case src & dest are not similarly aligned. i.e. If after aligning
812  * masters address with width requirements of transfer (by sending few byte by
813  * byte data), slave is still not aligned, then its width will be reduced to
814  * BYTE.
815  * - prefers the destination bus if both available
816  * - prefers bus with fixed address (i.e. peripheral)
817  */
818 static void pl08x_choose_master_bus(struct pl08x_lli_build_data *bd,
819 	struct pl08x_bus_data **mbus, struct pl08x_bus_data **sbus, u32 cctl)
820 {
821 	if (!(cctl & PL080_CONTROL_DST_INCR)) {
822 		*mbus = &bd->dstbus;
823 		*sbus = &bd->srcbus;
824 	} else if (!(cctl & PL080_CONTROL_SRC_INCR)) {
825 		*mbus = &bd->srcbus;
826 		*sbus = &bd->dstbus;
827 	} else {
828 		if (bd->dstbus.buswidth >= bd->srcbus.buswidth) {
829 			*mbus = &bd->dstbus;
830 			*sbus = &bd->srcbus;
831 		} else {
832 			*mbus = &bd->srcbus;
833 			*sbus = &bd->dstbus;
834 		}
835 	}
836 }
837 
838 /*
839  * Fills in one LLI for a certain transfer descriptor and advance the counter
840  */
841 static void pl08x_fill_lli_for_desc(struct pl08x_driver_data *pl08x,
842 				    struct pl08x_lli_build_data *bd,
843 				    int num_llis, int len, u32 cctl, u32 cctl2)
844 {
845 	u32 offset = num_llis * pl08x->lli_words;
846 	u32 *llis_va = bd->txd->llis_va + offset;
847 	dma_addr_t llis_bus = bd->txd->llis_bus;
848 
849 	BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS);
850 
851 	/* Advance the offset to next LLI. */
852 	offset += pl08x->lli_words;
853 
854 	llis_va[PL080_LLI_SRC] = bd->srcbus.addr;
855 	llis_va[PL080_LLI_DST] = bd->dstbus.addr;
856 	llis_va[PL080_LLI_LLI] = (llis_bus + sizeof(u32) * offset);
857 	llis_va[PL080_LLI_LLI] |= bd->lli_bus;
858 	llis_va[PL080_LLI_CCTL] = cctl;
859 	if (pl08x->vd->pl080s)
860 		llis_va[PL080S_LLI_CCTL2] = cctl2;
861 
862 	if (cctl & PL080_CONTROL_SRC_INCR)
863 		bd->srcbus.addr += len;
864 	if (cctl & PL080_CONTROL_DST_INCR)
865 		bd->dstbus.addr += len;
866 
867 	BUG_ON(bd->remainder < len);
868 
869 	bd->remainder -= len;
870 }
871 
872 static inline void prep_byte_width_lli(struct pl08x_driver_data *pl08x,
873 			struct pl08x_lli_build_data *bd, u32 *cctl, u32 len,
874 			int num_llis, size_t *total_bytes)
875 {
876 	*cctl = pl08x_cctl_bits(*cctl, 1, 1, len);
877 	pl08x_fill_lli_for_desc(pl08x, bd, num_llis, len, *cctl, len);
878 	(*total_bytes) += len;
879 }
880 
881 #ifdef VERBOSE_DEBUG
882 static void pl08x_dump_lli(struct pl08x_driver_data *pl08x,
883 			   const u32 *llis_va, int num_llis)
884 {
885 	int i;
886 
887 	if (pl08x->vd->pl080s) {
888 		dev_vdbg(&pl08x->adev->dev,
889 			"%-3s %-9s  %-10s %-10s %-10s %-10s %s\n",
890 			"lli", "", "csrc", "cdst", "clli", "cctl", "cctl2");
891 		for (i = 0; i < num_llis; i++) {
892 			dev_vdbg(&pl08x->adev->dev,
893 				"%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
894 				i, llis_va, llis_va[PL080_LLI_SRC],
895 				llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI],
896 				llis_va[PL080_LLI_CCTL],
897 				llis_va[PL080S_LLI_CCTL2]);
898 			llis_va += pl08x->lli_words;
899 		}
900 	} else {
901 		dev_vdbg(&pl08x->adev->dev,
902 			"%-3s %-9s  %-10s %-10s %-10s %s\n",
903 			"lli", "", "csrc", "cdst", "clli", "cctl");
904 		for (i = 0; i < num_llis; i++) {
905 			dev_vdbg(&pl08x->adev->dev,
906 				"%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x\n",
907 				i, llis_va, llis_va[PL080_LLI_SRC],
908 				llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI],
909 				llis_va[PL080_LLI_CCTL]);
910 			llis_va += pl08x->lli_words;
911 		}
912 	}
913 }
914 #else
915 static inline void pl08x_dump_lli(struct pl08x_driver_data *pl08x,
916 				  const u32 *llis_va, int num_llis) {}
917 #endif
918 
919 /*
920  * This fills in the table of LLIs for the transfer descriptor
921  * Note that we assume we never have to change the burst sizes
922  * Return 0 for error
923  */
924 static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
925 			      struct pl08x_txd *txd)
926 {
927 	struct pl08x_bus_data *mbus, *sbus;
928 	struct pl08x_lli_build_data bd;
929 	int num_llis = 0;
930 	u32 cctl, early_bytes = 0;
931 	size_t max_bytes_per_lli, total_bytes;
932 	u32 *llis_va, *last_lli;
933 	struct pl08x_sg *dsg;
934 
935 	txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus);
936 	if (!txd->llis_va) {
937 		dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__);
938 		return 0;
939 	}
940 
941 	bd.txd = txd;
942 	bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0;
943 	cctl = txd->cctl;
944 
945 	/* Find maximum width of the source bus */
946 	bd.srcbus.maxwidth =
947 		pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_SWIDTH_MASK) >>
948 				       PL080_CONTROL_SWIDTH_SHIFT);
949 
950 	/* Find maximum width of the destination bus */
951 	bd.dstbus.maxwidth =
952 		pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_DWIDTH_MASK) >>
953 				       PL080_CONTROL_DWIDTH_SHIFT);
954 
955 	list_for_each_entry(dsg, &txd->dsg_list, node) {
956 		total_bytes = 0;
957 		cctl = txd->cctl;
958 
959 		bd.srcbus.addr = dsg->src_addr;
960 		bd.dstbus.addr = dsg->dst_addr;
961 		bd.remainder = dsg->len;
962 		bd.srcbus.buswidth = bd.srcbus.maxwidth;
963 		bd.dstbus.buswidth = bd.dstbus.maxwidth;
964 
965 		pl08x_choose_master_bus(&bd, &mbus, &sbus, cctl);
966 
967 		dev_vdbg(&pl08x->adev->dev,
968 			"src=0x%08llx%s/%u dst=0x%08llx%s/%u len=%zu\n",
969 			(u64)bd.srcbus.addr,
970 			cctl & PL080_CONTROL_SRC_INCR ? "+" : "",
971 			bd.srcbus.buswidth,
972 			(u64)bd.dstbus.addr,
973 			cctl & PL080_CONTROL_DST_INCR ? "+" : "",
974 			bd.dstbus.buswidth,
975 			bd.remainder);
976 		dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n",
977 			mbus == &bd.srcbus ? "src" : "dst",
978 			sbus == &bd.srcbus ? "src" : "dst");
979 
980 		/*
981 		 * Zero length is only allowed if all these requirements are
982 		 * met:
983 		 * - flow controller is peripheral.
984 		 * - src.addr is aligned to src.width
985 		 * - dst.addr is aligned to dst.width
986 		 *
987 		 * sg_len == 1 should be true, as there can be two cases here:
988 		 *
989 		 * - Memory addresses are contiguous and are not scattered.
990 		 *   Here, Only one sg will be passed by user driver, with
991 		 *   memory address and zero length. We pass this to controller
992 		 *   and after the transfer it will receive the last burst
993 		 *   request from peripheral and so transfer finishes.
994 		 *
995 		 * - Memory addresses are scattered and are not contiguous.
996 		 *   Here, Obviously as DMA controller doesn't know when a lli's
997 		 *   transfer gets over, it can't load next lli. So in this
998 		 *   case, there has to be an assumption that only one lli is
999 		 *   supported. Thus, we can't have scattered addresses.
1000 		 */
1001 		if (!bd.remainder) {
1002 			u32 fc = (txd->ccfg & PL080_CONFIG_FLOW_CONTROL_MASK) >>
1003 				PL080_CONFIG_FLOW_CONTROL_SHIFT;
1004 			if (!((fc >= PL080_FLOW_SRC2DST_DST) &&
1005 					(fc <= PL080_FLOW_SRC2DST_SRC))) {
1006 				dev_err(&pl08x->adev->dev, "%s sg len can't be zero",
1007 					__func__);
1008 				return 0;
1009 			}
1010 
1011 			if (!IS_BUS_ALIGNED(&bd.srcbus) ||
1012 				!IS_BUS_ALIGNED(&bd.dstbus)) {
1013 				dev_err(&pl08x->adev->dev,
1014 					"%s src & dst address must be aligned to src"
1015 					" & dst width if peripheral is flow controller",
1016 					__func__);
1017 				return 0;
1018 			}
1019 
1020 			cctl = pl08x_cctl_bits(cctl, bd.srcbus.buswidth,
1021 					bd.dstbus.buswidth, 0);
1022 			pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++,
1023 					0, cctl, 0);
1024 			break;
1025 		}
1026 
1027 		/*
1028 		 * Send byte by byte for following cases
1029 		 * - Less than a bus width available
1030 		 * - until master bus is aligned
1031 		 */
1032 		if (bd.remainder < mbus->buswidth)
1033 			early_bytes = bd.remainder;
1034 		else if (!IS_BUS_ALIGNED(mbus)) {
1035 			early_bytes = mbus->buswidth -
1036 				(mbus->addr & (mbus->buswidth - 1));
1037 			if ((bd.remainder - early_bytes) < mbus->buswidth)
1038 				early_bytes = bd.remainder;
1039 		}
1040 
1041 		if (early_bytes) {
1042 			dev_vdbg(&pl08x->adev->dev,
1043 				"%s byte width LLIs (remain 0x%08zx)\n",
1044 				__func__, bd.remainder);
1045 			prep_byte_width_lli(pl08x, &bd, &cctl, early_bytes,
1046 				num_llis++, &total_bytes);
1047 		}
1048 
1049 		if (bd.remainder) {
1050 			/*
1051 			 * Master now aligned
1052 			 * - if slave is not then we must set its width down
1053 			 */
1054 			if (!IS_BUS_ALIGNED(sbus)) {
1055 				dev_dbg(&pl08x->adev->dev,
1056 					"%s set down bus width to one byte\n",
1057 					__func__);
1058 
1059 				sbus->buswidth = 1;
1060 			}
1061 
1062 			/*
1063 			 * Bytes transferred = tsize * src width, not
1064 			 * MIN(buswidths)
1065 			 */
1066 			max_bytes_per_lli = bd.srcbus.buswidth *
1067 						pl08x->vd->max_transfer_size;
1068 			dev_vdbg(&pl08x->adev->dev,
1069 				"%s max bytes per lli = %zu\n",
1070 				__func__, max_bytes_per_lli);
1071 
1072 			/*
1073 			 * Make largest possible LLIs until less than one bus
1074 			 * width left
1075 			 */
1076 			while (bd.remainder > (mbus->buswidth - 1)) {
1077 				size_t lli_len, tsize, width;
1078 
1079 				/*
1080 				 * If enough left try to send max possible,
1081 				 * otherwise try to send the remainder
1082 				 */
1083 				lli_len = min(bd.remainder, max_bytes_per_lli);
1084 
1085 				/*
1086 				 * Check against maximum bus alignment:
1087 				 * Calculate actual transfer size in relation to
1088 				 * bus width an get a maximum remainder of the
1089 				 * highest bus width - 1
1090 				 */
1091 				width = max(mbus->buswidth, sbus->buswidth);
1092 				lli_len = (lli_len / width) * width;
1093 				tsize = lli_len / bd.srcbus.buswidth;
1094 
1095 				dev_vdbg(&pl08x->adev->dev,
1096 					"%s fill lli with single lli chunk of "
1097 					"size 0x%08zx (remainder 0x%08zx)\n",
1098 					__func__, lli_len, bd.remainder);
1099 
1100 				cctl = pl08x_cctl_bits(cctl, bd.srcbus.buswidth,
1101 					bd.dstbus.buswidth, tsize);
1102 				pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++,
1103 						lli_len, cctl, tsize);
1104 				total_bytes += lli_len;
1105 			}
1106 
1107 			/*
1108 			 * Send any odd bytes
1109 			 */
1110 			if (bd.remainder) {
1111 				dev_vdbg(&pl08x->adev->dev,
1112 					"%s align with boundary, send odd bytes (remain %zu)\n",
1113 					__func__, bd.remainder);
1114 				prep_byte_width_lli(pl08x, &bd, &cctl,
1115 					bd.remainder, num_llis++, &total_bytes);
1116 			}
1117 		}
1118 
1119 		if (total_bytes != dsg->len) {
1120 			dev_err(&pl08x->adev->dev,
1121 				"%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n",
1122 				__func__, total_bytes, dsg->len);
1123 			return 0;
1124 		}
1125 
1126 		if (num_llis >= MAX_NUM_TSFR_LLIS) {
1127 			dev_err(&pl08x->adev->dev,
1128 				"%s need to increase MAX_NUM_TSFR_LLIS from 0x%08x\n",
1129 				__func__, MAX_NUM_TSFR_LLIS);
1130 			return 0;
1131 		}
1132 	}
1133 
1134 	llis_va = txd->llis_va;
1135 	last_lli = llis_va + (num_llis - 1) * pl08x->lli_words;
1136 
1137 	if (txd->cyclic) {
1138 		/* Link back to the first LLI. */
1139 		last_lli[PL080_LLI_LLI] = txd->llis_bus | bd.lli_bus;
1140 	} else {
1141 		/* The final LLI terminates the LLI. */
1142 		last_lli[PL080_LLI_LLI] = 0;
1143 		/* The final LLI element shall also fire an interrupt. */
1144 		last_lli[PL080_LLI_CCTL] |= PL080_CONTROL_TC_IRQ_EN;
1145 	}
1146 
1147 	pl08x_dump_lli(pl08x, llis_va, num_llis);
1148 
1149 	return num_llis;
1150 }
1151 
1152 static void pl08x_free_txd(struct pl08x_driver_data *pl08x,
1153 			   struct pl08x_txd *txd)
1154 {
1155 	struct pl08x_sg *dsg, *_dsg;
1156 
1157 	if (txd->llis_va)
1158 		dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus);
1159 
1160 	list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) {
1161 		list_del(&dsg->node);
1162 		kfree(dsg);
1163 	}
1164 
1165 	kfree(txd);
1166 }
1167 
1168 static void pl08x_desc_free(struct virt_dma_desc *vd)
1169 {
1170 	struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
1171 	struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan);
1172 
1173 	dma_descriptor_unmap(&vd->tx);
1174 	if (!txd->done)
1175 		pl08x_release_mux(plchan);
1176 
1177 	pl08x_free_txd(plchan->host, txd);
1178 }
1179 
1180 static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x,
1181 				struct pl08x_dma_chan *plchan)
1182 {
1183 	LIST_HEAD(head);
1184 
1185 	vchan_get_all_descriptors(&plchan->vc, &head);
1186 	vchan_dma_desc_free_list(&plchan->vc, &head);
1187 }
1188 
1189 /*
1190  * The DMA ENGINE API
1191  */
1192 static int pl08x_alloc_chan_resources(struct dma_chan *chan)
1193 {
1194 	return 0;
1195 }
1196 
1197 static void pl08x_free_chan_resources(struct dma_chan *chan)
1198 {
1199 	/* Ensure all queued descriptors are freed */
1200 	vchan_free_chan_resources(to_virt_chan(chan));
1201 }
1202 
1203 static struct dma_async_tx_descriptor *pl08x_prep_dma_interrupt(
1204 		struct dma_chan *chan, unsigned long flags)
1205 {
1206 	struct dma_async_tx_descriptor *retval = NULL;
1207 
1208 	return retval;
1209 }
1210 
1211 /*
1212  * Code accessing dma_async_is_complete() in a tight loop may give problems.
1213  * If slaves are relying on interrupts to signal completion this function
1214  * must not be called with interrupts disabled.
1215  */
1216 static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan,
1217 		dma_cookie_t cookie, struct dma_tx_state *txstate)
1218 {
1219 	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1220 	struct virt_dma_desc *vd;
1221 	unsigned long flags;
1222 	enum dma_status ret;
1223 	size_t bytes = 0;
1224 
1225 	ret = dma_cookie_status(chan, cookie, txstate);
1226 	if (ret == DMA_COMPLETE)
1227 		return ret;
1228 
1229 	/*
1230 	 * There's no point calculating the residue if there's
1231 	 * no txstate to store the value.
1232 	 */
1233 	if (!txstate) {
1234 		if (plchan->state == PL08X_CHAN_PAUSED)
1235 			ret = DMA_PAUSED;
1236 		return ret;
1237 	}
1238 
1239 	spin_lock_irqsave(&plchan->vc.lock, flags);
1240 	ret = dma_cookie_status(chan, cookie, txstate);
1241 	if (ret != DMA_COMPLETE) {
1242 		vd = vchan_find_desc(&plchan->vc, cookie);
1243 		if (vd) {
1244 			/* On the issued list, so hasn't been processed yet */
1245 			struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
1246 			struct pl08x_sg *dsg;
1247 
1248 			list_for_each_entry(dsg, &txd->dsg_list, node)
1249 				bytes += dsg->len;
1250 		} else {
1251 			bytes = pl08x_getbytes_chan(plchan);
1252 		}
1253 	}
1254 	spin_unlock_irqrestore(&plchan->vc.lock, flags);
1255 
1256 	/*
1257 	 * This cookie not complete yet
1258 	 * Get number of bytes left in the active transactions and queue
1259 	 */
1260 	dma_set_residue(txstate, bytes);
1261 
1262 	if (plchan->state == PL08X_CHAN_PAUSED && ret == DMA_IN_PROGRESS)
1263 		ret = DMA_PAUSED;
1264 
1265 	/* Whether waiting or running, we're in progress */
1266 	return ret;
1267 }
1268 
1269 /* PrimeCell DMA extension */
1270 struct burst_table {
1271 	u32 burstwords;
1272 	u32 reg;
1273 };
1274 
1275 static const struct burst_table burst_sizes[] = {
1276 	{
1277 		.burstwords = 256,
1278 		.reg = PL080_BSIZE_256,
1279 	},
1280 	{
1281 		.burstwords = 128,
1282 		.reg = PL080_BSIZE_128,
1283 	},
1284 	{
1285 		.burstwords = 64,
1286 		.reg = PL080_BSIZE_64,
1287 	},
1288 	{
1289 		.burstwords = 32,
1290 		.reg = PL080_BSIZE_32,
1291 	},
1292 	{
1293 		.burstwords = 16,
1294 		.reg = PL080_BSIZE_16,
1295 	},
1296 	{
1297 		.burstwords = 8,
1298 		.reg = PL080_BSIZE_8,
1299 	},
1300 	{
1301 		.burstwords = 4,
1302 		.reg = PL080_BSIZE_4,
1303 	},
1304 	{
1305 		.burstwords = 0,
1306 		.reg = PL080_BSIZE_1,
1307 	},
1308 };
1309 
1310 /*
1311  * Given the source and destination available bus masks, select which
1312  * will be routed to each port.  We try to have source and destination
1313  * on separate ports, but always respect the allowable settings.
1314  */
1315 static u32 pl08x_select_bus(u8 src, u8 dst)
1316 {
1317 	u32 cctl = 0;
1318 
1319 	if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1)))
1320 		cctl |= PL080_CONTROL_DST_AHB2;
1321 	if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2)))
1322 		cctl |= PL080_CONTROL_SRC_AHB2;
1323 
1324 	return cctl;
1325 }
1326 
1327 static u32 pl08x_cctl(u32 cctl)
1328 {
1329 	cctl &= ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 |
1330 		  PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR |
1331 		  PL080_CONTROL_PROT_MASK);
1332 
1333 	/* Access the cell in privileged mode, non-bufferable, non-cacheable */
1334 	return cctl | PL080_CONTROL_PROT_SYS;
1335 }
1336 
1337 static u32 pl08x_width(enum dma_slave_buswidth width)
1338 {
1339 	switch (width) {
1340 	case DMA_SLAVE_BUSWIDTH_1_BYTE:
1341 		return PL080_WIDTH_8BIT;
1342 	case DMA_SLAVE_BUSWIDTH_2_BYTES:
1343 		return PL080_WIDTH_16BIT;
1344 	case DMA_SLAVE_BUSWIDTH_4_BYTES:
1345 		return PL080_WIDTH_32BIT;
1346 	default:
1347 		return ~0;
1348 	}
1349 }
1350 
1351 static u32 pl08x_burst(u32 maxburst)
1352 {
1353 	int i;
1354 
1355 	for (i = 0; i < ARRAY_SIZE(burst_sizes); i++)
1356 		if (burst_sizes[i].burstwords <= maxburst)
1357 			break;
1358 
1359 	return burst_sizes[i].reg;
1360 }
1361 
1362 static u32 pl08x_get_cctl(struct pl08x_dma_chan *plchan,
1363 	enum dma_slave_buswidth addr_width, u32 maxburst)
1364 {
1365 	u32 width, burst, cctl = 0;
1366 
1367 	width = pl08x_width(addr_width);
1368 	if (width == ~0)
1369 		return ~0;
1370 
1371 	cctl |= width << PL080_CONTROL_SWIDTH_SHIFT;
1372 	cctl |= width << PL080_CONTROL_DWIDTH_SHIFT;
1373 
1374 	/*
1375 	 * If this channel will only request single transfers, set this
1376 	 * down to ONE element.  Also select one element if no maxburst
1377 	 * is specified.
1378 	 */
1379 	if (plchan->cd->single)
1380 		maxburst = 1;
1381 
1382 	burst = pl08x_burst(maxburst);
1383 	cctl |= burst << PL080_CONTROL_SB_SIZE_SHIFT;
1384 	cctl |= burst << PL080_CONTROL_DB_SIZE_SHIFT;
1385 
1386 	return pl08x_cctl(cctl);
1387 }
1388 
1389 /*
1390  * Slave transactions callback to the slave device to allow
1391  * synchronization of slave DMA signals with the DMAC enable
1392  */
1393 static void pl08x_issue_pending(struct dma_chan *chan)
1394 {
1395 	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1396 	unsigned long flags;
1397 
1398 	spin_lock_irqsave(&plchan->vc.lock, flags);
1399 	if (vchan_issue_pending(&plchan->vc)) {
1400 		if (!plchan->phychan && plchan->state != PL08X_CHAN_WAITING)
1401 			pl08x_phy_alloc_and_start(plchan);
1402 	}
1403 	spin_unlock_irqrestore(&plchan->vc.lock, flags);
1404 }
1405 
1406 static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan)
1407 {
1408 	struct pl08x_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
1409 
1410 	if (txd) {
1411 		INIT_LIST_HEAD(&txd->dsg_list);
1412 
1413 		/* Always enable error and terminal interrupts */
1414 		txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK |
1415 			    PL080_CONFIG_TC_IRQ_MASK;
1416 	}
1417 	return txd;
1418 }
1419 
1420 /*
1421  * Initialize a descriptor to be used by memcpy submit
1422  */
1423 static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy(
1424 		struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1425 		size_t len, unsigned long flags)
1426 {
1427 	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1428 	struct pl08x_driver_data *pl08x = plchan->host;
1429 	struct pl08x_txd *txd;
1430 	struct pl08x_sg *dsg;
1431 	int ret;
1432 
1433 	txd = pl08x_get_txd(plchan);
1434 	if (!txd) {
1435 		dev_err(&pl08x->adev->dev,
1436 			"%s no memory for descriptor\n", __func__);
1437 		return NULL;
1438 	}
1439 
1440 	dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);
1441 	if (!dsg) {
1442 		pl08x_free_txd(pl08x, txd);
1443 		dev_err(&pl08x->adev->dev, "%s no memory for pl080 sg\n",
1444 				__func__);
1445 		return NULL;
1446 	}
1447 	list_add_tail(&dsg->node, &txd->dsg_list);
1448 
1449 	dsg->src_addr = src;
1450 	dsg->dst_addr = dest;
1451 	dsg->len = len;
1452 
1453 	/* Set platform data for m2m */
1454 	txd->ccfg |= PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT;
1455 	txd->cctl = pl08x->pd->memcpy_channel.cctl_memcpy &
1456 			~(PL080_CONTROL_DST_AHB2 | PL080_CONTROL_SRC_AHB2);
1457 
1458 	/* Both to be incremented or the code will break */
1459 	txd->cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR;
1460 
1461 	if (pl08x->vd->dualmaster)
1462 		txd->cctl |= pl08x_select_bus(pl08x->mem_buses,
1463 					      pl08x->mem_buses);
1464 
1465 	ret = pl08x_fill_llis_for_desc(plchan->host, txd);
1466 	if (!ret) {
1467 		pl08x_free_txd(pl08x, txd);
1468 		return NULL;
1469 	}
1470 
1471 	return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
1472 }
1473 
1474 static struct pl08x_txd *pl08x_init_txd(
1475 		struct dma_chan *chan,
1476 		enum dma_transfer_direction direction,
1477 		dma_addr_t *slave_addr)
1478 {
1479 	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1480 	struct pl08x_driver_data *pl08x = plchan->host;
1481 	struct pl08x_txd *txd;
1482 	enum dma_slave_buswidth addr_width;
1483 	int ret, tmp;
1484 	u8 src_buses, dst_buses;
1485 	u32 maxburst, cctl;
1486 
1487 	txd = pl08x_get_txd(plchan);
1488 	if (!txd) {
1489 		dev_err(&pl08x->adev->dev, "%s no txd\n", __func__);
1490 		return NULL;
1491 	}
1492 
1493 	/*
1494 	 * Set up addresses, the PrimeCell configured address
1495 	 * will take precedence since this may configure the
1496 	 * channel target address dynamically at runtime.
1497 	 */
1498 	if (direction == DMA_MEM_TO_DEV) {
1499 		cctl = PL080_CONTROL_SRC_INCR;
1500 		*slave_addr = plchan->cfg.dst_addr;
1501 		addr_width = plchan->cfg.dst_addr_width;
1502 		maxburst = plchan->cfg.dst_maxburst;
1503 		src_buses = pl08x->mem_buses;
1504 		dst_buses = plchan->cd->periph_buses;
1505 	} else if (direction == DMA_DEV_TO_MEM) {
1506 		cctl = PL080_CONTROL_DST_INCR;
1507 		*slave_addr = plchan->cfg.src_addr;
1508 		addr_width = plchan->cfg.src_addr_width;
1509 		maxburst = plchan->cfg.src_maxburst;
1510 		src_buses = plchan->cd->periph_buses;
1511 		dst_buses = pl08x->mem_buses;
1512 	} else {
1513 		pl08x_free_txd(pl08x, txd);
1514 		dev_err(&pl08x->adev->dev,
1515 			"%s direction unsupported\n", __func__);
1516 		return NULL;
1517 	}
1518 
1519 	cctl |= pl08x_get_cctl(plchan, addr_width, maxburst);
1520 	if (cctl == ~0) {
1521 		pl08x_free_txd(pl08x, txd);
1522 		dev_err(&pl08x->adev->dev,
1523 			"DMA slave configuration botched?\n");
1524 		return NULL;
1525 	}
1526 
1527 	txd->cctl = cctl | pl08x_select_bus(src_buses, dst_buses);
1528 
1529 	if (plchan->cfg.device_fc)
1530 		tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER_PER :
1531 			PL080_FLOW_PER2MEM_PER;
1532 	else
1533 		tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER :
1534 			PL080_FLOW_PER2MEM;
1535 
1536 	txd->ccfg |= tmp << PL080_CONFIG_FLOW_CONTROL_SHIFT;
1537 
1538 	ret = pl08x_request_mux(plchan);
1539 	if (ret < 0) {
1540 		pl08x_free_txd(pl08x, txd);
1541 		dev_dbg(&pl08x->adev->dev,
1542 			"unable to mux for transfer on %s due to platform restrictions\n",
1543 			plchan->name);
1544 		return NULL;
1545 	}
1546 
1547 	dev_dbg(&pl08x->adev->dev, "allocated DMA request signal %d for xfer on %s\n",
1548 		 plchan->signal, plchan->name);
1549 
1550 	/* Assign the flow control signal to this channel */
1551 	if (direction == DMA_MEM_TO_DEV)
1552 		txd->ccfg |= plchan->signal << PL080_CONFIG_DST_SEL_SHIFT;
1553 	else
1554 		txd->ccfg |= plchan->signal << PL080_CONFIG_SRC_SEL_SHIFT;
1555 
1556 	return txd;
1557 }
1558 
1559 static int pl08x_tx_add_sg(struct pl08x_txd *txd,
1560 			   enum dma_transfer_direction direction,
1561 			   dma_addr_t slave_addr,
1562 			   dma_addr_t buf_addr,
1563 			   unsigned int len)
1564 {
1565 	struct pl08x_sg *dsg;
1566 
1567 	dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);
1568 	if (!dsg)
1569 		return -ENOMEM;
1570 
1571 	list_add_tail(&dsg->node, &txd->dsg_list);
1572 
1573 	dsg->len = len;
1574 	if (direction == DMA_MEM_TO_DEV) {
1575 		dsg->src_addr = buf_addr;
1576 		dsg->dst_addr = slave_addr;
1577 	} else {
1578 		dsg->src_addr = slave_addr;
1579 		dsg->dst_addr = buf_addr;
1580 	}
1581 
1582 	return 0;
1583 }
1584 
1585 static struct dma_async_tx_descriptor *pl08x_prep_slave_sg(
1586 		struct dma_chan *chan, struct scatterlist *sgl,
1587 		unsigned int sg_len, enum dma_transfer_direction direction,
1588 		unsigned long flags, void *context)
1589 {
1590 	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1591 	struct pl08x_driver_data *pl08x = plchan->host;
1592 	struct pl08x_txd *txd;
1593 	struct scatterlist *sg;
1594 	int ret, tmp;
1595 	dma_addr_t slave_addr;
1596 
1597 	dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n",
1598 			__func__, sg_dma_len(sgl), plchan->name);
1599 
1600 	txd = pl08x_init_txd(chan, direction, &slave_addr);
1601 	if (!txd)
1602 		return NULL;
1603 
1604 	for_each_sg(sgl, sg, sg_len, tmp) {
1605 		ret = pl08x_tx_add_sg(txd, direction, slave_addr,
1606 				      sg_dma_address(sg),
1607 				      sg_dma_len(sg));
1608 		if (ret) {
1609 			pl08x_release_mux(plchan);
1610 			pl08x_free_txd(pl08x, txd);
1611 			dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n",
1612 					__func__);
1613 			return NULL;
1614 		}
1615 	}
1616 
1617 	ret = pl08x_fill_llis_for_desc(plchan->host, txd);
1618 	if (!ret) {
1619 		pl08x_release_mux(plchan);
1620 		pl08x_free_txd(pl08x, txd);
1621 		return NULL;
1622 	}
1623 
1624 	return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
1625 }
1626 
1627 static struct dma_async_tx_descriptor *pl08x_prep_dma_cyclic(
1628 		struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
1629 		size_t period_len, enum dma_transfer_direction direction,
1630 		unsigned long flags)
1631 {
1632 	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1633 	struct pl08x_driver_data *pl08x = plchan->host;
1634 	struct pl08x_txd *txd;
1635 	int ret, tmp;
1636 	dma_addr_t slave_addr;
1637 
1638 	dev_dbg(&pl08x->adev->dev,
1639 		"%s prepare cyclic transaction of %zd/%zd bytes %s %s\n",
1640 		__func__, period_len, buf_len,
1641 		direction == DMA_MEM_TO_DEV ? "to" : "from",
1642 		plchan->name);
1643 
1644 	txd = pl08x_init_txd(chan, direction, &slave_addr);
1645 	if (!txd)
1646 		return NULL;
1647 
1648 	txd->cyclic = true;
1649 	txd->cctl |= PL080_CONTROL_TC_IRQ_EN;
1650 	for (tmp = 0; tmp < buf_len; tmp += period_len) {
1651 		ret = pl08x_tx_add_sg(txd, direction, slave_addr,
1652 				      buf_addr + tmp, period_len);
1653 		if (ret) {
1654 			pl08x_release_mux(plchan);
1655 			pl08x_free_txd(pl08x, txd);
1656 			return NULL;
1657 		}
1658 	}
1659 
1660 	ret = pl08x_fill_llis_for_desc(plchan->host, txd);
1661 	if (!ret) {
1662 		pl08x_release_mux(plchan);
1663 		pl08x_free_txd(pl08x, txd);
1664 		return NULL;
1665 	}
1666 
1667 	return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
1668 }
1669 
1670 static int pl08x_config(struct dma_chan *chan,
1671 			struct dma_slave_config *config)
1672 {
1673 	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1674 	struct pl08x_driver_data *pl08x = plchan->host;
1675 
1676 	if (!plchan->slave)
1677 		return -EINVAL;
1678 
1679 	/* Reject definitely invalid configurations */
1680 	if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
1681 	    config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
1682 		return -EINVAL;
1683 
1684 	if (config->device_fc && pl08x->vd->pl080s) {
1685 		dev_err(&pl08x->adev->dev,
1686 			"%s: PL080S does not support peripheral flow control\n",
1687 			__func__);
1688 		return -EINVAL;
1689 	}
1690 
1691 	plchan->cfg = *config;
1692 
1693 	return 0;
1694 }
1695 
1696 static int pl08x_terminate_all(struct dma_chan *chan)
1697 {
1698 	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1699 	struct pl08x_driver_data *pl08x = plchan->host;
1700 	unsigned long flags;
1701 
1702 	spin_lock_irqsave(&plchan->vc.lock, flags);
1703 	if (!plchan->phychan && !plchan->at) {
1704 		spin_unlock_irqrestore(&plchan->vc.lock, flags);
1705 		return 0;
1706 	}
1707 
1708 	plchan->state = PL08X_CHAN_IDLE;
1709 
1710 	if (plchan->phychan) {
1711 		/*
1712 		 * Mark physical channel as free and free any slave
1713 		 * signal
1714 		 */
1715 		pl08x_phy_free(plchan);
1716 	}
1717 	/* Dequeue jobs and free LLIs */
1718 	if (plchan->at) {
1719 		pl08x_desc_free(&plchan->at->vd);
1720 		plchan->at = NULL;
1721 	}
1722 	/* Dequeue jobs not yet fired as well */
1723 	pl08x_free_txd_list(pl08x, plchan);
1724 
1725 	spin_unlock_irqrestore(&plchan->vc.lock, flags);
1726 
1727 	return 0;
1728 }
1729 
1730 static int pl08x_pause(struct dma_chan *chan)
1731 {
1732 	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1733 	unsigned long flags;
1734 
1735 	/*
1736 	 * Anything succeeds on channels with no physical allocation and
1737 	 * no queued transfers.
1738 	 */
1739 	spin_lock_irqsave(&plchan->vc.lock, flags);
1740 	if (!plchan->phychan && !plchan->at) {
1741 		spin_unlock_irqrestore(&plchan->vc.lock, flags);
1742 		return 0;
1743 	}
1744 
1745 	pl08x_pause_phy_chan(plchan->phychan);
1746 	plchan->state = PL08X_CHAN_PAUSED;
1747 
1748 	spin_unlock_irqrestore(&plchan->vc.lock, flags);
1749 
1750 	return 0;
1751 }
1752 
1753 static int pl08x_resume(struct dma_chan *chan)
1754 {
1755 	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1756 	unsigned long flags;
1757 
1758 	/*
1759 	 * Anything succeeds on channels with no physical allocation and
1760 	 * no queued transfers.
1761 	 */
1762 	spin_lock_irqsave(&plchan->vc.lock, flags);
1763 	if (!plchan->phychan && !plchan->at) {
1764 		spin_unlock_irqrestore(&plchan->vc.lock, flags);
1765 		return 0;
1766 	}
1767 
1768 	pl08x_resume_phy_chan(plchan->phychan);
1769 	plchan->state = PL08X_CHAN_RUNNING;
1770 
1771 	spin_unlock_irqrestore(&plchan->vc.lock, flags);
1772 
1773 	return 0;
1774 }
1775 
1776 bool pl08x_filter_id(struct dma_chan *chan, void *chan_id)
1777 {
1778 	struct pl08x_dma_chan *plchan;
1779 	char *name = chan_id;
1780 
1781 	/* Reject channels for devices not bound to this driver */
1782 	if (chan->device->dev->driver != &pl08x_amba_driver.drv)
1783 		return false;
1784 
1785 	plchan = to_pl08x_chan(chan);
1786 
1787 	/* Check that the channel is not taken! */
1788 	if (!strcmp(plchan->name, name))
1789 		return true;
1790 
1791 	return false;
1792 }
1793 EXPORT_SYMBOL_GPL(pl08x_filter_id);
1794 
1795 /*
1796  * Just check that the device is there and active
1797  * TODO: turn this bit on/off depending on the number of physical channels
1798  * actually used, if it is zero... well shut it off. That will save some
1799  * power. Cut the clock at the same time.
1800  */
1801 static void pl08x_ensure_on(struct pl08x_driver_data *pl08x)
1802 {
1803 	/* The Nomadik variant does not have the config register */
1804 	if (pl08x->vd->nomadik)
1805 		return;
1806 	writel(PL080_CONFIG_ENABLE, pl08x->base + PL080_CONFIG);
1807 }
1808 
1809 static irqreturn_t pl08x_irq(int irq, void *dev)
1810 {
1811 	struct pl08x_driver_data *pl08x = dev;
1812 	u32 mask = 0, err, tc, i;
1813 
1814 	/* check & clear - ERR & TC interrupts */
1815 	err = readl(pl08x->base + PL080_ERR_STATUS);
1816 	if (err) {
1817 		dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n",
1818 			__func__, err);
1819 		writel(err, pl08x->base + PL080_ERR_CLEAR);
1820 	}
1821 	tc = readl(pl08x->base + PL080_TC_STATUS);
1822 	if (tc)
1823 		writel(tc, pl08x->base + PL080_TC_CLEAR);
1824 
1825 	if (!err && !tc)
1826 		return IRQ_NONE;
1827 
1828 	for (i = 0; i < pl08x->vd->channels; i++) {
1829 		if (((1 << i) & err) || ((1 << i) & tc)) {
1830 			/* Locate physical channel */
1831 			struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i];
1832 			struct pl08x_dma_chan *plchan = phychan->serving;
1833 			struct pl08x_txd *tx;
1834 
1835 			if (!plchan) {
1836 				dev_err(&pl08x->adev->dev,
1837 					"%s Error TC interrupt on unused channel: 0x%08x\n",
1838 					__func__, i);
1839 				continue;
1840 			}
1841 
1842 			spin_lock(&plchan->vc.lock);
1843 			tx = plchan->at;
1844 			if (tx && tx->cyclic) {
1845 				vchan_cyclic_callback(&tx->vd);
1846 			} else if (tx) {
1847 				plchan->at = NULL;
1848 				/*
1849 				 * This descriptor is done, release its mux
1850 				 * reservation.
1851 				 */
1852 				pl08x_release_mux(plchan);
1853 				tx->done = true;
1854 				vchan_cookie_complete(&tx->vd);
1855 
1856 				/*
1857 				 * And start the next descriptor (if any),
1858 				 * otherwise free this channel.
1859 				 */
1860 				if (vchan_next_desc(&plchan->vc))
1861 					pl08x_start_next_txd(plchan);
1862 				else
1863 					pl08x_phy_free(plchan);
1864 			}
1865 			spin_unlock(&plchan->vc.lock);
1866 
1867 			mask |= (1 << i);
1868 		}
1869 	}
1870 
1871 	return mask ? IRQ_HANDLED : IRQ_NONE;
1872 }
1873 
1874 static void pl08x_dma_slave_init(struct pl08x_dma_chan *chan)
1875 {
1876 	chan->slave = true;
1877 	chan->name = chan->cd->bus_id;
1878 	chan->cfg.src_addr = chan->cd->addr;
1879 	chan->cfg.dst_addr = chan->cd->addr;
1880 }
1881 
1882 /*
1883  * Initialise the DMAC memcpy/slave channels.
1884  * Make a local wrapper to hold required data
1885  */
1886 static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x,
1887 		struct dma_device *dmadev, unsigned int channels, bool slave)
1888 {
1889 	struct pl08x_dma_chan *chan;
1890 	int i;
1891 
1892 	INIT_LIST_HEAD(&dmadev->channels);
1893 
1894 	/*
1895 	 * Register as many many memcpy as we have physical channels,
1896 	 * we won't always be able to use all but the code will have
1897 	 * to cope with that situation.
1898 	 */
1899 	for (i = 0; i < channels; i++) {
1900 		chan = kzalloc(sizeof(*chan), GFP_KERNEL);
1901 		if (!chan) {
1902 			dev_err(&pl08x->adev->dev,
1903 				"%s no memory for channel\n", __func__);
1904 			return -ENOMEM;
1905 		}
1906 
1907 		chan->host = pl08x;
1908 		chan->state = PL08X_CHAN_IDLE;
1909 		chan->signal = -1;
1910 
1911 		if (slave) {
1912 			chan->cd = &pl08x->pd->slave_channels[i];
1913 			pl08x_dma_slave_init(chan);
1914 		} else {
1915 			chan->cd = &pl08x->pd->memcpy_channel;
1916 			chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i);
1917 			if (!chan->name) {
1918 				kfree(chan);
1919 				return -ENOMEM;
1920 			}
1921 		}
1922 		dev_dbg(&pl08x->adev->dev,
1923 			 "initialize virtual channel \"%s\"\n",
1924 			 chan->name);
1925 
1926 		chan->vc.desc_free = pl08x_desc_free;
1927 		vchan_init(&chan->vc, dmadev);
1928 	}
1929 	dev_info(&pl08x->adev->dev, "initialized %d virtual %s channels\n",
1930 		 i, slave ? "slave" : "memcpy");
1931 	return i;
1932 }
1933 
1934 static void pl08x_free_virtual_channels(struct dma_device *dmadev)
1935 {
1936 	struct pl08x_dma_chan *chan = NULL;
1937 	struct pl08x_dma_chan *next;
1938 
1939 	list_for_each_entry_safe(chan,
1940 				 next, &dmadev->channels, vc.chan.device_node) {
1941 		list_del(&chan->vc.chan.device_node);
1942 		kfree(chan);
1943 	}
1944 }
1945 
1946 #ifdef CONFIG_DEBUG_FS
1947 static const char *pl08x_state_str(enum pl08x_dma_chan_state state)
1948 {
1949 	switch (state) {
1950 	case PL08X_CHAN_IDLE:
1951 		return "idle";
1952 	case PL08X_CHAN_RUNNING:
1953 		return "running";
1954 	case PL08X_CHAN_PAUSED:
1955 		return "paused";
1956 	case PL08X_CHAN_WAITING:
1957 		return "waiting";
1958 	default:
1959 		break;
1960 	}
1961 	return "UNKNOWN STATE";
1962 }
1963 
1964 static int pl08x_debugfs_show(struct seq_file *s, void *data)
1965 {
1966 	struct pl08x_driver_data *pl08x = s->private;
1967 	struct pl08x_dma_chan *chan;
1968 	struct pl08x_phy_chan *ch;
1969 	unsigned long flags;
1970 	int i;
1971 
1972 	seq_printf(s, "PL08x physical channels:\n");
1973 	seq_printf(s, "CHANNEL:\tUSER:\n");
1974 	seq_printf(s, "--------\t-----\n");
1975 	for (i = 0; i < pl08x->vd->channels; i++) {
1976 		struct pl08x_dma_chan *virt_chan;
1977 
1978 		ch = &pl08x->phy_chans[i];
1979 
1980 		spin_lock_irqsave(&ch->lock, flags);
1981 		virt_chan = ch->serving;
1982 
1983 		seq_printf(s, "%d\t\t%s%s\n",
1984 			   ch->id,
1985 			   virt_chan ? virt_chan->name : "(none)",
1986 			   ch->locked ? " LOCKED" : "");
1987 
1988 		spin_unlock_irqrestore(&ch->lock, flags);
1989 	}
1990 
1991 	seq_printf(s, "\nPL08x virtual memcpy channels:\n");
1992 	seq_printf(s, "CHANNEL:\tSTATE:\n");
1993 	seq_printf(s, "--------\t------\n");
1994 	list_for_each_entry(chan, &pl08x->memcpy.channels, vc.chan.device_node) {
1995 		seq_printf(s, "%s\t\t%s\n", chan->name,
1996 			   pl08x_state_str(chan->state));
1997 	}
1998 
1999 	seq_printf(s, "\nPL08x virtual slave channels:\n");
2000 	seq_printf(s, "CHANNEL:\tSTATE:\n");
2001 	seq_printf(s, "--------\t------\n");
2002 	list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) {
2003 		seq_printf(s, "%s\t\t%s\n", chan->name,
2004 			   pl08x_state_str(chan->state));
2005 	}
2006 
2007 	return 0;
2008 }
2009 
2010 static int pl08x_debugfs_open(struct inode *inode, struct file *file)
2011 {
2012 	return single_open(file, pl08x_debugfs_show, inode->i_private);
2013 }
2014 
2015 static const struct file_operations pl08x_debugfs_operations = {
2016 	.open		= pl08x_debugfs_open,
2017 	.read		= seq_read,
2018 	.llseek		= seq_lseek,
2019 	.release	= single_release,
2020 };
2021 
2022 static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
2023 {
2024 	/* Expose a simple debugfs interface to view all clocks */
2025 	(void) debugfs_create_file(dev_name(&pl08x->adev->dev),
2026 			S_IFREG | S_IRUGO, NULL, pl08x,
2027 			&pl08x_debugfs_operations);
2028 }
2029 
2030 #else
2031 static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
2032 {
2033 }
2034 #endif
2035 
2036 static int pl08x_probe(struct amba_device *adev, const struct amba_id *id)
2037 {
2038 	struct pl08x_driver_data *pl08x;
2039 	const struct vendor_data *vd = id->data;
2040 	u32 tsfr_size;
2041 	int ret = 0;
2042 	int i;
2043 
2044 	ret = amba_request_regions(adev, NULL);
2045 	if (ret)
2046 		return ret;
2047 
2048 	/* Ensure that we can do DMA */
2049 	ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32));
2050 	if (ret)
2051 		goto out_no_pl08x;
2052 
2053 	/* Create the driver state holder */
2054 	pl08x = kzalloc(sizeof(*pl08x), GFP_KERNEL);
2055 	if (!pl08x) {
2056 		ret = -ENOMEM;
2057 		goto out_no_pl08x;
2058 	}
2059 
2060 	/* Initialize memcpy engine */
2061 	dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask);
2062 	pl08x->memcpy.dev = &adev->dev;
2063 	pl08x->memcpy.device_alloc_chan_resources = pl08x_alloc_chan_resources;
2064 	pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources;
2065 	pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy;
2066 	pl08x->memcpy.device_prep_dma_interrupt = pl08x_prep_dma_interrupt;
2067 	pl08x->memcpy.device_tx_status = pl08x_dma_tx_status;
2068 	pl08x->memcpy.device_issue_pending = pl08x_issue_pending;
2069 	pl08x->memcpy.device_config = pl08x_config;
2070 	pl08x->memcpy.device_pause = pl08x_pause;
2071 	pl08x->memcpy.device_resume = pl08x_resume;
2072 	pl08x->memcpy.device_terminate_all = pl08x_terminate_all;
2073 
2074 	/* Initialize slave engine */
2075 	dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask);
2076 	dma_cap_set(DMA_CYCLIC, pl08x->slave.cap_mask);
2077 	pl08x->slave.dev = &adev->dev;
2078 	pl08x->slave.device_alloc_chan_resources = pl08x_alloc_chan_resources;
2079 	pl08x->slave.device_free_chan_resources = pl08x_free_chan_resources;
2080 	pl08x->slave.device_prep_dma_interrupt = pl08x_prep_dma_interrupt;
2081 	pl08x->slave.device_tx_status = pl08x_dma_tx_status;
2082 	pl08x->slave.device_issue_pending = pl08x_issue_pending;
2083 	pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg;
2084 	pl08x->slave.device_prep_dma_cyclic = pl08x_prep_dma_cyclic;
2085 	pl08x->slave.device_config = pl08x_config;
2086 	pl08x->slave.device_pause = pl08x_pause;
2087 	pl08x->slave.device_resume = pl08x_resume;
2088 	pl08x->slave.device_terminate_all = pl08x_terminate_all;
2089 
2090 	/* Get the platform data */
2091 	pl08x->pd = dev_get_platdata(&adev->dev);
2092 	if (!pl08x->pd) {
2093 		dev_err(&adev->dev, "no platform data supplied\n");
2094 		ret = -EINVAL;
2095 		goto out_no_platdata;
2096 	}
2097 
2098 	/* Assign useful pointers to the driver state */
2099 	pl08x->adev = adev;
2100 	pl08x->vd = vd;
2101 
2102 	/* By default, AHB1 only.  If dualmaster, from platform */
2103 	pl08x->lli_buses = PL08X_AHB1;
2104 	pl08x->mem_buses = PL08X_AHB1;
2105 	if (pl08x->vd->dualmaster) {
2106 		pl08x->lli_buses = pl08x->pd->lli_buses;
2107 		pl08x->mem_buses = pl08x->pd->mem_buses;
2108 	}
2109 
2110 	if (vd->pl080s)
2111 		pl08x->lli_words = PL080S_LLI_WORDS;
2112 	else
2113 		pl08x->lli_words = PL080_LLI_WORDS;
2114 	tsfr_size = MAX_NUM_TSFR_LLIS * pl08x->lli_words * sizeof(u32);
2115 
2116 	/* A DMA memory pool for LLIs, align on 1-byte boundary */
2117 	pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev,
2118 						tsfr_size, PL08X_ALIGN, 0);
2119 	if (!pl08x->pool) {
2120 		ret = -ENOMEM;
2121 		goto out_no_lli_pool;
2122 	}
2123 
2124 	pl08x->base = ioremap(adev->res.start, resource_size(&adev->res));
2125 	if (!pl08x->base) {
2126 		ret = -ENOMEM;
2127 		goto out_no_ioremap;
2128 	}
2129 
2130 	/* Turn on the PL08x */
2131 	pl08x_ensure_on(pl08x);
2132 
2133 	/* Attach the interrupt handler */
2134 	writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR);
2135 	writel(0x000000FF, pl08x->base + PL080_TC_CLEAR);
2136 
2137 	ret = request_irq(adev->irq[0], pl08x_irq, 0, DRIVER_NAME, pl08x);
2138 	if (ret) {
2139 		dev_err(&adev->dev, "%s failed to request interrupt %d\n",
2140 			__func__, adev->irq[0]);
2141 		goto out_no_irq;
2142 	}
2143 
2144 	/* Initialize physical channels */
2145 	pl08x->phy_chans = kzalloc((vd->channels * sizeof(*pl08x->phy_chans)),
2146 			GFP_KERNEL);
2147 	if (!pl08x->phy_chans) {
2148 		dev_err(&adev->dev, "%s failed to allocate "
2149 			"physical channel holders\n",
2150 			__func__);
2151 		ret = -ENOMEM;
2152 		goto out_no_phychans;
2153 	}
2154 
2155 	for (i = 0; i < vd->channels; i++) {
2156 		struct pl08x_phy_chan *ch = &pl08x->phy_chans[i];
2157 
2158 		ch->id = i;
2159 		ch->base = pl08x->base + PL080_Cx_BASE(i);
2160 		ch->reg_config = ch->base + vd->config_offset;
2161 		spin_lock_init(&ch->lock);
2162 
2163 		/*
2164 		 * Nomadik variants can have channels that are locked
2165 		 * down for the secure world only. Lock up these channels
2166 		 * by perpetually serving a dummy virtual channel.
2167 		 */
2168 		if (vd->nomadik) {
2169 			u32 val;
2170 
2171 			val = readl(ch->reg_config);
2172 			if (val & (PL080N_CONFIG_ITPROT | PL080N_CONFIG_SECPROT)) {
2173 				dev_info(&adev->dev, "physical channel %d reserved for secure access only\n", i);
2174 				ch->locked = true;
2175 			}
2176 		}
2177 
2178 		dev_dbg(&adev->dev, "physical channel %d is %s\n",
2179 			i, pl08x_phy_channel_busy(ch) ? "BUSY" : "FREE");
2180 	}
2181 
2182 	/* Register as many memcpy channels as there are physical channels */
2183 	ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->memcpy,
2184 					      pl08x->vd->channels, false);
2185 	if (ret <= 0) {
2186 		dev_warn(&pl08x->adev->dev,
2187 			 "%s failed to enumerate memcpy channels - %d\n",
2188 			 __func__, ret);
2189 		goto out_no_memcpy;
2190 	}
2191 
2192 	/* Register slave channels */
2193 	ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave,
2194 			pl08x->pd->num_slave_channels, true);
2195 	if (ret < 0) {
2196 		dev_warn(&pl08x->adev->dev,
2197 			"%s failed to enumerate slave channels - %d\n",
2198 				__func__, ret);
2199 		goto out_no_slave;
2200 	}
2201 
2202 	ret = dma_async_device_register(&pl08x->memcpy);
2203 	if (ret) {
2204 		dev_warn(&pl08x->adev->dev,
2205 			"%s failed to register memcpy as an async device - %d\n",
2206 			__func__, ret);
2207 		goto out_no_memcpy_reg;
2208 	}
2209 
2210 	ret = dma_async_device_register(&pl08x->slave);
2211 	if (ret) {
2212 		dev_warn(&pl08x->adev->dev,
2213 			"%s failed to register slave as an async device - %d\n",
2214 			__func__, ret);
2215 		goto out_no_slave_reg;
2216 	}
2217 
2218 	amba_set_drvdata(adev, pl08x);
2219 	init_pl08x_debugfs(pl08x);
2220 	dev_info(&pl08x->adev->dev, "DMA: PL%03x%s rev%u at 0x%08llx irq %d\n",
2221 		 amba_part(adev), pl08x->vd->pl080s ? "s" : "", amba_rev(adev),
2222 		 (unsigned long long)adev->res.start, adev->irq[0]);
2223 
2224 	return 0;
2225 
2226 out_no_slave_reg:
2227 	dma_async_device_unregister(&pl08x->memcpy);
2228 out_no_memcpy_reg:
2229 	pl08x_free_virtual_channels(&pl08x->slave);
2230 out_no_slave:
2231 	pl08x_free_virtual_channels(&pl08x->memcpy);
2232 out_no_memcpy:
2233 	kfree(pl08x->phy_chans);
2234 out_no_phychans:
2235 	free_irq(adev->irq[0], pl08x);
2236 out_no_irq:
2237 	iounmap(pl08x->base);
2238 out_no_ioremap:
2239 	dma_pool_destroy(pl08x->pool);
2240 out_no_lli_pool:
2241 out_no_platdata:
2242 	kfree(pl08x);
2243 out_no_pl08x:
2244 	amba_release_regions(adev);
2245 	return ret;
2246 }
2247 
2248 /* PL080 has 8 channels and the PL080 have just 2 */
2249 static struct vendor_data vendor_pl080 = {
2250 	.config_offset = PL080_CH_CONFIG,
2251 	.channels = 8,
2252 	.dualmaster = true,
2253 	.max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
2254 };
2255 
2256 static struct vendor_data vendor_nomadik = {
2257 	.config_offset = PL080_CH_CONFIG,
2258 	.channels = 8,
2259 	.dualmaster = true,
2260 	.nomadik = true,
2261 	.max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
2262 };
2263 
2264 static struct vendor_data vendor_pl080s = {
2265 	.config_offset = PL080S_CH_CONFIG,
2266 	.channels = 8,
2267 	.pl080s = true,
2268 	.max_transfer_size = PL080S_CONTROL_TRANSFER_SIZE_MASK,
2269 };
2270 
2271 static struct vendor_data vendor_pl081 = {
2272 	.config_offset = PL080_CH_CONFIG,
2273 	.channels = 2,
2274 	.dualmaster = false,
2275 	.max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
2276 };
2277 
2278 static struct amba_id pl08x_ids[] = {
2279 	/* Samsung PL080S variant */
2280 	{
2281 		.id	= 0x0a141080,
2282 		.mask	= 0xffffffff,
2283 		.data	= &vendor_pl080s,
2284 	},
2285 	/* PL080 */
2286 	{
2287 		.id	= 0x00041080,
2288 		.mask	= 0x000fffff,
2289 		.data	= &vendor_pl080,
2290 	},
2291 	/* PL081 */
2292 	{
2293 		.id	= 0x00041081,
2294 		.mask	= 0x000fffff,
2295 		.data	= &vendor_pl081,
2296 	},
2297 	/* Nomadik 8815 PL080 variant */
2298 	{
2299 		.id	= 0x00280080,
2300 		.mask	= 0x00ffffff,
2301 		.data	= &vendor_nomadik,
2302 	},
2303 	{ 0, 0 },
2304 };
2305 
2306 MODULE_DEVICE_TABLE(amba, pl08x_ids);
2307 
2308 static struct amba_driver pl08x_amba_driver = {
2309 	.drv.name	= DRIVER_NAME,
2310 	.id_table	= pl08x_ids,
2311 	.probe		= pl08x_probe,
2312 };
2313 
2314 static int __init pl08x_init(void)
2315 {
2316 	int retval;
2317 	retval = amba_driver_register(&pl08x_amba_driver);
2318 	if (retval)
2319 		printk(KERN_WARNING DRIVER_NAME
2320 		       "failed to register as an AMBA device (%d)\n",
2321 		       retval);
2322 	return retval;
2323 }
2324 subsys_initcall(pl08x_init);
2325