xref: /openbmc/linux/drivers/dma/ep93xx_dma.c (revision 799fb82aa132fa3a3886b7872997a5a84e820062)
1  // SPDX-License-Identifier: GPL-2.0-or-later
2  /*
3   * Driver for the Cirrus Logic EP93xx DMA Controller
4   *
5   * Copyright (C) 2011 Mika Westerberg
6   *
7   * DMA M2P implementation is based on the original
8   * arch/arm/mach-ep93xx/dma-m2p.c which has following copyrights:
9   *
10   *   Copyright (C) 2006 Lennert Buytenhek <buytenh@wantstofly.org>
11   *   Copyright (C) 2006 Applied Data Systems
12   *   Copyright (C) 2009 Ryan Mallon <rmallon@gmail.com>
13   *
14   * This driver is based on dw_dmac and amba-pl08x drivers.
15   */
16  
17  #include <linux/clk.h>
18  #include <linux/init.h>
19  #include <linux/interrupt.h>
20  #include <linux/dmaengine.h>
21  #include <linux/module.h>
22  #include <linux/mod_devicetable.h>
23  #include <linux/platform_device.h>
24  #include <linux/slab.h>
25  
26  #include <linux/platform_data/dma-ep93xx.h>
27  
28  #include "dmaengine.h"
29  
30  /* M2P registers */
31  #define M2P_CONTROL			0x0000
32  #define M2P_CONTROL_STALLINT		BIT(0)
33  #define M2P_CONTROL_NFBINT		BIT(1)
34  #define M2P_CONTROL_CH_ERROR_INT	BIT(3)
35  #define M2P_CONTROL_ENABLE		BIT(4)
36  #define M2P_CONTROL_ICE			BIT(6)
37  
38  #define M2P_INTERRUPT			0x0004
39  #define M2P_INTERRUPT_STALL		BIT(0)
40  #define M2P_INTERRUPT_NFB		BIT(1)
41  #define M2P_INTERRUPT_ERROR		BIT(3)
42  
43  #define M2P_PPALLOC			0x0008
44  #define M2P_STATUS			0x000c
45  
46  #define M2P_MAXCNT0			0x0020
47  #define M2P_BASE0			0x0024
48  #define M2P_MAXCNT1			0x0030
49  #define M2P_BASE1			0x0034
50  
51  #define M2P_STATE_IDLE			0
52  #define M2P_STATE_STALL			1
53  #define M2P_STATE_ON			2
54  #define M2P_STATE_NEXT			3
55  
56  /* M2M registers */
57  #define M2M_CONTROL			0x0000
58  #define M2M_CONTROL_DONEINT		BIT(2)
59  #define M2M_CONTROL_ENABLE		BIT(3)
60  #define M2M_CONTROL_START		BIT(4)
61  #define M2M_CONTROL_DAH			BIT(11)
62  #define M2M_CONTROL_SAH			BIT(12)
63  #define M2M_CONTROL_PW_SHIFT		9
64  #define M2M_CONTROL_PW_8		(0 << M2M_CONTROL_PW_SHIFT)
65  #define M2M_CONTROL_PW_16		(1 << M2M_CONTROL_PW_SHIFT)
66  #define M2M_CONTROL_PW_32		(2 << M2M_CONTROL_PW_SHIFT)
67  #define M2M_CONTROL_PW_MASK		(3 << M2M_CONTROL_PW_SHIFT)
68  #define M2M_CONTROL_TM_SHIFT		13
69  #define M2M_CONTROL_TM_TX		(1 << M2M_CONTROL_TM_SHIFT)
70  #define M2M_CONTROL_TM_RX		(2 << M2M_CONTROL_TM_SHIFT)
71  #define M2M_CONTROL_NFBINT		BIT(21)
72  #define M2M_CONTROL_RSS_SHIFT		22
73  #define M2M_CONTROL_RSS_SSPRX		(1 << M2M_CONTROL_RSS_SHIFT)
74  #define M2M_CONTROL_RSS_SSPTX		(2 << M2M_CONTROL_RSS_SHIFT)
75  #define M2M_CONTROL_RSS_IDE		(3 << M2M_CONTROL_RSS_SHIFT)
76  #define M2M_CONTROL_NO_HDSK		BIT(24)
77  #define M2M_CONTROL_PWSC_SHIFT		25
78  
79  #define M2M_INTERRUPT			0x0004
80  #define M2M_INTERRUPT_MASK		6
81  
82  #define M2M_STATUS			0x000c
83  #define M2M_STATUS_CTL_SHIFT		1
84  #define M2M_STATUS_CTL_IDLE		(0 << M2M_STATUS_CTL_SHIFT)
85  #define M2M_STATUS_CTL_STALL		(1 << M2M_STATUS_CTL_SHIFT)
86  #define M2M_STATUS_CTL_MEMRD		(2 << M2M_STATUS_CTL_SHIFT)
87  #define M2M_STATUS_CTL_MEMWR		(3 << M2M_STATUS_CTL_SHIFT)
88  #define M2M_STATUS_CTL_BWCWAIT		(4 << M2M_STATUS_CTL_SHIFT)
89  #define M2M_STATUS_CTL_MASK		(7 << M2M_STATUS_CTL_SHIFT)
90  #define M2M_STATUS_BUF_SHIFT		4
91  #define M2M_STATUS_BUF_NO		(0 << M2M_STATUS_BUF_SHIFT)
92  #define M2M_STATUS_BUF_ON		(1 << M2M_STATUS_BUF_SHIFT)
93  #define M2M_STATUS_BUF_NEXT		(2 << M2M_STATUS_BUF_SHIFT)
94  #define M2M_STATUS_BUF_MASK		(3 << M2M_STATUS_BUF_SHIFT)
95  #define M2M_STATUS_DONE			BIT(6)
96  
97  #define M2M_BCR0			0x0010
98  #define M2M_BCR1			0x0014
99  #define M2M_SAR_BASE0			0x0018
100  #define M2M_SAR_BASE1			0x001c
101  #define M2M_DAR_BASE0			0x002c
102  #define M2M_DAR_BASE1			0x0030
103  
104  #define DMA_MAX_CHAN_BYTES		0xffff
105  #define DMA_MAX_CHAN_DESCRIPTORS	32
106  
107  struct ep93xx_dma_engine;
108  static int ep93xx_dma_slave_config_write(struct dma_chan *chan,
109  					 enum dma_transfer_direction dir,
110  					 struct dma_slave_config *config);
111  
112  /**
113   * struct ep93xx_dma_desc - EP93xx specific transaction descriptor
114   * @src_addr: source address of the transaction
115   * @dst_addr: destination address of the transaction
116   * @size: size of the transaction (in bytes)
117   * @complete: this descriptor is completed
118   * @txd: dmaengine API descriptor
119   * @tx_list: list of linked descriptors
120   * @node: link used for putting this into a channel queue
121   */
122  struct ep93xx_dma_desc {
123  	u32				src_addr;
124  	u32				dst_addr;
125  	size_t				size;
126  	bool				complete;
127  	struct dma_async_tx_descriptor	txd;
128  	struct list_head		tx_list;
129  	struct list_head		node;
130  };
131  
132  /**
133   * struct ep93xx_dma_chan - an EP93xx DMA M2P/M2M channel
134   * @chan: dmaengine API channel
135   * @edma: pointer to the engine device
136   * @regs: memory mapped registers
137   * @irq: interrupt number of the channel
138   * @clk: clock used by this channel
139   * @tasklet: channel specific tasklet used for callbacks
140   * @lock: lock protecting the fields following
141   * @flags: flags for the channel
142   * @buffer: which buffer to use next (0/1)
143   * @active: flattened chain of descriptors currently being processed
144   * @queue: pending descriptors which are handled next
145   * @free_list: list of free descriptors which can be used
146   * @runtime_addr: physical address currently used as dest/src (M2M only). This
147   *                is set via .device_config before slave operation is
148   *                prepared
149   * @runtime_ctrl: M2M runtime values for the control register.
150   * @slave_config: slave configuration
151   *
152   * As EP93xx DMA controller doesn't support real chained DMA descriptors we
153   * will have slightly different scheme here: @active points to a head of
154   * flattened DMA descriptor chain.
155   *
156   * @queue holds pending transactions. These are linked through the first
157   * descriptor in the chain. When a descriptor is moved to the @active queue,
158   * the first and chained descriptors are flattened into a single list.
159   *
160   * @chan.private holds pointer to &struct ep93xx_dma_data which contains
161   * necessary channel configuration information. For memcpy channels this must
162   * be %NULL.
163   */
164  struct ep93xx_dma_chan {
165  	struct dma_chan			chan;
166  	const struct ep93xx_dma_engine	*edma;
167  	void __iomem			*regs;
168  	int				irq;
169  	struct clk			*clk;
170  	struct tasklet_struct		tasklet;
171  	/* protects the fields following */
172  	spinlock_t			lock;
173  	unsigned long			flags;
174  /* Channel is configured for cyclic transfers */
175  #define EP93XX_DMA_IS_CYCLIC		0
176  
177  	int				buffer;
178  	struct list_head		active;
179  	struct list_head		queue;
180  	struct list_head		free_list;
181  	u32				runtime_addr;
182  	u32				runtime_ctrl;
183  	struct dma_slave_config		slave_config;
184  };
185  
186  /**
187   * struct ep93xx_dma_engine - the EP93xx DMA engine instance
188   * @dma_dev: holds the dmaengine device
189   * @m2m: is this an M2M or M2P device
190   * @hw_setup: method which sets the channel up for operation
191   * @hw_synchronize: synchronizes DMA channel termination to current context
192   * @hw_shutdown: shuts the channel down and flushes whatever is left
193   * @hw_submit: pushes active descriptor(s) to the hardware
194   * @hw_interrupt: handle the interrupt
195   * @num_channels: number of channels for this instance
196   * @channels: array of channels
197   *
198   * There is one instance of this struct for the M2P channels and one for the
199   * M2M channels. hw_xxx() methods are used to perform operations which are
200   * different on M2M and M2P channels. These methods are called with channel
201   * lock held and interrupts disabled so they cannot sleep.
202   */
203  struct ep93xx_dma_engine {
204  	struct dma_device	dma_dev;
205  	bool			m2m;
206  	int			(*hw_setup)(struct ep93xx_dma_chan *);
207  	void			(*hw_synchronize)(struct ep93xx_dma_chan *);
208  	void			(*hw_shutdown)(struct ep93xx_dma_chan *);
209  	void			(*hw_submit)(struct ep93xx_dma_chan *);
210  	int			(*hw_interrupt)(struct ep93xx_dma_chan *);
211  #define INTERRUPT_UNKNOWN	0
212  #define INTERRUPT_DONE		1
213  #define INTERRUPT_NEXT_BUFFER	2
214  
215  	size_t			num_channels;
216  	struct ep93xx_dma_chan	channels[];
217  };
218  
219  static inline struct device *chan2dev(struct ep93xx_dma_chan *edmac)
220  {
221  	return &edmac->chan.dev->device;
222  }
223  
224  static struct ep93xx_dma_chan *to_ep93xx_dma_chan(struct dma_chan *chan)
225  {
226  	return container_of(chan, struct ep93xx_dma_chan, chan);
227  }
228  
229  /**
230   * ep93xx_dma_set_active - set new active descriptor chain
231   * @edmac: channel
232   * @desc: head of the new active descriptor chain
233   *
234   * Sets @desc to be the head of the new active descriptor chain. This is the
235   * chain which is processed next. The active list must be empty before calling
236   * this function.
237   *
238   * Called with @edmac->lock held and interrupts disabled.
239   */
240  static void ep93xx_dma_set_active(struct ep93xx_dma_chan *edmac,
241  				  struct ep93xx_dma_desc *desc)
242  {
243  	BUG_ON(!list_empty(&edmac->active));
244  
245  	list_add_tail(&desc->node, &edmac->active);
246  
247  	/* Flatten the @desc->tx_list chain into @edmac->active list */
248  	while (!list_empty(&desc->tx_list)) {
249  		struct ep93xx_dma_desc *d = list_first_entry(&desc->tx_list,
250  			struct ep93xx_dma_desc, node);
251  
252  		/*
253  		 * We copy the callback parameters from the first descriptor
254  		 * to all the chained descriptors. This way we can call the
255  		 * callback without having to find out the first descriptor in
256  		 * the chain. Useful for cyclic transfers.
257  		 */
258  		d->txd.callback = desc->txd.callback;
259  		d->txd.callback_param = desc->txd.callback_param;
260  
261  		list_move_tail(&d->node, &edmac->active);
262  	}
263  }
264  
265  /* Called with @edmac->lock held and interrupts disabled */
266  static struct ep93xx_dma_desc *
267  ep93xx_dma_get_active(struct ep93xx_dma_chan *edmac)
268  {
269  	return list_first_entry_or_null(&edmac->active,
270  					struct ep93xx_dma_desc, node);
271  }
272  
273  /**
274   * ep93xx_dma_advance_active - advances to the next active descriptor
275   * @edmac: channel
276   *
277   * Function advances active descriptor to the next in the @edmac->active and
278   * returns %true if we still have descriptors in the chain to process.
279   * Otherwise returns %false.
280   *
281   * When the channel is in cyclic mode always returns %true.
282   *
283   * Called with @edmac->lock held and interrupts disabled.
284   */
285  static bool ep93xx_dma_advance_active(struct ep93xx_dma_chan *edmac)
286  {
287  	struct ep93xx_dma_desc *desc;
288  
289  	list_rotate_left(&edmac->active);
290  
291  	if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags))
292  		return true;
293  
294  	desc = ep93xx_dma_get_active(edmac);
295  	if (!desc)
296  		return false;
297  
298  	/*
299  	 * If txd.cookie is set it means that we are back in the first
300  	 * descriptor in the chain and hence done with it.
301  	 */
302  	return !desc->txd.cookie;
303  }
304  
305  /*
306   * M2P DMA implementation
307   */
308  
309  static void m2p_set_control(struct ep93xx_dma_chan *edmac, u32 control)
310  {
311  	writel(control, edmac->regs + M2P_CONTROL);
312  	/*
313  	 * EP93xx User's Guide states that we must perform a dummy read after
314  	 * write to the control register.
315  	 */
316  	readl(edmac->regs + M2P_CONTROL);
317  }
318  
319  static int m2p_hw_setup(struct ep93xx_dma_chan *edmac)
320  {
321  	struct ep93xx_dma_data *data = edmac->chan.private;
322  	u32 control;
323  
324  	writel(data->port & 0xf, edmac->regs + M2P_PPALLOC);
325  
326  	control = M2P_CONTROL_CH_ERROR_INT | M2P_CONTROL_ICE
327  		| M2P_CONTROL_ENABLE;
328  	m2p_set_control(edmac, control);
329  
330  	edmac->buffer = 0;
331  
332  	return 0;
333  }
334  
335  static inline u32 m2p_channel_state(struct ep93xx_dma_chan *edmac)
336  {
337  	return (readl(edmac->regs + M2P_STATUS) >> 4) & 0x3;
338  }
339  
340  static void m2p_hw_synchronize(struct ep93xx_dma_chan *edmac)
341  {
342  	unsigned long flags;
343  	u32 control;
344  
345  	spin_lock_irqsave(&edmac->lock, flags);
346  	control = readl(edmac->regs + M2P_CONTROL);
347  	control &= ~(M2P_CONTROL_STALLINT | M2P_CONTROL_NFBINT);
348  	m2p_set_control(edmac, control);
349  	spin_unlock_irqrestore(&edmac->lock, flags);
350  
351  	while (m2p_channel_state(edmac) >= M2P_STATE_ON)
352  		schedule();
353  }
354  
355  static void m2p_hw_shutdown(struct ep93xx_dma_chan *edmac)
356  {
357  	m2p_set_control(edmac, 0);
358  
359  	while (m2p_channel_state(edmac) != M2P_STATE_IDLE)
360  		dev_warn(chan2dev(edmac), "M2P: Not yet IDLE\n");
361  }
362  
363  static void m2p_fill_desc(struct ep93xx_dma_chan *edmac)
364  {
365  	struct ep93xx_dma_desc *desc;
366  	u32 bus_addr;
367  
368  	desc = ep93xx_dma_get_active(edmac);
369  	if (!desc) {
370  		dev_warn(chan2dev(edmac), "M2P: empty descriptor list\n");
371  		return;
372  	}
373  
374  	if (ep93xx_dma_chan_direction(&edmac->chan) == DMA_MEM_TO_DEV)
375  		bus_addr = desc->src_addr;
376  	else
377  		bus_addr = desc->dst_addr;
378  
379  	if (edmac->buffer == 0) {
380  		writel(desc->size, edmac->regs + M2P_MAXCNT0);
381  		writel(bus_addr, edmac->regs + M2P_BASE0);
382  	} else {
383  		writel(desc->size, edmac->regs + M2P_MAXCNT1);
384  		writel(bus_addr, edmac->regs + M2P_BASE1);
385  	}
386  
387  	edmac->buffer ^= 1;
388  }
389  
390  static void m2p_hw_submit(struct ep93xx_dma_chan *edmac)
391  {
392  	u32 control = readl(edmac->regs + M2P_CONTROL);
393  
394  	m2p_fill_desc(edmac);
395  	control |= M2P_CONTROL_STALLINT;
396  
397  	if (ep93xx_dma_advance_active(edmac)) {
398  		m2p_fill_desc(edmac);
399  		control |= M2P_CONTROL_NFBINT;
400  	}
401  
402  	m2p_set_control(edmac, control);
403  }
404  
405  static int m2p_hw_interrupt(struct ep93xx_dma_chan *edmac)
406  {
407  	u32 irq_status = readl(edmac->regs + M2P_INTERRUPT);
408  	u32 control;
409  
410  	if (irq_status & M2P_INTERRUPT_ERROR) {
411  		struct ep93xx_dma_desc *desc = ep93xx_dma_get_active(edmac);
412  
413  		/* Clear the error interrupt */
414  		writel(1, edmac->regs + M2P_INTERRUPT);
415  
416  		/*
417  		 * It seems that there is no easy way of reporting errors back
418  		 * to client so we just report the error here and continue as
419  		 * usual.
420  		 *
421  		 * Revisit this when there is a mechanism to report back the
422  		 * errors.
423  		 */
424  		dev_err(chan2dev(edmac),
425  			"DMA transfer failed! Details:\n"
426  			"\tcookie	: %d\n"
427  			"\tsrc_addr	: 0x%08x\n"
428  			"\tdst_addr	: 0x%08x\n"
429  			"\tsize		: %zu\n",
430  			desc->txd.cookie, desc->src_addr, desc->dst_addr,
431  			desc->size);
432  	}
433  
434  	/*
435  	 * Even latest E2 silicon revision sometimes assert STALL interrupt
436  	 * instead of NFB. Therefore we treat them equally, basing on the
437  	 * amount of data we still have to transfer.
438  	 */
439  	if (!(irq_status & (M2P_INTERRUPT_STALL | M2P_INTERRUPT_NFB)))
440  		return INTERRUPT_UNKNOWN;
441  
442  	if (ep93xx_dma_advance_active(edmac)) {
443  		m2p_fill_desc(edmac);
444  		return INTERRUPT_NEXT_BUFFER;
445  	}
446  
447  	/* Disable interrupts */
448  	control = readl(edmac->regs + M2P_CONTROL);
449  	control &= ~(M2P_CONTROL_STALLINT | M2P_CONTROL_NFBINT);
450  	m2p_set_control(edmac, control);
451  
452  	return INTERRUPT_DONE;
453  }
454  
455  /*
456   * M2M DMA implementation
457   */
458  
459  static int m2m_hw_setup(struct ep93xx_dma_chan *edmac)
460  {
461  	const struct ep93xx_dma_data *data = edmac->chan.private;
462  	u32 control = 0;
463  
464  	if (!data) {
465  		/* This is memcpy channel, nothing to configure */
466  		writel(control, edmac->regs + M2M_CONTROL);
467  		return 0;
468  	}
469  
470  	switch (data->port) {
471  	case EP93XX_DMA_SSP:
472  		/*
473  		 * This was found via experimenting - anything less than 5
474  		 * causes the channel to perform only a partial transfer which
475  		 * leads to problems since we don't get DONE interrupt then.
476  		 */
477  		control = (5 << M2M_CONTROL_PWSC_SHIFT);
478  		control |= M2M_CONTROL_NO_HDSK;
479  
480  		if (data->direction == DMA_MEM_TO_DEV) {
481  			control |= M2M_CONTROL_DAH;
482  			control |= M2M_CONTROL_TM_TX;
483  			control |= M2M_CONTROL_RSS_SSPTX;
484  		} else {
485  			control |= M2M_CONTROL_SAH;
486  			control |= M2M_CONTROL_TM_RX;
487  			control |= M2M_CONTROL_RSS_SSPRX;
488  		}
489  		break;
490  
491  	case EP93XX_DMA_IDE:
492  		/*
493  		 * This IDE part is totally untested. Values below are taken
494  		 * from the EP93xx Users's Guide and might not be correct.
495  		 */
496  		if (data->direction == DMA_MEM_TO_DEV) {
497  			/* Worst case from the UG */
498  			control = (3 << M2M_CONTROL_PWSC_SHIFT);
499  			control |= M2M_CONTROL_DAH;
500  			control |= M2M_CONTROL_TM_TX;
501  		} else {
502  			control = (2 << M2M_CONTROL_PWSC_SHIFT);
503  			control |= M2M_CONTROL_SAH;
504  			control |= M2M_CONTROL_TM_RX;
505  		}
506  
507  		control |= M2M_CONTROL_NO_HDSK;
508  		control |= M2M_CONTROL_RSS_IDE;
509  		control |= M2M_CONTROL_PW_16;
510  		break;
511  
512  	default:
513  		return -EINVAL;
514  	}
515  
516  	writel(control, edmac->regs + M2M_CONTROL);
517  	return 0;
518  }
519  
520  static void m2m_hw_shutdown(struct ep93xx_dma_chan *edmac)
521  {
522  	/* Just disable the channel */
523  	writel(0, edmac->regs + M2M_CONTROL);
524  }
525  
526  static void m2m_fill_desc(struct ep93xx_dma_chan *edmac)
527  {
528  	struct ep93xx_dma_desc *desc;
529  
530  	desc = ep93xx_dma_get_active(edmac);
531  	if (!desc) {
532  		dev_warn(chan2dev(edmac), "M2M: empty descriptor list\n");
533  		return;
534  	}
535  
536  	if (edmac->buffer == 0) {
537  		writel(desc->src_addr, edmac->regs + M2M_SAR_BASE0);
538  		writel(desc->dst_addr, edmac->regs + M2M_DAR_BASE0);
539  		writel(desc->size, edmac->regs + M2M_BCR0);
540  	} else {
541  		writel(desc->src_addr, edmac->regs + M2M_SAR_BASE1);
542  		writel(desc->dst_addr, edmac->regs + M2M_DAR_BASE1);
543  		writel(desc->size, edmac->regs + M2M_BCR1);
544  	}
545  
546  	edmac->buffer ^= 1;
547  }
548  
549  static void m2m_hw_submit(struct ep93xx_dma_chan *edmac)
550  {
551  	struct ep93xx_dma_data *data = edmac->chan.private;
552  	u32 control = readl(edmac->regs + M2M_CONTROL);
553  
554  	/*
555  	 * Since we allow clients to configure PW (peripheral width) we always
556  	 * clear PW bits here and then set them according what is given in
557  	 * the runtime configuration.
558  	 */
559  	control &= ~M2M_CONTROL_PW_MASK;
560  	control |= edmac->runtime_ctrl;
561  
562  	m2m_fill_desc(edmac);
563  	control |= M2M_CONTROL_DONEINT;
564  
565  	if (ep93xx_dma_advance_active(edmac)) {
566  		m2m_fill_desc(edmac);
567  		control |= M2M_CONTROL_NFBINT;
568  	}
569  
570  	/*
571  	 * Now we can finally enable the channel. For M2M channel this must be
572  	 * done _after_ the BCRx registers are programmed.
573  	 */
574  	control |= M2M_CONTROL_ENABLE;
575  	writel(control, edmac->regs + M2M_CONTROL);
576  
577  	if (!data) {
578  		/*
579  		 * For memcpy channels the software trigger must be asserted
580  		 * in order to start the memcpy operation.
581  		 */
582  		control |= M2M_CONTROL_START;
583  		writel(control, edmac->regs + M2M_CONTROL);
584  	}
585  }
586  
587  /*
588   * According to EP93xx User's Guide, we should receive DONE interrupt when all
589   * M2M DMA controller transactions complete normally. This is not always the
590   * case - sometimes EP93xx M2M DMA asserts DONE interrupt when the DMA channel
591   * is still running (channel Buffer FSM in DMA_BUF_ON state, and channel
592   * Control FSM in DMA_MEM_RD state, observed at least in IDE-DMA operation).
593   * In effect, disabling the channel when only DONE bit is set could stop
594   * currently running DMA transfer. To avoid this, we use Buffer FSM and
595   * Control FSM to check current state of DMA channel.
596   */
597  static int m2m_hw_interrupt(struct ep93xx_dma_chan *edmac)
598  {
599  	u32 status = readl(edmac->regs + M2M_STATUS);
600  	u32 ctl_fsm = status & M2M_STATUS_CTL_MASK;
601  	u32 buf_fsm = status & M2M_STATUS_BUF_MASK;
602  	bool done = status & M2M_STATUS_DONE;
603  	bool last_done;
604  	u32 control;
605  	struct ep93xx_dma_desc *desc;
606  
607  	/* Accept only DONE and NFB interrupts */
608  	if (!(readl(edmac->regs + M2M_INTERRUPT) & M2M_INTERRUPT_MASK))
609  		return INTERRUPT_UNKNOWN;
610  
611  	if (done) {
612  		/* Clear the DONE bit */
613  		writel(0, edmac->regs + M2M_INTERRUPT);
614  	}
615  
616  	/*
617  	 * Check whether we are done with descriptors or not. This, together
618  	 * with DMA channel state, determines action to take in interrupt.
619  	 */
620  	desc = ep93xx_dma_get_active(edmac);
621  	last_done = !desc || desc->txd.cookie;
622  
623  	/*
624  	 * Use M2M DMA Buffer FSM and Control FSM to check current state of
625  	 * DMA channel. Using DONE and NFB bits from channel status register
626  	 * or bits from channel interrupt register is not reliable.
627  	 */
628  	if (!last_done &&
629  	    (buf_fsm == M2M_STATUS_BUF_NO ||
630  	     buf_fsm == M2M_STATUS_BUF_ON)) {
631  		/*
632  		 * Two buffers are ready for update when Buffer FSM is in
633  		 * DMA_NO_BUF state. Only one buffer can be prepared without
634  		 * disabling the channel or polling the DONE bit.
635  		 * To simplify things, always prepare only one buffer.
636  		 */
637  		if (ep93xx_dma_advance_active(edmac)) {
638  			m2m_fill_desc(edmac);
639  			if (done && !edmac->chan.private) {
640  				/* Software trigger for memcpy channel */
641  				control = readl(edmac->regs + M2M_CONTROL);
642  				control |= M2M_CONTROL_START;
643  				writel(control, edmac->regs + M2M_CONTROL);
644  			}
645  			return INTERRUPT_NEXT_BUFFER;
646  		} else {
647  			last_done = true;
648  		}
649  	}
650  
651  	/*
652  	 * Disable the channel only when Buffer FSM is in DMA_NO_BUF state
653  	 * and Control FSM is in DMA_STALL state.
654  	 */
655  	if (last_done &&
656  	    buf_fsm == M2M_STATUS_BUF_NO &&
657  	    ctl_fsm == M2M_STATUS_CTL_STALL) {
658  		/* Disable interrupts and the channel */
659  		control = readl(edmac->regs + M2M_CONTROL);
660  		control &= ~(M2M_CONTROL_DONEINT | M2M_CONTROL_NFBINT
661  			    | M2M_CONTROL_ENABLE);
662  		writel(control, edmac->regs + M2M_CONTROL);
663  		return INTERRUPT_DONE;
664  	}
665  
666  	/*
667  	 * Nothing to do this time.
668  	 */
669  	return INTERRUPT_NEXT_BUFFER;
670  }
671  
672  /*
673   * DMA engine API implementation
674   */
675  
676  static struct ep93xx_dma_desc *
677  ep93xx_dma_desc_get(struct ep93xx_dma_chan *edmac)
678  {
679  	struct ep93xx_dma_desc *desc, *_desc;
680  	struct ep93xx_dma_desc *ret = NULL;
681  	unsigned long flags;
682  
683  	spin_lock_irqsave(&edmac->lock, flags);
684  	list_for_each_entry_safe(desc, _desc, &edmac->free_list, node) {
685  		if (async_tx_test_ack(&desc->txd)) {
686  			list_del_init(&desc->node);
687  
688  			/* Re-initialize the descriptor */
689  			desc->src_addr = 0;
690  			desc->dst_addr = 0;
691  			desc->size = 0;
692  			desc->complete = false;
693  			desc->txd.cookie = 0;
694  			desc->txd.callback = NULL;
695  			desc->txd.callback_param = NULL;
696  
697  			ret = desc;
698  			break;
699  		}
700  	}
701  	spin_unlock_irqrestore(&edmac->lock, flags);
702  	return ret;
703  }
704  
705  static void ep93xx_dma_desc_put(struct ep93xx_dma_chan *edmac,
706  				struct ep93xx_dma_desc *desc)
707  {
708  	if (desc) {
709  		unsigned long flags;
710  
711  		spin_lock_irqsave(&edmac->lock, flags);
712  		list_splice_init(&desc->tx_list, &edmac->free_list);
713  		list_add(&desc->node, &edmac->free_list);
714  		spin_unlock_irqrestore(&edmac->lock, flags);
715  	}
716  }
717  
718  /**
719   * ep93xx_dma_advance_work - start processing the next pending transaction
720   * @edmac: channel
721   *
722   * If we have pending transactions queued and we are currently idling, this
723   * function takes the next queued transaction from the @edmac->queue and
724   * pushes it to the hardware for execution.
725   */
726  static void ep93xx_dma_advance_work(struct ep93xx_dma_chan *edmac)
727  {
728  	struct ep93xx_dma_desc *new;
729  	unsigned long flags;
730  
731  	spin_lock_irqsave(&edmac->lock, flags);
732  	if (!list_empty(&edmac->active) || list_empty(&edmac->queue)) {
733  		spin_unlock_irqrestore(&edmac->lock, flags);
734  		return;
735  	}
736  
737  	/* Take the next descriptor from the pending queue */
738  	new = list_first_entry(&edmac->queue, struct ep93xx_dma_desc, node);
739  	list_del_init(&new->node);
740  
741  	ep93xx_dma_set_active(edmac, new);
742  
743  	/* Push it to the hardware */
744  	edmac->edma->hw_submit(edmac);
745  	spin_unlock_irqrestore(&edmac->lock, flags);
746  }
747  
748  static void ep93xx_dma_tasklet(struct tasklet_struct *t)
749  {
750  	struct ep93xx_dma_chan *edmac = from_tasklet(edmac, t, tasklet);
751  	struct ep93xx_dma_desc *desc, *d;
752  	struct dmaengine_desc_callback cb;
753  	LIST_HEAD(list);
754  
755  	memset(&cb, 0, sizeof(cb));
756  	spin_lock_irq(&edmac->lock);
757  	/*
758  	 * If dma_terminate_all() was called before we get to run, the active
759  	 * list has become empty. If that happens we aren't supposed to do
760  	 * anything more than call ep93xx_dma_advance_work().
761  	 */
762  	desc = ep93xx_dma_get_active(edmac);
763  	if (desc) {
764  		if (desc->complete) {
765  			/* mark descriptor complete for non cyclic case only */
766  			if (!test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags))
767  				dma_cookie_complete(&desc->txd);
768  			list_splice_init(&edmac->active, &list);
769  		}
770  		dmaengine_desc_get_callback(&desc->txd, &cb);
771  	}
772  	spin_unlock_irq(&edmac->lock);
773  
774  	/* Pick up the next descriptor from the queue */
775  	ep93xx_dma_advance_work(edmac);
776  
777  	/* Now we can release all the chained descriptors */
778  	list_for_each_entry_safe(desc, d, &list, node) {
779  		dma_descriptor_unmap(&desc->txd);
780  		ep93xx_dma_desc_put(edmac, desc);
781  	}
782  
783  	dmaengine_desc_callback_invoke(&cb, NULL);
784  }
785  
786  static irqreturn_t ep93xx_dma_interrupt(int irq, void *dev_id)
787  {
788  	struct ep93xx_dma_chan *edmac = dev_id;
789  	struct ep93xx_dma_desc *desc;
790  	irqreturn_t ret = IRQ_HANDLED;
791  
792  	spin_lock(&edmac->lock);
793  
794  	desc = ep93xx_dma_get_active(edmac);
795  	if (!desc) {
796  		dev_warn(chan2dev(edmac),
797  			 "got interrupt while active list is empty\n");
798  		spin_unlock(&edmac->lock);
799  		return IRQ_NONE;
800  	}
801  
802  	switch (edmac->edma->hw_interrupt(edmac)) {
803  	case INTERRUPT_DONE:
804  		desc->complete = true;
805  		tasklet_schedule(&edmac->tasklet);
806  		break;
807  
808  	case INTERRUPT_NEXT_BUFFER:
809  		if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags))
810  			tasklet_schedule(&edmac->tasklet);
811  		break;
812  
813  	default:
814  		dev_warn(chan2dev(edmac), "unknown interrupt!\n");
815  		ret = IRQ_NONE;
816  		break;
817  	}
818  
819  	spin_unlock(&edmac->lock);
820  	return ret;
821  }
822  
823  /**
824   * ep93xx_dma_tx_submit - set the prepared descriptor(s) to be executed
825   * @tx: descriptor to be executed
826   *
827   * Function will execute given descriptor on the hardware or if the hardware
828   * is busy, queue the descriptor to be executed later on. Returns cookie which
829   * can be used to poll the status of the descriptor.
830   */
831  static dma_cookie_t ep93xx_dma_tx_submit(struct dma_async_tx_descriptor *tx)
832  {
833  	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(tx->chan);
834  	struct ep93xx_dma_desc *desc;
835  	dma_cookie_t cookie;
836  	unsigned long flags;
837  
838  	spin_lock_irqsave(&edmac->lock, flags);
839  	cookie = dma_cookie_assign(tx);
840  
841  	desc = container_of(tx, struct ep93xx_dma_desc, txd);
842  
843  	/*
844  	 * If nothing is currently prosessed, we push this descriptor
845  	 * directly to the hardware. Otherwise we put the descriptor
846  	 * to the pending queue.
847  	 */
848  	if (list_empty(&edmac->active)) {
849  		ep93xx_dma_set_active(edmac, desc);
850  		edmac->edma->hw_submit(edmac);
851  	} else {
852  		list_add_tail(&desc->node, &edmac->queue);
853  	}
854  
855  	spin_unlock_irqrestore(&edmac->lock, flags);
856  	return cookie;
857  }
858  
859  /**
860   * ep93xx_dma_alloc_chan_resources - allocate resources for the channel
861   * @chan: channel to allocate resources
862   *
863   * Function allocates necessary resources for the given DMA channel and
864   * returns number of allocated descriptors for the channel. Negative errno
865   * is returned in case of failure.
866   */
867  static int ep93xx_dma_alloc_chan_resources(struct dma_chan *chan)
868  {
869  	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
870  	struct ep93xx_dma_data *data = chan->private;
871  	const char *name = dma_chan_name(chan);
872  	int ret, i;
873  
874  	/* Sanity check the channel parameters */
875  	if (!edmac->edma->m2m) {
876  		if (!data)
877  			return -EINVAL;
878  		if (data->port < EP93XX_DMA_I2S1 ||
879  		    data->port > EP93XX_DMA_IRDA)
880  			return -EINVAL;
881  		if (data->direction != ep93xx_dma_chan_direction(chan))
882  			return -EINVAL;
883  	} else {
884  		if (data) {
885  			switch (data->port) {
886  			case EP93XX_DMA_SSP:
887  			case EP93XX_DMA_IDE:
888  				if (!is_slave_direction(data->direction))
889  					return -EINVAL;
890  				break;
891  			default:
892  				return -EINVAL;
893  			}
894  		}
895  	}
896  
897  	if (data && data->name)
898  		name = data->name;
899  
900  	ret = clk_prepare_enable(edmac->clk);
901  	if (ret)
902  		return ret;
903  
904  	ret = request_irq(edmac->irq, ep93xx_dma_interrupt, 0, name, edmac);
905  	if (ret)
906  		goto fail_clk_disable;
907  
908  	spin_lock_irq(&edmac->lock);
909  	dma_cookie_init(&edmac->chan);
910  	ret = edmac->edma->hw_setup(edmac);
911  	spin_unlock_irq(&edmac->lock);
912  
913  	if (ret)
914  		goto fail_free_irq;
915  
916  	for (i = 0; i < DMA_MAX_CHAN_DESCRIPTORS; i++) {
917  		struct ep93xx_dma_desc *desc;
918  
919  		desc = kzalloc(sizeof(*desc), GFP_KERNEL);
920  		if (!desc) {
921  			dev_warn(chan2dev(edmac), "not enough descriptors\n");
922  			break;
923  		}
924  
925  		INIT_LIST_HEAD(&desc->tx_list);
926  
927  		dma_async_tx_descriptor_init(&desc->txd, chan);
928  		desc->txd.flags = DMA_CTRL_ACK;
929  		desc->txd.tx_submit = ep93xx_dma_tx_submit;
930  
931  		ep93xx_dma_desc_put(edmac, desc);
932  	}
933  
934  	return i;
935  
936  fail_free_irq:
937  	free_irq(edmac->irq, edmac);
938  fail_clk_disable:
939  	clk_disable_unprepare(edmac->clk);
940  
941  	return ret;
942  }
943  
944  /**
945   * ep93xx_dma_free_chan_resources - release resources for the channel
946   * @chan: channel
947   *
948   * Function releases all the resources allocated for the given channel.
949   * The channel must be idle when this is called.
950   */
951  static void ep93xx_dma_free_chan_resources(struct dma_chan *chan)
952  {
953  	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
954  	struct ep93xx_dma_desc *desc, *d;
955  	unsigned long flags;
956  	LIST_HEAD(list);
957  
958  	BUG_ON(!list_empty(&edmac->active));
959  	BUG_ON(!list_empty(&edmac->queue));
960  
961  	spin_lock_irqsave(&edmac->lock, flags);
962  	edmac->edma->hw_shutdown(edmac);
963  	edmac->runtime_addr = 0;
964  	edmac->runtime_ctrl = 0;
965  	edmac->buffer = 0;
966  	list_splice_init(&edmac->free_list, &list);
967  	spin_unlock_irqrestore(&edmac->lock, flags);
968  
969  	list_for_each_entry_safe(desc, d, &list, node)
970  		kfree(desc);
971  
972  	clk_disable_unprepare(edmac->clk);
973  	free_irq(edmac->irq, edmac);
974  }
975  
976  /**
977   * ep93xx_dma_prep_dma_memcpy - prepare a memcpy DMA operation
978   * @chan: channel
979   * @dest: destination bus address
980   * @src: source bus address
981   * @len: size of the transaction
982   * @flags: flags for the descriptor
983   *
984   * Returns a valid DMA descriptor or %NULL in case of failure.
985   */
986  static struct dma_async_tx_descriptor *
987  ep93xx_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest,
988  			   dma_addr_t src, size_t len, unsigned long flags)
989  {
990  	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
991  	struct ep93xx_dma_desc *desc, *first;
992  	size_t bytes, offset;
993  
994  	first = NULL;
995  	for (offset = 0; offset < len; offset += bytes) {
996  		desc = ep93xx_dma_desc_get(edmac);
997  		if (!desc) {
998  			dev_warn(chan2dev(edmac), "couldn't get descriptor\n");
999  			goto fail;
1000  		}
1001  
1002  		bytes = min_t(size_t, len - offset, DMA_MAX_CHAN_BYTES);
1003  
1004  		desc->src_addr = src + offset;
1005  		desc->dst_addr = dest + offset;
1006  		desc->size = bytes;
1007  
1008  		if (!first)
1009  			first = desc;
1010  		else
1011  			list_add_tail(&desc->node, &first->tx_list);
1012  	}
1013  
1014  	first->txd.cookie = -EBUSY;
1015  	first->txd.flags = flags;
1016  
1017  	return &first->txd;
1018  fail:
1019  	ep93xx_dma_desc_put(edmac, first);
1020  	return NULL;
1021  }
1022  
1023  /**
1024   * ep93xx_dma_prep_slave_sg - prepare a slave DMA operation
1025   * @chan: channel
1026   * @sgl: list of buffers to transfer
1027   * @sg_len: number of entries in @sgl
1028   * @dir: direction of tha DMA transfer
1029   * @flags: flags for the descriptor
1030   * @context: operation context (ignored)
1031   *
1032   * Returns a valid DMA descriptor or %NULL in case of failure.
1033   */
1034  static struct dma_async_tx_descriptor *
1035  ep93xx_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
1036  			 unsigned int sg_len, enum dma_transfer_direction dir,
1037  			 unsigned long flags, void *context)
1038  {
1039  	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
1040  	struct ep93xx_dma_desc *desc, *first;
1041  	struct scatterlist *sg;
1042  	int i;
1043  
1044  	if (!edmac->edma->m2m && dir != ep93xx_dma_chan_direction(chan)) {
1045  		dev_warn(chan2dev(edmac),
1046  			 "channel was configured with different direction\n");
1047  		return NULL;
1048  	}
1049  
1050  	if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) {
1051  		dev_warn(chan2dev(edmac),
1052  			 "channel is already used for cyclic transfers\n");
1053  		return NULL;
1054  	}
1055  
1056  	ep93xx_dma_slave_config_write(chan, dir, &edmac->slave_config);
1057  
1058  	first = NULL;
1059  	for_each_sg(sgl, sg, sg_len, i) {
1060  		size_t len = sg_dma_len(sg);
1061  
1062  		if (len > DMA_MAX_CHAN_BYTES) {
1063  			dev_warn(chan2dev(edmac), "too big transfer size %zu\n",
1064  				 len);
1065  			goto fail;
1066  		}
1067  
1068  		desc = ep93xx_dma_desc_get(edmac);
1069  		if (!desc) {
1070  			dev_warn(chan2dev(edmac), "couldn't get descriptor\n");
1071  			goto fail;
1072  		}
1073  
1074  		if (dir == DMA_MEM_TO_DEV) {
1075  			desc->src_addr = sg_dma_address(sg);
1076  			desc->dst_addr = edmac->runtime_addr;
1077  		} else {
1078  			desc->src_addr = edmac->runtime_addr;
1079  			desc->dst_addr = sg_dma_address(sg);
1080  		}
1081  		desc->size = len;
1082  
1083  		if (!first)
1084  			first = desc;
1085  		else
1086  			list_add_tail(&desc->node, &first->tx_list);
1087  	}
1088  
1089  	first->txd.cookie = -EBUSY;
1090  	first->txd.flags = flags;
1091  
1092  	return &first->txd;
1093  
1094  fail:
1095  	ep93xx_dma_desc_put(edmac, first);
1096  	return NULL;
1097  }
1098  
1099  /**
1100   * ep93xx_dma_prep_dma_cyclic - prepare a cyclic DMA operation
1101   * @chan: channel
1102   * @dma_addr: DMA mapped address of the buffer
1103   * @buf_len: length of the buffer (in bytes)
1104   * @period_len: length of a single period
1105   * @dir: direction of the operation
1106   * @flags: tx descriptor status flags
1107   *
1108   * Prepares a descriptor for cyclic DMA operation. This means that once the
1109   * descriptor is submitted, we will be submitting in a @period_len sized
1110   * buffers and calling callback once the period has been elapsed. Transfer
1111   * terminates only when client calls dmaengine_terminate_all() for this
1112   * channel.
1113   *
1114   * Returns a valid DMA descriptor or %NULL in case of failure.
1115   */
1116  static struct dma_async_tx_descriptor *
1117  ep93xx_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr,
1118  			   size_t buf_len, size_t period_len,
1119  			   enum dma_transfer_direction dir, unsigned long flags)
1120  {
1121  	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
1122  	struct ep93xx_dma_desc *desc, *first;
1123  	size_t offset = 0;
1124  
1125  	if (!edmac->edma->m2m && dir != ep93xx_dma_chan_direction(chan)) {
1126  		dev_warn(chan2dev(edmac),
1127  			 "channel was configured with different direction\n");
1128  		return NULL;
1129  	}
1130  
1131  	if (test_and_set_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) {
1132  		dev_warn(chan2dev(edmac),
1133  			 "channel is already used for cyclic transfers\n");
1134  		return NULL;
1135  	}
1136  
1137  	if (period_len > DMA_MAX_CHAN_BYTES) {
1138  		dev_warn(chan2dev(edmac), "too big period length %zu\n",
1139  			 period_len);
1140  		return NULL;
1141  	}
1142  
1143  	ep93xx_dma_slave_config_write(chan, dir, &edmac->slave_config);
1144  
1145  	/* Split the buffer into period size chunks */
1146  	first = NULL;
1147  	for (offset = 0; offset < buf_len; offset += period_len) {
1148  		desc = ep93xx_dma_desc_get(edmac);
1149  		if (!desc) {
1150  			dev_warn(chan2dev(edmac), "couldn't get descriptor\n");
1151  			goto fail;
1152  		}
1153  
1154  		if (dir == DMA_MEM_TO_DEV) {
1155  			desc->src_addr = dma_addr + offset;
1156  			desc->dst_addr = edmac->runtime_addr;
1157  		} else {
1158  			desc->src_addr = edmac->runtime_addr;
1159  			desc->dst_addr = dma_addr + offset;
1160  		}
1161  
1162  		desc->size = period_len;
1163  
1164  		if (!first)
1165  			first = desc;
1166  		else
1167  			list_add_tail(&desc->node, &first->tx_list);
1168  	}
1169  
1170  	first->txd.cookie = -EBUSY;
1171  
1172  	return &first->txd;
1173  
1174  fail:
1175  	ep93xx_dma_desc_put(edmac, first);
1176  	return NULL;
1177  }
1178  
1179  /**
1180   * ep93xx_dma_synchronize - Synchronizes the termination of transfers to the
1181   * current context.
1182   * @chan: channel
1183   *
1184   * Synchronizes the DMA channel termination to the current context. When this
1185   * function returns it is guaranteed that all transfers for previously issued
1186   * descriptors have stopped and it is safe to free the memory associated
1187   * with them. Furthermore it is guaranteed that all complete callback functions
1188   * for a previously submitted descriptor have finished running and it is safe to
1189   * free resources accessed from within the complete callbacks.
1190   */
1191  static void ep93xx_dma_synchronize(struct dma_chan *chan)
1192  {
1193  	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
1194  
1195  	if (edmac->edma->hw_synchronize)
1196  		edmac->edma->hw_synchronize(edmac);
1197  }
1198  
1199  /**
1200   * ep93xx_dma_terminate_all - terminate all transactions
1201   * @chan: channel
1202   *
1203   * Stops all DMA transactions. All descriptors are put back to the
1204   * @edmac->free_list and callbacks are _not_ called.
1205   */
1206  static int ep93xx_dma_terminate_all(struct dma_chan *chan)
1207  {
1208  	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
1209  	struct ep93xx_dma_desc *desc, *_d;
1210  	unsigned long flags;
1211  	LIST_HEAD(list);
1212  
1213  	spin_lock_irqsave(&edmac->lock, flags);
1214  	/* First we disable and flush the DMA channel */
1215  	edmac->edma->hw_shutdown(edmac);
1216  	clear_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags);
1217  	list_splice_init(&edmac->active, &list);
1218  	list_splice_init(&edmac->queue, &list);
1219  	/*
1220  	 * We then re-enable the channel. This way we can continue submitting
1221  	 * the descriptors by just calling ->hw_submit() again.
1222  	 */
1223  	edmac->edma->hw_setup(edmac);
1224  	spin_unlock_irqrestore(&edmac->lock, flags);
1225  
1226  	list_for_each_entry_safe(desc, _d, &list, node)
1227  		ep93xx_dma_desc_put(edmac, desc);
1228  
1229  	return 0;
1230  }
1231  
1232  static int ep93xx_dma_slave_config(struct dma_chan *chan,
1233  				   struct dma_slave_config *config)
1234  {
1235  	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
1236  
1237  	memcpy(&edmac->slave_config, config, sizeof(*config));
1238  
1239  	return 0;
1240  }
1241  
1242  static int ep93xx_dma_slave_config_write(struct dma_chan *chan,
1243  					 enum dma_transfer_direction dir,
1244  					 struct dma_slave_config *config)
1245  {
1246  	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
1247  	enum dma_slave_buswidth width;
1248  	unsigned long flags;
1249  	u32 addr, ctrl;
1250  
1251  	if (!edmac->edma->m2m)
1252  		return -EINVAL;
1253  
1254  	switch (dir) {
1255  	case DMA_DEV_TO_MEM:
1256  		width = config->src_addr_width;
1257  		addr = config->src_addr;
1258  		break;
1259  
1260  	case DMA_MEM_TO_DEV:
1261  		width = config->dst_addr_width;
1262  		addr = config->dst_addr;
1263  		break;
1264  
1265  	default:
1266  		return -EINVAL;
1267  	}
1268  
1269  	switch (width) {
1270  	case DMA_SLAVE_BUSWIDTH_1_BYTE:
1271  		ctrl = 0;
1272  		break;
1273  	case DMA_SLAVE_BUSWIDTH_2_BYTES:
1274  		ctrl = M2M_CONTROL_PW_16;
1275  		break;
1276  	case DMA_SLAVE_BUSWIDTH_4_BYTES:
1277  		ctrl = M2M_CONTROL_PW_32;
1278  		break;
1279  	default:
1280  		return -EINVAL;
1281  	}
1282  
1283  	spin_lock_irqsave(&edmac->lock, flags);
1284  	edmac->runtime_addr = addr;
1285  	edmac->runtime_ctrl = ctrl;
1286  	spin_unlock_irqrestore(&edmac->lock, flags);
1287  
1288  	return 0;
1289  }
1290  
1291  /**
1292   * ep93xx_dma_tx_status - check if a transaction is completed
1293   * @chan: channel
1294   * @cookie: transaction specific cookie
1295   * @state: state of the transaction is stored here if given
1296   *
1297   * This function can be used to query state of a given transaction.
1298   */
1299  static enum dma_status ep93xx_dma_tx_status(struct dma_chan *chan,
1300  					    dma_cookie_t cookie,
1301  					    struct dma_tx_state *state)
1302  {
1303  	return dma_cookie_status(chan, cookie, state);
1304  }
1305  
1306  /**
1307   * ep93xx_dma_issue_pending - push pending transactions to the hardware
1308   * @chan: channel
1309   *
1310   * When this function is called, all pending transactions are pushed to the
1311   * hardware and executed.
1312   */
1313  static void ep93xx_dma_issue_pending(struct dma_chan *chan)
1314  {
1315  	ep93xx_dma_advance_work(to_ep93xx_dma_chan(chan));
1316  }
1317  
1318  static int __init ep93xx_dma_probe(struct platform_device *pdev)
1319  {
1320  	struct ep93xx_dma_platform_data *pdata = dev_get_platdata(&pdev->dev);
1321  	struct ep93xx_dma_engine *edma;
1322  	struct dma_device *dma_dev;
1323  	size_t edma_size;
1324  	int ret, i;
1325  
1326  	edma_size = pdata->num_channels * sizeof(struct ep93xx_dma_chan);
1327  	edma = kzalloc(sizeof(*edma) + edma_size, GFP_KERNEL);
1328  	if (!edma)
1329  		return -ENOMEM;
1330  
1331  	dma_dev = &edma->dma_dev;
1332  	edma->m2m = platform_get_device_id(pdev)->driver_data;
1333  	edma->num_channels = pdata->num_channels;
1334  
1335  	INIT_LIST_HEAD(&dma_dev->channels);
1336  	for (i = 0; i < pdata->num_channels; i++) {
1337  		const struct ep93xx_dma_chan_data *cdata = &pdata->channels[i];
1338  		struct ep93xx_dma_chan *edmac = &edma->channels[i];
1339  
1340  		edmac->chan.device = dma_dev;
1341  		edmac->regs = cdata->base;
1342  		edmac->irq = cdata->irq;
1343  		edmac->edma = edma;
1344  
1345  		edmac->clk = clk_get(NULL, cdata->name);
1346  		if (IS_ERR(edmac->clk)) {
1347  			dev_warn(&pdev->dev, "failed to get clock for %s\n",
1348  				 cdata->name);
1349  			continue;
1350  		}
1351  
1352  		spin_lock_init(&edmac->lock);
1353  		INIT_LIST_HEAD(&edmac->active);
1354  		INIT_LIST_HEAD(&edmac->queue);
1355  		INIT_LIST_HEAD(&edmac->free_list);
1356  		tasklet_setup(&edmac->tasklet, ep93xx_dma_tasklet);
1357  
1358  		list_add_tail(&edmac->chan.device_node,
1359  			      &dma_dev->channels);
1360  	}
1361  
1362  	dma_cap_zero(dma_dev->cap_mask);
1363  	dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
1364  	dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask);
1365  
1366  	dma_dev->dev = &pdev->dev;
1367  	dma_dev->device_alloc_chan_resources = ep93xx_dma_alloc_chan_resources;
1368  	dma_dev->device_free_chan_resources = ep93xx_dma_free_chan_resources;
1369  	dma_dev->device_prep_slave_sg = ep93xx_dma_prep_slave_sg;
1370  	dma_dev->device_prep_dma_cyclic = ep93xx_dma_prep_dma_cyclic;
1371  	dma_dev->device_config = ep93xx_dma_slave_config;
1372  	dma_dev->device_synchronize = ep93xx_dma_synchronize;
1373  	dma_dev->device_terminate_all = ep93xx_dma_terminate_all;
1374  	dma_dev->device_issue_pending = ep93xx_dma_issue_pending;
1375  	dma_dev->device_tx_status = ep93xx_dma_tx_status;
1376  
1377  	dma_set_max_seg_size(dma_dev->dev, DMA_MAX_CHAN_BYTES);
1378  
1379  	if (edma->m2m) {
1380  		dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
1381  		dma_dev->device_prep_dma_memcpy = ep93xx_dma_prep_dma_memcpy;
1382  
1383  		edma->hw_setup = m2m_hw_setup;
1384  		edma->hw_shutdown = m2m_hw_shutdown;
1385  		edma->hw_submit = m2m_hw_submit;
1386  		edma->hw_interrupt = m2m_hw_interrupt;
1387  	} else {
1388  		dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
1389  
1390  		edma->hw_synchronize = m2p_hw_synchronize;
1391  		edma->hw_setup = m2p_hw_setup;
1392  		edma->hw_shutdown = m2p_hw_shutdown;
1393  		edma->hw_submit = m2p_hw_submit;
1394  		edma->hw_interrupt = m2p_hw_interrupt;
1395  	}
1396  
1397  	ret = dma_async_device_register(dma_dev);
1398  	if (unlikely(ret)) {
1399  		for (i = 0; i < edma->num_channels; i++) {
1400  			struct ep93xx_dma_chan *edmac = &edma->channels[i];
1401  			if (!IS_ERR_OR_NULL(edmac->clk))
1402  				clk_put(edmac->clk);
1403  		}
1404  		kfree(edma);
1405  	} else {
1406  		dev_info(dma_dev->dev, "EP93xx M2%s DMA ready\n",
1407  			 edma->m2m ? "M" : "P");
1408  	}
1409  
1410  	return ret;
1411  }
1412  
1413  static const struct platform_device_id ep93xx_dma_driver_ids[] = {
1414  	{ "ep93xx-dma-m2p", 0 },
1415  	{ "ep93xx-dma-m2m", 1 },
1416  	{ },
1417  };
1418  
1419  static struct platform_driver ep93xx_dma_driver = {
1420  	.driver		= {
1421  		.name	= "ep93xx-dma",
1422  	},
1423  	.id_table	= ep93xx_dma_driver_ids,
1424  };
1425  
1426  static int __init ep93xx_dma_module_init(void)
1427  {
1428  	return platform_driver_probe(&ep93xx_dma_driver, ep93xx_dma_probe);
1429  }
1430  subsys_initcall(ep93xx_dma_module_init);
1431  
1432  MODULE_AUTHOR("Mika Westerberg <mika.westerberg@iki.fi>");
1433  MODULE_DESCRIPTION("EP93xx DMA driver");
1434  MODULE_LICENSE("GPL");
1435