xref: /openbmc/linux/drivers/dma/owl-dma.c (revision 3053b681)
1 // SPDX-License-Identifier: GPL-2.0+
2 //
3 // Actions Semi Owl SoCs DMA driver
4 //
5 // Copyright (c) 2014 Actions Semi Inc.
6 // Author: David Liu <liuwei@actions-semi.com>
7 //
8 // Copyright (c) 2018 Linaro Ltd.
9 // Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
10 
11 #include <linux/bitops.h>
12 #include <linux/clk.h>
13 #include <linux/delay.h>
14 #include <linux/dmaengine.h>
15 #include <linux/dma-mapping.h>
16 #include <linux/dmapool.h>
17 #include <linux/err.h>
18 #include <linux/init.h>
19 #include <linux/interrupt.h>
20 #include <linux/io.h>
21 #include <linux/mm.h>
22 #include <linux/module.h>
23 #include <linux/of.h>
24 #include <linux/of_dma.h>
25 #include <linux/platform_device.h>
26 #include <linux/slab.h>
27 #include "virt-dma.h"
28 
29 #define OWL_DMA_FRAME_MAX_LENGTH		0xfffff
30 
31 /* Global DMA Controller Registers */
32 #define OWL_DMA_IRQ_PD0				0x00
33 #define OWL_DMA_IRQ_PD1				0x04
34 #define OWL_DMA_IRQ_PD2				0x08
35 #define OWL_DMA_IRQ_PD3				0x0C
36 #define OWL_DMA_IRQ_EN0				0x10
37 #define OWL_DMA_IRQ_EN1				0x14
38 #define OWL_DMA_IRQ_EN2				0x18
39 #define OWL_DMA_IRQ_EN3				0x1C
40 #define OWL_DMA_SECURE_ACCESS_CTL		0x20
41 #define OWL_DMA_NIC_QOS				0x24
42 #define OWL_DMA_DBGSEL				0x28
43 #define OWL_DMA_IDLE_STAT			0x2C
44 
45 /* Channel Registers */
46 #define OWL_DMA_CHAN_BASE(i)			(0x100 + (i) * 0x100)
47 #define OWL_DMAX_MODE				0x00
48 #define OWL_DMAX_SOURCE				0x04
49 #define OWL_DMAX_DESTINATION			0x08
50 #define OWL_DMAX_FRAME_LEN			0x0C
51 #define OWL_DMAX_FRAME_CNT			0x10
52 #define OWL_DMAX_REMAIN_FRAME_CNT		0x14
53 #define OWL_DMAX_REMAIN_CNT			0x18
54 #define OWL_DMAX_SOURCE_STRIDE			0x1C
55 #define OWL_DMAX_DESTINATION_STRIDE		0x20
56 #define OWL_DMAX_START				0x24
57 #define OWL_DMAX_PAUSE				0x28
58 #define OWL_DMAX_CHAINED_CTL			0x2C
59 #define OWL_DMAX_CONSTANT			0x30
60 #define OWL_DMAX_LINKLIST_CTL			0x34
61 #define OWL_DMAX_NEXT_DESCRIPTOR		0x38
62 #define OWL_DMAX_CURRENT_DESCRIPTOR_NUM		0x3C
63 #define OWL_DMAX_INT_CTL			0x40
64 #define OWL_DMAX_INT_STATUS			0x44
65 #define OWL_DMAX_CURRENT_SOURCE_POINTER		0x48
66 #define OWL_DMAX_CURRENT_DESTINATION_POINTER	0x4C
67 
68 /* OWL_DMAX_MODE Bits */
69 #define OWL_DMA_MODE_TS(x)			(((x) & GENMASK(5, 0)) << 0)
70 #define OWL_DMA_MODE_ST(x)			(((x) & GENMASK(1, 0)) << 8)
71 #define	OWL_DMA_MODE_ST_DEV			OWL_DMA_MODE_ST(0)
72 #define	OWL_DMA_MODE_ST_DCU			OWL_DMA_MODE_ST(2)
73 #define	OWL_DMA_MODE_ST_SRAM			OWL_DMA_MODE_ST(3)
74 #define OWL_DMA_MODE_DT(x)			(((x) & GENMASK(1, 0)) << 10)
75 #define	OWL_DMA_MODE_DT_DEV			OWL_DMA_MODE_DT(0)
76 #define	OWL_DMA_MODE_DT_DCU			OWL_DMA_MODE_DT(2)
77 #define	OWL_DMA_MODE_DT_SRAM			OWL_DMA_MODE_DT(3)
78 #define OWL_DMA_MODE_SAM(x)			(((x) & GENMASK(1, 0)) << 16)
79 #define	OWL_DMA_MODE_SAM_CONST			OWL_DMA_MODE_SAM(0)
80 #define	OWL_DMA_MODE_SAM_INC			OWL_DMA_MODE_SAM(1)
81 #define	OWL_DMA_MODE_SAM_STRIDE			OWL_DMA_MODE_SAM(2)
82 #define OWL_DMA_MODE_DAM(x)			(((x) & GENMASK(1, 0)) << 18)
83 #define	OWL_DMA_MODE_DAM_CONST			OWL_DMA_MODE_DAM(0)
84 #define	OWL_DMA_MODE_DAM_INC			OWL_DMA_MODE_DAM(1)
85 #define	OWL_DMA_MODE_DAM_STRIDE			OWL_DMA_MODE_DAM(2)
86 #define OWL_DMA_MODE_PW(x)			(((x) & GENMASK(2, 0)) << 20)
87 #define OWL_DMA_MODE_CB				BIT(23)
88 #define OWL_DMA_MODE_NDDBW(x)			(((x) & 0x1) << 28)
89 #define	OWL_DMA_MODE_NDDBW_32BIT		OWL_DMA_MODE_NDDBW(0)
90 #define	OWL_DMA_MODE_NDDBW_8BIT			OWL_DMA_MODE_NDDBW(1)
91 #define OWL_DMA_MODE_CFE			BIT(29)
92 #define OWL_DMA_MODE_LME			BIT(30)
93 #define OWL_DMA_MODE_CME			BIT(31)
94 
95 /* OWL_DMAX_LINKLIST_CTL Bits */
96 #define OWL_DMA_LLC_SAV(x)			(((x) & GENMASK(1, 0)) << 8)
97 #define	OWL_DMA_LLC_SAV_INC			OWL_DMA_LLC_SAV(0)
98 #define	OWL_DMA_LLC_SAV_LOAD_NEXT		OWL_DMA_LLC_SAV(1)
99 #define	OWL_DMA_LLC_SAV_LOAD_PREV		OWL_DMA_LLC_SAV(2)
100 #define OWL_DMA_LLC_DAV(x)			(((x) & GENMASK(1, 0)) << 10)
101 #define	OWL_DMA_LLC_DAV_INC			OWL_DMA_LLC_DAV(0)
102 #define	OWL_DMA_LLC_DAV_LOAD_NEXT		OWL_DMA_LLC_DAV(1)
103 #define	OWL_DMA_LLC_DAV_LOAD_PREV		OWL_DMA_LLC_DAV(2)
104 #define OWL_DMA_LLC_SUSPEND			BIT(16)
105 
106 /* OWL_DMAX_INT_CTL Bits */
107 #define OWL_DMA_INTCTL_BLOCK			BIT(0)
108 #define OWL_DMA_INTCTL_SUPER_BLOCK		BIT(1)
109 #define OWL_DMA_INTCTL_FRAME			BIT(2)
110 #define OWL_DMA_INTCTL_HALF_FRAME		BIT(3)
111 #define OWL_DMA_INTCTL_LAST_FRAME		BIT(4)
112 
113 /* OWL_DMAX_INT_STATUS Bits */
114 #define OWL_DMA_INTSTAT_BLOCK			BIT(0)
115 #define OWL_DMA_INTSTAT_SUPER_BLOCK		BIT(1)
116 #define OWL_DMA_INTSTAT_FRAME			BIT(2)
117 #define OWL_DMA_INTSTAT_HALF_FRAME		BIT(3)
118 #define OWL_DMA_INTSTAT_LAST_FRAME		BIT(4)
119 
120 /* Pack shift and newshift in a single word */
121 #define BIT_FIELD(val, width, shift, newshift)	\
122 		((((val) >> (shift)) & ((BIT(width)) - 1)) << (newshift))
123 
124 /* Frame count value is fixed as 1 */
125 #define FCNT_VAL				0x1
126 
127 /**
128  * enum owl_dmadesc_offsets - Describe DMA descriptor, hardware link
129  * list for dma transfer
130  * @OWL_DMADESC_NEXT_LLI: physical address of the next link list
131  * @OWL_DMADESC_SADDR: source physical address
132  * @OWL_DMADESC_DADDR: destination physical address
133  * @OWL_DMADESC_FLEN: frame length
134  * @OWL_DMADESC_SRC_STRIDE: source stride
135  * @OWL_DMADESC_DST_STRIDE: destination stride
136  * @OWL_DMADESC_CTRLA: dma_mode and linklist ctrl config
137  * @OWL_DMADESC_CTRLB: interrupt config
138  * @OWL_DMADESC_CONST_NUM: data for constant fill
139  * @OWL_DMADESC_SIZE: max size of this enum
140  */
141 enum owl_dmadesc_offsets {
142 	OWL_DMADESC_NEXT_LLI = 0,
143 	OWL_DMADESC_SADDR,
144 	OWL_DMADESC_DADDR,
145 	OWL_DMADESC_FLEN,
146 	OWL_DMADESC_SRC_STRIDE,
147 	OWL_DMADESC_DST_STRIDE,
148 	OWL_DMADESC_CTRLA,
149 	OWL_DMADESC_CTRLB,
150 	OWL_DMADESC_CONST_NUM,
151 	OWL_DMADESC_SIZE
152 };
153 
154 enum owl_dma_id {
155 	S900_DMA,
156 	S700_DMA,
157 };
158 
159 /**
160  * struct owl_dma_lli - Link list for dma transfer
161  * @hw: hardware link list
162  * @phys: physical address of hardware link list
163  * @node: node for txd's lli_list
164  */
165 struct owl_dma_lli {
166 	u32			hw[OWL_DMADESC_SIZE];
167 	dma_addr_t		phys;
168 	struct list_head	node;
169 };
170 
171 /**
172  * struct owl_dma_txd - Wrapper for struct dma_async_tx_descriptor
173  * @vd: virtual DMA descriptor
174  * @lli_list: link list of lli nodes
175  * @cyclic: flag to indicate cyclic transfers
176  */
177 struct owl_dma_txd {
178 	struct virt_dma_desc	vd;
179 	struct list_head	lli_list;
180 	bool			cyclic;
181 };
182 
183 /**
184  * struct owl_dma_pchan - Holder for the physical channels
185  * @id: physical index to this channel
186  * @base: virtual memory base for the dma channel
187  * @vchan: the virtual channel currently being served by this physical channel
188  */
189 struct owl_dma_pchan {
190 	u32			id;
191 	void __iomem		*base;
192 	struct owl_dma_vchan	*vchan;
193 };
194 
195 /**
196  * struct owl_dma_vchan - Wrapper for DMA ENGINE channel
197  * @vc: wrapped virtual channel
198  * @pchan: the physical channel utilized by this channel
199  * @txd: active transaction on this channel
200  * @cfg: slave configuration for this channel
201  * @drq: physical DMA request ID for this channel
202  */
203 struct owl_dma_vchan {
204 	struct virt_dma_chan	vc;
205 	struct owl_dma_pchan	*pchan;
206 	struct owl_dma_txd	*txd;
207 	struct dma_slave_config cfg;
208 	u8			drq;
209 };
210 
211 /**
212  * struct owl_dma - Holder for the Owl DMA controller
213  * @dma: dma engine for this instance
214  * @base: virtual memory base for the DMA controller
215  * @clk: clock for the DMA controller
216  * @lock: a lock to use when change DMA controller global register
217  * @lli_pool: a pool for the LLI descriptors
218  * @irq: interrupt ID for the DMA controller
219  * @nr_pchans: the number of physical channels
220  * @pchans: array of data for the physical channels
221  * @nr_vchans: the number of physical channels
222  * @vchans: array of data for the physical channels
223  * @devid: device id based on OWL SoC
224  */
225 struct owl_dma {
226 	struct dma_device	dma;
227 	void __iomem		*base;
228 	struct clk		*clk;
229 	spinlock_t		lock;
230 	struct dma_pool		*lli_pool;
231 	int			irq;
232 
233 	unsigned int		nr_pchans;
234 	struct owl_dma_pchan	*pchans;
235 
236 	unsigned int		nr_vchans;
237 	struct owl_dma_vchan	*vchans;
238 	enum owl_dma_id		devid;
239 };
240 
pchan_update(struct owl_dma_pchan * pchan,u32 reg,u32 val,bool state)241 static void pchan_update(struct owl_dma_pchan *pchan, u32 reg,
242 			 u32 val, bool state)
243 {
244 	u32 regval;
245 
246 	regval = readl(pchan->base + reg);
247 
248 	if (state)
249 		regval |= val;
250 	else
251 		regval &= ~val;
252 
253 	writel(regval, pchan->base + reg);
254 }
255 
pchan_writel(struct owl_dma_pchan * pchan,u32 reg,u32 data)256 static void pchan_writel(struct owl_dma_pchan *pchan, u32 reg, u32 data)
257 {
258 	writel(data, pchan->base + reg);
259 }
260 
pchan_readl(struct owl_dma_pchan * pchan,u32 reg)261 static u32 pchan_readl(struct owl_dma_pchan *pchan, u32 reg)
262 {
263 	return readl(pchan->base + reg);
264 }
265 
dma_update(struct owl_dma * od,u32 reg,u32 val,bool state)266 static void dma_update(struct owl_dma *od, u32 reg, u32 val, bool state)
267 {
268 	u32 regval;
269 
270 	regval = readl(od->base + reg);
271 
272 	if (state)
273 		regval |= val;
274 	else
275 		regval &= ~val;
276 
277 	writel(regval, od->base + reg);
278 }
279 
dma_writel(struct owl_dma * od,u32 reg,u32 data)280 static void dma_writel(struct owl_dma *od, u32 reg, u32 data)
281 {
282 	writel(data, od->base + reg);
283 }
284 
dma_readl(struct owl_dma * od,u32 reg)285 static u32 dma_readl(struct owl_dma *od, u32 reg)
286 {
287 	return readl(od->base + reg);
288 }
289 
to_owl_dma(struct dma_device * dd)290 static inline struct owl_dma *to_owl_dma(struct dma_device *dd)
291 {
292 	return container_of(dd, struct owl_dma, dma);
293 }
294 
chan2dev(struct dma_chan * chan)295 static struct device *chan2dev(struct dma_chan *chan)
296 {
297 	return &chan->dev->device;
298 }
299 
to_owl_vchan(struct dma_chan * chan)300 static inline struct owl_dma_vchan *to_owl_vchan(struct dma_chan *chan)
301 {
302 	return container_of(chan, struct owl_dma_vchan, vc.chan);
303 }
304 
to_owl_txd(struct dma_async_tx_descriptor * tx)305 static inline struct owl_dma_txd *to_owl_txd(struct dma_async_tx_descriptor *tx)
306 {
307 	return container_of(tx, struct owl_dma_txd, vd.tx);
308 }
309 
llc_hw_ctrla(u32 mode,u32 llc_ctl)310 static inline u32 llc_hw_ctrla(u32 mode, u32 llc_ctl)
311 {
312 	u32 ctl;
313 
314 	ctl = BIT_FIELD(mode, 4, 28, 28) |
315 	      BIT_FIELD(mode, 8, 16, 20) |
316 	      BIT_FIELD(mode, 4, 8, 16) |
317 	      BIT_FIELD(mode, 6, 0, 10) |
318 	      BIT_FIELD(llc_ctl, 2, 10, 8) |
319 	      BIT_FIELD(llc_ctl, 2, 8, 6);
320 
321 	return ctl;
322 }
323 
llc_hw_ctrlb(u32 int_ctl)324 static inline u32 llc_hw_ctrlb(u32 int_ctl)
325 {
326 	u32 ctl;
327 
328 	/*
329 	 * Irrespective of the SoC, ctrlb value starts filling from
330 	 * bit 18.
331 	 */
332 	ctl = BIT_FIELD(int_ctl, 7, 0, 18);
333 
334 	return ctl;
335 }
336 
llc_hw_flen(struct owl_dma_lli * lli)337 static u32 llc_hw_flen(struct owl_dma_lli *lli)
338 {
339 	return lli->hw[OWL_DMADESC_FLEN] & GENMASK(19, 0);
340 }
341 
owl_dma_free_lli(struct owl_dma * od,struct owl_dma_lli * lli)342 static void owl_dma_free_lli(struct owl_dma *od,
343 			     struct owl_dma_lli *lli)
344 {
345 	list_del(&lli->node);
346 	dma_pool_free(od->lli_pool, lli, lli->phys);
347 }
348 
owl_dma_alloc_lli(struct owl_dma * od)349 static struct owl_dma_lli *owl_dma_alloc_lli(struct owl_dma *od)
350 {
351 	struct owl_dma_lli *lli;
352 	dma_addr_t phys;
353 
354 	lli = dma_pool_alloc(od->lli_pool, GFP_NOWAIT, &phys);
355 	if (!lli)
356 		return NULL;
357 
358 	INIT_LIST_HEAD(&lli->node);
359 	lli->phys = phys;
360 
361 	return lli;
362 }
363 
owl_dma_add_lli(struct owl_dma_txd * txd,struct owl_dma_lli * prev,struct owl_dma_lli * next,bool is_cyclic)364 static struct owl_dma_lli *owl_dma_add_lli(struct owl_dma_txd *txd,
365 					   struct owl_dma_lli *prev,
366 					   struct owl_dma_lli *next,
367 					   bool is_cyclic)
368 {
369 	if (!is_cyclic)
370 		list_add_tail(&next->node, &txd->lli_list);
371 
372 	if (prev) {
373 		prev->hw[OWL_DMADESC_NEXT_LLI] = next->phys;
374 		prev->hw[OWL_DMADESC_CTRLA] |=
375 					llc_hw_ctrla(OWL_DMA_MODE_LME, 0);
376 	}
377 
378 	return next;
379 }
380 
owl_dma_cfg_lli(struct owl_dma_vchan * vchan,struct owl_dma_lli * lli,dma_addr_t src,dma_addr_t dst,u32 len,enum dma_transfer_direction dir,struct dma_slave_config * sconfig,bool is_cyclic)381 static inline int owl_dma_cfg_lli(struct owl_dma_vchan *vchan,
382 				  struct owl_dma_lli *lli,
383 				  dma_addr_t src, dma_addr_t dst,
384 				  u32 len, enum dma_transfer_direction dir,
385 				  struct dma_slave_config *sconfig,
386 				  bool is_cyclic)
387 {
388 	struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);
389 	u32 mode, ctrlb;
390 
391 	mode = OWL_DMA_MODE_PW(0);
392 
393 	switch (dir) {
394 	case DMA_MEM_TO_MEM:
395 		mode |= OWL_DMA_MODE_TS(0) | OWL_DMA_MODE_ST_DCU |
396 			OWL_DMA_MODE_DT_DCU | OWL_DMA_MODE_SAM_INC |
397 			OWL_DMA_MODE_DAM_INC;
398 
399 		break;
400 	case DMA_MEM_TO_DEV:
401 		mode |= OWL_DMA_MODE_TS(vchan->drq)
402 			| OWL_DMA_MODE_ST_DCU | OWL_DMA_MODE_DT_DEV
403 			| OWL_DMA_MODE_SAM_INC | OWL_DMA_MODE_DAM_CONST;
404 
405 		/*
406 		 * Hardware only supports 32bit and 8bit buswidth. Since the
407 		 * default is 32bit, select 8bit only when requested.
408 		 */
409 		if (sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_1_BYTE)
410 			mode |= OWL_DMA_MODE_NDDBW_8BIT;
411 
412 		break;
413 	case DMA_DEV_TO_MEM:
414 		 mode |= OWL_DMA_MODE_TS(vchan->drq)
415 			| OWL_DMA_MODE_ST_DEV | OWL_DMA_MODE_DT_DCU
416 			| OWL_DMA_MODE_SAM_CONST | OWL_DMA_MODE_DAM_INC;
417 
418 		/*
419 		 * Hardware only supports 32bit and 8bit buswidth. Since the
420 		 * default is 32bit, select 8bit only when requested.
421 		 */
422 		if (sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_1_BYTE)
423 			mode |= OWL_DMA_MODE_NDDBW_8BIT;
424 
425 		break;
426 	default:
427 		return -EINVAL;
428 	}
429 
430 	lli->hw[OWL_DMADESC_CTRLA] = llc_hw_ctrla(mode,
431 						  OWL_DMA_LLC_SAV_LOAD_NEXT |
432 						  OWL_DMA_LLC_DAV_LOAD_NEXT);
433 
434 	if (is_cyclic)
435 		ctrlb = llc_hw_ctrlb(OWL_DMA_INTCTL_BLOCK);
436 	else
437 		ctrlb = llc_hw_ctrlb(OWL_DMA_INTCTL_SUPER_BLOCK);
438 
439 	lli->hw[OWL_DMADESC_NEXT_LLI] = 0; /* One link list by default */
440 	lli->hw[OWL_DMADESC_SADDR] = src;
441 	lli->hw[OWL_DMADESC_DADDR] = dst;
442 	lli->hw[OWL_DMADESC_SRC_STRIDE] = 0;
443 	lli->hw[OWL_DMADESC_DST_STRIDE] = 0;
444 
445 	if (od->devid == S700_DMA) {
446 		/* Max frame length is 1MB */
447 		lli->hw[OWL_DMADESC_FLEN] = len;
448 		/*
449 		 * On S700, word starts from offset 0x1C is shared between
450 		 * frame count and ctrlb, where first 12 bits are for frame
451 		 * count and rest of 20 bits are for ctrlb.
452 		 */
453 		lli->hw[OWL_DMADESC_CTRLB] = FCNT_VAL | ctrlb;
454 	} else {
455 		/*
456 		 * On S900, word starts from offset 0xC is shared between
457 		 * frame length (max frame length is 1MB) and frame count,
458 		 * where first 20 bits are for frame length and rest of
459 		 * 12 bits are for frame count.
460 		 */
461 		lli->hw[OWL_DMADESC_FLEN] = len | FCNT_VAL << 20;
462 		lli->hw[OWL_DMADESC_CTRLB] = ctrlb;
463 	}
464 
465 	return 0;
466 }
467 
owl_dma_get_pchan(struct owl_dma * od,struct owl_dma_vchan * vchan)468 static struct owl_dma_pchan *owl_dma_get_pchan(struct owl_dma *od,
469 					       struct owl_dma_vchan *vchan)
470 {
471 	struct owl_dma_pchan *pchan = NULL;
472 	unsigned long flags;
473 	int i;
474 
475 	for (i = 0; i < od->nr_pchans; i++) {
476 		pchan = &od->pchans[i];
477 
478 		spin_lock_irqsave(&od->lock, flags);
479 		if (!pchan->vchan) {
480 			pchan->vchan = vchan;
481 			spin_unlock_irqrestore(&od->lock, flags);
482 			break;
483 		}
484 
485 		spin_unlock_irqrestore(&od->lock, flags);
486 	}
487 
488 	return pchan;
489 }
490 
owl_dma_pchan_busy(struct owl_dma * od,struct owl_dma_pchan * pchan)491 static int owl_dma_pchan_busy(struct owl_dma *od, struct owl_dma_pchan *pchan)
492 {
493 	unsigned int val;
494 
495 	val = dma_readl(od, OWL_DMA_IDLE_STAT);
496 
497 	return !(val & (1 << pchan->id));
498 }
499 
owl_dma_terminate_pchan(struct owl_dma * od,struct owl_dma_pchan * pchan)500 static void owl_dma_terminate_pchan(struct owl_dma *od,
501 				    struct owl_dma_pchan *pchan)
502 {
503 	unsigned long flags;
504 	u32 irq_pd;
505 
506 	pchan_writel(pchan, OWL_DMAX_START, 0);
507 	pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
508 
509 	spin_lock_irqsave(&od->lock, flags);
510 	dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), false);
511 
512 	irq_pd = dma_readl(od, OWL_DMA_IRQ_PD0);
513 	if (irq_pd & (1 << pchan->id)) {
514 		dev_warn(od->dma.dev,
515 			 "terminating pchan %d that still has pending irq\n",
516 			 pchan->id);
517 		dma_writel(od, OWL_DMA_IRQ_PD0, (1 << pchan->id));
518 	}
519 
520 	pchan->vchan = NULL;
521 
522 	spin_unlock_irqrestore(&od->lock, flags);
523 }
524 
owl_dma_pause_pchan(struct owl_dma_pchan * pchan)525 static void owl_dma_pause_pchan(struct owl_dma_pchan *pchan)
526 {
527 	pchan_writel(pchan, 1, OWL_DMAX_PAUSE);
528 }
529 
owl_dma_resume_pchan(struct owl_dma_pchan * pchan)530 static void owl_dma_resume_pchan(struct owl_dma_pchan *pchan)
531 {
532 	pchan_writel(pchan, 0, OWL_DMAX_PAUSE);
533 }
534 
owl_dma_start_next_txd(struct owl_dma_vchan * vchan)535 static int owl_dma_start_next_txd(struct owl_dma_vchan *vchan)
536 {
537 	struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);
538 	struct virt_dma_desc *vd = vchan_next_desc(&vchan->vc);
539 	struct owl_dma_pchan *pchan = vchan->pchan;
540 	struct owl_dma_txd *txd = to_owl_txd(&vd->tx);
541 	struct owl_dma_lli *lli;
542 	unsigned long flags;
543 	u32 int_ctl;
544 
545 	list_del(&vd->node);
546 
547 	vchan->txd = txd;
548 
549 	/* Wait for channel inactive */
550 	while (owl_dma_pchan_busy(od, pchan))
551 		cpu_relax();
552 
553 	lli = list_first_entry(&txd->lli_list,
554 			       struct owl_dma_lli, node);
555 
556 	if (txd->cyclic)
557 		int_ctl = OWL_DMA_INTCTL_BLOCK;
558 	else
559 		int_ctl = OWL_DMA_INTCTL_SUPER_BLOCK;
560 
561 	pchan_writel(pchan, OWL_DMAX_MODE, OWL_DMA_MODE_LME);
562 	pchan_writel(pchan, OWL_DMAX_LINKLIST_CTL,
563 		     OWL_DMA_LLC_SAV_LOAD_NEXT | OWL_DMA_LLC_DAV_LOAD_NEXT);
564 	pchan_writel(pchan, OWL_DMAX_NEXT_DESCRIPTOR, lli->phys);
565 	pchan_writel(pchan, OWL_DMAX_INT_CTL, int_ctl);
566 
567 	/* Clear IRQ status for this pchan */
568 	pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
569 
570 	spin_lock_irqsave(&od->lock, flags);
571 
572 	dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), true);
573 
574 	spin_unlock_irqrestore(&od->lock, flags);
575 
576 	dev_dbg(chan2dev(&vchan->vc.chan), "starting pchan %d\n", pchan->id);
577 
578 	/* Start DMA transfer for this pchan */
579 	pchan_writel(pchan, OWL_DMAX_START, 0x1);
580 
581 	return 0;
582 }
583 
owl_dma_phy_free(struct owl_dma * od,struct owl_dma_vchan * vchan)584 static void owl_dma_phy_free(struct owl_dma *od, struct owl_dma_vchan *vchan)
585 {
586 	/* Ensure that the physical channel is stopped */
587 	owl_dma_terminate_pchan(od, vchan->pchan);
588 
589 	vchan->pchan = NULL;
590 }
591 
owl_dma_interrupt(int irq,void * dev_id)592 static irqreturn_t owl_dma_interrupt(int irq, void *dev_id)
593 {
594 	struct owl_dma *od = dev_id;
595 	struct owl_dma_vchan *vchan;
596 	struct owl_dma_pchan *pchan;
597 	unsigned long pending;
598 	int i;
599 	unsigned int global_irq_pending, chan_irq_pending;
600 
601 	spin_lock(&od->lock);
602 
603 	pending = dma_readl(od, OWL_DMA_IRQ_PD0);
604 
605 	/* Clear IRQ status for each pchan */
606 	for_each_set_bit(i, &pending, od->nr_pchans) {
607 		pchan = &od->pchans[i];
608 		pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
609 	}
610 
611 	/* Clear pending IRQ */
612 	dma_writel(od, OWL_DMA_IRQ_PD0, pending);
613 
614 	/* Check missed pending IRQ */
615 	for (i = 0; i < od->nr_pchans; i++) {
616 		pchan = &od->pchans[i];
617 		chan_irq_pending = pchan_readl(pchan, OWL_DMAX_INT_CTL) &
618 				   pchan_readl(pchan, OWL_DMAX_INT_STATUS);
619 
620 		/* Dummy read to ensure OWL_DMA_IRQ_PD0 value is updated */
621 		dma_readl(od, OWL_DMA_IRQ_PD0);
622 
623 		global_irq_pending = dma_readl(od, OWL_DMA_IRQ_PD0);
624 
625 		if (chan_irq_pending && !(global_irq_pending & BIT(i))) {
626 			dev_dbg(od->dma.dev,
627 				"global and channel IRQ pending match err\n");
628 
629 			/* Clear IRQ status for this pchan */
630 			pchan_update(pchan, OWL_DMAX_INT_STATUS,
631 				     0xff, false);
632 
633 			/* Update global IRQ pending */
634 			pending |= BIT(i);
635 		}
636 	}
637 
638 	spin_unlock(&od->lock);
639 
640 	for_each_set_bit(i, &pending, od->nr_pchans) {
641 		struct owl_dma_txd *txd;
642 
643 		pchan = &od->pchans[i];
644 
645 		vchan = pchan->vchan;
646 		if (!vchan) {
647 			dev_warn(od->dma.dev, "no vchan attached on pchan %d\n",
648 				 pchan->id);
649 			continue;
650 		}
651 
652 		spin_lock(&vchan->vc.lock);
653 
654 		txd = vchan->txd;
655 		if (txd) {
656 			vchan->txd = NULL;
657 
658 			vchan_cookie_complete(&txd->vd);
659 
660 			/*
661 			 * Start the next descriptor (if any),
662 			 * otherwise free this channel.
663 			 */
664 			if (vchan_next_desc(&vchan->vc))
665 				owl_dma_start_next_txd(vchan);
666 			else
667 				owl_dma_phy_free(od, vchan);
668 		}
669 
670 		spin_unlock(&vchan->vc.lock);
671 	}
672 
673 	return IRQ_HANDLED;
674 }
675 
owl_dma_free_txd(struct owl_dma * od,struct owl_dma_txd * txd)676 static void owl_dma_free_txd(struct owl_dma *od, struct owl_dma_txd *txd)
677 {
678 	struct owl_dma_lli *lli, *_lli;
679 
680 	if (unlikely(!txd))
681 		return;
682 
683 	list_for_each_entry_safe(lli, _lli, &txd->lli_list, node)
684 		owl_dma_free_lli(od, lli);
685 
686 	kfree(txd);
687 }
688 
owl_dma_desc_free(struct virt_dma_desc * vd)689 static void owl_dma_desc_free(struct virt_dma_desc *vd)
690 {
691 	struct owl_dma *od = to_owl_dma(vd->tx.chan->device);
692 	struct owl_dma_txd *txd = to_owl_txd(&vd->tx);
693 
694 	owl_dma_free_txd(od, txd);
695 }
696 
owl_dma_terminate_all(struct dma_chan * chan)697 static int owl_dma_terminate_all(struct dma_chan *chan)
698 {
699 	struct owl_dma *od = to_owl_dma(chan->device);
700 	struct owl_dma_vchan *vchan = to_owl_vchan(chan);
701 	unsigned long flags;
702 	LIST_HEAD(head);
703 
704 	spin_lock_irqsave(&vchan->vc.lock, flags);
705 
706 	if (vchan->pchan)
707 		owl_dma_phy_free(od, vchan);
708 
709 	if (vchan->txd) {
710 		owl_dma_desc_free(&vchan->txd->vd);
711 		vchan->txd = NULL;
712 	}
713 
714 	vchan_get_all_descriptors(&vchan->vc, &head);
715 
716 	spin_unlock_irqrestore(&vchan->vc.lock, flags);
717 
718 	vchan_dma_desc_free_list(&vchan->vc, &head);
719 
720 	return 0;
721 }
722 
owl_dma_config(struct dma_chan * chan,struct dma_slave_config * config)723 static int owl_dma_config(struct dma_chan *chan,
724 			  struct dma_slave_config *config)
725 {
726 	struct owl_dma_vchan *vchan = to_owl_vchan(chan);
727 
728 	/* Reject definitely invalid configurations */
729 	if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
730 	    config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
731 		return -EINVAL;
732 
733 	memcpy(&vchan->cfg, config, sizeof(struct dma_slave_config));
734 
735 	return 0;
736 }
737 
owl_dma_pause(struct dma_chan * chan)738 static int owl_dma_pause(struct dma_chan *chan)
739 {
740 	struct owl_dma_vchan *vchan = to_owl_vchan(chan);
741 	unsigned long flags;
742 
743 	spin_lock_irqsave(&vchan->vc.lock, flags);
744 
745 	owl_dma_pause_pchan(vchan->pchan);
746 
747 	spin_unlock_irqrestore(&vchan->vc.lock, flags);
748 
749 	return 0;
750 }
751 
owl_dma_resume(struct dma_chan * chan)752 static int owl_dma_resume(struct dma_chan *chan)
753 {
754 	struct owl_dma_vchan *vchan = to_owl_vchan(chan);
755 	unsigned long flags;
756 
757 	if (!vchan->pchan && !vchan->txd)
758 		return 0;
759 
760 	dev_dbg(chan2dev(chan), "vchan %p: resume\n", &vchan->vc);
761 
762 	spin_lock_irqsave(&vchan->vc.lock, flags);
763 
764 	owl_dma_resume_pchan(vchan->pchan);
765 
766 	spin_unlock_irqrestore(&vchan->vc.lock, flags);
767 
768 	return 0;
769 }
770 
owl_dma_getbytes_chan(struct owl_dma_vchan * vchan)771 static u32 owl_dma_getbytes_chan(struct owl_dma_vchan *vchan)
772 {
773 	struct owl_dma_pchan *pchan;
774 	struct owl_dma_txd *txd;
775 	struct owl_dma_lli *lli;
776 	unsigned int next_lli_phy;
777 	size_t bytes;
778 
779 	pchan = vchan->pchan;
780 	txd = vchan->txd;
781 
782 	if (!pchan || !txd)
783 		return 0;
784 
785 	/* Get remain count of current node in link list */
786 	bytes = pchan_readl(pchan, OWL_DMAX_REMAIN_CNT);
787 
788 	/* Loop through the preceding nodes to get total remaining bytes */
789 	if (pchan_readl(pchan, OWL_DMAX_MODE) & OWL_DMA_MODE_LME) {
790 		next_lli_phy = pchan_readl(pchan, OWL_DMAX_NEXT_DESCRIPTOR);
791 		list_for_each_entry(lli, &txd->lli_list, node) {
792 			/* Start from the next active node */
793 			if (lli->phys == next_lli_phy) {
794 				list_for_each_entry(lli, &txd->lli_list, node)
795 					bytes += llc_hw_flen(lli);
796 				break;
797 			}
798 		}
799 	}
800 
801 	return bytes;
802 }
803 
owl_dma_tx_status(struct dma_chan * chan,dma_cookie_t cookie,struct dma_tx_state * state)804 static enum dma_status owl_dma_tx_status(struct dma_chan *chan,
805 					 dma_cookie_t cookie,
806 					 struct dma_tx_state *state)
807 {
808 	struct owl_dma_vchan *vchan = to_owl_vchan(chan);
809 	struct owl_dma_lli *lli;
810 	struct virt_dma_desc *vd;
811 	struct owl_dma_txd *txd;
812 	enum dma_status ret;
813 	unsigned long flags;
814 	size_t bytes = 0;
815 
816 	ret = dma_cookie_status(chan, cookie, state);
817 	if (ret == DMA_COMPLETE || !state)
818 		return ret;
819 
820 	spin_lock_irqsave(&vchan->vc.lock, flags);
821 
822 	vd = vchan_find_desc(&vchan->vc, cookie);
823 	if (vd) {
824 		txd = to_owl_txd(&vd->tx);
825 		list_for_each_entry(lli, &txd->lli_list, node)
826 			bytes += llc_hw_flen(lli);
827 	} else {
828 		bytes = owl_dma_getbytes_chan(vchan);
829 	}
830 
831 	spin_unlock_irqrestore(&vchan->vc.lock, flags);
832 
833 	dma_set_residue(state, bytes);
834 
835 	return ret;
836 }
837 
owl_dma_phy_alloc_and_start(struct owl_dma_vchan * vchan)838 static void owl_dma_phy_alloc_and_start(struct owl_dma_vchan *vchan)
839 {
840 	struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);
841 	struct owl_dma_pchan *pchan;
842 
843 	pchan = owl_dma_get_pchan(od, vchan);
844 	if (!pchan)
845 		return;
846 
847 	dev_dbg(od->dma.dev, "allocated pchan %d\n", pchan->id);
848 
849 	vchan->pchan = pchan;
850 	owl_dma_start_next_txd(vchan);
851 }
852 
owl_dma_issue_pending(struct dma_chan * chan)853 static void owl_dma_issue_pending(struct dma_chan *chan)
854 {
855 	struct owl_dma_vchan *vchan = to_owl_vchan(chan);
856 	unsigned long flags;
857 
858 	spin_lock_irqsave(&vchan->vc.lock, flags);
859 	if (vchan_issue_pending(&vchan->vc)) {
860 		if (!vchan->pchan)
861 			owl_dma_phy_alloc_and_start(vchan);
862 	}
863 	spin_unlock_irqrestore(&vchan->vc.lock, flags);
864 }
865 
866 static struct dma_async_tx_descriptor
owl_dma_prep_memcpy(struct dma_chan * chan,dma_addr_t dst,dma_addr_t src,size_t len,unsigned long flags)867 		*owl_dma_prep_memcpy(struct dma_chan *chan,
868 				     dma_addr_t dst, dma_addr_t src,
869 				     size_t len, unsigned long flags)
870 {
871 	struct owl_dma *od = to_owl_dma(chan->device);
872 	struct owl_dma_vchan *vchan = to_owl_vchan(chan);
873 	struct owl_dma_txd *txd;
874 	struct owl_dma_lli *lli, *prev = NULL;
875 	size_t offset, bytes;
876 	int ret;
877 
878 	if (!len)
879 		return NULL;
880 
881 	txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
882 	if (!txd)
883 		return NULL;
884 
885 	INIT_LIST_HEAD(&txd->lli_list);
886 
887 	/* Process the transfer as frame by frame */
888 	for (offset = 0; offset < len; offset += bytes) {
889 		lli = owl_dma_alloc_lli(od);
890 		if (!lli) {
891 			dev_warn(chan2dev(chan), "failed to allocate lli\n");
892 			goto err_txd_free;
893 		}
894 
895 		bytes = min_t(size_t, (len - offset), OWL_DMA_FRAME_MAX_LENGTH);
896 
897 		ret = owl_dma_cfg_lli(vchan, lli, src + offset, dst + offset,
898 				      bytes, DMA_MEM_TO_MEM,
899 				      &vchan->cfg, txd->cyclic);
900 		if (ret) {
901 			dev_warn(chan2dev(chan), "failed to config lli\n");
902 			goto err_txd_free;
903 		}
904 
905 		prev = owl_dma_add_lli(txd, prev, lli, false);
906 	}
907 
908 	return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
909 
910 err_txd_free:
911 	owl_dma_free_txd(od, txd);
912 	return NULL;
913 }
914 
915 static struct dma_async_tx_descriptor
owl_dma_prep_slave_sg(struct dma_chan * chan,struct scatterlist * sgl,unsigned int sg_len,enum dma_transfer_direction dir,unsigned long flags,void * context)916 		*owl_dma_prep_slave_sg(struct dma_chan *chan,
917 				       struct scatterlist *sgl,
918 				       unsigned int sg_len,
919 				       enum dma_transfer_direction dir,
920 				       unsigned long flags, void *context)
921 {
922 	struct owl_dma *od = to_owl_dma(chan->device);
923 	struct owl_dma_vchan *vchan = to_owl_vchan(chan);
924 	struct dma_slave_config *sconfig = &vchan->cfg;
925 	struct owl_dma_txd *txd;
926 	struct owl_dma_lli *lli, *prev = NULL;
927 	struct scatterlist *sg;
928 	dma_addr_t addr, src = 0, dst = 0;
929 	size_t len;
930 	int ret, i;
931 
932 	txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
933 	if (!txd)
934 		return NULL;
935 
936 	INIT_LIST_HEAD(&txd->lli_list);
937 
938 	for_each_sg(sgl, sg, sg_len, i) {
939 		addr = sg_dma_address(sg);
940 		len = sg_dma_len(sg);
941 
942 		if (len > OWL_DMA_FRAME_MAX_LENGTH) {
943 			dev_err(od->dma.dev,
944 				"frame length exceeds max supported length");
945 			goto err_txd_free;
946 		}
947 
948 		lli = owl_dma_alloc_lli(od);
949 		if (!lli) {
950 			dev_err(chan2dev(chan), "failed to allocate lli");
951 			goto err_txd_free;
952 		}
953 
954 		if (dir == DMA_MEM_TO_DEV) {
955 			src = addr;
956 			dst = sconfig->dst_addr;
957 		} else {
958 			src = sconfig->src_addr;
959 			dst = addr;
960 		}
961 
962 		ret = owl_dma_cfg_lli(vchan, lli, src, dst, len, dir, sconfig,
963 				      txd->cyclic);
964 		if (ret) {
965 			dev_warn(chan2dev(chan), "failed to config lli");
966 			goto err_txd_free;
967 		}
968 
969 		prev = owl_dma_add_lli(txd, prev, lli, false);
970 	}
971 
972 	return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
973 
974 err_txd_free:
975 	owl_dma_free_txd(od, txd);
976 
977 	return NULL;
978 }
979 
980 static struct dma_async_tx_descriptor
owl_prep_dma_cyclic(struct dma_chan * chan,dma_addr_t buf_addr,size_t buf_len,size_t period_len,enum dma_transfer_direction dir,unsigned long flags)981 		*owl_prep_dma_cyclic(struct dma_chan *chan,
982 				     dma_addr_t buf_addr, size_t buf_len,
983 				     size_t period_len,
984 				     enum dma_transfer_direction dir,
985 				     unsigned long flags)
986 {
987 	struct owl_dma *od = to_owl_dma(chan->device);
988 	struct owl_dma_vchan *vchan = to_owl_vchan(chan);
989 	struct dma_slave_config *sconfig = &vchan->cfg;
990 	struct owl_dma_txd *txd;
991 	struct owl_dma_lli *lli, *prev = NULL, *first = NULL;
992 	dma_addr_t src = 0, dst = 0;
993 	unsigned int periods = buf_len / period_len;
994 	int ret, i;
995 
996 	txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
997 	if (!txd)
998 		return NULL;
999 
1000 	INIT_LIST_HEAD(&txd->lli_list);
1001 	txd->cyclic = true;
1002 
1003 	for (i = 0; i < periods; i++) {
1004 		lli = owl_dma_alloc_lli(od);
1005 		if (!lli) {
1006 			dev_warn(chan2dev(chan), "failed to allocate lli");
1007 			goto err_txd_free;
1008 		}
1009 
1010 		if (dir == DMA_MEM_TO_DEV) {
1011 			src = buf_addr + (period_len * i);
1012 			dst = sconfig->dst_addr;
1013 		} else if (dir == DMA_DEV_TO_MEM) {
1014 			src = sconfig->src_addr;
1015 			dst = buf_addr + (period_len * i);
1016 		}
1017 
1018 		ret = owl_dma_cfg_lli(vchan, lli, src, dst, period_len,
1019 				      dir, sconfig, txd->cyclic);
1020 		if (ret) {
1021 			dev_warn(chan2dev(chan), "failed to config lli");
1022 			goto err_txd_free;
1023 		}
1024 
1025 		if (!first)
1026 			first = lli;
1027 
1028 		prev = owl_dma_add_lli(txd, prev, lli, false);
1029 	}
1030 
1031 	/* close the cyclic list */
1032 	owl_dma_add_lli(txd, prev, first, true);
1033 
1034 	return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
1035 
1036 err_txd_free:
1037 	owl_dma_free_txd(od, txd);
1038 
1039 	return NULL;
1040 }
1041 
owl_dma_free_chan_resources(struct dma_chan * chan)1042 static void owl_dma_free_chan_resources(struct dma_chan *chan)
1043 {
1044 	struct owl_dma_vchan *vchan = to_owl_vchan(chan);
1045 
1046 	/* Ensure all queued descriptors are freed */
1047 	vchan_free_chan_resources(&vchan->vc);
1048 }
1049 
owl_dma_free(struct owl_dma * od)1050 static inline void owl_dma_free(struct owl_dma *od)
1051 {
1052 	struct owl_dma_vchan *vchan = NULL;
1053 	struct owl_dma_vchan *next;
1054 
1055 	list_for_each_entry_safe(vchan,
1056 				 next, &od->dma.channels, vc.chan.device_node) {
1057 		list_del(&vchan->vc.chan.device_node);
1058 		tasklet_kill(&vchan->vc.task);
1059 	}
1060 }
1061 
owl_dma_of_xlate(struct of_phandle_args * dma_spec,struct of_dma * ofdma)1062 static struct dma_chan *owl_dma_of_xlate(struct of_phandle_args *dma_spec,
1063 					 struct of_dma *ofdma)
1064 {
1065 	struct owl_dma *od = ofdma->of_dma_data;
1066 	struct owl_dma_vchan *vchan;
1067 	struct dma_chan *chan;
1068 	u8 drq = dma_spec->args[0];
1069 
1070 	if (drq > od->nr_vchans)
1071 		return NULL;
1072 
1073 	chan = dma_get_any_slave_channel(&od->dma);
1074 	if (!chan)
1075 		return NULL;
1076 
1077 	vchan = to_owl_vchan(chan);
1078 	vchan->drq = drq;
1079 
1080 	return chan;
1081 }
1082 
1083 static const struct of_device_id owl_dma_match[] = {
1084 	{ .compatible = "actions,s500-dma", .data = (void *)S900_DMA,},
1085 	{ .compatible = "actions,s700-dma", .data = (void *)S700_DMA,},
1086 	{ .compatible = "actions,s900-dma", .data = (void *)S900_DMA,},
1087 	{ /* sentinel */ },
1088 };
1089 MODULE_DEVICE_TABLE(of, owl_dma_match);
1090 
owl_dma_probe(struct platform_device * pdev)1091 static int owl_dma_probe(struct platform_device *pdev)
1092 {
1093 	struct device_node *np = pdev->dev.of_node;
1094 	struct owl_dma *od;
1095 	int ret, i, nr_channels, nr_requests;
1096 
1097 	od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
1098 	if (!od)
1099 		return -ENOMEM;
1100 
1101 	od->base = devm_platform_ioremap_resource(pdev, 0);
1102 	if (IS_ERR(od->base))
1103 		return PTR_ERR(od->base);
1104 
1105 	ret = of_property_read_u32(np, "dma-channels", &nr_channels);
1106 	if (ret) {
1107 		dev_err(&pdev->dev, "can't get dma-channels\n");
1108 		return ret;
1109 	}
1110 
1111 	ret = of_property_read_u32(np, "dma-requests", &nr_requests);
1112 	if (ret) {
1113 		dev_err(&pdev->dev, "can't get dma-requests\n");
1114 		return ret;
1115 	}
1116 
1117 	dev_info(&pdev->dev, "dma-channels %d, dma-requests %d\n",
1118 		 nr_channels, nr_requests);
1119 
1120 	od->devid = (uintptr_t)of_device_get_match_data(&pdev->dev);
1121 
1122 	od->nr_pchans = nr_channels;
1123 	od->nr_vchans = nr_requests;
1124 
1125 	pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32);
1126 
1127 	platform_set_drvdata(pdev, od);
1128 	spin_lock_init(&od->lock);
1129 
1130 	dma_cap_set(DMA_MEMCPY, od->dma.cap_mask);
1131 	dma_cap_set(DMA_SLAVE, od->dma.cap_mask);
1132 	dma_cap_set(DMA_CYCLIC, od->dma.cap_mask);
1133 
1134 	od->dma.dev = &pdev->dev;
1135 	od->dma.device_free_chan_resources = owl_dma_free_chan_resources;
1136 	od->dma.device_tx_status = owl_dma_tx_status;
1137 	od->dma.device_issue_pending = owl_dma_issue_pending;
1138 	od->dma.device_prep_dma_memcpy = owl_dma_prep_memcpy;
1139 	od->dma.device_prep_slave_sg = owl_dma_prep_slave_sg;
1140 	od->dma.device_prep_dma_cyclic = owl_prep_dma_cyclic;
1141 	od->dma.device_config = owl_dma_config;
1142 	od->dma.device_pause = owl_dma_pause;
1143 	od->dma.device_resume = owl_dma_resume;
1144 	od->dma.device_terminate_all = owl_dma_terminate_all;
1145 	od->dma.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1146 	od->dma.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1147 	od->dma.directions = BIT(DMA_MEM_TO_MEM);
1148 	od->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1149 
1150 	INIT_LIST_HEAD(&od->dma.channels);
1151 
1152 	od->clk = devm_clk_get(&pdev->dev, NULL);
1153 	if (IS_ERR(od->clk)) {
1154 		dev_err(&pdev->dev, "unable to get clock\n");
1155 		return PTR_ERR(od->clk);
1156 	}
1157 
1158 	/*
1159 	 * Eventhough the DMA controller is capable of generating 4
1160 	 * IRQ's for DMA priority feature, we only use 1 IRQ for
1161 	 * simplification.
1162 	 */
1163 	od->irq = platform_get_irq(pdev, 0);
1164 	ret = devm_request_irq(&pdev->dev, od->irq, owl_dma_interrupt, 0,
1165 			       dev_name(&pdev->dev), od);
1166 	if (ret) {
1167 		dev_err(&pdev->dev, "unable to request IRQ\n");
1168 		return ret;
1169 	}
1170 
1171 	/* Init physical channel */
1172 	od->pchans = devm_kcalloc(&pdev->dev, od->nr_pchans,
1173 				  sizeof(struct owl_dma_pchan), GFP_KERNEL);
1174 	if (!od->pchans)
1175 		return -ENOMEM;
1176 
1177 	for (i = 0; i < od->nr_pchans; i++) {
1178 		struct owl_dma_pchan *pchan = &od->pchans[i];
1179 
1180 		pchan->id = i;
1181 		pchan->base = od->base + OWL_DMA_CHAN_BASE(i);
1182 	}
1183 
1184 	/* Init virtual channel */
1185 	od->vchans = devm_kcalloc(&pdev->dev, od->nr_vchans,
1186 				  sizeof(struct owl_dma_vchan), GFP_KERNEL);
1187 	if (!od->vchans)
1188 		return -ENOMEM;
1189 
1190 	for (i = 0; i < od->nr_vchans; i++) {
1191 		struct owl_dma_vchan *vchan = &od->vchans[i];
1192 
1193 		vchan->vc.desc_free = owl_dma_desc_free;
1194 		vchan_init(&vchan->vc, &od->dma);
1195 	}
1196 
1197 	/* Create a pool of consistent memory blocks for hardware descriptors */
1198 	od->lli_pool = dma_pool_create(dev_name(od->dma.dev), od->dma.dev,
1199 				       sizeof(struct owl_dma_lli),
1200 				       __alignof__(struct owl_dma_lli),
1201 				       0);
1202 	if (!od->lli_pool) {
1203 		dev_err(&pdev->dev, "unable to allocate DMA descriptor pool\n");
1204 		return -ENOMEM;
1205 	}
1206 
1207 	clk_prepare_enable(od->clk);
1208 
1209 	ret = dma_async_device_register(&od->dma);
1210 	if (ret) {
1211 		dev_err(&pdev->dev, "failed to register DMA engine device\n");
1212 		goto err_pool_free;
1213 	}
1214 
1215 	/* Device-tree DMA controller registration */
1216 	ret = of_dma_controller_register(pdev->dev.of_node,
1217 					 owl_dma_of_xlate, od);
1218 	if (ret) {
1219 		dev_err(&pdev->dev, "of_dma_controller_register failed\n");
1220 		goto err_dma_unregister;
1221 	}
1222 
1223 	return 0;
1224 
1225 err_dma_unregister:
1226 	dma_async_device_unregister(&od->dma);
1227 err_pool_free:
1228 	clk_disable_unprepare(od->clk);
1229 	dma_pool_destroy(od->lli_pool);
1230 
1231 	return ret;
1232 }
1233 
owl_dma_remove(struct platform_device * pdev)1234 static int owl_dma_remove(struct platform_device *pdev)
1235 {
1236 	struct owl_dma *od = platform_get_drvdata(pdev);
1237 
1238 	of_dma_controller_free(pdev->dev.of_node);
1239 	dma_async_device_unregister(&od->dma);
1240 
1241 	/* Mask all interrupts for this execution environment */
1242 	dma_writel(od, OWL_DMA_IRQ_EN0, 0x0);
1243 
1244 	/* Make sure we won't have any further interrupts */
1245 	devm_free_irq(od->dma.dev, od->irq, od);
1246 
1247 	owl_dma_free(od);
1248 
1249 	clk_disable_unprepare(od->clk);
1250 	dma_pool_destroy(od->lli_pool);
1251 
1252 	return 0;
1253 }
1254 
1255 static struct platform_driver owl_dma_driver = {
1256 	.probe	= owl_dma_probe,
1257 	.remove	= owl_dma_remove,
1258 	.driver = {
1259 		.name = "dma-owl",
1260 		.of_match_table = of_match_ptr(owl_dma_match),
1261 	},
1262 };
1263 
owl_dma_init(void)1264 static int owl_dma_init(void)
1265 {
1266 	return platform_driver_register(&owl_dma_driver);
1267 }
1268 subsys_initcall(owl_dma_init);
1269 
owl_dma_exit(void)1270 static void __exit owl_dma_exit(void)
1271 {
1272 	platform_driver_unregister(&owl_dma_driver);
1273 }
1274 module_exit(owl_dma_exit);
1275 
1276 MODULE_AUTHOR("David Liu <liuwei@actions-semi.com>");
1277 MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>");
1278 MODULE_DESCRIPTION("Actions Semi Owl SoCs DMA driver");
1279 MODULE_LICENSE("GPL");
1280