xref: /openbmc/linux/drivers/dma/mediatek/mtk-cqdma.c (revision 897500c7)
1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2018-2019 MediaTek Inc.
3 
4 /*
5  * Driver for MediaTek Command-Queue DMA Controller
6  *
7  * Author: Shun-Chih Yu <shun-chih.yu@mediatek.com>
8  *
9  */
10 
11 #include <linux/bitops.h>
12 #include <linux/clk.h>
13 #include <linux/dmaengine.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/err.h>
16 #include <linux/iopoll.h>
17 #include <linux/interrupt.h>
18 #include <linux/list.h>
19 #include <linux/module.h>
20 #include <linux/of.h>
21 #include <linux/of_dma.h>
22 #include <linux/platform_device.h>
23 #include <linux/pm_runtime.h>
24 #include <linux/refcount.h>
25 #include <linux/slab.h>
26 
27 #include "../virt-dma.h"
28 
29 #define MTK_CQDMA_USEC_POLL		10
30 #define MTK_CQDMA_TIMEOUT_POLL		1000
31 #define MTK_CQDMA_DMA_BUSWIDTHS		BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
32 #define MTK_CQDMA_ALIGN_SIZE		1
33 
34 /* The default number of virtual channel */
35 #define MTK_CQDMA_NR_VCHANS		32
36 
37 /* The default number of physical channel */
38 #define MTK_CQDMA_NR_PCHANS		3
39 
40 /* Registers for underlying dma manipulation */
41 #define MTK_CQDMA_INT_FLAG		0x0
42 #define MTK_CQDMA_INT_EN		0x4
43 #define MTK_CQDMA_EN			0x8
44 #define MTK_CQDMA_RESET			0xc
45 #define MTK_CQDMA_FLUSH			0x14
46 #define MTK_CQDMA_SRC			0x1c
47 #define MTK_CQDMA_DST			0x20
48 #define MTK_CQDMA_LEN1			0x24
49 #define MTK_CQDMA_LEN2			0x28
50 #define MTK_CQDMA_SRC2			0x60
51 #define MTK_CQDMA_DST2			0x64
52 
53 /* Registers setting */
54 #define MTK_CQDMA_EN_BIT		BIT(0)
55 #define MTK_CQDMA_INT_FLAG_BIT		BIT(0)
56 #define MTK_CQDMA_INT_EN_BIT		BIT(0)
57 #define MTK_CQDMA_FLUSH_BIT		BIT(0)
58 
59 #define MTK_CQDMA_WARM_RST_BIT		BIT(0)
60 #define MTK_CQDMA_HARD_RST_BIT		BIT(1)
61 
62 #define MTK_CQDMA_MAX_LEN		GENMASK(27, 0)
63 #define MTK_CQDMA_ADDR_LIMIT		GENMASK(31, 0)
64 #define MTK_CQDMA_ADDR2_SHFIT		(32)
65 
66 /**
67  * struct mtk_cqdma_vdesc - The struct holding info describing virtual
68  *                         descriptor (CVD)
69  * @vd:                    An instance for struct virt_dma_desc
70  * @len:                   The total data size device wants to move
71  * @residue:               The remaining data size device will move
72  * @dest:                  The destination address device wants to move to
73  * @src:                   The source address device wants to move from
74  * @ch:                    The pointer to the corresponding dma channel
75  * @node:                  The lise_head struct to build link-list for VDs
76  * @parent:                The pointer to the parent CVD
77  */
78 struct mtk_cqdma_vdesc {
79 	struct virt_dma_desc vd;
80 	size_t len;
81 	size_t residue;
82 	dma_addr_t dest;
83 	dma_addr_t src;
84 	struct dma_chan *ch;
85 
86 	struct list_head node;
87 	struct mtk_cqdma_vdesc *parent;
88 };
89 
90 /**
91  * struct mtk_cqdma_pchan - The struct holding info describing physical
92  *                         channel (PC)
93  * @queue:                 Queue for the PDs issued to this PC
94  * @base:                  The mapped register I/O base of this PC
95  * @irq:                   The IRQ that this PC are using
96  * @refcnt:                Track how many VCs are using this PC
97  * @tasklet:               Tasklet for this PC
98  * @lock:                  Lock protect agaisting multiple VCs access PC
99  */
100 struct mtk_cqdma_pchan {
101 	struct list_head queue;
102 	void __iomem *base;
103 	u32 irq;
104 
105 	refcount_t refcnt;
106 
107 	struct tasklet_struct tasklet;
108 
109 	/* lock to protect PC */
110 	spinlock_t lock;
111 };
112 
113 /**
114  * struct mtk_cqdma_vchan - The struct holding info describing virtual
115  *                         channel (VC)
116  * @vc:                    An instance for struct virt_dma_chan
117  * @pc:                    The pointer to the underlying PC
118  * @issue_completion:	   The wait for all issued descriptors completited
119  * @issue_synchronize:	   Bool indicating channel synchronization starts
120  */
121 struct mtk_cqdma_vchan {
122 	struct virt_dma_chan vc;
123 	struct mtk_cqdma_pchan *pc;
124 	struct completion issue_completion;
125 	bool issue_synchronize;
126 };
127 
128 /**
129  * struct mtk_cqdma_device - The struct holding info describing CQDMA
130  *                          device
131  * @ddev:                   An instance for struct dma_device
132  * @clk:                    The clock that device internal is using
133  * @dma_requests:           The number of VCs the device supports to
134  * @dma_channels:           The number of PCs the device supports to
135  * @vc:                     The pointer to all available VCs
136  * @pc:                     The pointer to all the underlying PCs
137  */
138 struct mtk_cqdma_device {
139 	struct dma_device ddev;
140 	struct clk *clk;
141 
142 	u32 dma_requests;
143 	u32 dma_channels;
144 	struct mtk_cqdma_vchan *vc;
145 	struct mtk_cqdma_pchan **pc;
146 };
147 
to_cqdma_dev(struct dma_chan * chan)148 static struct mtk_cqdma_device *to_cqdma_dev(struct dma_chan *chan)
149 {
150 	return container_of(chan->device, struct mtk_cqdma_device, ddev);
151 }
152 
to_cqdma_vchan(struct dma_chan * chan)153 static struct mtk_cqdma_vchan *to_cqdma_vchan(struct dma_chan *chan)
154 {
155 	return container_of(chan, struct mtk_cqdma_vchan, vc.chan);
156 }
157 
to_cqdma_vdesc(struct virt_dma_desc * vd)158 static struct mtk_cqdma_vdesc *to_cqdma_vdesc(struct virt_dma_desc *vd)
159 {
160 	return container_of(vd, struct mtk_cqdma_vdesc, vd);
161 }
162 
cqdma2dev(struct mtk_cqdma_device * cqdma)163 static struct device *cqdma2dev(struct mtk_cqdma_device *cqdma)
164 {
165 	return cqdma->ddev.dev;
166 }
167 
mtk_dma_read(struct mtk_cqdma_pchan * pc,u32 reg)168 static u32 mtk_dma_read(struct mtk_cqdma_pchan *pc, u32 reg)
169 {
170 	return readl(pc->base + reg);
171 }
172 
mtk_dma_write(struct mtk_cqdma_pchan * pc,u32 reg,u32 val)173 static void mtk_dma_write(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
174 {
175 	writel_relaxed(val, pc->base + reg);
176 }
177 
mtk_dma_rmw(struct mtk_cqdma_pchan * pc,u32 reg,u32 mask,u32 set)178 static void mtk_dma_rmw(struct mtk_cqdma_pchan *pc, u32 reg,
179 			u32 mask, u32 set)
180 {
181 	u32 val;
182 
183 	val = mtk_dma_read(pc, reg);
184 	val &= ~mask;
185 	val |= set;
186 	mtk_dma_write(pc, reg, val);
187 }
188 
mtk_dma_set(struct mtk_cqdma_pchan * pc,u32 reg,u32 val)189 static void mtk_dma_set(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
190 {
191 	mtk_dma_rmw(pc, reg, 0, val);
192 }
193 
mtk_dma_clr(struct mtk_cqdma_pchan * pc,u32 reg,u32 val)194 static void mtk_dma_clr(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
195 {
196 	mtk_dma_rmw(pc, reg, val, 0);
197 }
198 
mtk_cqdma_vdesc_free(struct virt_dma_desc * vd)199 static void mtk_cqdma_vdesc_free(struct virt_dma_desc *vd)
200 {
201 	kfree(to_cqdma_vdesc(vd));
202 }
203 
mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan * pc,bool atomic)204 static int mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan *pc, bool atomic)
205 {
206 	u32 status = 0;
207 
208 	if (!atomic)
209 		return readl_poll_timeout(pc->base + MTK_CQDMA_EN,
210 					  status,
211 					  !(status & MTK_CQDMA_EN_BIT),
212 					  MTK_CQDMA_USEC_POLL,
213 					  MTK_CQDMA_TIMEOUT_POLL);
214 
215 	return readl_poll_timeout_atomic(pc->base + MTK_CQDMA_EN,
216 					 status,
217 					 !(status & MTK_CQDMA_EN_BIT),
218 					 MTK_CQDMA_USEC_POLL,
219 					 MTK_CQDMA_TIMEOUT_POLL);
220 }
221 
mtk_cqdma_hard_reset(struct mtk_cqdma_pchan * pc)222 static int mtk_cqdma_hard_reset(struct mtk_cqdma_pchan *pc)
223 {
224 	mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
225 	mtk_dma_clr(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
226 
227 	return mtk_cqdma_poll_engine_done(pc, true);
228 }
229 
mtk_cqdma_start(struct mtk_cqdma_pchan * pc,struct mtk_cqdma_vdesc * cvd)230 static void mtk_cqdma_start(struct mtk_cqdma_pchan *pc,
231 			    struct mtk_cqdma_vdesc *cvd)
232 {
233 	/* wait for the previous transaction done */
234 	if (mtk_cqdma_poll_engine_done(pc, true) < 0)
235 		dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma wait transaction timeout\n");
236 
237 	/* warm reset the dma engine for the new transaction */
238 	mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_WARM_RST_BIT);
239 	if (mtk_cqdma_poll_engine_done(pc, true) < 0)
240 		dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma warm reset timeout\n");
241 
242 	/* setup the source */
243 	mtk_dma_set(pc, MTK_CQDMA_SRC, cvd->src & MTK_CQDMA_ADDR_LIMIT);
244 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
245 	mtk_dma_set(pc, MTK_CQDMA_SRC2, cvd->src >> MTK_CQDMA_ADDR2_SHFIT);
246 #else
247 	mtk_dma_set(pc, MTK_CQDMA_SRC2, 0);
248 #endif
249 
250 	/* setup the destination */
251 	mtk_dma_set(pc, MTK_CQDMA_DST, cvd->dest & MTK_CQDMA_ADDR_LIMIT);
252 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
253 	mtk_dma_set(pc, MTK_CQDMA_DST2, cvd->dest >> MTK_CQDMA_ADDR2_SHFIT);
254 #else
255 	mtk_dma_set(pc, MTK_CQDMA_DST2, 0);
256 #endif
257 
258 	/* setup the length */
259 	mtk_dma_set(pc, MTK_CQDMA_LEN1, cvd->len);
260 
261 	/* start dma engine */
262 	mtk_dma_set(pc, MTK_CQDMA_EN, MTK_CQDMA_EN_BIT);
263 }
264 
mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan * cvc)265 static void mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan *cvc)
266 {
267 	struct virt_dma_desc *vd, *vd2;
268 	struct mtk_cqdma_pchan *pc = cvc->pc;
269 	struct mtk_cqdma_vdesc *cvd;
270 	bool trigger_engine = false;
271 
272 	lockdep_assert_held(&cvc->vc.lock);
273 	lockdep_assert_held(&pc->lock);
274 
275 	list_for_each_entry_safe(vd, vd2, &cvc->vc.desc_issued, node) {
276 		/* need to trigger dma engine if PC's queue is empty */
277 		if (list_empty(&pc->queue))
278 			trigger_engine = true;
279 
280 		cvd = to_cqdma_vdesc(vd);
281 
282 		/* add VD into PC's queue */
283 		list_add_tail(&cvd->node, &pc->queue);
284 
285 		/* start the dma engine */
286 		if (trigger_engine)
287 			mtk_cqdma_start(pc, cvd);
288 
289 		/* remove VD from list desc_issued */
290 		list_del(&vd->node);
291 	}
292 }
293 
294 /*
295  * return true if this VC is active,
296  * meaning that there are VDs under processing by the PC
297  */
mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan * cvc)298 static bool mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan *cvc)
299 {
300 	struct mtk_cqdma_vdesc *cvd;
301 
302 	list_for_each_entry(cvd, &cvc->pc->queue, node)
303 		if (cvc == to_cqdma_vchan(cvd->ch))
304 			return true;
305 
306 	return false;
307 }
308 
309 /*
310  * return the pointer of the CVD that is just consumed by the PC
311  */
312 static struct mtk_cqdma_vdesc
mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan * pc)313 *mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan *pc)
314 {
315 	struct mtk_cqdma_vchan *cvc;
316 	struct mtk_cqdma_vdesc *cvd, *ret = NULL;
317 
318 	/* consume a CVD from PC's queue */
319 	cvd = list_first_entry_or_null(&pc->queue,
320 				       struct mtk_cqdma_vdesc, node);
321 	if (unlikely(!cvd || !cvd->parent))
322 		return NULL;
323 
324 	cvc = to_cqdma_vchan(cvd->ch);
325 	ret = cvd;
326 
327 	/* update residue of the parent CVD */
328 	cvd->parent->residue -= cvd->len;
329 
330 	/* delete CVD from PC's queue */
331 	list_del(&cvd->node);
332 
333 	spin_lock(&cvc->vc.lock);
334 
335 	/* check whether all the child CVDs completed */
336 	if (!cvd->parent->residue) {
337 		/* add the parent VD into list desc_completed */
338 		vchan_cookie_complete(&cvd->parent->vd);
339 
340 		/* setup completion if this VC is under synchronization */
341 		if (cvc->issue_synchronize && !mtk_cqdma_is_vchan_active(cvc)) {
342 			complete(&cvc->issue_completion);
343 			cvc->issue_synchronize = false;
344 		}
345 	}
346 
347 	spin_unlock(&cvc->vc.lock);
348 
349 	/* start transaction for next CVD in the queue */
350 	cvd = list_first_entry_or_null(&pc->queue,
351 				       struct mtk_cqdma_vdesc, node);
352 	if (cvd)
353 		mtk_cqdma_start(pc, cvd);
354 
355 	return ret;
356 }
357 
mtk_cqdma_tasklet_cb(struct tasklet_struct * t)358 static void mtk_cqdma_tasklet_cb(struct tasklet_struct *t)
359 {
360 	struct mtk_cqdma_pchan *pc = from_tasklet(pc, t, tasklet);
361 	struct mtk_cqdma_vdesc *cvd = NULL;
362 	unsigned long flags;
363 
364 	spin_lock_irqsave(&pc->lock, flags);
365 	/* consume the queue */
366 	cvd = mtk_cqdma_consume_work_queue(pc);
367 	spin_unlock_irqrestore(&pc->lock, flags);
368 
369 	/* submit the next CVD */
370 	if (cvd) {
371 		dma_run_dependencies(&cvd->vd.tx);
372 
373 		/*
374 		 * free child CVD after completion.
375 		 * the parent CVD would be freed with desc_free by user.
376 		 */
377 		if (cvd->parent != cvd)
378 			kfree(cvd);
379 	}
380 
381 	/* re-enable interrupt before leaving tasklet */
382 	enable_irq(pc->irq);
383 }
384 
mtk_cqdma_irq(int irq,void * devid)385 static irqreturn_t mtk_cqdma_irq(int irq, void *devid)
386 {
387 	struct mtk_cqdma_device *cqdma = devid;
388 	irqreturn_t ret = IRQ_NONE;
389 	bool schedule_tasklet = false;
390 	u32 i;
391 
392 	/* clear interrupt flags for each PC */
393 	for (i = 0; i < cqdma->dma_channels; ++i, schedule_tasklet = false) {
394 		spin_lock(&cqdma->pc[i]->lock);
395 		if (mtk_dma_read(cqdma->pc[i],
396 				 MTK_CQDMA_INT_FLAG) & MTK_CQDMA_INT_FLAG_BIT) {
397 			/* clear interrupt */
398 			mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_FLAG,
399 				    MTK_CQDMA_INT_FLAG_BIT);
400 
401 			schedule_tasklet = true;
402 			ret = IRQ_HANDLED;
403 		}
404 		spin_unlock(&cqdma->pc[i]->lock);
405 
406 		if (schedule_tasklet) {
407 			/* disable interrupt */
408 			disable_irq_nosync(cqdma->pc[i]->irq);
409 
410 			/* schedule the tasklet to handle the transactions */
411 			tasklet_schedule(&cqdma->pc[i]->tasklet);
412 		}
413 	}
414 
415 	return ret;
416 }
417 
mtk_cqdma_find_active_desc(struct dma_chan * c,dma_cookie_t cookie)418 static struct virt_dma_desc *mtk_cqdma_find_active_desc(struct dma_chan *c,
419 							dma_cookie_t cookie)
420 {
421 	struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
422 	struct virt_dma_desc *vd;
423 	unsigned long flags;
424 
425 	spin_lock_irqsave(&cvc->pc->lock, flags);
426 	list_for_each_entry(vd, &cvc->pc->queue, node)
427 		if (vd->tx.cookie == cookie) {
428 			spin_unlock_irqrestore(&cvc->pc->lock, flags);
429 			return vd;
430 		}
431 	spin_unlock_irqrestore(&cvc->pc->lock, flags);
432 
433 	list_for_each_entry(vd, &cvc->vc.desc_issued, node)
434 		if (vd->tx.cookie == cookie)
435 			return vd;
436 
437 	return NULL;
438 }
439 
mtk_cqdma_tx_status(struct dma_chan * c,dma_cookie_t cookie,struct dma_tx_state * txstate)440 static enum dma_status mtk_cqdma_tx_status(struct dma_chan *c,
441 					   dma_cookie_t cookie,
442 					   struct dma_tx_state *txstate)
443 {
444 	struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
445 	struct mtk_cqdma_vdesc *cvd;
446 	struct virt_dma_desc *vd;
447 	enum dma_status ret;
448 	unsigned long flags;
449 	size_t bytes = 0;
450 
451 	ret = dma_cookie_status(c, cookie, txstate);
452 	if (ret == DMA_COMPLETE || !txstate)
453 		return ret;
454 
455 	spin_lock_irqsave(&cvc->vc.lock, flags);
456 	vd = mtk_cqdma_find_active_desc(c, cookie);
457 	spin_unlock_irqrestore(&cvc->vc.lock, flags);
458 
459 	if (vd) {
460 		cvd = to_cqdma_vdesc(vd);
461 		bytes = cvd->residue;
462 	}
463 
464 	dma_set_residue(txstate, bytes);
465 
466 	return ret;
467 }
468 
mtk_cqdma_issue_pending(struct dma_chan * c)469 static void mtk_cqdma_issue_pending(struct dma_chan *c)
470 {
471 	struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
472 	unsigned long pc_flags;
473 	unsigned long vc_flags;
474 
475 	/* acquire PC's lock before VS's lock for lock dependency in tasklet */
476 	spin_lock_irqsave(&cvc->pc->lock, pc_flags);
477 	spin_lock_irqsave(&cvc->vc.lock, vc_flags);
478 
479 	if (vchan_issue_pending(&cvc->vc))
480 		mtk_cqdma_issue_vchan_pending(cvc);
481 
482 	spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);
483 	spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);
484 }
485 
486 static struct dma_async_tx_descriptor *
mtk_cqdma_prep_dma_memcpy(struct dma_chan * c,dma_addr_t dest,dma_addr_t src,size_t len,unsigned long flags)487 mtk_cqdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest,
488 			  dma_addr_t src, size_t len, unsigned long flags)
489 {
490 	struct mtk_cqdma_vdesc **cvd;
491 	struct dma_async_tx_descriptor *tx = NULL, *prev_tx = NULL;
492 	size_t i, tlen, nr_vd;
493 
494 	/*
495 	 * In the case that trsanction length is larger than the
496 	 * DMA engine supports, a single memcpy transaction needs
497 	 * to be separated into several DMA transactions.
498 	 * Each DMA transaction would be described by a CVD,
499 	 * and the first one is referred as the parent CVD,
500 	 * while the others are child CVDs.
501 	 * The parent CVD's tx descriptor is the only tx descriptor
502 	 * returned to the DMA user, and it should not be completed
503 	 * until all the child CVDs completed.
504 	 */
505 	nr_vd = DIV_ROUND_UP(len, MTK_CQDMA_MAX_LEN);
506 	cvd = kcalloc(nr_vd, sizeof(*cvd), GFP_NOWAIT);
507 	if (!cvd)
508 		return NULL;
509 
510 	for (i = 0; i < nr_vd; ++i) {
511 		cvd[i] = kzalloc(sizeof(*cvd[i]), GFP_NOWAIT);
512 		if (!cvd[i]) {
513 			for (; i > 0; --i)
514 				kfree(cvd[i - 1]);
515 			return NULL;
516 		}
517 
518 		/* setup dma channel */
519 		cvd[i]->ch = c;
520 
521 		/* setup sourece, destination, and length */
522 		tlen = (len > MTK_CQDMA_MAX_LEN) ? MTK_CQDMA_MAX_LEN : len;
523 		cvd[i]->len = tlen;
524 		cvd[i]->src = src;
525 		cvd[i]->dest = dest;
526 
527 		/* setup tx descriptor */
528 		tx = vchan_tx_prep(to_virt_chan(c), &cvd[i]->vd, flags);
529 		tx->next = NULL;
530 
531 		if (!i) {
532 			cvd[0]->residue = len;
533 		} else {
534 			prev_tx->next = tx;
535 			cvd[i]->residue = tlen;
536 		}
537 
538 		cvd[i]->parent = cvd[0];
539 
540 		/* update the src, dest, len, prev_tx for the next CVD */
541 		src += tlen;
542 		dest += tlen;
543 		len -= tlen;
544 		prev_tx = tx;
545 	}
546 
547 	return &cvd[0]->vd.tx;
548 }
549 
mtk_cqdma_free_inactive_desc(struct dma_chan * c)550 static void mtk_cqdma_free_inactive_desc(struct dma_chan *c)
551 {
552 	struct virt_dma_chan *vc = to_virt_chan(c);
553 	unsigned long flags;
554 	LIST_HEAD(head);
555 
556 	/*
557 	 * set desc_allocated, desc_submitted,
558 	 * and desc_issued as the candicates to be freed
559 	 */
560 	spin_lock_irqsave(&vc->lock, flags);
561 	list_splice_tail_init(&vc->desc_allocated, &head);
562 	list_splice_tail_init(&vc->desc_submitted, &head);
563 	list_splice_tail_init(&vc->desc_issued, &head);
564 	spin_unlock_irqrestore(&vc->lock, flags);
565 
566 	/* free descriptor lists */
567 	vchan_dma_desc_free_list(vc, &head);
568 }
569 
mtk_cqdma_free_active_desc(struct dma_chan * c)570 static void mtk_cqdma_free_active_desc(struct dma_chan *c)
571 {
572 	struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
573 	bool sync_needed = false;
574 	unsigned long pc_flags;
575 	unsigned long vc_flags;
576 
577 	/* acquire PC's lock first due to lock dependency in dma ISR */
578 	spin_lock_irqsave(&cvc->pc->lock, pc_flags);
579 	spin_lock_irqsave(&cvc->vc.lock, vc_flags);
580 
581 	/* synchronization is required if this VC is active */
582 	if (mtk_cqdma_is_vchan_active(cvc)) {
583 		cvc->issue_synchronize = true;
584 		sync_needed = true;
585 	}
586 
587 	spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);
588 	spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);
589 
590 	/* waiting for the completion of this VC */
591 	if (sync_needed)
592 		wait_for_completion(&cvc->issue_completion);
593 
594 	/* free all descriptors in list desc_completed */
595 	vchan_synchronize(&cvc->vc);
596 
597 	WARN_ONCE(!list_empty(&cvc->vc.desc_completed),
598 		  "Desc pending still in list desc_completed\n");
599 }
600 
mtk_cqdma_terminate_all(struct dma_chan * c)601 static int mtk_cqdma_terminate_all(struct dma_chan *c)
602 {
603 	/* free descriptors not processed yet by hardware */
604 	mtk_cqdma_free_inactive_desc(c);
605 
606 	/* free descriptors being processed by hardware */
607 	mtk_cqdma_free_active_desc(c);
608 
609 	return 0;
610 }
611 
mtk_cqdma_alloc_chan_resources(struct dma_chan * c)612 static int mtk_cqdma_alloc_chan_resources(struct dma_chan *c)
613 {
614 	struct mtk_cqdma_device *cqdma = to_cqdma_dev(c);
615 	struct mtk_cqdma_vchan *vc = to_cqdma_vchan(c);
616 	struct mtk_cqdma_pchan *pc = NULL;
617 	u32 i, min_refcnt = U32_MAX, refcnt;
618 	unsigned long flags;
619 
620 	/* allocate PC with the minimun refcount */
621 	for (i = 0; i < cqdma->dma_channels; ++i) {
622 		refcnt = refcount_read(&cqdma->pc[i]->refcnt);
623 		if (refcnt < min_refcnt) {
624 			pc = cqdma->pc[i];
625 			min_refcnt = refcnt;
626 		}
627 	}
628 
629 	if (!pc)
630 		return -ENOSPC;
631 
632 	spin_lock_irqsave(&pc->lock, flags);
633 
634 	if (!refcount_read(&pc->refcnt)) {
635 		/* allocate PC when the refcount is zero */
636 		mtk_cqdma_hard_reset(pc);
637 
638 		/* enable interrupt for this PC */
639 		mtk_dma_set(pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
640 
641 		/*
642 		 * refcount_inc would complain increment on 0; use-after-free.
643 		 * Thus, we need to explicitly set it as 1 initially.
644 		 */
645 		refcount_set(&pc->refcnt, 1);
646 	} else {
647 		refcount_inc(&pc->refcnt);
648 	}
649 
650 	spin_unlock_irqrestore(&pc->lock, flags);
651 
652 	vc->pc = pc;
653 
654 	return 0;
655 }
656 
mtk_cqdma_free_chan_resources(struct dma_chan * c)657 static void mtk_cqdma_free_chan_resources(struct dma_chan *c)
658 {
659 	struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
660 	unsigned long flags;
661 
662 	/* free all descriptors in all lists on the VC */
663 	mtk_cqdma_terminate_all(c);
664 
665 	spin_lock_irqsave(&cvc->pc->lock, flags);
666 
667 	/* PC is not freed until there is no VC mapped to it */
668 	if (refcount_dec_and_test(&cvc->pc->refcnt)) {
669 		/* start the flush operation and stop the engine */
670 		mtk_dma_set(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
671 
672 		/* wait for the completion of flush operation */
673 		if (mtk_cqdma_poll_engine_done(cvc->pc, true) < 0)
674 			dev_err(cqdma2dev(to_cqdma_dev(c)), "cqdma flush timeout\n");
675 
676 		/* clear the flush bit and interrupt flag */
677 		mtk_dma_clr(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
678 		mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_FLAG,
679 			    MTK_CQDMA_INT_FLAG_BIT);
680 
681 		/* disable interrupt for this PC */
682 		mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
683 	}
684 
685 	spin_unlock_irqrestore(&cvc->pc->lock, flags);
686 }
687 
mtk_cqdma_hw_init(struct mtk_cqdma_device * cqdma)688 static int mtk_cqdma_hw_init(struct mtk_cqdma_device *cqdma)
689 {
690 	unsigned long flags;
691 	int err;
692 	u32 i;
693 
694 	pm_runtime_enable(cqdma2dev(cqdma));
695 	pm_runtime_get_sync(cqdma2dev(cqdma));
696 
697 	err = clk_prepare_enable(cqdma->clk);
698 
699 	if (err) {
700 		pm_runtime_put_sync(cqdma2dev(cqdma));
701 		pm_runtime_disable(cqdma2dev(cqdma));
702 		return err;
703 	}
704 
705 	/* reset all PCs */
706 	for (i = 0; i < cqdma->dma_channels; ++i) {
707 		spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
708 		if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0) {
709 			dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
710 			spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
711 
712 			clk_disable_unprepare(cqdma->clk);
713 			pm_runtime_put_sync(cqdma2dev(cqdma));
714 			pm_runtime_disable(cqdma2dev(cqdma));
715 			return -EINVAL;
716 		}
717 		spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
718 	}
719 
720 	return 0;
721 }
722 
mtk_cqdma_hw_deinit(struct mtk_cqdma_device * cqdma)723 static void mtk_cqdma_hw_deinit(struct mtk_cqdma_device *cqdma)
724 {
725 	unsigned long flags;
726 	u32 i;
727 
728 	/* reset all PCs */
729 	for (i = 0; i < cqdma->dma_channels; ++i) {
730 		spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
731 		if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0)
732 			dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
733 		spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
734 	}
735 
736 	clk_disable_unprepare(cqdma->clk);
737 
738 	pm_runtime_put_sync(cqdma2dev(cqdma));
739 	pm_runtime_disable(cqdma2dev(cqdma));
740 }
741 
742 static const struct of_device_id mtk_cqdma_match[] = {
743 	{ .compatible = "mediatek,mt6765-cqdma" },
744 	{ /* sentinel */ }
745 };
746 MODULE_DEVICE_TABLE(of, mtk_cqdma_match);
747 
mtk_cqdma_probe(struct platform_device * pdev)748 static int mtk_cqdma_probe(struct platform_device *pdev)
749 {
750 	struct mtk_cqdma_device *cqdma;
751 	struct mtk_cqdma_vchan *vc;
752 	struct dma_device *dd;
753 	int err;
754 	u32 i;
755 
756 	cqdma = devm_kzalloc(&pdev->dev, sizeof(*cqdma), GFP_KERNEL);
757 	if (!cqdma)
758 		return -ENOMEM;
759 
760 	dd = &cqdma->ddev;
761 
762 	cqdma->clk = devm_clk_get(&pdev->dev, "cqdma");
763 	if (IS_ERR(cqdma->clk)) {
764 		dev_err(&pdev->dev, "No clock for %s\n",
765 			dev_name(&pdev->dev));
766 		return PTR_ERR(cqdma->clk);
767 	}
768 
769 	dma_cap_set(DMA_MEMCPY, dd->cap_mask);
770 
771 	dd->copy_align = MTK_CQDMA_ALIGN_SIZE;
772 	dd->device_alloc_chan_resources = mtk_cqdma_alloc_chan_resources;
773 	dd->device_free_chan_resources = mtk_cqdma_free_chan_resources;
774 	dd->device_tx_status = mtk_cqdma_tx_status;
775 	dd->device_issue_pending = mtk_cqdma_issue_pending;
776 	dd->device_prep_dma_memcpy = mtk_cqdma_prep_dma_memcpy;
777 	dd->device_terminate_all = mtk_cqdma_terminate_all;
778 	dd->src_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
779 	dd->dst_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
780 	dd->directions = BIT(DMA_MEM_TO_MEM);
781 	dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
782 	dd->dev = &pdev->dev;
783 	INIT_LIST_HEAD(&dd->channels);
784 
785 	if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
786 						      "dma-requests",
787 						      &cqdma->dma_requests)) {
788 		dev_info(&pdev->dev,
789 			 "Using %u as missing dma-requests property\n",
790 			 MTK_CQDMA_NR_VCHANS);
791 
792 		cqdma->dma_requests = MTK_CQDMA_NR_VCHANS;
793 	}
794 
795 	if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
796 						      "dma-channels",
797 						      &cqdma->dma_channels)) {
798 		dev_info(&pdev->dev,
799 			 "Using %u as missing dma-channels property\n",
800 			 MTK_CQDMA_NR_PCHANS);
801 
802 		cqdma->dma_channels = MTK_CQDMA_NR_PCHANS;
803 	}
804 
805 	cqdma->pc = devm_kcalloc(&pdev->dev, cqdma->dma_channels,
806 				 sizeof(*cqdma->pc), GFP_KERNEL);
807 	if (!cqdma->pc)
808 		return -ENOMEM;
809 
810 	/* initialization for PCs */
811 	for (i = 0; i < cqdma->dma_channels; ++i) {
812 		cqdma->pc[i] = devm_kcalloc(&pdev->dev, 1,
813 					    sizeof(**cqdma->pc), GFP_KERNEL);
814 		if (!cqdma->pc[i])
815 			return -ENOMEM;
816 
817 		INIT_LIST_HEAD(&cqdma->pc[i]->queue);
818 		spin_lock_init(&cqdma->pc[i]->lock);
819 		refcount_set(&cqdma->pc[i]->refcnt, 0);
820 		cqdma->pc[i]->base = devm_platform_ioremap_resource(pdev, i);
821 		if (IS_ERR(cqdma->pc[i]->base))
822 			return PTR_ERR(cqdma->pc[i]->base);
823 
824 		/* allocate IRQ resource */
825 		err = platform_get_irq(pdev, i);
826 		if (err < 0)
827 			return err;
828 		cqdma->pc[i]->irq = err;
829 
830 		err = devm_request_irq(&pdev->dev, cqdma->pc[i]->irq,
831 				       mtk_cqdma_irq, 0, dev_name(&pdev->dev),
832 				       cqdma);
833 		if (err) {
834 			dev_err(&pdev->dev,
835 				"request_irq failed with err %d\n", err);
836 			return -EINVAL;
837 		}
838 	}
839 
840 	/* allocate resource for VCs */
841 	cqdma->vc = devm_kcalloc(&pdev->dev, cqdma->dma_requests,
842 				 sizeof(*cqdma->vc), GFP_KERNEL);
843 	if (!cqdma->vc)
844 		return -ENOMEM;
845 
846 	for (i = 0; i < cqdma->dma_requests; i++) {
847 		vc = &cqdma->vc[i];
848 		vc->vc.desc_free = mtk_cqdma_vdesc_free;
849 		vchan_init(&vc->vc, dd);
850 		init_completion(&vc->issue_completion);
851 	}
852 
853 	err = dma_async_device_register(dd);
854 	if (err)
855 		return err;
856 
857 	err = of_dma_controller_register(pdev->dev.of_node,
858 					 of_dma_xlate_by_chan_id, cqdma);
859 	if (err) {
860 		dev_err(&pdev->dev,
861 			"MediaTek CQDMA OF registration failed %d\n", err);
862 		goto err_unregister;
863 	}
864 
865 	err = mtk_cqdma_hw_init(cqdma);
866 	if (err) {
867 		dev_err(&pdev->dev,
868 			"MediaTek CQDMA HW initialization failed %d\n", err);
869 		goto err_unregister;
870 	}
871 
872 	platform_set_drvdata(pdev, cqdma);
873 
874 	/* initialize tasklet for each PC */
875 	for (i = 0; i < cqdma->dma_channels; ++i)
876 		tasklet_setup(&cqdma->pc[i]->tasklet, mtk_cqdma_tasklet_cb);
877 
878 	dev_info(&pdev->dev, "MediaTek CQDMA driver registered\n");
879 
880 	return 0;
881 
882 err_unregister:
883 	dma_async_device_unregister(dd);
884 
885 	return err;
886 }
887 
mtk_cqdma_remove(struct platform_device * pdev)888 static int mtk_cqdma_remove(struct platform_device *pdev)
889 {
890 	struct mtk_cqdma_device *cqdma = platform_get_drvdata(pdev);
891 	struct mtk_cqdma_vchan *vc;
892 	unsigned long flags;
893 	int i;
894 
895 	/* kill VC task */
896 	for (i = 0; i < cqdma->dma_requests; i++) {
897 		vc = &cqdma->vc[i];
898 
899 		list_del(&vc->vc.chan.device_node);
900 		tasklet_kill(&vc->vc.task);
901 	}
902 
903 	/* disable interrupt */
904 	for (i = 0; i < cqdma->dma_channels; i++) {
905 		spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
906 		mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_EN,
907 			    MTK_CQDMA_INT_EN_BIT);
908 		spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
909 
910 		/* Waits for any pending IRQ handlers to complete */
911 		synchronize_irq(cqdma->pc[i]->irq);
912 
913 		tasklet_kill(&cqdma->pc[i]->tasklet);
914 	}
915 
916 	/* disable hardware */
917 	mtk_cqdma_hw_deinit(cqdma);
918 
919 	dma_async_device_unregister(&cqdma->ddev);
920 	of_dma_controller_free(pdev->dev.of_node);
921 
922 	return 0;
923 }
924 
925 static struct platform_driver mtk_cqdma_driver = {
926 	.probe = mtk_cqdma_probe,
927 	.remove = mtk_cqdma_remove,
928 	.driver = {
929 		.name           = KBUILD_MODNAME,
930 		.of_match_table = mtk_cqdma_match,
931 	},
932 };
933 module_platform_driver(mtk_cqdma_driver);
934 
935 MODULE_DESCRIPTION("MediaTek CQDMA Controller Driver");
936 MODULE_AUTHOR("Shun-Chih Yu <shun-chih.yu@mediatek.com>");
937 MODULE_LICENSE("GPL v2");
938