xref: /openbmc/linux/drivers/dma/sa11x0-dma.c (revision 07c609cc)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * SA11x0 DMAengine support
4  *
5  * Copyright (C) 2012 Russell King
6  *   Derived in part from arch/arm/mach-sa1100/dma.c,
7  *   Copyright (C) 2000, 2001 by Nicolas Pitre
8  */
9 #include <linux/sched.h>
10 #include <linux/device.h>
11 #include <linux/dmaengine.h>
12 #include <linux/init.h>
13 #include <linux/interrupt.h>
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/platform_device.h>
17 #include <linux/slab.h>
18 #include <linux/spinlock.h>
19 
20 #include "virt-dma.h"
21 
22 #define NR_PHY_CHAN	6
23 #define DMA_ALIGN	3
24 #define DMA_MAX_SIZE	0x1fff
25 #define DMA_CHUNK_SIZE	0x1000
26 
27 #define DMA_DDAR	0x00
28 #define DMA_DCSR_S	0x04
29 #define DMA_DCSR_C	0x08
30 #define DMA_DCSR_R	0x0c
31 #define DMA_DBSA	0x10
32 #define DMA_DBTA	0x14
33 #define DMA_DBSB	0x18
34 #define DMA_DBTB	0x1c
35 #define DMA_SIZE	0x20
36 
37 #define DCSR_RUN	(1 << 0)
38 #define DCSR_IE		(1 << 1)
39 #define DCSR_ERROR	(1 << 2)
40 #define DCSR_DONEA	(1 << 3)
41 #define DCSR_STRTA	(1 << 4)
42 #define DCSR_DONEB	(1 << 5)
43 #define DCSR_STRTB	(1 << 6)
44 #define DCSR_BIU	(1 << 7)
45 
46 #define DDAR_RW		(1 << 0)	/* 0 = W, 1 = R */
47 #define DDAR_E		(1 << 1)	/* 0 = LE, 1 = BE */
48 #define DDAR_BS		(1 << 2)	/* 0 = BS4, 1 = BS8 */
49 #define DDAR_DW		(1 << 3)	/* 0 = 8b, 1 = 16b */
50 #define DDAR_Ser0UDCTr	(0x0 << 4)
51 #define DDAR_Ser0UDCRc	(0x1 << 4)
52 #define DDAR_Ser1SDLCTr	(0x2 << 4)
53 #define DDAR_Ser1SDLCRc	(0x3 << 4)
54 #define DDAR_Ser1UARTTr	(0x4 << 4)
55 #define DDAR_Ser1UARTRc	(0x5 << 4)
56 #define DDAR_Ser2ICPTr	(0x6 << 4)
57 #define DDAR_Ser2ICPRc	(0x7 << 4)
58 #define DDAR_Ser3UARTTr	(0x8 << 4)
59 #define DDAR_Ser3UARTRc	(0x9 << 4)
60 #define DDAR_Ser4MCP0Tr	(0xa << 4)
61 #define DDAR_Ser4MCP0Rc	(0xb << 4)
62 #define DDAR_Ser4MCP1Tr	(0xc << 4)
63 #define DDAR_Ser4MCP1Rc	(0xd << 4)
64 #define DDAR_Ser4SSPTr	(0xe << 4)
65 #define DDAR_Ser4SSPRc	(0xf << 4)
66 
67 struct sa11x0_dma_sg {
68 	u32			addr;
69 	u32			len;
70 };
71 
72 struct sa11x0_dma_desc {
73 	struct virt_dma_desc	vd;
74 
75 	u32			ddar;
76 	size_t			size;
77 	unsigned		period;
78 	bool			cyclic;
79 
80 	unsigned		sglen;
81 	struct sa11x0_dma_sg	sg[];
82 };
83 
84 struct sa11x0_dma_phy;
85 
86 struct sa11x0_dma_chan {
87 	struct virt_dma_chan	vc;
88 
89 	/* protected by c->vc.lock */
90 	struct sa11x0_dma_phy	*phy;
91 	enum dma_status		status;
92 
93 	/* protected by d->lock */
94 	struct list_head	node;
95 
96 	u32			ddar;
97 	const char		*name;
98 };
99 
100 struct sa11x0_dma_phy {
101 	void __iomem		*base;
102 	struct sa11x0_dma_dev	*dev;
103 	unsigned		num;
104 
105 	struct sa11x0_dma_chan	*vchan;
106 
107 	/* Protected by c->vc.lock */
108 	unsigned		sg_load;
109 	struct sa11x0_dma_desc	*txd_load;
110 	unsigned		sg_done;
111 	struct sa11x0_dma_desc	*txd_done;
112 	u32			dbs[2];
113 	u32			dbt[2];
114 	u32			dcsr;
115 };
116 
117 struct sa11x0_dma_dev {
118 	struct dma_device	slave;
119 	void __iomem		*base;
120 	spinlock_t		lock;
121 	struct tasklet_struct	task;
122 	struct list_head	chan_pending;
123 	struct sa11x0_dma_phy	phy[NR_PHY_CHAN];
124 };
125 
to_sa11x0_dma_chan(struct dma_chan * chan)126 static struct sa11x0_dma_chan *to_sa11x0_dma_chan(struct dma_chan *chan)
127 {
128 	return container_of(chan, struct sa11x0_dma_chan, vc.chan);
129 }
130 
to_sa11x0_dma(struct dma_device * dmadev)131 static struct sa11x0_dma_dev *to_sa11x0_dma(struct dma_device *dmadev)
132 {
133 	return container_of(dmadev, struct sa11x0_dma_dev, slave);
134 }
135 
sa11x0_dma_next_desc(struct sa11x0_dma_chan * c)136 static struct sa11x0_dma_desc *sa11x0_dma_next_desc(struct sa11x0_dma_chan *c)
137 {
138 	struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
139 
140 	return vd ? container_of(vd, struct sa11x0_dma_desc, vd) : NULL;
141 }
142 
sa11x0_dma_free_desc(struct virt_dma_desc * vd)143 static void sa11x0_dma_free_desc(struct virt_dma_desc *vd)
144 {
145 	kfree(container_of(vd, struct sa11x0_dma_desc, vd));
146 }
147 
sa11x0_dma_start_desc(struct sa11x0_dma_phy * p,struct sa11x0_dma_desc * txd)148 static void sa11x0_dma_start_desc(struct sa11x0_dma_phy *p, struct sa11x0_dma_desc *txd)
149 {
150 	list_del(&txd->vd.node);
151 	p->txd_load = txd;
152 	p->sg_load = 0;
153 
154 	dev_vdbg(p->dev->slave.dev, "pchan %u: txd %p[%x]: starting: DDAR:%x\n",
155 		p->num, &txd->vd, txd->vd.tx.cookie, txd->ddar);
156 }
157 
sa11x0_dma_start_sg(struct sa11x0_dma_phy * p,struct sa11x0_dma_chan * c)158 static void noinline sa11x0_dma_start_sg(struct sa11x0_dma_phy *p,
159 	struct sa11x0_dma_chan *c)
160 {
161 	struct sa11x0_dma_desc *txd = p->txd_load;
162 	struct sa11x0_dma_sg *sg;
163 	void __iomem *base = p->base;
164 	unsigned dbsx, dbtx;
165 	u32 dcsr;
166 
167 	if (!txd)
168 		return;
169 
170 	dcsr = readl_relaxed(base + DMA_DCSR_R);
171 
172 	/* Don't try to load the next transfer if both buffers are started */
173 	if ((dcsr & (DCSR_STRTA | DCSR_STRTB)) == (DCSR_STRTA | DCSR_STRTB))
174 		return;
175 
176 	if (p->sg_load == txd->sglen) {
177 		if (!txd->cyclic) {
178 			struct sa11x0_dma_desc *txn = sa11x0_dma_next_desc(c);
179 
180 			/*
181 			 * We have reached the end of the current descriptor.
182 			 * Peek at the next descriptor, and if compatible with
183 			 * the current, start processing it.
184 			 */
185 			if (txn && txn->ddar == txd->ddar) {
186 				txd = txn;
187 				sa11x0_dma_start_desc(p, txn);
188 			} else {
189 				p->txd_load = NULL;
190 				return;
191 			}
192 		} else {
193 			/* Cyclic: reset back to beginning */
194 			p->sg_load = 0;
195 		}
196 	}
197 
198 	sg = &txd->sg[p->sg_load++];
199 
200 	/* Select buffer to load according to channel status */
201 	if (((dcsr & (DCSR_BIU | DCSR_STRTB)) == (DCSR_BIU | DCSR_STRTB)) ||
202 	    ((dcsr & (DCSR_BIU | DCSR_STRTA)) == 0)) {
203 		dbsx = DMA_DBSA;
204 		dbtx = DMA_DBTA;
205 		dcsr = DCSR_STRTA | DCSR_IE | DCSR_RUN;
206 	} else {
207 		dbsx = DMA_DBSB;
208 		dbtx = DMA_DBTB;
209 		dcsr = DCSR_STRTB | DCSR_IE | DCSR_RUN;
210 	}
211 
212 	writel_relaxed(sg->addr, base + dbsx);
213 	writel_relaxed(sg->len, base + dbtx);
214 	writel(dcsr, base + DMA_DCSR_S);
215 
216 	dev_dbg(p->dev->slave.dev, "pchan %u: load: DCSR:%02x DBS%c:%08x DBT%c:%08x\n",
217 		p->num, dcsr,
218 		'A' + (dbsx == DMA_DBSB), sg->addr,
219 		'A' + (dbtx == DMA_DBTB), sg->len);
220 }
221 
sa11x0_dma_complete(struct sa11x0_dma_phy * p,struct sa11x0_dma_chan * c)222 static void noinline sa11x0_dma_complete(struct sa11x0_dma_phy *p,
223 	struct sa11x0_dma_chan *c)
224 {
225 	struct sa11x0_dma_desc *txd = p->txd_done;
226 
227 	if (++p->sg_done == txd->sglen) {
228 		if (!txd->cyclic) {
229 			vchan_cookie_complete(&txd->vd);
230 
231 			p->sg_done = 0;
232 			p->txd_done = p->txd_load;
233 
234 			if (!p->txd_done)
235 				tasklet_schedule(&p->dev->task);
236 		} else {
237 			if ((p->sg_done % txd->period) == 0)
238 				vchan_cyclic_callback(&txd->vd);
239 
240 			/* Cyclic: reset back to beginning */
241 			p->sg_done = 0;
242 		}
243 	}
244 
245 	sa11x0_dma_start_sg(p, c);
246 }
247 
sa11x0_dma_irq(int irq,void * dev_id)248 static irqreturn_t sa11x0_dma_irq(int irq, void *dev_id)
249 {
250 	struct sa11x0_dma_phy *p = dev_id;
251 	struct sa11x0_dma_dev *d = p->dev;
252 	struct sa11x0_dma_chan *c;
253 	u32 dcsr;
254 
255 	dcsr = readl_relaxed(p->base + DMA_DCSR_R);
256 	if (!(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB)))
257 		return IRQ_NONE;
258 
259 	/* Clear reported status bits */
260 	writel_relaxed(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB),
261 		p->base + DMA_DCSR_C);
262 
263 	dev_dbg(d->slave.dev, "pchan %u: irq: DCSR:%02x\n", p->num, dcsr);
264 
265 	if (dcsr & DCSR_ERROR) {
266 		dev_err(d->slave.dev, "pchan %u: error. DCSR:%02x DDAR:%08x DBSA:%08x DBTA:%08x DBSB:%08x DBTB:%08x\n",
267 			p->num, dcsr,
268 			readl_relaxed(p->base + DMA_DDAR),
269 			readl_relaxed(p->base + DMA_DBSA),
270 			readl_relaxed(p->base + DMA_DBTA),
271 			readl_relaxed(p->base + DMA_DBSB),
272 			readl_relaxed(p->base + DMA_DBTB));
273 	}
274 
275 	c = p->vchan;
276 	if (c) {
277 		unsigned long flags;
278 
279 		spin_lock_irqsave(&c->vc.lock, flags);
280 		/*
281 		 * Now that we're holding the lock, check that the vchan
282 		 * really is associated with this pchan before touching the
283 		 * hardware.  This should always succeed, because we won't
284 		 * change p->vchan or c->phy while the channel is actively
285 		 * transferring.
286 		 */
287 		if (c->phy == p) {
288 			if (dcsr & DCSR_DONEA)
289 				sa11x0_dma_complete(p, c);
290 			if (dcsr & DCSR_DONEB)
291 				sa11x0_dma_complete(p, c);
292 		}
293 		spin_unlock_irqrestore(&c->vc.lock, flags);
294 	}
295 
296 	return IRQ_HANDLED;
297 }
298 
sa11x0_dma_start_txd(struct sa11x0_dma_chan * c)299 static void sa11x0_dma_start_txd(struct sa11x0_dma_chan *c)
300 {
301 	struct sa11x0_dma_desc *txd = sa11x0_dma_next_desc(c);
302 
303 	/* If the issued list is empty, we have no further txds to process */
304 	if (txd) {
305 		struct sa11x0_dma_phy *p = c->phy;
306 
307 		sa11x0_dma_start_desc(p, txd);
308 		p->txd_done = txd;
309 		p->sg_done = 0;
310 
311 		/* The channel should not have any transfers started */
312 		WARN_ON(readl_relaxed(p->base + DMA_DCSR_R) &
313 				      (DCSR_STRTA | DCSR_STRTB));
314 
315 		/* Clear the run and start bits before changing DDAR */
316 		writel_relaxed(DCSR_RUN | DCSR_STRTA | DCSR_STRTB,
317 			       p->base + DMA_DCSR_C);
318 		writel_relaxed(txd->ddar, p->base + DMA_DDAR);
319 
320 		/* Try to start both buffers */
321 		sa11x0_dma_start_sg(p, c);
322 		sa11x0_dma_start_sg(p, c);
323 	}
324 }
325 
sa11x0_dma_tasklet(struct tasklet_struct * t)326 static void sa11x0_dma_tasklet(struct tasklet_struct *t)
327 {
328 	struct sa11x0_dma_dev *d = from_tasklet(d, t, task);
329 	struct sa11x0_dma_phy *p;
330 	struct sa11x0_dma_chan *c;
331 	unsigned pch, pch_alloc = 0;
332 
333 	dev_dbg(d->slave.dev, "tasklet enter\n");
334 
335 	list_for_each_entry(c, &d->slave.channels, vc.chan.device_node) {
336 		spin_lock_irq(&c->vc.lock);
337 		p = c->phy;
338 		if (p && !p->txd_done) {
339 			sa11x0_dma_start_txd(c);
340 			if (!p->txd_done) {
341 				/* No current txd associated with this channel */
342 				dev_dbg(d->slave.dev, "pchan %u: free\n", p->num);
343 
344 				/* Mark this channel free */
345 				c->phy = NULL;
346 				p->vchan = NULL;
347 			}
348 		}
349 		spin_unlock_irq(&c->vc.lock);
350 	}
351 
352 	spin_lock_irq(&d->lock);
353 	for (pch = 0; pch < NR_PHY_CHAN; pch++) {
354 		p = &d->phy[pch];
355 
356 		if (p->vchan == NULL && !list_empty(&d->chan_pending)) {
357 			c = list_first_entry(&d->chan_pending,
358 				struct sa11x0_dma_chan, node);
359 			list_del_init(&c->node);
360 
361 			pch_alloc |= 1 << pch;
362 
363 			/* Mark this channel allocated */
364 			p->vchan = c;
365 
366 			dev_dbg(d->slave.dev, "pchan %u: alloc vchan %p\n", pch, &c->vc);
367 		}
368 	}
369 	spin_unlock_irq(&d->lock);
370 
371 	for (pch = 0; pch < NR_PHY_CHAN; pch++) {
372 		if (pch_alloc & (1 << pch)) {
373 			p = &d->phy[pch];
374 			c = p->vchan;
375 
376 			spin_lock_irq(&c->vc.lock);
377 			c->phy = p;
378 
379 			sa11x0_dma_start_txd(c);
380 			spin_unlock_irq(&c->vc.lock);
381 		}
382 	}
383 
384 	dev_dbg(d->slave.dev, "tasklet exit\n");
385 }
386 
387 
sa11x0_dma_free_chan_resources(struct dma_chan * chan)388 static void sa11x0_dma_free_chan_resources(struct dma_chan *chan)
389 {
390 	struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
391 	struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
392 	unsigned long flags;
393 
394 	spin_lock_irqsave(&d->lock, flags);
395 	list_del_init(&c->node);
396 	spin_unlock_irqrestore(&d->lock, flags);
397 
398 	vchan_free_chan_resources(&c->vc);
399 }
400 
sa11x0_dma_pos(struct sa11x0_dma_phy * p)401 static dma_addr_t sa11x0_dma_pos(struct sa11x0_dma_phy *p)
402 {
403 	unsigned reg;
404 	u32 dcsr;
405 
406 	dcsr = readl_relaxed(p->base + DMA_DCSR_R);
407 
408 	if ((dcsr & (DCSR_BIU | DCSR_STRTA)) == DCSR_STRTA ||
409 	    (dcsr & (DCSR_BIU | DCSR_STRTB)) == DCSR_BIU)
410 		reg = DMA_DBSA;
411 	else
412 		reg = DMA_DBSB;
413 
414 	return readl_relaxed(p->base + reg);
415 }
416 
sa11x0_dma_tx_status(struct dma_chan * chan,dma_cookie_t cookie,struct dma_tx_state * state)417 static enum dma_status sa11x0_dma_tx_status(struct dma_chan *chan,
418 	dma_cookie_t cookie, struct dma_tx_state *state)
419 {
420 	struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
421 	struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
422 	struct sa11x0_dma_phy *p;
423 	struct virt_dma_desc *vd;
424 	unsigned long flags;
425 	enum dma_status ret;
426 
427 	ret = dma_cookie_status(&c->vc.chan, cookie, state);
428 	if (ret == DMA_COMPLETE)
429 		return ret;
430 
431 	if (!state)
432 		return c->status;
433 
434 	spin_lock_irqsave(&c->vc.lock, flags);
435 	p = c->phy;
436 
437 	/*
438 	 * If the cookie is on our issue queue, then the residue is
439 	 * its total size.
440 	 */
441 	vd = vchan_find_desc(&c->vc, cookie);
442 	if (vd) {
443 		state->residue = container_of(vd, struct sa11x0_dma_desc, vd)->size;
444 	} else if (!p) {
445 		state->residue = 0;
446 	} else {
447 		struct sa11x0_dma_desc *txd;
448 		size_t bytes = 0;
449 
450 		if (p->txd_done && p->txd_done->vd.tx.cookie == cookie)
451 			txd = p->txd_done;
452 		else if (p->txd_load && p->txd_load->vd.tx.cookie == cookie)
453 			txd = p->txd_load;
454 		else
455 			txd = NULL;
456 
457 		ret = c->status;
458 		if (txd) {
459 			dma_addr_t addr = sa11x0_dma_pos(p);
460 			unsigned i;
461 
462 			dev_vdbg(d->slave.dev, "tx_status: addr:%pad\n", &addr);
463 
464 			for (i = 0; i < txd->sglen; i++) {
465 				dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x\n",
466 					i, txd->sg[i].addr, txd->sg[i].len);
467 				if (addr >= txd->sg[i].addr &&
468 				    addr < txd->sg[i].addr + txd->sg[i].len) {
469 					unsigned len;
470 
471 					len = txd->sg[i].len -
472 						(addr - txd->sg[i].addr);
473 					dev_vdbg(d->slave.dev, "tx_status: [%u] +%x\n",
474 						i, len);
475 					bytes += len;
476 					i++;
477 					break;
478 				}
479 			}
480 			for (; i < txd->sglen; i++) {
481 				dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x ++\n",
482 					i, txd->sg[i].addr, txd->sg[i].len);
483 				bytes += txd->sg[i].len;
484 			}
485 		}
486 		state->residue = bytes;
487 	}
488 	spin_unlock_irqrestore(&c->vc.lock, flags);
489 
490 	dev_vdbg(d->slave.dev, "tx_status: bytes 0x%x\n", state->residue);
491 
492 	return ret;
493 }
494 
495 /*
496  * Move pending txds to the issued list, and re-init pending list.
497  * If not already pending, add this channel to the list of pending
498  * channels and trigger the tasklet to run.
499  */
sa11x0_dma_issue_pending(struct dma_chan * chan)500 static void sa11x0_dma_issue_pending(struct dma_chan *chan)
501 {
502 	struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
503 	struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
504 	unsigned long flags;
505 
506 	spin_lock_irqsave(&c->vc.lock, flags);
507 	if (vchan_issue_pending(&c->vc)) {
508 		if (!c->phy) {
509 			spin_lock(&d->lock);
510 			if (list_empty(&c->node)) {
511 				list_add_tail(&c->node, &d->chan_pending);
512 				tasklet_schedule(&d->task);
513 				dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc);
514 			}
515 			spin_unlock(&d->lock);
516 		}
517 	} else
518 		dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc);
519 	spin_unlock_irqrestore(&c->vc.lock, flags);
520 }
521 
sa11x0_dma_prep_slave_sg(struct dma_chan * chan,struct scatterlist * sg,unsigned int sglen,enum dma_transfer_direction dir,unsigned long flags,void * context)522 static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg(
523 	struct dma_chan *chan, struct scatterlist *sg, unsigned int sglen,
524 	enum dma_transfer_direction dir, unsigned long flags, void *context)
525 {
526 	struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
527 	struct sa11x0_dma_desc *txd;
528 	struct scatterlist *sgent;
529 	unsigned i, j = sglen;
530 	size_t size = 0;
531 
532 	/* SA11x0 channels can only operate in their native direction */
533 	if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
534 		dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
535 			&c->vc, c->ddar, dir);
536 		return NULL;
537 	}
538 
539 	/* Do not allow zero-sized txds */
540 	if (sglen == 0)
541 		return NULL;
542 
543 	for_each_sg(sg, sgent, sglen, i) {
544 		dma_addr_t addr = sg_dma_address(sgent);
545 		unsigned int len = sg_dma_len(sgent);
546 
547 		if (len > DMA_MAX_SIZE)
548 			j += DIV_ROUND_UP(len, DMA_MAX_SIZE & ~DMA_ALIGN) - 1;
549 		if (addr & DMA_ALIGN) {
550 			dev_dbg(chan->device->dev, "vchan %p: bad buffer alignment: %pad\n",
551 				&c->vc, &addr);
552 			return NULL;
553 		}
554 	}
555 
556 	txd = kzalloc(struct_size(txd, sg, j), GFP_ATOMIC);
557 	if (!txd) {
558 		dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
559 		return NULL;
560 	}
561 
562 	j = 0;
563 	for_each_sg(sg, sgent, sglen, i) {
564 		dma_addr_t addr = sg_dma_address(sgent);
565 		unsigned len = sg_dma_len(sgent);
566 
567 		size += len;
568 
569 		do {
570 			unsigned tlen = len;
571 
572 			/*
573 			 * Check whether the transfer will fit.  If not, try
574 			 * to split the transfer up such that we end up with
575 			 * equal chunks - but make sure that we preserve the
576 			 * alignment.  This avoids small segments.
577 			 */
578 			if (tlen > DMA_MAX_SIZE) {
579 				unsigned mult = DIV_ROUND_UP(tlen,
580 					DMA_MAX_SIZE & ~DMA_ALIGN);
581 
582 				tlen = (tlen / mult) & ~DMA_ALIGN;
583 			}
584 
585 			txd->sg[j].addr = addr;
586 			txd->sg[j].len = tlen;
587 
588 			addr += tlen;
589 			len -= tlen;
590 			j++;
591 		} while (len);
592 	}
593 
594 	txd->ddar = c->ddar;
595 	txd->size = size;
596 	txd->sglen = j;
597 
598 	dev_dbg(chan->device->dev, "vchan %p: txd %p: size %zu nr %u\n",
599 		&c->vc, &txd->vd, txd->size, txd->sglen);
600 
601 	return vchan_tx_prep(&c->vc, &txd->vd, flags);
602 }
603 
sa11x0_dma_prep_dma_cyclic(struct dma_chan * chan,dma_addr_t addr,size_t size,size_t period,enum dma_transfer_direction dir,unsigned long flags)604 static struct dma_async_tx_descriptor *sa11x0_dma_prep_dma_cyclic(
605 	struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period,
606 	enum dma_transfer_direction dir, unsigned long flags)
607 {
608 	struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
609 	struct sa11x0_dma_desc *txd;
610 	unsigned i, j, k, sglen, sgperiod;
611 
612 	/* SA11x0 channels can only operate in their native direction */
613 	if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
614 		dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
615 			&c->vc, c->ddar, dir);
616 		return NULL;
617 	}
618 
619 	sgperiod = DIV_ROUND_UP(period, DMA_MAX_SIZE & ~DMA_ALIGN);
620 	sglen = size * sgperiod / period;
621 
622 	/* Do not allow zero-sized txds */
623 	if (sglen == 0)
624 		return NULL;
625 
626 	txd = kzalloc(struct_size(txd, sg, sglen), GFP_ATOMIC);
627 	if (!txd) {
628 		dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
629 		return NULL;
630 	}
631 
632 	for (i = k = 0; i < size / period; i++) {
633 		size_t tlen, len = period;
634 
635 		for (j = 0; j < sgperiod; j++, k++) {
636 			tlen = len;
637 
638 			if (tlen > DMA_MAX_SIZE) {
639 				unsigned mult = DIV_ROUND_UP(tlen, DMA_MAX_SIZE & ~DMA_ALIGN);
640 				tlen = (tlen / mult) & ~DMA_ALIGN;
641 			}
642 
643 			txd->sg[k].addr = addr;
644 			txd->sg[k].len = tlen;
645 			addr += tlen;
646 			len -= tlen;
647 		}
648 
649 		WARN_ON(len != 0);
650 	}
651 
652 	WARN_ON(k != sglen);
653 
654 	txd->ddar = c->ddar;
655 	txd->size = size;
656 	txd->sglen = sglen;
657 	txd->cyclic = 1;
658 	txd->period = sgperiod;
659 
660 	return vchan_tx_prep(&c->vc, &txd->vd, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
661 }
662 
sa11x0_dma_device_config(struct dma_chan * chan,struct dma_slave_config * cfg)663 static int sa11x0_dma_device_config(struct dma_chan *chan,
664 				    struct dma_slave_config *cfg)
665 {
666 	struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
667 	u32 ddar = c->ddar & ((0xf << 4) | DDAR_RW);
668 	dma_addr_t addr;
669 	enum dma_slave_buswidth width;
670 	u32 maxburst;
671 
672 	if (ddar & DDAR_RW) {
673 		addr = cfg->src_addr;
674 		width = cfg->src_addr_width;
675 		maxburst = cfg->src_maxburst;
676 	} else {
677 		addr = cfg->dst_addr;
678 		width = cfg->dst_addr_width;
679 		maxburst = cfg->dst_maxburst;
680 	}
681 
682 	if ((width != DMA_SLAVE_BUSWIDTH_1_BYTE &&
683 	     width != DMA_SLAVE_BUSWIDTH_2_BYTES) ||
684 	    (maxburst != 4 && maxburst != 8))
685 		return -EINVAL;
686 
687 	if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
688 		ddar |= DDAR_DW;
689 	if (maxburst == 8)
690 		ddar |= DDAR_BS;
691 
692 	dev_dbg(c->vc.chan.device->dev, "vchan %p: dma_slave_config addr %pad width %u burst %u\n",
693 		&c->vc, &addr, width, maxburst);
694 
695 	c->ddar = ddar | (addr & 0xf0000000) | (addr & 0x003ffffc) << 6;
696 
697 	return 0;
698 }
699 
sa11x0_dma_device_pause(struct dma_chan * chan)700 static int sa11x0_dma_device_pause(struct dma_chan *chan)
701 {
702 	struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
703 	struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
704 	struct sa11x0_dma_phy *p;
705 	unsigned long flags;
706 
707 	dev_dbg(d->slave.dev, "vchan %p: pause\n", &c->vc);
708 	spin_lock_irqsave(&c->vc.lock, flags);
709 	if (c->status == DMA_IN_PROGRESS) {
710 		c->status = DMA_PAUSED;
711 
712 		p = c->phy;
713 		if (p) {
714 			writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
715 		} else {
716 			spin_lock(&d->lock);
717 			list_del_init(&c->node);
718 			spin_unlock(&d->lock);
719 		}
720 	}
721 	spin_unlock_irqrestore(&c->vc.lock, flags);
722 
723 	return 0;
724 }
725 
sa11x0_dma_device_resume(struct dma_chan * chan)726 static int sa11x0_dma_device_resume(struct dma_chan *chan)
727 {
728 	struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
729 	struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
730 	struct sa11x0_dma_phy *p;
731 	unsigned long flags;
732 
733 	dev_dbg(d->slave.dev, "vchan %p: resume\n", &c->vc);
734 	spin_lock_irqsave(&c->vc.lock, flags);
735 	if (c->status == DMA_PAUSED) {
736 		c->status = DMA_IN_PROGRESS;
737 
738 		p = c->phy;
739 		if (p) {
740 			writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_S);
741 		} else if (!list_empty(&c->vc.desc_issued)) {
742 			spin_lock(&d->lock);
743 			list_add_tail(&c->node, &d->chan_pending);
744 			spin_unlock(&d->lock);
745 		}
746 	}
747 	spin_unlock_irqrestore(&c->vc.lock, flags);
748 
749 	return 0;
750 }
751 
sa11x0_dma_device_terminate_all(struct dma_chan * chan)752 static int sa11x0_dma_device_terminate_all(struct dma_chan *chan)
753 {
754 	struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
755 	struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
756 	struct sa11x0_dma_phy *p;
757 	LIST_HEAD(head);
758 	unsigned long flags;
759 
760 	dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc);
761 	/* Clear the tx descriptor lists */
762 	spin_lock_irqsave(&c->vc.lock, flags);
763 	vchan_get_all_descriptors(&c->vc, &head);
764 
765 	p = c->phy;
766 	if (p) {
767 		dev_dbg(d->slave.dev, "pchan %u: terminating\n", p->num);
768 		/* vchan is assigned to a pchan - stop the channel */
769 		writel(DCSR_RUN | DCSR_IE |
770 		       DCSR_STRTA | DCSR_DONEA |
771 		       DCSR_STRTB | DCSR_DONEB,
772 		       p->base + DMA_DCSR_C);
773 
774 		if (p->txd_load) {
775 			if (p->txd_load != p->txd_done)
776 				list_add_tail(&p->txd_load->vd.node, &head);
777 			p->txd_load = NULL;
778 		}
779 		if (p->txd_done) {
780 			list_add_tail(&p->txd_done->vd.node, &head);
781 			p->txd_done = NULL;
782 		}
783 		c->phy = NULL;
784 		spin_lock(&d->lock);
785 		p->vchan = NULL;
786 		spin_unlock(&d->lock);
787 		tasklet_schedule(&d->task);
788 	}
789 	spin_unlock_irqrestore(&c->vc.lock, flags);
790 	vchan_dma_desc_free_list(&c->vc, &head);
791 
792 	return 0;
793 }
794 
795 struct sa11x0_dma_channel_desc {
796 	u32 ddar;
797 	const char *name;
798 };
799 
800 #define CD(d1, d2) { .ddar = DDAR_##d1 | d2, .name = #d1 }
801 static const struct sa11x0_dma_channel_desc chan_desc[] = {
802 	CD(Ser0UDCTr, 0),
803 	CD(Ser0UDCRc, DDAR_RW),
804 	CD(Ser1SDLCTr, 0),
805 	CD(Ser1SDLCRc, DDAR_RW),
806 	CD(Ser1UARTTr, 0),
807 	CD(Ser1UARTRc, DDAR_RW),
808 	CD(Ser2ICPTr, 0),
809 	CD(Ser2ICPRc, DDAR_RW),
810 	CD(Ser3UARTTr, 0),
811 	CD(Ser3UARTRc, DDAR_RW),
812 	CD(Ser4MCP0Tr, 0),
813 	CD(Ser4MCP0Rc, DDAR_RW),
814 	CD(Ser4MCP1Tr, 0),
815 	CD(Ser4MCP1Rc, DDAR_RW),
816 	CD(Ser4SSPTr, 0),
817 	CD(Ser4SSPRc, DDAR_RW),
818 };
819 
820 static const struct dma_slave_map sa11x0_dma_map[] = {
821 	{ "sa11x0-ir", "tx", "Ser2ICPTr" },
822 	{ "sa11x0-ir", "rx", "Ser2ICPRc" },
823 	{ "sa11x0-ssp", "tx", "Ser4SSPTr" },
824 	{ "sa11x0-ssp", "rx", "Ser4SSPRc" },
825 };
826 
sa11x0_dma_filter_fn(struct dma_chan * chan,void * param)827 static bool sa11x0_dma_filter_fn(struct dma_chan *chan, void *param)
828 {
829 	struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
830 	const char *p = param;
831 
832 	return !strcmp(c->name, p);
833 }
834 
sa11x0_dma_init_dmadev(struct dma_device * dmadev,struct device * dev)835 static int sa11x0_dma_init_dmadev(struct dma_device *dmadev,
836 	struct device *dev)
837 {
838 	unsigned i;
839 
840 	INIT_LIST_HEAD(&dmadev->channels);
841 	dmadev->dev = dev;
842 	dmadev->device_free_chan_resources = sa11x0_dma_free_chan_resources;
843 	dmadev->device_config = sa11x0_dma_device_config;
844 	dmadev->device_pause = sa11x0_dma_device_pause;
845 	dmadev->device_resume = sa11x0_dma_device_resume;
846 	dmadev->device_terminate_all = sa11x0_dma_device_terminate_all;
847 	dmadev->device_tx_status = sa11x0_dma_tx_status;
848 	dmadev->device_issue_pending = sa11x0_dma_issue_pending;
849 
850 	for (i = 0; i < ARRAY_SIZE(chan_desc); i++) {
851 		struct sa11x0_dma_chan *c;
852 
853 		c = kzalloc(sizeof(*c), GFP_KERNEL);
854 		if (!c) {
855 			dev_err(dev, "no memory for channel %u\n", i);
856 			return -ENOMEM;
857 		}
858 
859 		c->status = DMA_IN_PROGRESS;
860 		c->ddar = chan_desc[i].ddar;
861 		c->name = chan_desc[i].name;
862 		INIT_LIST_HEAD(&c->node);
863 
864 		c->vc.desc_free = sa11x0_dma_free_desc;
865 		vchan_init(&c->vc, dmadev);
866 	}
867 
868 	return dma_async_device_register(dmadev);
869 }
870 
sa11x0_dma_request_irq(struct platform_device * pdev,int nr,void * data)871 static int sa11x0_dma_request_irq(struct platform_device *pdev, int nr,
872 	void *data)
873 {
874 	int irq = platform_get_irq(pdev, nr);
875 
876 	if (irq <= 0)
877 		return -ENXIO;
878 
879 	return request_irq(irq, sa11x0_dma_irq, 0, dev_name(&pdev->dev), data);
880 }
881 
sa11x0_dma_free_irq(struct platform_device * pdev,int nr,void * data)882 static void sa11x0_dma_free_irq(struct platform_device *pdev, int nr,
883 	void *data)
884 {
885 	int irq = platform_get_irq(pdev, nr);
886 	if (irq > 0)
887 		free_irq(irq, data);
888 }
889 
sa11x0_dma_free_channels(struct dma_device * dmadev)890 static void sa11x0_dma_free_channels(struct dma_device *dmadev)
891 {
892 	struct sa11x0_dma_chan *c, *cn;
893 
894 	list_for_each_entry_safe(c, cn, &dmadev->channels, vc.chan.device_node) {
895 		list_del(&c->vc.chan.device_node);
896 		tasklet_kill(&c->vc.task);
897 		kfree(c);
898 	}
899 }
900 
sa11x0_dma_probe(struct platform_device * pdev)901 static int sa11x0_dma_probe(struct platform_device *pdev)
902 {
903 	struct sa11x0_dma_dev *d;
904 	struct resource *res;
905 	unsigned i;
906 	int ret;
907 
908 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
909 	if (!res)
910 		return -ENXIO;
911 
912 	d = kzalloc(sizeof(*d), GFP_KERNEL);
913 	if (!d) {
914 		ret = -ENOMEM;
915 		goto err_alloc;
916 	}
917 
918 	spin_lock_init(&d->lock);
919 	INIT_LIST_HEAD(&d->chan_pending);
920 
921 	d->slave.filter.fn = sa11x0_dma_filter_fn;
922 	d->slave.filter.mapcnt = ARRAY_SIZE(sa11x0_dma_map);
923 	d->slave.filter.map = sa11x0_dma_map;
924 
925 	d->base = ioremap(res->start, resource_size(res));
926 	if (!d->base) {
927 		ret = -ENOMEM;
928 		goto err_ioremap;
929 	}
930 
931 	tasklet_setup(&d->task, sa11x0_dma_tasklet);
932 
933 	for (i = 0; i < NR_PHY_CHAN; i++) {
934 		struct sa11x0_dma_phy *p = &d->phy[i];
935 
936 		p->dev = d;
937 		p->num = i;
938 		p->base = d->base + i * DMA_SIZE;
939 		writel_relaxed(DCSR_RUN | DCSR_IE | DCSR_ERROR |
940 			DCSR_DONEA | DCSR_STRTA | DCSR_DONEB | DCSR_STRTB,
941 			p->base + DMA_DCSR_C);
942 		writel_relaxed(0, p->base + DMA_DDAR);
943 
944 		ret = sa11x0_dma_request_irq(pdev, i, p);
945 		if (ret) {
946 			while (i) {
947 				i--;
948 				sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
949 			}
950 			goto err_irq;
951 		}
952 	}
953 
954 	dma_cap_set(DMA_SLAVE, d->slave.cap_mask);
955 	dma_cap_set(DMA_CYCLIC, d->slave.cap_mask);
956 	d->slave.device_prep_slave_sg = sa11x0_dma_prep_slave_sg;
957 	d->slave.device_prep_dma_cyclic = sa11x0_dma_prep_dma_cyclic;
958 	d->slave.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
959 	d->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
960 	d->slave.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
961 				   BIT(DMA_SLAVE_BUSWIDTH_2_BYTES);
962 	d->slave.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
963 				   BIT(DMA_SLAVE_BUSWIDTH_2_BYTES);
964 	ret = sa11x0_dma_init_dmadev(&d->slave, &pdev->dev);
965 	if (ret) {
966 		dev_warn(d->slave.dev, "failed to register slave async device: %d\n",
967 			ret);
968 		goto err_slave_reg;
969 	}
970 
971 	platform_set_drvdata(pdev, d);
972 	return 0;
973 
974  err_slave_reg:
975 	sa11x0_dma_free_channels(&d->slave);
976 	for (i = 0; i < NR_PHY_CHAN; i++)
977 		sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
978  err_irq:
979 	tasklet_kill(&d->task);
980 	iounmap(d->base);
981  err_ioremap:
982 	kfree(d);
983  err_alloc:
984 	return ret;
985 }
986 
sa11x0_dma_remove(struct platform_device * pdev)987 static int sa11x0_dma_remove(struct platform_device *pdev)
988 {
989 	struct sa11x0_dma_dev *d = platform_get_drvdata(pdev);
990 	unsigned pch;
991 
992 	dma_async_device_unregister(&d->slave);
993 
994 	sa11x0_dma_free_channels(&d->slave);
995 	for (pch = 0; pch < NR_PHY_CHAN; pch++)
996 		sa11x0_dma_free_irq(pdev, pch, &d->phy[pch]);
997 	tasklet_kill(&d->task);
998 	iounmap(d->base);
999 	kfree(d);
1000 
1001 	return 0;
1002 }
1003 
sa11x0_dma_suspend(struct device * dev)1004 static __maybe_unused int sa11x0_dma_suspend(struct device *dev)
1005 {
1006 	struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
1007 	unsigned pch;
1008 
1009 	for (pch = 0; pch < NR_PHY_CHAN; pch++) {
1010 		struct sa11x0_dma_phy *p = &d->phy[pch];
1011 		u32 dcsr, saved_dcsr;
1012 
1013 		dcsr = saved_dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1014 		if (dcsr & DCSR_RUN) {
1015 			writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
1016 			dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1017 		}
1018 
1019 		saved_dcsr &= DCSR_RUN | DCSR_IE;
1020 		if (dcsr & DCSR_BIU) {
1021 			p->dbs[0] = readl_relaxed(p->base + DMA_DBSB);
1022 			p->dbt[0] = readl_relaxed(p->base + DMA_DBTB);
1023 			p->dbs[1] = readl_relaxed(p->base + DMA_DBSA);
1024 			p->dbt[1] = readl_relaxed(p->base + DMA_DBTA);
1025 			saved_dcsr |= (dcsr & DCSR_STRTA ? DCSR_STRTB : 0) |
1026 				      (dcsr & DCSR_STRTB ? DCSR_STRTA : 0);
1027 		} else {
1028 			p->dbs[0] = readl_relaxed(p->base + DMA_DBSA);
1029 			p->dbt[0] = readl_relaxed(p->base + DMA_DBTA);
1030 			p->dbs[1] = readl_relaxed(p->base + DMA_DBSB);
1031 			p->dbt[1] = readl_relaxed(p->base + DMA_DBTB);
1032 			saved_dcsr |= dcsr & (DCSR_STRTA | DCSR_STRTB);
1033 		}
1034 		p->dcsr = saved_dcsr;
1035 
1036 		writel(DCSR_STRTA | DCSR_STRTB, p->base + DMA_DCSR_C);
1037 	}
1038 
1039 	return 0;
1040 }
1041 
sa11x0_dma_resume(struct device * dev)1042 static __maybe_unused int sa11x0_dma_resume(struct device *dev)
1043 {
1044 	struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
1045 	unsigned pch;
1046 
1047 	for (pch = 0; pch < NR_PHY_CHAN; pch++) {
1048 		struct sa11x0_dma_phy *p = &d->phy[pch];
1049 		struct sa11x0_dma_desc *txd = NULL;
1050 		u32 dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1051 
1052 		WARN_ON(dcsr & (DCSR_BIU | DCSR_STRTA | DCSR_STRTB | DCSR_RUN));
1053 
1054 		if (p->txd_done)
1055 			txd = p->txd_done;
1056 		else if (p->txd_load)
1057 			txd = p->txd_load;
1058 
1059 		if (!txd)
1060 			continue;
1061 
1062 		writel_relaxed(txd->ddar, p->base + DMA_DDAR);
1063 
1064 		writel_relaxed(p->dbs[0], p->base + DMA_DBSA);
1065 		writel_relaxed(p->dbt[0], p->base + DMA_DBTA);
1066 		writel_relaxed(p->dbs[1], p->base + DMA_DBSB);
1067 		writel_relaxed(p->dbt[1], p->base + DMA_DBTB);
1068 		writel_relaxed(p->dcsr, p->base + DMA_DCSR_S);
1069 	}
1070 
1071 	return 0;
1072 }
1073 
1074 static const struct dev_pm_ops sa11x0_dma_pm_ops = {
1075 	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(sa11x0_dma_suspend, sa11x0_dma_resume)
1076 };
1077 
1078 static struct platform_driver sa11x0_dma_driver = {
1079 	.driver = {
1080 		.name	= "sa11x0-dma",
1081 		.pm	= &sa11x0_dma_pm_ops,
1082 	},
1083 	.probe		= sa11x0_dma_probe,
1084 	.remove		= sa11x0_dma_remove,
1085 };
1086 
sa11x0_dma_init(void)1087 static int __init sa11x0_dma_init(void)
1088 {
1089 	return platform_driver_register(&sa11x0_dma_driver);
1090 }
1091 subsys_initcall(sa11x0_dma_init);
1092 
sa11x0_dma_exit(void)1093 static void __exit sa11x0_dma_exit(void)
1094 {
1095 	platform_driver_unregister(&sa11x0_dma_driver);
1096 }
1097 module_exit(sa11x0_dma_exit);
1098 
1099 MODULE_AUTHOR("Russell King");
1100 MODULE_DESCRIPTION("SA-11x0 DMA driver");
1101 MODULE_LICENSE("GPL v2");
1102 MODULE_ALIAS("platform:sa11x0-dma");
1103