xref: /openbmc/linux/drivers/dma/apple-admac.c (revision 27e45f2e)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Driver for Audio DMA Controller (ADMAC) on t8103 (M1) and other Apple chips
4  *
5  * Copyright (C) The Asahi Linux Contributors
6  */
7 
8 #include <linux/bits.h>
9 #include <linux/bitfield.h>
10 #include <linux/device.h>
11 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/of_device.h>
14 #include <linux/of_dma.h>
15 #include <linux/reset.h>
16 #include <linux/spinlock.h>
17 #include <linux/interrupt.h>
18 
19 #include "dmaengine.h"
20 
21 #define NCHANNELS_MAX	64
22 #define IRQ_NOUTPUTS	4
23 
24 /*
25  * For allocation purposes we split the cache
26  * memory into blocks of fixed size (given in bytes).
27  */
28 #define SRAM_BLOCK	2048
29 
30 #define RING_WRITE_SLOT		GENMASK(1, 0)
31 #define RING_READ_SLOT		GENMASK(5, 4)
32 #define RING_FULL		BIT(9)
33 #define RING_EMPTY		BIT(8)
34 #define RING_ERR		BIT(10)
35 
36 #define STATUS_DESC_DONE	BIT(0)
37 #define STATUS_ERR		BIT(6)
38 
39 #define FLAG_DESC_NOTIFY	BIT(16)
40 
41 #define REG_TX_START		0x0000
42 #define REG_TX_STOP		0x0004
43 #define REG_RX_START		0x0008
44 #define REG_RX_STOP		0x000c
45 #define REG_IMPRINT		0x0090
46 #define REG_TX_SRAM_SIZE	0x0094
47 #define REG_RX_SRAM_SIZE	0x0098
48 
49 #define REG_CHAN_CTL(ch)	(0x8000 + (ch) * 0x200)
50 #define REG_CHAN_CTL_RST_RINGS	BIT(0)
51 
52 #define REG_DESC_RING(ch)	(0x8070 + (ch) * 0x200)
53 #define REG_REPORT_RING(ch)	(0x8074 + (ch) * 0x200)
54 
55 #define REG_RESIDUE(ch)		(0x8064 + (ch) * 0x200)
56 
57 #define REG_BUS_WIDTH(ch)	(0x8040 + (ch) * 0x200)
58 
59 #define BUS_WIDTH_8BIT		0x00
60 #define BUS_WIDTH_16BIT		0x01
61 #define BUS_WIDTH_32BIT		0x02
62 #define BUS_WIDTH_FRAME_2_WORDS	0x10
63 #define BUS_WIDTH_FRAME_4_WORDS	0x20
64 
65 #define REG_CHAN_SRAM_CARVEOUT(ch)	(0x8050 + (ch) * 0x200)
66 #define CHAN_SRAM_CARVEOUT_SIZE		GENMASK(31, 16)
67 #define CHAN_SRAM_CARVEOUT_BASE		GENMASK(15, 0)
68 
69 #define REG_CHAN_FIFOCTL(ch)	(0x8054 + (ch) * 0x200)
70 #define CHAN_FIFOCTL_LIMIT	GENMASK(31, 16)
71 #define CHAN_FIFOCTL_THRESHOLD	GENMASK(15, 0)
72 
73 #define REG_DESC_WRITE(ch)	(0x10000 + ((ch) / 2) * 0x4 + ((ch) & 1) * 0x4000)
74 #define REG_REPORT_READ(ch)	(0x10100 + ((ch) / 2) * 0x4 + ((ch) & 1) * 0x4000)
75 
76 #define REG_TX_INTSTATE(idx)		(0x0030 + (idx) * 4)
77 #define REG_RX_INTSTATE(idx)		(0x0040 + (idx) * 4)
78 #define REG_CHAN_INTSTATUS(ch, idx)	(0x8010 + (ch) * 0x200 + (idx) * 4)
79 #define REG_CHAN_INTMASK(ch, idx)	(0x8020 + (ch) * 0x200 + (idx) * 4)
80 
81 struct admac_data;
82 struct admac_tx;
83 
84 struct admac_chan {
85 	unsigned int no;
86 	struct admac_data *host;
87 	struct dma_chan chan;
88 	struct tasklet_struct tasklet;
89 
90 	u32 carveout;
91 
92 	spinlock_t lock;
93 	struct admac_tx *current_tx;
94 	int nperiod_acks;
95 
96 	/*
97 	 * We maintain a 'submitted' and 'issued' list mainly for interface
98 	 * correctness. Typical use of the driver (per channel) will be
99 	 * prepping, submitting and issuing a single cyclic transaction which
100 	 * will stay current until terminate_all is called.
101 	 */
102 	struct list_head submitted;
103 	struct list_head issued;
104 
105 	struct list_head to_free;
106 };
107 
108 struct admac_sram {
109 	u32 size;
110 	/*
111 	 * SRAM_CARVEOUT has 16-bit fields, so the SRAM cannot be larger than
112 	 * 64K and a 32-bit bitfield over 2K blocks covers it.
113 	 */
114 	u32 allocated;
115 };
116 
117 struct admac_data {
118 	struct dma_device dma;
119 	struct device *dev;
120 	__iomem void *base;
121 	struct reset_control *rstc;
122 
123 	struct mutex cache_alloc_lock;
124 	struct admac_sram txcache, rxcache;
125 
126 	int irq;
127 	int irq_index;
128 	int nchannels;
129 	struct admac_chan channels[];
130 };
131 
132 struct admac_tx {
133 	struct dma_async_tx_descriptor tx;
134 	bool cyclic;
135 	dma_addr_t buf_addr;
136 	dma_addr_t buf_end;
137 	size_t buf_len;
138 	size_t period_len;
139 
140 	size_t submitted_pos;
141 	size_t reclaimed_pos;
142 
143 	struct list_head node;
144 };
145 
146 static int admac_alloc_sram_carveout(struct admac_data *ad,
147 				     enum dma_transfer_direction dir,
148 				     u32 *out)
149 {
150 	struct admac_sram *sram;
151 	int i, ret = 0, nblocks;
152 
153 	if (dir == DMA_MEM_TO_DEV)
154 		sram = &ad->txcache;
155 	else
156 		sram = &ad->rxcache;
157 
158 	mutex_lock(&ad->cache_alloc_lock);
159 
160 	nblocks = sram->size / SRAM_BLOCK;
161 	for (i = 0; i < nblocks; i++)
162 		if (!(sram->allocated & BIT(i)))
163 			break;
164 
165 	if (i < nblocks) {
166 		*out = FIELD_PREP(CHAN_SRAM_CARVEOUT_BASE, i * SRAM_BLOCK) |
167 			FIELD_PREP(CHAN_SRAM_CARVEOUT_SIZE, SRAM_BLOCK);
168 		sram->allocated |= BIT(i);
169 	} else {
170 		ret = -EBUSY;
171 	}
172 
173 	mutex_unlock(&ad->cache_alloc_lock);
174 
175 	return ret;
176 }
177 
178 static void admac_free_sram_carveout(struct admac_data *ad,
179 				     enum dma_transfer_direction dir,
180 				     u32 carveout)
181 {
182 	struct admac_sram *sram;
183 	u32 base = FIELD_GET(CHAN_SRAM_CARVEOUT_BASE, carveout);
184 	int i;
185 
186 	if (dir == DMA_MEM_TO_DEV)
187 		sram = &ad->txcache;
188 	else
189 		sram = &ad->rxcache;
190 
191 	if (WARN_ON(base >= sram->size))
192 		return;
193 
194 	mutex_lock(&ad->cache_alloc_lock);
195 	i = base / SRAM_BLOCK;
196 	sram->allocated &= ~BIT(i);
197 	mutex_unlock(&ad->cache_alloc_lock);
198 }
199 
200 static void admac_modify(struct admac_data *ad, int reg, u32 mask, u32 val)
201 {
202 	void __iomem *addr = ad->base + reg;
203 	u32 curr = readl_relaxed(addr);
204 
205 	writel_relaxed((curr & ~mask) | (val & mask), addr);
206 }
207 
208 static struct admac_chan *to_admac_chan(struct dma_chan *chan)
209 {
210 	return container_of(chan, struct admac_chan, chan);
211 }
212 
213 static struct admac_tx *to_admac_tx(struct dma_async_tx_descriptor *tx)
214 {
215 	return container_of(tx, struct admac_tx, tx);
216 }
217 
218 static enum dma_transfer_direction admac_chan_direction(int channo)
219 {
220 	/* Channel directions are hardwired */
221 	return (channo & 1) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV;
222 }
223 
224 static dma_cookie_t admac_tx_submit(struct dma_async_tx_descriptor *tx)
225 {
226 	struct admac_tx *adtx = to_admac_tx(tx);
227 	struct admac_chan *adchan = to_admac_chan(tx->chan);
228 	unsigned long flags;
229 	dma_cookie_t cookie;
230 
231 	spin_lock_irqsave(&adchan->lock, flags);
232 	cookie = dma_cookie_assign(tx);
233 	list_add_tail(&adtx->node, &adchan->submitted);
234 	spin_unlock_irqrestore(&adchan->lock, flags);
235 
236 	return cookie;
237 }
238 
239 static int admac_desc_free(struct dma_async_tx_descriptor *tx)
240 {
241 	kfree(to_admac_tx(tx));
242 
243 	return 0;
244 }
245 
246 static struct dma_async_tx_descriptor *admac_prep_dma_cyclic(
247 		struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
248 		size_t period_len, enum dma_transfer_direction direction,
249 		unsigned long flags)
250 {
251 	struct admac_chan *adchan = container_of(chan, struct admac_chan, chan);
252 	struct admac_tx *adtx;
253 
254 	if (direction != admac_chan_direction(adchan->no))
255 		return NULL;
256 
257 	adtx = kzalloc(sizeof(*adtx), GFP_NOWAIT);
258 	if (!adtx)
259 		return NULL;
260 
261 	adtx->cyclic = true;
262 
263 	adtx->buf_addr = buf_addr;
264 	adtx->buf_len = buf_len;
265 	adtx->buf_end = buf_addr + buf_len;
266 	adtx->period_len = period_len;
267 
268 	adtx->submitted_pos = 0;
269 	adtx->reclaimed_pos = 0;
270 
271 	dma_async_tx_descriptor_init(&adtx->tx, chan);
272 	adtx->tx.tx_submit = admac_tx_submit;
273 	adtx->tx.desc_free = admac_desc_free;
274 
275 	return &adtx->tx;
276 }
277 
278 /*
279  * Write one hardware descriptor for a dmaengine cyclic transaction.
280  */
281 static void admac_cyclic_write_one_desc(struct admac_data *ad, int channo,
282 					struct admac_tx *tx)
283 {
284 	dma_addr_t addr;
285 
286 	addr = tx->buf_addr + (tx->submitted_pos % tx->buf_len);
287 
288 	/* If happens means we have buggy code */
289 	WARN_ON_ONCE(addr + tx->period_len > tx->buf_end);
290 
291 	dev_dbg(ad->dev, "ch%d descriptor: addr=0x%pad len=0x%zx flags=0x%lx\n",
292 		channo, &addr, tx->period_len, FLAG_DESC_NOTIFY);
293 
294 	writel_relaxed(lower_32_bits(addr), ad->base + REG_DESC_WRITE(channo));
295 	writel_relaxed(upper_32_bits(addr), ad->base + REG_DESC_WRITE(channo));
296 	writel_relaxed(tx->period_len,      ad->base + REG_DESC_WRITE(channo));
297 	writel_relaxed(FLAG_DESC_NOTIFY,    ad->base + REG_DESC_WRITE(channo));
298 
299 	tx->submitted_pos += tx->period_len;
300 	tx->submitted_pos %= 2 * tx->buf_len;
301 }
302 
303 /*
304  * Write all the hardware descriptors for a dmaengine cyclic
305  * transaction there is space for.
306  */
307 static void admac_cyclic_write_desc(struct admac_data *ad, int channo,
308 				    struct admac_tx *tx)
309 {
310 	int i;
311 
312 	for (i = 0; i < 4; i++) {
313 		if (readl_relaxed(ad->base + REG_DESC_RING(channo)) & RING_FULL)
314 			break;
315 		admac_cyclic_write_one_desc(ad, channo, tx);
316 	}
317 }
318 
319 static int admac_ring_noccupied_slots(int ringval)
320 {
321 	int wrslot = FIELD_GET(RING_WRITE_SLOT, ringval);
322 	int rdslot = FIELD_GET(RING_READ_SLOT, ringval);
323 
324 	if (wrslot != rdslot) {
325 		return (wrslot + 4 - rdslot) % 4;
326 	} else {
327 		WARN_ON((ringval & (RING_FULL | RING_EMPTY)) == 0);
328 
329 		if (ringval & RING_FULL)
330 			return 4;
331 		else
332 			return 0;
333 	}
334 }
335 
336 /*
337  * Read from hardware the residue of a cyclic dmaengine transaction.
338  */
339 static u32 admac_cyclic_read_residue(struct admac_data *ad, int channo,
340 				     struct admac_tx *adtx)
341 {
342 	u32 ring1, ring2;
343 	u32 residue1, residue2;
344 	int nreports;
345 	size_t pos;
346 
347 	ring1 =    readl_relaxed(ad->base + REG_REPORT_RING(channo));
348 	residue1 = readl_relaxed(ad->base + REG_RESIDUE(channo));
349 	ring2 =    readl_relaxed(ad->base + REG_REPORT_RING(channo));
350 	residue2 = readl_relaxed(ad->base + REG_RESIDUE(channo));
351 
352 	if (residue2 > residue1) {
353 		/*
354 		 * Controller must have loaded next descriptor between
355 		 * the two residue reads
356 		 */
357 		nreports = admac_ring_noccupied_slots(ring1) + 1;
358 	} else {
359 		/* No descriptor load between the two reads, ring2 is safe to use */
360 		nreports = admac_ring_noccupied_slots(ring2);
361 	}
362 
363 	pos = adtx->reclaimed_pos + adtx->period_len * (nreports + 1) - residue2;
364 
365 	return adtx->buf_len - pos % adtx->buf_len;
366 }
367 
368 static enum dma_status admac_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
369 				       struct dma_tx_state *txstate)
370 {
371 	struct admac_chan *adchan = to_admac_chan(chan);
372 	struct admac_data *ad = adchan->host;
373 	struct admac_tx *adtx;
374 
375 	enum dma_status ret;
376 	size_t residue;
377 	unsigned long flags;
378 
379 	ret = dma_cookie_status(chan, cookie, txstate);
380 	if (ret == DMA_COMPLETE || !txstate)
381 		return ret;
382 
383 	spin_lock_irqsave(&adchan->lock, flags);
384 	adtx = adchan->current_tx;
385 
386 	if (adtx && adtx->tx.cookie == cookie) {
387 		ret = DMA_IN_PROGRESS;
388 		residue = admac_cyclic_read_residue(ad, adchan->no, adtx);
389 	} else {
390 		ret = DMA_IN_PROGRESS;
391 		residue = 0;
392 		list_for_each_entry(adtx, &adchan->issued, node) {
393 			if (adtx->tx.cookie == cookie) {
394 				residue = adtx->buf_len;
395 				break;
396 			}
397 		}
398 	}
399 	spin_unlock_irqrestore(&adchan->lock, flags);
400 
401 	dma_set_residue(txstate, residue);
402 	return ret;
403 }
404 
405 static void admac_start_chan(struct admac_chan *adchan)
406 {
407 	struct admac_data *ad = adchan->host;
408 	u32 startbit = 1 << (adchan->no / 2);
409 
410 	writel_relaxed(STATUS_DESC_DONE | STATUS_ERR,
411 		       ad->base + REG_CHAN_INTSTATUS(adchan->no, ad->irq_index));
412 	writel_relaxed(STATUS_DESC_DONE | STATUS_ERR,
413 		       ad->base + REG_CHAN_INTMASK(adchan->no, ad->irq_index));
414 
415 	switch (admac_chan_direction(adchan->no)) {
416 	case DMA_MEM_TO_DEV:
417 		writel_relaxed(startbit, ad->base + REG_TX_START);
418 		break;
419 	case DMA_DEV_TO_MEM:
420 		writel_relaxed(startbit, ad->base + REG_RX_START);
421 		break;
422 	default:
423 		break;
424 	}
425 	dev_dbg(adchan->host->dev, "ch%d start\n", adchan->no);
426 }
427 
428 static void admac_stop_chan(struct admac_chan *adchan)
429 {
430 	struct admac_data *ad = adchan->host;
431 	u32 stopbit = 1 << (adchan->no / 2);
432 
433 	switch (admac_chan_direction(adchan->no)) {
434 	case DMA_MEM_TO_DEV:
435 		writel_relaxed(stopbit, ad->base + REG_TX_STOP);
436 		break;
437 	case DMA_DEV_TO_MEM:
438 		writel_relaxed(stopbit, ad->base + REG_RX_STOP);
439 		break;
440 	default:
441 		break;
442 	}
443 	dev_dbg(adchan->host->dev, "ch%d stop\n", adchan->no);
444 }
445 
446 static void admac_reset_rings(struct admac_chan *adchan)
447 {
448 	struct admac_data *ad = adchan->host;
449 
450 	writel_relaxed(REG_CHAN_CTL_RST_RINGS,
451 		       ad->base + REG_CHAN_CTL(adchan->no));
452 	writel_relaxed(0, ad->base + REG_CHAN_CTL(adchan->no));
453 }
454 
455 static void admac_start_current_tx(struct admac_chan *adchan)
456 {
457 	struct admac_data *ad = adchan->host;
458 	int ch = adchan->no;
459 
460 	admac_reset_rings(adchan);
461 	writel_relaxed(0, ad->base + REG_CHAN_CTL(ch));
462 
463 	admac_cyclic_write_one_desc(ad, ch, adchan->current_tx);
464 	admac_start_chan(adchan);
465 	admac_cyclic_write_desc(ad, ch, adchan->current_tx);
466 }
467 
468 static void admac_issue_pending(struct dma_chan *chan)
469 {
470 	struct admac_chan *adchan = to_admac_chan(chan);
471 	struct admac_tx *tx;
472 	unsigned long flags;
473 
474 	spin_lock_irqsave(&adchan->lock, flags);
475 	list_splice_tail_init(&adchan->submitted, &adchan->issued);
476 	if (!list_empty(&adchan->issued) && !adchan->current_tx) {
477 		tx = list_first_entry(&adchan->issued, struct admac_tx, node);
478 		list_del(&tx->node);
479 
480 		adchan->current_tx = tx;
481 		adchan->nperiod_acks = 0;
482 		admac_start_current_tx(adchan);
483 	}
484 	spin_unlock_irqrestore(&adchan->lock, flags);
485 }
486 
487 static int admac_pause(struct dma_chan *chan)
488 {
489 	struct admac_chan *adchan = to_admac_chan(chan);
490 
491 	admac_stop_chan(adchan);
492 
493 	return 0;
494 }
495 
496 static int admac_resume(struct dma_chan *chan)
497 {
498 	struct admac_chan *adchan = to_admac_chan(chan);
499 
500 	admac_start_chan(adchan);
501 
502 	return 0;
503 }
504 
505 static int admac_terminate_all(struct dma_chan *chan)
506 {
507 	struct admac_chan *adchan = to_admac_chan(chan);
508 	unsigned long flags;
509 
510 	spin_lock_irqsave(&adchan->lock, flags);
511 	admac_stop_chan(adchan);
512 	admac_reset_rings(adchan);
513 
514 	adchan->current_tx = NULL;
515 	/*
516 	 * Descriptors can only be freed after the tasklet
517 	 * has been killed (in admac_synchronize).
518 	 */
519 	list_splice_tail_init(&adchan->submitted, &adchan->to_free);
520 	list_splice_tail_init(&adchan->issued, &adchan->to_free);
521 	spin_unlock_irqrestore(&adchan->lock, flags);
522 
523 	return 0;
524 }
525 
526 static void admac_synchronize(struct dma_chan *chan)
527 {
528 	struct admac_chan *adchan = to_admac_chan(chan);
529 	struct admac_tx *adtx, *_adtx;
530 	unsigned long flags;
531 	LIST_HEAD(head);
532 
533 	spin_lock_irqsave(&adchan->lock, flags);
534 	list_splice_tail_init(&adchan->to_free, &head);
535 	spin_unlock_irqrestore(&adchan->lock, flags);
536 
537 	tasklet_kill(&adchan->tasklet);
538 
539 	list_for_each_entry_safe(adtx, _adtx, &head, node) {
540 		list_del(&adtx->node);
541 		admac_desc_free(&adtx->tx);
542 	}
543 }
544 
545 static int admac_alloc_chan_resources(struct dma_chan *chan)
546 {
547 	struct admac_chan *adchan = to_admac_chan(chan);
548 	struct admac_data *ad = adchan->host;
549 	int ret;
550 
551 	dma_cookie_init(&adchan->chan);
552 	ret = admac_alloc_sram_carveout(ad, admac_chan_direction(adchan->no),
553 					&adchan->carveout);
554 	if (ret < 0)
555 		return ret;
556 
557 	writel_relaxed(adchan->carveout,
558 		       ad->base + REG_CHAN_SRAM_CARVEOUT(adchan->no));
559 	return 0;
560 }
561 
562 static void admac_free_chan_resources(struct dma_chan *chan)
563 {
564 	struct admac_chan *adchan = to_admac_chan(chan);
565 
566 	admac_terminate_all(chan);
567 	admac_synchronize(chan);
568 	admac_free_sram_carveout(adchan->host, admac_chan_direction(adchan->no),
569 				 adchan->carveout);
570 }
571 
572 static struct dma_chan *admac_dma_of_xlate(struct of_phandle_args *dma_spec,
573 					   struct of_dma *ofdma)
574 {
575 	struct admac_data *ad = (struct admac_data *) ofdma->of_dma_data;
576 	unsigned int index;
577 
578 	if (dma_spec->args_count != 1)
579 		return NULL;
580 
581 	index = dma_spec->args[0];
582 
583 	if (index >= ad->nchannels) {
584 		dev_err(ad->dev, "channel index %u out of bounds\n", index);
585 		return NULL;
586 	}
587 
588 	return dma_get_slave_channel(&ad->channels[index].chan);
589 }
590 
591 static int admac_drain_reports(struct admac_data *ad, int channo)
592 {
593 	int count;
594 
595 	for (count = 0; count < 4; count++) {
596 		u32 countval_hi, countval_lo, unk1, flags;
597 
598 		if (readl_relaxed(ad->base + REG_REPORT_RING(channo)) & RING_EMPTY)
599 			break;
600 
601 		countval_lo = readl_relaxed(ad->base + REG_REPORT_READ(channo));
602 		countval_hi = readl_relaxed(ad->base + REG_REPORT_READ(channo));
603 		unk1 =        readl_relaxed(ad->base + REG_REPORT_READ(channo));
604 		flags =       readl_relaxed(ad->base + REG_REPORT_READ(channo));
605 
606 		dev_dbg(ad->dev, "ch%d report: countval=0x%llx unk1=0x%x flags=0x%x\n",
607 			channo, ((u64) countval_hi) << 32 | countval_lo, unk1, flags);
608 	}
609 
610 	return count;
611 }
612 
613 static void admac_handle_status_err(struct admac_data *ad, int channo)
614 {
615 	bool handled = false;
616 
617 	if (readl_relaxed(ad->base + REG_DESC_RING(channo)) & RING_ERR) {
618 		writel_relaxed(RING_ERR, ad->base + REG_DESC_RING(channo));
619 		dev_err_ratelimited(ad->dev, "ch%d descriptor ring error\n", channo);
620 		handled = true;
621 	}
622 
623 	if (readl_relaxed(ad->base + REG_REPORT_RING(channo)) & RING_ERR) {
624 		writel_relaxed(RING_ERR, ad->base + REG_REPORT_RING(channo));
625 		dev_err_ratelimited(ad->dev, "ch%d report ring error\n", channo);
626 		handled = true;
627 	}
628 
629 	if (unlikely(!handled)) {
630 		dev_err(ad->dev, "ch%d unknown error, masking errors as cause of IRQs\n", channo);
631 		admac_modify(ad, REG_CHAN_INTMASK(channo, ad->irq_index),
632 			     STATUS_ERR, 0);
633 	}
634 }
635 
636 static void admac_handle_status_desc_done(struct admac_data *ad, int channo)
637 {
638 	struct admac_chan *adchan = &ad->channels[channo];
639 	unsigned long flags;
640 	int nreports;
641 
642 	writel_relaxed(STATUS_DESC_DONE,
643 		       ad->base + REG_CHAN_INTSTATUS(channo, ad->irq_index));
644 
645 	spin_lock_irqsave(&adchan->lock, flags);
646 	nreports = admac_drain_reports(ad, channo);
647 
648 	if (adchan->current_tx) {
649 		struct admac_tx *tx = adchan->current_tx;
650 
651 		adchan->nperiod_acks += nreports;
652 		tx->reclaimed_pos += nreports * tx->period_len;
653 		tx->reclaimed_pos %= 2 * tx->buf_len;
654 
655 		admac_cyclic_write_desc(ad, channo, tx);
656 		tasklet_schedule(&adchan->tasklet);
657 	}
658 	spin_unlock_irqrestore(&adchan->lock, flags);
659 }
660 
661 static void admac_handle_chan_int(struct admac_data *ad, int no)
662 {
663 	u32 cause = readl_relaxed(ad->base + REG_CHAN_INTSTATUS(no, ad->irq_index));
664 
665 	if (cause & STATUS_ERR)
666 		admac_handle_status_err(ad, no);
667 
668 	if (cause & STATUS_DESC_DONE)
669 		admac_handle_status_desc_done(ad, no);
670 }
671 
672 static irqreturn_t admac_interrupt(int irq, void *devid)
673 {
674 	struct admac_data *ad = devid;
675 	u32 rx_intstate, tx_intstate;
676 	int i;
677 
678 	rx_intstate = readl_relaxed(ad->base + REG_RX_INTSTATE(ad->irq_index));
679 	tx_intstate = readl_relaxed(ad->base + REG_TX_INTSTATE(ad->irq_index));
680 
681 	if (!tx_intstate && !rx_intstate)
682 		return IRQ_NONE;
683 
684 	for (i = 0; i < ad->nchannels; i += 2) {
685 		if (tx_intstate & 1)
686 			admac_handle_chan_int(ad, i);
687 		tx_intstate >>= 1;
688 	}
689 
690 	for (i = 1; i < ad->nchannels; i += 2) {
691 		if (rx_intstate & 1)
692 			admac_handle_chan_int(ad, i);
693 		rx_intstate >>= 1;
694 	}
695 
696 	return IRQ_HANDLED;
697 }
698 
699 static void admac_chan_tasklet(struct tasklet_struct *t)
700 {
701 	struct admac_chan *adchan = from_tasklet(adchan, t, tasklet);
702 	struct admac_tx *adtx;
703 	struct dmaengine_desc_callback cb;
704 	struct dmaengine_result tx_result;
705 	int nacks;
706 
707 	spin_lock_irq(&adchan->lock);
708 	adtx = adchan->current_tx;
709 	nacks = adchan->nperiod_acks;
710 	adchan->nperiod_acks = 0;
711 	spin_unlock_irq(&adchan->lock);
712 
713 	if (!adtx || !nacks)
714 		return;
715 
716 	tx_result.result = DMA_TRANS_NOERROR;
717 	tx_result.residue = 0;
718 
719 	dmaengine_desc_get_callback(&adtx->tx, &cb);
720 	while (nacks--)
721 		dmaengine_desc_callback_invoke(&cb, &tx_result);
722 }
723 
724 static int admac_device_config(struct dma_chan *chan,
725 			       struct dma_slave_config *config)
726 {
727 	struct admac_chan *adchan = to_admac_chan(chan);
728 	struct admac_data *ad = adchan->host;
729 	bool is_tx = admac_chan_direction(adchan->no) == DMA_MEM_TO_DEV;
730 	int wordsize = 0;
731 	u32 bus_width = 0;
732 
733 	switch (is_tx ? config->dst_addr_width : config->src_addr_width) {
734 	case DMA_SLAVE_BUSWIDTH_1_BYTE:
735 		wordsize = 1;
736 		bus_width |= BUS_WIDTH_8BIT;
737 		break;
738 	case DMA_SLAVE_BUSWIDTH_2_BYTES:
739 		wordsize = 2;
740 		bus_width |= BUS_WIDTH_16BIT;
741 		break;
742 	case DMA_SLAVE_BUSWIDTH_4_BYTES:
743 		wordsize = 4;
744 		bus_width |= BUS_WIDTH_32BIT;
745 		break;
746 	default:
747 		return -EINVAL;
748 	}
749 
750 	/*
751 	 * We take port_window_size to be the number of words in a frame.
752 	 *
753 	 * The controller has some means of out-of-band signalling, to the peripheral,
754 	 * of words position in a frame. That's where the importance of this control
755 	 * comes from.
756 	 */
757 	switch (is_tx ? config->dst_port_window_size : config->src_port_window_size) {
758 	case 0 ... 1:
759 		break;
760 	case 2:
761 		bus_width |= BUS_WIDTH_FRAME_2_WORDS;
762 		break;
763 	case 4:
764 		bus_width |= BUS_WIDTH_FRAME_4_WORDS;
765 		break;
766 	default:
767 		return -EINVAL;
768 	}
769 
770 	writel_relaxed(bus_width, ad->base + REG_BUS_WIDTH(adchan->no));
771 
772 	/*
773 	 * By FIFOCTL_LIMIT we seem to set the maximal number of bytes allowed to be
774 	 * held in controller's per-channel FIFO. Transfers seem to be triggered
775 	 * around the time FIFO occupancy touches FIFOCTL_THRESHOLD.
776 	 *
777 	 * The numbers we set are more or less arbitrary.
778 	 */
779 	writel_relaxed(FIELD_PREP(CHAN_FIFOCTL_LIMIT, 0x30 * wordsize)
780 		       | FIELD_PREP(CHAN_FIFOCTL_THRESHOLD, 0x18 * wordsize),
781 		       ad->base + REG_CHAN_FIFOCTL(adchan->no));
782 
783 	return 0;
784 }
785 
786 static int admac_probe(struct platform_device *pdev)
787 {
788 	struct device_node *np = pdev->dev.of_node;
789 	struct admac_data *ad;
790 	struct dma_device *dma;
791 	int nchannels;
792 	int err, irq, i;
793 
794 	err = of_property_read_u32(np, "dma-channels", &nchannels);
795 	if (err || nchannels > NCHANNELS_MAX) {
796 		dev_err(&pdev->dev, "missing or invalid dma-channels property\n");
797 		return -EINVAL;
798 	}
799 
800 	ad = devm_kzalloc(&pdev->dev, struct_size(ad, channels, nchannels), GFP_KERNEL);
801 	if (!ad)
802 		return -ENOMEM;
803 
804 	platform_set_drvdata(pdev, ad);
805 	ad->dev = &pdev->dev;
806 	ad->nchannels = nchannels;
807 	mutex_init(&ad->cache_alloc_lock);
808 
809 	/*
810 	 * The controller has 4 IRQ outputs. Try them all until
811 	 * we find one we can use.
812 	 */
813 	for (i = 0; i < IRQ_NOUTPUTS; i++) {
814 		irq = platform_get_irq_optional(pdev, i);
815 		if (irq >= 0) {
816 			ad->irq_index = i;
817 			break;
818 		}
819 	}
820 
821 	if (irq < 0)
822 		return dev_err_probe(&pdev->dev, irq, "no usable interrupt\n");
823 	ad->irq = irq;
824 
825 	ad->base = devm_platform_ioremap_resource(pdev, 0);
826 	if (IS_ERR(ad->base))
827 		return dev_err_probe(&pdev->dev, PTR_ERR(ad->base),
828 				     "unable to obtain MMIO resource\n");
829 
830 	ad->rstc = devm_reset_control_get_optional_shared(&pdev->dev, NULL);
831 	if (IS_ERR(ad->rstc))
832 		return PTR_ERR(ad->rstc);
833 
834 	dma = &ad->dma;
835 
836 	dma_cap_set(DMA_PRIVATE, dma->cap_mask);
837 	dma_cap_set(DMA_CYCLIC, dma->cap_mask);
838 
839 	dma->dev = &pdev->dev;
840 	dma->device_alloc_chan_resources = admac_alloc_chan_resources;
841 	dma->device_free_chan_resources = admac_free_chan_resources;
842 	dma->device_tx_status = admac_tx_status;
843 	dma->device_issue_pending = admac_issue_pending;
844 	dma->device_terminate_all = admac_terminate_all;
845 	dma->device_synchronize = admac_synchronize;
846 	dma->device_prep_dma_cyclic = admac_prep_dma_cyclic;
847 	dma->device_config = admac_device_config;
848 	dma->device_pause = admac_pause;
849 	dma->device_resume = admac_resume;
850 
851 	dma->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM);
852 	dma->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
853 	dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
854 			BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
855 			BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
856 
857 	INIT_LIST_HEAD(&dma->channels);
858 	for (i = 0; i < nchannels; i++) {
859 		struct admac_chan *adchan = &ad->channels[i];
860 
861 		adchan->host = ad;
862 		adchan->no = i;
863 		adchan->chan.device = &ad->dma;
864 		spin_lock_init(&adchan->lock);
865 		INIT_LIST_HEAD(&adchan->submitted);
866 		INIT_LIST_HEAD(&adchan->issued);
867 		INIT_LIST_HEAD(&adchan->to_free);
868 		list_add_tail(&adchan->chan.device_node, &dma->channels);
869 		tasklet_setup(&adchan->tasklet, admac_chan_tasklet);
870 	}
871 
872 	err = reset_control_reset(ad->rstc);
873 	if (err)
874 		return dev_err_probe(&pdev->dev, err,
875 				     "unable to trigger reset\n");
876 
877 	err = request_irq(irq, admac_interrupt, 0, dev_name(&pdev->dev), ad);
878 	if (err) {
879 		dev_err_probe(&pdev->dev, err,
880 				"unable to register interrupt\n");
881 		goto free_reset;
882 	}
883 
884 	err = dma_async_device_register(&ad->dma);
885 	if (err) {
886 		dev_err_probe(&pdev->dev, err, "failed to register DMA device\n");
887 		goto free_irq;
888 	}
889 
890 	err = of_dma_controller_register(pdev->dev.of_node, admac_dma_of_xlate, ad);
891 	if (err) {
892 		dma_async_device_unregister(&ad->dma);
893 		dev_err_probe(&pdev->dev, err, "failed to register with OF\n");
894 		goto free_irq;
895 	}
896 
897 	ad->txcache.size = readl_relaxed(ad->base + REG_TX_SRAM_SIZE);
898 	ad->rxcache.size = readl_relaxed(ad->base + REG_RX_SRAM_SIZE);
899 
900 	dev_info(&pdev->dev, "Audio DMA Controller\n");
901 	dev_info(&pdev->dev, "imprint %x TX cache %u RX cache %u\n",
902 		 readl_relaxed(ad->base + REG_IMPRINT), ad->txcache.size, ad->rxcache.size);
903 
904 	return 0;
905 
906 free_irq:
907 	free_irq(ad->irq, ad);
908 free_reset:
909 	reset_control_rearm(ad->rstc);
910 	return err;
911 }
912 
913 static int admac_remove(struct platform_device *pdev)
914 {
915 	struct admac_data *ad = platform_get_drvdata(pdev);
916 
917 	of_dma_controller_free(pdev->dev.of_node);
918 	dma_async_device_unregister(&ad->dma);
919 	free_irq(ad->irq, ad);
920 	reset_control_rearm(ad->rstc);
921 
922 	return 0;
923 }
924 
925 static const struct of_device_id admac_of_match[] = {
926 	{ .compatible = "apple,admac", },
927 	{ }
928 };
929 MODULE_DEVICE_TABLE(of, admac_of_match);
930 
931 static struct platform_driver apple_admac_driver = {
932 	.driver = {
933 		.name = "apple-admac",
934 		.of_match_table = admac_of_match,
935 	},
936 	.probe = admac_probe,
937 	.remove = admac_remove,
938 };
939 module_platform_driver(apple_admac_driver);
940 
941 MODULE_AUTHOR("Martin Povišer <povik+lin@cutebit.org>");
942 MODULE_DESCRIPTION("Driver for Audio DMA Controller (ADMAC) on Apple SoCs");
943 MODULE_LICENSE("GPL");
944