1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * IMG Multi-threaded DMA Controller (MDC)
4 *
5 * Copyright (C) 2009,2012,2013 Imagination Technologies Ltd.
6 * Copyright (C) 2014 Google, Inc.
7 */
8
9 #include <linux/clk.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/dmaengine.h>
12 #include <linux/dmapool.h>
13 #include <linux/interrupt.h>
14 #include <linux/io.h>
15 #include <linux/irq.h>
16 #include <linux/kernel.h>
17 #include <linux/mfd/syscon.h>
18 #include <linux/module.h>
19 #include <linux/of.h>
20 #include <linux/of_dma.h>
21 #include <linux/platform_device.h>
22 #include <linux/pm_runtime.h>
23 #include <linux/regmap.h>
24 #include <linux/slab.h>
25 #include <linux/spinlock.h>
26
27 #include "dmaengine.h"
28 #include "virt-dma.h"
29
30 #define MDC_MAX_DMA_CHANNELS 32
31
32 #define MDC_GENERAL_CONFIG 0x000
33 #define MDC_GENERAL_CONFIG_LIST_IEN BIT(31)
34 #define MDC_GENERAL_CONFIG_IEN BIT(29)
35 #define MDC_GENERAL_CONFIG_LEVEL_INT BIT(28)
36 #define MDC_GENERAL_CONFIG_INC_W BIT(12)
37 #define MDC_GENERAL_CONFIG_INC_R BIT(8)
38 #define MDC_GENERAL_CONFIG_PHYSICAL_W BIT(7)
39 #define MDC_GENERAL_CONFIG_WIDTH_W_SHIFT 4
40 #define MDC_GENERAL_CONFIG_WIDTH_W_MASK 0x7
41 #define MDC_GENERAL_CONFIG_PHYSICAL_R BIT(3)
42 #define MDC_GENERAL_CONFIG_WIDTH_R_SHIFT 0
43 #define MDC_GENERAL_CONFIG_WIDTH_R_MASK 0x7
44
45 #define MDC_READ_PORT_CONFIG 0x004
46 #define MDC_READ_PORT_CONFIG_STHREAD_SHIFT 28
47 #define MDC_READ_PORT_CONFIG_STHREAD_MASK 0xf
48 #define MDC_READ_PORT_CONFIG_RTHREAD_SHIFT 24
49 #define MDC_READ_PORT_CONFIG_RTHREAD_MASK 0xf
50 #define MDC_READ_PORT_CONFIG_WTHREAD_SHIFT 16
51 #define MDC_READ_PORT_CONFIG_WTHREAD_MASK 0xf
52 #define MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT 4
53 #define MDC_READ_PORT_CONFIG_BURST_SIZE_MASK 0xff
54 #define MDC_READ_PORT_CONFIG_DREQ_ENABLE BIT(1)
55
56 #define MDC_READ_ADDRESS 0x008
57
58 #define MDC_WRITE_ADDRESS 0x00c
59
60 #define MDC_TRANSFER_SIZE 0x010
61 #define MDC_TRANSFER_SIZE_MASK 0xffffff
62
63 #define MDC_LIST_NODE_ADDRESS 0x014
64
65 #define MDC_CMDS_PROCESSED 0x018
66 #define MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT 16
67 #define MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK 0x3f
68 #define MDC_CMDS_PROCESSED_INT_ACTIVE BIT(8)
69 #define MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT 0
70 #define MDC_CMDS_PROCESSED_CMDS_DONE_MASK 0x3f
71
72 #define MDC_CONTROL_AND_STATUS 0x01c
73 #define MDC_CONTROL_AND_STATUS_CANCEL BIT(20)
74 #define MDC_CONTROL_AND_STATUS_LIST_EN BIT(4)
75 #define MDC_CONTROL_AND_STATUS_EN BIT(0)
76
77 #define MDC_ACTIVE_TRANSFER_SIZE 0x030
78
79 #define MDC_GLOBAL_CONFIG_A 0x900
80 #define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT 16
81 #define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK 0xff
82 #define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT 8
83 #define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK 0xff
84 #define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT 0
85 #define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK 0xff
86
87 struct mdc_hw_list_desc {
88 u32 gen_conf;
89 u32 readport_conf;
90 u32 read_addr;
91 u32 write_addr;
92 u32 xfer_size;
93 u32 node_addr;
94 u32 cmds_done;
95 u32 ctrl_status;
96 /*
97 * Not part of the list descriptor, but instead used by the CPU to
98 * traverse the list.
99 */
100 struct mdc_hw_list_desc *next_desc;
101 };
102
103 struct mdc_tx_desc {
104 struct mdc_chan *chan;
105 struct virt_dma_desc vd;
106 dma_addr_t list_phys;
107 struct mdc_hw_list_desc *list;
108 bool cyclic;
109 bool cmd_loaded;
110 unsigned int list_len;
111 unsigned int list_period_len;
112 size_t list_xfer_size;
113 unsigned int list_cmds_done;
114 };
115
116 struct mdc_chan {
117 struct mdc_dma *mdma;
118 struct virt_dma_chan vc;
119 struct dma_slave_config config;
120 struct mdc_tx_desc *desc;
121 int irq;
122 unsigned int periph;
123 unsigned int thread;
124 unsigned int chan_nr;
125 };
126
127 struct mdc_dma_soc_data {
128 void (*enable_chan)(struct mdc_chan *mchan);
129 void (*disable_chan)(struct mdc_chan *mchan);
130 };
131
132 struct mdc_dma {
133 struct dma_device dma_dev;
134 void __iomem *regs;
135 struct clk *clk;
136 struct dma_pool *desc_pool;
137 struct regmap *periph_regs;
138 spinlock_t lock;
139 unsigned int nr_threads;
140 unsigned int nr_channels;
141 unsigned int bus_width;
142 unsigned int max_burst_mult;
143 unsigned int max_xfer_size;
144 const struct mdc_dma_soc_data *soc;
145 struct mdc_chan channels[MDC_MAX_DMA_CHANNELS];
146 };
147
mdc_readl(struct mdc_dma * mdma,u32 reg)148 static inline u32 mdc_readl(struct mdc_dma *mdma, u32 reg)
149 {
150 return readl(mdma->regs + reg);
151 }
152
mdc_writel(struct mdc_dma * mdma,u32 val,u32 reg)153 static inline void mdc_writel(struct mdc_dma *mdma, u32 val, u32 reg)
154 {
155 writel(val, mdma->regs + reg);
156 }
157
mdc_chan_readl(struct mdc_chan * mchan,u32 reg)158 static inline u32 mdc_chan_readl(struct mdc_chan *mchan, u32 reg)
159 {
160 return mdc_readl(mchan->mdma, mchan->chan_nr * 0x040 + reg);
161 }
162
mdc_chan_writel(struct mdc_chan * mchan,u32 val,u32 reg)163 static inline void mdc_chan_writel(struct mdc_chan *mchan, u32 val, u32 reg)
164 {
165 mdc_writel(mchan->mdma, val, mchan->chan_nr * 0x040 + reg);
166 }
167
to_mdc_chan(struct dma_chan * c)168 static inline struct mdc_chan *to_mdc_chan(struct dma_chan *c)
169 {
170 return container_of(to_virt_chan(c), struct mdc_chan, vc);
171 }
172
to_mdc_desc(struct dma_async_tx_descriptor * t)173 static inline struct mdc_tx_desc *to_mdc_desc(struct dma_async_tx_descriptor *t)
174 {
175 struct virt_dma_desc *vdesc = container_of(t, struct virt_dma_desc, tx);
176
177 return container_of(vdesc, struct mdc_tx_desc, vd);
178 }
179
mdma2dev(struct mdc_dma * mdma)180 static inline struct device *mdma2dev(struct mdc_dma *mdma)
181 {
182 return mdma->dma_dev.dev;
183 }
184
to_mdc_width(unsigned int bytes)185 static inline unsigned int to_mdc_width(unsigned int bytes)
186 {
187 return ffs(bytes) - 1;
188 }
189
mdc_set_read_width(struct mdc_hw_list_desc * ldesc,unsigned int bytes)190 static inline void mdc_set_read_width(struct mdc_hw_list_desc *ldesc,
191 unsigned int bytes)
192 {
193 ldesc->gen_conf |= to_mdc_width(bytes) <<
194 MDC_GENERAL_CONFIG_WIDTH_R_SHIFT;
195 }
196
mdc_set_write_width(struct mdc_hw_list_desc * ldesc,unsigned int bytes)197 static inline void mdc_set_write_width(struct mdc_hw_list_desc *ldesc,
198 unsigned int bytes)
199 {
200 ldesc->gen_conf |= to_mdc_width(bytes) <<
201 MDC_GENERAL_CONFIG_WIDTH_W_SHIFT;
202 }
203
mdc_list_desc_config(struct mdc_chan * mchan,struct mdc_hw_list_desc * ldesc,enum dma_transfer_direction dir,dma_addr_t src,dma_addr_t dst,size_t len)204 static void mdc_list_desc_config(struct mdc_chan *mchan,
205 struct mdc_hw_list_desc *ldesc,
206 enum dma_transfer_direction dir,
207 dma_addr_t src, dma_addr_t dst, size_t len)
208 {
209 struct mdc_dma *mdma = mchan->mdma;
210 unsigned int max_burst, burst_size;
211
212 ldesc->gen_conf = MDC_GENERAL_CONFIG_IEN | MDC_GENERAL_CONFIG_LIST_IEN |
213 MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |
214 MDC_GENERAL_CONFIG_PHYSICAL_R;
215 ldesc->readport_conf =
216 (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |
217 (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |
218 (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);
219 ldesc->read_addr = src;
220 ldesc->write_addr = dst;
221 ldesc->xfer_size = len - 1;
222 ldesc->node_addr = 0;
223 ldesc->cmds_done = 0;
224 ldesc->ctrl_status = MDC_CONTROL_AND_STATUS_LIST_EN |
225 MDC_CONTROL_AND_STATUS_EN;
226 ldesc->next_desc = NULL;
227
228 if (IS_ALIGNED(dst, mdma->bus_width) &&
229 IS_ALIGNED(src, mdma->bus_width))
230 max_burst = mdma->bus_width * mdma->max_burst_mult;
231 else
232 max_burst = mdma->bus_width * (mdma->max_burst_mult - 1);
233
234 if (dir == DMA_MEM_TO_DEV) {
235 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R;
236 ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;
237 mdc_set_read_width(ldesc, mdma->bus_width);
238 mdc_set_write_width(ldesc, mchan->config.dst_addr_width);
239 burst_size = min(max_burst, mchan->config.dst_maxburst *
240 mchan->config.dst_addr_width);
241 } else if (dir == DMA_DEV_TO_MEM) {
242 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_W;
243 ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;
244 mdc_set_read_width(ldesc, mchan->config.src_addr_width);
245 mdc_set_write_width(ldesc, mdma->bus_width);
246 burst_size = min(max_burst, mchan->config.src_maxburst *
247 mchan->config.src_addr_width);
248 } else {
249 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R |
250 MDC_GENERAL_CONFIG_INC_W;
251 mdc_set_read_width(ldesc, mdma->bus_width);
252 mdc_set_write_width(ldesc, mdma->bus_width);
253 burst_size = max_burst;
254 }
255 ldesc->readport_conf |= (burst_size - 1) <<
256 MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT;
257 }
258
mdc_list_desc_free(struct mdc_tx_desc * mdesc)259 static void mdc_list_desc_free(struct mdc_tx_desc *mdesc)
260 {
261 struct mdc_dma *mdma = mdesc->chan->mdma;
262 struct mdc_hw_list_desc *curr, *next;
263 dma_addr_t curr_phys, next_phys;
264
265 curr = mdesc->list;
266 curr_phys = mdesc->list_phys;
267 while (curr) {
268 next = curr->next_desc;
269 next_phys = curr->node_addr;
270 dma_pool_free(mdma->desc_pool, curr, curr_phys);
271 curr = next;
272 curr_phys = next_phys;
273 }
274 }
275
mdc_desc_free(struct virt_dma_desc * vd)276 static void mdc_desc_free(struct virt_dma_desc *vd)
277 {
278 struct mdc_tx_desc *mdesc = to_mdc_desc(&vd->tx);
279
280 mdc_list_desc_free(mdesc);
281 kfree(mdesc);
282 }
283
mdc_prep_dma_memcpy(struct dma_chan * chan,dma_addr_t dest,dma_addr_t src,size_t len,unsigned long flags)284 static struct dma_async_tx_descriptor *mdc_prep_dma_memcpy(
285 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len,
286 unsigned long flags)
287 {
288 struct mdc_chan *mchan = to_mdc_chan(chan);
289 struct mdc_dma *mdma = mchan->mdma;
290 struct mdc_tx_desc *mdesc;
291 struct mdc_hw_list_desc *curr, *prev = NULL;
292 dma_addr_t curr_phys;
293
294 if (!len)
295 return NULL;
296
297 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
298 if (!mdesc)
299 return NULL;
300 mdesc->chan = mchan;
301 mdesc->list_xfer_size = len;
302
303 while (len > 0) {
304 size_t xfer_size;
305
306 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, &curr_phys);
307 if (!curr)
308 goto free_desc;
309
310 if (prev) {
311 prev->node_addr = curr_phys;
312 prev->next_desc = curr;
313 } else {
314 mdesc->list_phys = curr_phys;
315 mdesc->list = curr;
316 }
317
318 xfer_size = min_t(size_t, mdma->max_xfer_size, len);
319
320 mdc_list_desc_config(mchan, curr, DMA_MEM_TO_MEM, src, dest,
321 xfer_size);
322
323 prev = curr;
324
325 mdesc->list_len++;
326 src += xfer_size;
327 dest += xfer_size;
328 len -= xfer_size;
329 }
330
331 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
332
333 free_desc:
334 mdc_desc_free(&mdesc->vd);
335
336 return NULL;
337 }
338
mdc_check_slave_width(struct mdc_chan * mchan,enum dma_transfer_direction dir)339 static int mdc_check_slave_width(struct mdc_chan *mchan,
340 enum dma_transfer_direction dir)
341 {
342 enum dma_slave_buswidth width;
343
344 if (dir == DMA_MEM_TO_DEV)
345 width = mchan->config.dst_addr_width;
346 else
347 width = mchan->config.src_addr_width;
348
349 switch (width) {
350 case DMA_SLAVE_BUSWIDTH_1_BYTE:
351 case DMA_SLAVE_BUSWIDTH_2_BYTES:
352 case DMA_SLAVE_BUSWIDTH_4_BYTES:
353 case DMA_SLAVE_BUSWIDTH_8_BYTES:
354 break;
355 default:
356 return -EINVAL;
357 }
358
359 if (width > mchan->mdma->bus_width)
360 return -EINVAL;
361
362 return 0;
363 }
364
mdc_prep_dma_cyclic(struct dma_chan * chan,dma_addr_t buf_addr,size_t buf_len,size_t period_len,enum dma_transfer_direction dir,unsigned long flags)365 static struct dma_async_tx_descriptor *mdc_prep_dma_cyclic(
366 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
367 size_t period_len, enum dma_transfer_direction dir,
368 unsigned long flags)
369 {
370 struct mdc_chan *mchan = to_mdc_chan(chan);
371 struct mdc_dma *mdma = mchan->mdma;
372 struct mdc_tx_desc *mdesc;
373 struct mdc_hw_list_desc *curr, *prev = NULL;
374 dma_addr_t curr_phys;
375
376 if (!buf_len && !period_len)
377 return NULL;
378
379 if (!is_slave_direction(dir))
380 return NULL;
381
382 if (mdc_check_slave_width(mchan, dir) < 0)
383 return NULL;
384
385 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
386 if (!mdesc)
387 return NULL;
388 mdesc->chan = mchan;
389 mdesc->cyclic = true;
390 mdesc->list_xfer_size = buf_len;
391 mdesc->list_period_len = DIV_ROUND_UP(period_len,
392 mdma->max_xfer_size);
393
394 while (buf_len > 0) {
395 size_t remainder = min(period_len, buf_len);
396
397 while (remainder > 0) {
398 size_t xfer_size;
399
400 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
401 &curr_phys);
402 if (!curr)
403 goto free_desc;
404
405 if (!prev) {
406 mdesc->list_phys = curr_phys;
407 mdesc->list = curr;
408 } else {
409 prev->node_addr = curr_phys;
410 prev->next_desc = curr;
411 }
412
413 xfer_size = min_t(size_t, mdma->max_xfer_size,
414 remainder);
415
416 if (dir == DMA_MEM_TO_DEV) {
417 mdc_list_desc_config(mchan, curr, dir,
418 buf_addr,
419 mchan->config.dst_addr,
420 xfer_size);
421 } else {
422 mdc_list_desc_config(mchan, curr, dir,
423 mchan->config.src_addr,
424 buf_addr,
425 xfer_size);
426 }
427
428 prev = curr;
429
430 mdesc->list_len++;
431 buf_addr += xfer_size;
432 buf_len -= xfer_size;
433 remainder -= xfer_size;
434 }
435 }
436 prev->node_addr = mdesc->list_phys;
437
438 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
439
440 free_desc:
441 mdc_desc_free(&mdesc->vd);
442
443 return NULL;
444 }
445
mdc_prep_slave_sg(struct dma_chan * chan,struct scatterlist * sgl,unsigned int sg_len,enum dma_transfer_direction dir,unsigned long flags,void * context)446 static struct dma_async_tx_descriptor *mdc_prep_slave_sg(
447 struct dma_chan *chan, struct scatterlist *sgl,
448 unsigned int sg_len, enum dma_transfer_direction dir,
449 unsigned long flags, void *context)
450 {
451 struct mdc_chan *mchan = to_mdc_chan(chan);
452 struct mdc_dma *mdma = mchan->mdma;
453 struct mdc_tx_desc *mdesc;
454 struct scatterlist *sg;
455 struct mdc_hw_list_desc *curr, *prev = NULL;
456 dma_addr_t curr_phys;
457 unsigned int i;
458
459 if (!sgl)
460 return NULL;
461
462 if (!is_slave_direction(dir))
463 return NULL;
464
465 if (mdc_check_slave_width(mchan, dir) < 0)
466 return NULL;
467
468 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
469 if (!mdesc)
470 return NULL;
471 mdesc->chan = mchan;
472
473 for_each_sg(sgl, sg, sg_len, i) {
474 dma_addr_t buf = sg_dma_address(sg);
475 size_t buf_len = sg_dma_len(sg);
476
477 while (buf_len > 0) {
478 size_t xfer_size;
479
480 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
481 &curr_phys);
482 if (!curr)
483 goto free_desc;
484
485 if (!prev) {
486 mdesc->list_phys = curr_phys;
487 mdesc->list = curr;
488 } else {
489 prev->node_addr = curr_phys;
490 prev->next_desc = curr;
491 }
492
493 xfer_size = min_t(size_t, mdma->max_xfer_size,
494 buf_len);
495
496 if (dir == DMA_MEM_TO_DEV) {
497 mdc_list_desc_config(mchan, curr, dir, buf,
498 mchan->config.dst_addr,
499 xfer_size);
500 } else {
501 mdc_list_desc_config(mchan, curr, dir,
502 mchan->config.src_addr,
503 buf, xfer_size);
504 }
505
506 prev = curr;
507
508 mdesc->list_len++;
509 mdesc->list_xfer_size += xfer_size;
510 buf += xfer_size;
511 buf_len -= xfer_size;
512 }
513 }
514
515 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
516
517 free_desc:
518 mdc_desc_free(&mdesc->vd);
519
520 return NULL;
521 }
522
mdc_issue_desc(struct mdc_chan * mchan)523 static void mdc_issue_desc(struct mdc_chan *mchan)
524 {
525 struct mdc_dma *mdma = mchan->mdma;
526 struct virt_dma_desc *vd;
527 struct mdc_tx_desc *mdesc;
528 u32 val;
529
530 vd = vchan_next_desc(&mchan->vc);
531 if (!vd)
532 return;
533
534 list_del(&vd->node);
535
536 mdesc = to_mdc_desc(&vd->tx);
537 mchan->desc = mdesc;
538
539 dev_dbg(mdma2dev(mdma), "Issuing descriptor on channel %d\n",
540 mchan->chan_nr);
541
542 mdma->soc->enable_chan(mchan);
543
544 val = mdc_chan_readl(mchan, MDC_GENERAL_CONFIG);
545 val |= MDC_GENERAL_CONFIG_LIST_IEN | MDC_GENERAL_CONFIG_IEN |
546 MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |
547 MDC_GENERAL_CONFIG_PHYSICAL_R;
548 mdc_chan_writel(mchan, val, MDC_GENERAL_CONFIG);
549 val = (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |
550 (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |
551 (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);
552 mdc_chan_writel(mchan, val, MDC_READ_PORT_CONFIG);
553 mdc_chan_writel(mchan, mdesc->list_phys, MDC_LIST_NODE_ADDRESS);
554 val = mdc_chan_readl(mchan, MDC_CONTROL_AND_STATUS);
555 val |= MDC_CONTROL_AND_STATUS_LIST_EN;
556 mdc_chan_writel(mchan, val, MDC_CONTROL_AND_STATUS);
557 }
558
mdc_issue_pending(struct dma_chan * chan)559 static void mdc_issue_pending(struct dma_chan *chan)
560 {
561 struct mdc_chan *mchan = to_mdc_chan(chan);
562 unsigned long flags;
563
564 spin_lock_irqsave(&mchan->vc.lock, flags);
565 if (vchan_issue_pending(&mchan->vc) && !mchan->desc)
566 mdc_issue_desc(mchan);
567 spin_unlock_irqrestore(&mchan->vc.lock, flags);
568 }
569
mdc_tx_status(struct dma_chan * chan,dma_cookie_t cookie,struct dma_tx_state * txstate)570 static enum dma_status mdc_tx_status(struct dma_chan *chan,
571 dma_cookie_t cookie, struct dma_tx_state *txstate)
572 {
573 struct mdc_chan *mchan = to_mdc_chan(chan);
574 struct mdc_tx_desc *mdesc;
575 struct virt_dma_desc *vd;
576 unsigned long flags;
577 size_t bytes = 0;
578 int ret;
579
580 ret = dma_cookie_status(chan, cookie, txstate);
581 if (ret == DMA_COMPLETE)
582 return ret;
583
584 if (!txstate)
585 return ret;
586
587 spin_lock_irqsave(&mchan->vc.lock, flags);
588 vd = vchan_find_desc(&mchan->vc, cookie);
589 if (vd) {
590 mdesc = to_mdc_desc(&vd->tx);
591 bytes = mdesc->list_xfer_size;
592 } else if (mchan->desc && mchan->desc->vd.tx.cookie == cookie) {
593 struct mdc_hw_list_desc *ldesc;
594 u32 val1, val2, done, processed, residue;
595 int i, cmds;
596
597 mdesc = mchan->desc;
598
599 /*
600 * Determine the number of commands that haven't been
601 * processed (handled by the IRQ handler) yet.
602 */
603 do {
604 val1 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &
605 ~MDC_CMDS_PROCESSED_INT_ACTIVE;
606 residue = mdc_chan_readl(mchan,
607 MDC_ACTIVE_TRANSFER_SIZE);
608 val2 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &
609 ~MDC_CMDS_PROCESSED_INT_ACTIVE;
610 } while (val1 != val2);
611
612 done = (val1 >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
613 MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
614 processed = (val1 >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &
615 MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;
616 cmds = (done - processed) %
617 (MDC_CMDS_PROCESSED_CMDS_DONE_MASK + 1);
618
619 /*
620 * If the command loaded event hasn't been processed yet, then
621 * the difference above includes an extra command.
622 */
623 if (!mdesc->cmd_loaded)
624 cmds--;
625 else
626 cmds += mdesc->list_cmds_done;
627
628 bytes = mdesc->list_xfer_size;
629 ldesc = mdesc->list;
630 for (i = 0; i < cmds; i++) {
631 bytes -= ldesc->xfer_size + 1;
632 ldesc = ldesc->next_desc;
633 }
634 if (ldesc) {
635 if (residue != MDC_TRANSFER_SIZE_MASK)
636 bytes -= ldesc->xfer_size - residue;
637 else
638 bytes -= ldesc->xfer_size + 1;
639 }
640 }
641 spin_unlock_irqrestore(&mchan->vc.lock, flags);
642
643 dma_set_residue(txstate, bytes);
644
645 return ret;
646 }
647
mdc_get_new_events(struct mdc_chan * mchan)648 static unsigned int mdc_get_new_events(struct mdc_chan *mchan)
649 {
650 u32 val, processed, done1, done2;
651 unsigned int ret;
652
653 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
654 processed = (val >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &
655 MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;
656 /*
657 * CMDS_DONE may have incremented between reading CMDS_PROCESSED
658 * and clearing INT_ACTIVE. Re-read CMDS_PROCESSED to ensure we
659 * didn't miss a command completion.
660 */
661 do {
662 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
663
664 done1 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
665 MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
666
667 val &= ~((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK <<
668 MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) |
669 MDC_CMDS_PROCESSED_INT_ACTIVE);
670
671 val |= done1 << MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT;
672
673 mdc_chan_writel(mchan, val, MDC_CMDS_PROCESSED);
674
675 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
676
677 done2 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
678 MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
679 } while (done1 != done2);
680
681 if (done1 >= processed)
682 ret = done1 - processed;
683 else
684 ret = ((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK + 1) -
685 processed) + done1;
686
687 return ret;
688 }
689
mdc_terminate_all(struct dma_chan * chan)690 static int mdc_terminate_all(struct dma_chan *chan)
691 {
692 struct mdc_chan *mchan = to_mdc_chan(chan);
693 unsigned long flags;
694 LIST_HEAD(head);
695
696 spin_lock_irqsave(&mchan->vc.lock, flags);
697
698 mdc_chan_writel(mchan, MDC_CONTROL_AND_STATUS_CANCEL,
699 MDC_CONTROL_AND_STATUS);
700
701 if (mchan->desc) {
702 vchan_terminate_vdesc(&mchan->desc->vd);
703 mchan->desc = NULL;
704 }
705 vchan_get_all_descriptors(&mchan->vc, &head);
706
707 mdc_get_new_events(mchan);
708
709 spin_unlock_irqrestore(&mchan->vc.lock, flags);
710
711 vchan_dma_desc_free_list(&mchan->vc, &head);
712
713 return 0;
714 }
715
mdc_synchronize(struct dma_chan * chan)716 static void mdc_synchronize(struct dma_chan *chan)
717 {
718 struct mdc_chan *mchan = to_mdc_chan(chan);
719
720 vchan_synchronize(&mchan->vc);
721 }
722
mdc_slave_config(struct dma_chan * chan,struct dma_slave_config * config)723 static int mdc_slave_config(struct dma_chan *chan,
724 struct dma_slave_config *config)
725 {
726 struct mdc_chan *mchan = to_mdc_chan(chan);
727 unsigned long flags;
728
729 spin_lock_irqsave(&mchan->vc.lock, flags);
730 mchan->config = *config;
731 spin_unlock_irqrestore(&mchan->vc.lock, flags);
732
733 return 0;
734 }
735
mdc_alloc_chan_resources(struct dma_chan * chan)736 static int mdc_alloc_chan_resources(struct dma_chan *chan)
737 {
738 struct mdc_chan *mchan = to_mdc_chan(chan);
739 struct device *dev = mdma2dev(mchan->mdma);
740
741 return pm_runtime_get_sync(dev);
742 }
743
mdc_free_chan_resources(struct dma_chan * chan)744 static void mdc_free_chan_resources(struct dma_chan *chan)
745 {
746 struct mdc_chan *mchan = to_mdc_chan(chan);
747 struct mdc_dma *mdma = mchan->mdma;
748 struct device *dev = mdma2dev(mdma);
749
750 mdc_terminate_all(chan);
751 mdma->soc->disable_chan(mchan);
752 pm_runtime_put(dev);
753 }
754
mdc_chan_irq(int irq,void * dev_id)755 static irqreturn_t mdc_chan_irq(int irq, void *dev_id)
756 {
757 struct mdc_chan *mchan = (struct mdc_chan *)dev_id;
758 struct mdc_tx_desc *mdesc;
759 unsigned int i, new_events;
760
761 spin_lock(&mchan->vc.lock);
762
763 dev_dbg(mdma2dev(mchan->mdma), "IRQ on channel %d\n", mchan->chan_nr);
764
765 new_events = mdc_get_new_events(mchan);
766
767 if (!new_events)
768 goto out;
769
770 mdesc = mchan->desc;
771 if (!mdesc) {
772 dev_warn(mdma2dev(mchan->mdma),
773 "IRQ with no active descriptor on channel %d\n",
774 mchan->chan_nr);
775 goto out;
776 }
777
778 for (i = 0; i < new_events; i++) {
779 /*
780 * The first interrupt in a transfer indicates that the
781 * command list has been loaded, not that a command has
782 * been completed.
783 */
784 if (!mdesc->cmd_loaded) {
785 mdesc->cmd_loaded = true;
786 continue;
787 }
788
789 mdesc->list_cmds_done++;
790 if (mdesc->cyclic) {
791 mdesc->list_cmds_done %= mdesc->list_len;
792 if (mdesc->list_cmds_done % mdesc->list_period_len == 0)
793 vchan_cyclic_callback(&mdesc->vd);
794 } else if (mdesc->list_cmds_done == mdesc->list_len) {
795 mchan->desc = NULL;
796 vchan_cookie_complete(&mdesc->vd);
797 mdc_issue_desc(mchan);
798 break;
799 }
800 }
801 out:
802 spin_unlock(&mchan->vc.lock);
803
804 return IRQ_HANDLED;
805 }
806
mdc_of_xlate(struct of_phandle_args * dma_spec,struct of_dma * ofdma)807 static struct dma_chan *mdc_of_xlate(struct of_phandle_args *dma_spec,
808 struct of_dma *ofdma)
809 {
810 struct mdc_dma *mdma = ofdma->of_dma_data;
811 struct dma_chan *chan;
812
813 if (dma_spec->args_count != 3)
814 return NULL;
815
816 list_for_each_entry(chan, &mdma->dma_dev.channels, device_node) {
817 struct mdc_chan *mchan = to_mdc_chan(chan);
818
819 if (!(dma_spec->args[1] & BIT(mchan->chan_nr)))
820 continue;
821 if (dma_get_slave_channel(chan)) {
822 mchan->periph = dma_spec->args[0];
823 mchan->thread = dma_spec->args[2];
824 return chan;
825 }
826 }
827
828 return NULL;
829 }
830
831 #define PISTACHIO_CR_PERIPH_DMA_ROUTE(ch) (0x120 + 0x4 * ((ch) / 4))
832 #define PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(ch) (8 * ((ch) % 4))
833 #define PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK 0x3f
834
pistachio_mdc_enable_chan(struct mdc_chan * mchan)835 static void pistachio_mdc_enable_chan(struct mdc_chan *mchan)
836 {
837 struct mdc_dma *mdma = mchan->mdma;
838
839 regmap_update_bits(mdma->periph_regs,
840 PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),
841 PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<
842 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),
843 mchan->periph <<
844 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr));
845 }
846
pistachio_mdc_disable_chan(struct mdc_chan * mchan)847 static void pistachio_mdc_disable_chan(struct mdc_chan *mchan)
848 {
849 struct mdc_dma *mdma = mchan->mdma;
850
851 regmap_update_bits(mdma->periph_regs,
852 PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),
853 PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<
854 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),
855 0);
856 }
857
858 static const struct mdc_dma_soc_data pistachio_mdc_data = {
859 .enable_chan = pistachio_mdc_enable_chan,
860 .disable_chan = pistachio_mdc_disable_chan,
861 };
862
863 static const struct of_device_id mdc_dma_of_match[] = {
864 { .compatible = "img,pistachio-mdc-dma", .data = &pistachio_mdc_data, },
865 { },
866 };
867 MODULE_DEVICE_TABLE(of, mdc_dma_of_match);
868
img_mdc_runtime_suspend(struct device * dev)869 static int img_mdc_runtime_suspend(struct device *dev)
870 {
871 struct mdc_dma *mdma = dev_get_drvdata(dev);
872
873 clk_disable_unprepare(mdma->clk);
874
875 return 0;
876 }
877
img_mdc_runtime_resume(struct device * dev)878 static int img_mdc_runtime_resume(struct device *dev)
879 {
880 struct mdc_dma *mdma = dev_get_drvdata(dev);
881
882 return clk_prepare_enable(mdma->clk);
883 }
884
mdc_dma_probe(struct platform_device * pdev)885 static int mdc_dma_probe(struct platform_device *pdev)
886 {
887 struct mdc_dma *mdma;
888 unsigned int i;
889 u32 val;
890 int ret;
891
892 mdma = devm_kzalloc(&pdev->dev, sizeof(*mdma), GFP_KERNEL);
893 if (!mdma)
894 return -ENOMEM;
895 platform_set_drvdata(pdev, mdma);
896
897 mdma->soc = of_device_get_match_data(&pdev->dev);
898
899 mdma->regs = devm_platform_ioremap_resource(pdev, 0);
900 if (IS_ERR(mdma->regs))
901 return PTR_ERR(mdma->regs);
902
903 mdma->periph_regs = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
904 "img,cr-periph");
905 if (IS_ERR(mdma->periph_regs))
906 return PTR_ERR(mdma->periph_regs);
907
908 mdma->clk = devm_clk_get(&pdev->dev, "sys");
909 if (IS_ERR(mdma->clk))
910 return PTR_ERR(mdma->clk);
911
912 dma_cap_zero(mdma->dma_dev.cap_mask);
913 dma_cap_set(DMA_SLAVE, mdma->dma_dev.cap_mask);
914 dma_cap_set(DMA_PRIVATE, mdma->dma_dev.cap_mask);
915 dma_cap_set(DMA_CYCLIC, mdma->dma_dev.cap_mask);
916 dma_cap_set(DMA_MEMCPY, mdma->dma_dev.cap_mask);
917
918 val = mdc_readl(mdma, MDC_GLOBAL_CONFIG_A);
919 mdma->nr_channels = (val >> MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT) &
920 MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK;
921 mdma->nr_threads =
922 1 << ((val >> MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT) &
923 MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK);
924 mdma->bus_width =
925 (1 << ((val >> MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT) &
926 MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK)) / 8;
927 /*
928 * Although transfer sizes of up to MDC_TRANSFER_SIZE_MASK + 1 bytes
929 * are supported, this makes it possible for the value reported in
930 * MDC_ACTIVE_TRANSFER_SIZE to be ambiguous - an active transfer size
931 * of MDC_TRANSFER_SIZE_MASK may indicate either that 0 bytes or
932 * MDC_TRANSFER_SIZE_MASK + 1 bytes are remaining. To eliminate this
933 * ambiguity, restrict transfer sizes to one bus-width less than the
934 * actual maximum.
935 */
936 mdma->max_xfer_size = MDC_TRANSFER_SIZE_MASK + 1 - mdma->bus_width;
937
938 of_property_read_u32(pdev->dev.of_node, "dma-channels",
939 &mdma->nr_channels);
940 ret = of_property_read_u32(pdev->dev.of_node,
941 "img,max-burst-multiplier",
942 &mdma->max_burst_mult);
943 if (ret)
944 return ret;
945
946 mdma->dma_dev.dev = &pdev->dev;
947 mdma->dma_dev.device_prep_slave_sg = mdc_prep_slave_sg;
948 mdma->dma_dev.device_prep_dma_cyclic = mdc_prep_dma_cyclic;
949 mdma->dma_dev.device_prep_dma_memcpy = mdc_prep_dma_memcpy;
950 mdma->dma_dev.device_alloc_chan_resources = mdc_alloc_chan_resources;
951 mdma->dma_dev.device_free_chan_resources = mdc_free_chan_resources;
952 mdma->dma_dev.device_tx_status = mdc_tx_status;
953 mdma->dma_dev.device_issue_pending = mdc_issue_pending;
954 mdma->dma_dev.device_terminate_all = mdc_terminate_all;
955 mdma->dma_dev.device_synchronize = mdc_synchronize;
956 mdma->dma_dev.device_config = mdc_slave_config;
957
958 mdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
959 mdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
960 for (i = 1; i <= mdma->bus_width; i <<= 1) {
961 mdma->dma_dev.src_addr_widths |= BIT(i);
962 mdma->dma_dev.dst_addr_widths |= BIT(i);
963 }
964
965 INIT_LIST_HEAD(&mdma->dma_dev.channels);
966 for (i = 0; i < mdma->nr_channels; i++) {
967 struct mdc_chan *mchan = &mdma->channels[i];
968
969 mchan->mdma = mdma;
970 mchan->chan_nr = i;
971 mchan->irq = platform_get_irq(pdev, i);
972 if (mchan->irq < 0)
973 return mchan->irq;
974
975 ret = devm_request_irq(&pdev->dev, mchan->irq, mdc_chan_irq,
976 IRQ_TYPE_LEVEL_HIGH,
977 dev_name(&pdev->dev), mchan);
978 if (ret < 0)
979 return ret;
980
981 mchan->vc.desc_free = mdc_desc_free;
982 vchan_init(&mchan->vc, &mdma->dma_dev);
983 }
984
985 mdma->desc_pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev,
986 sizeof(struct mdc_hw_list_desc),
987 4, 0);
988 if (!mdma->desc_pool)
989 return -ENOMEM;
990
991 pm_runtime_enable(&pdev->dev);
992 if (!pm_runtime_enabled(&pdev->dev)) {
993 ret = img_mdc_runtime_resume(&pdev->dev);
994 if (ret)
995 return ret;
996 }
997
998 ret = dma_async_device_register(&mdma->dma_dev);
999 if (ret)
1000 goto suspend;
1001
1002 ret = of_dma_controller_register(pdev->dev.of_node, mdc_of_xlate, mdma);
1003 if (ret)
1004 goto unregister;
1005
1006 dev_info(&pdev->dev, "MDC with %u channels and %u threads\n",
1007 mdma->nr_channels, mdma->nr_threads);
1008
1009 return 0;
1010
1011 unregister:
1012 dma_async_device_unregister(&mdma->dma_dev);
1013 suspend:
1014 if (!pm_runtime_enabled(&pdev->dev))
1015 img_mdc_runtime_suspend(&pdev->dev);
1016 pm_runtime_disable(&pdev->dev);
1017 return ret;
1018 }
1019
mdc_dma_remove(struct platform_device * pdev)1020 static int mdc_dma_remove(struct platform_device *pdev)
1021 {
1022 struct mdc_dma *mdma = platform_get_drvdata(pdev);
1023 struct mdc_chan *mchan, *next;
1024
1025 of_dma_controller_free(pdev->dev.of_node);
1026 dma_async_device_unregister(&mdma->dma_dev);
1027
1028 list_for_each_entry_safe(mchan, next, &mdma->dma_dev.channels,
1029 vc.chan.device_node) {
1030 list_del(&mchan->vc.chan.device_node);
1031
1032 devm_free_irq(&pdev->dev, mchan->irq, mchan);
1033
1034 tasklet_kill(&mchan->vc.task);
1035 }
1036
1037 pm_runtime_disable(&pdev->dev);
1038 if (!pm_runtime_status_suspended(&pdev->dev))
1039 img_mdc_runtime_suspend(&pdev->dev);
1040
1041 return 0;
1042 }
1043
1044 #ifdef CONFIG_PM_SLEEP
img_mdc_suspend_late(struct device * dev)1045 static int img_mdc_suspend_late(struct device *dev)
1046 {
1047 struct mdc_dma *mdma = dev_get_drvdata(dev);
1048 int i;
1049
1050 /* Check that all channels are idle */
1051 for (i = 0; i < mdma->nr_channels; i++) {
1052 struct mdc_chan *mchan = &mdma->channels[i];
1053
1054 if (unlikely(mchan->desc))
1055 return -EBUSY;
1056 }
1057
1058 return pm_runtime_force_suspend(dev);
1059 }
1060
img_mdc_resume_early(struct device * dev)1061 static int img_mdc_resume_early(struct device *dev)
1062 {
1063 return pm_runtime_force_resume(dev);
1064 }
1065 #endif /* CONFIG_PM_SLEEP */
1066
1067 static const struct dev_pm_ops img_mdc_pm_ops = {
1068 SET_RUNTIME_PM_OPS(img_mdc_runtime_suspend,
1069 img_mdc_runtime_resume, NULL)
1070 SET_LATE_SYSTEM_SLEEP_PM_OPS(img_mdc_suspend_late,
1071 img_mdc_resume_early)
1072 };
1073
1074 static struct platform_driver mdc_dma_driver = {
1075 .driver = {
1076 .name = "img-mdc-dma",
1077 .pm = &img_mdc_pm_ops,
1078 .of_match_table = of_match_ptr(mdc_dma_of_match),
1079 },
1080 .probe = mdc_dma_probe,
1081 .remove = mdc_dma_remove,
1082 };
1083 module_platform_driver(mdc_dma_driver);
1084
1085 MODULE_DESCRIPTION("IMG Multi-threaded DMA Controller (MDC) driver");
1086 MODULE_AUTHOR("Andrew Bresticker <abrestic@chromium.org>");
1087 MODULE_LICENSE("GPL v2");
1088