1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3 * Freescale i.MX28 APBH DMA driver
4 *
5 * Copyright (C) 2011 Marek Vasut <marek.vasut@gmail.com>
6 * on behalf of DENX Software Engineering GmbH
7 *
8 * Based on code from LTIB:
9 * Copyright (C) 2010 Freescale Semiconductor, Inc. All Rights Reserved.
10 */
11
12 #include <linux/list.h>
13
14 #include <common.h>
15 #include <malloc.h>
16 #include <linux/errno.h>
17 #include <asm/io.h>
18 #include <asm/arch/clock.h>
19 #include <asm/arch/imx-regs.h>
20 #include <asm/arch/sys_proto.h>
21 #include <asm/mach-imx/dma.h>
22 #include <asm/mach-imx/regs-apbh.h>
23
24 static struct mxs_dma_chan mxs_dma_channels[MXS_MAX_DMA_CHANNELS];
25
26 /*
27 * Test is the DMA channel is valid channel
28 */
mxs_dma_validate_chan(int channel)29 int mxs_dma_validate_chan(int channel)
30 {
31 struct mxs_dma_chan *pchan;
32
33 if ((channel < 0) || (channel >= MXS_MAX_DMA_CHANNELS))
34 return -EINVAL;
35
36 pchan = mxs_dma_channels + channel;
37 if (!(pchan->flags & MXS_DMA_FLAGS_ALLOCATED))
38 return -EINVAL;
39
40 return 0;
41 }
42
43 /*
44 * Return the address of the command within a descriptor.
45 */
mxs_dma_cmd_address(struct mxs_dma_desc * desc)46 static unsigned int mxs_dma_cmd_address(struct mxs_dma_desc *desc)
47 {
48 return desc->address + offsetof(struct mxs_dma_desc, cmd);
49 }
50
51 /*
52 * Read a DMA channel's hardware semaphore.
53 *
54 * As used by the MXS platform's DMA software, the DMA channel's hardware
55 * semaphore reflects the number of DMA commands the hardware will process, but
56 * has not yet finished. This is a volatile value read directly from hardware,
57 * so it must be be viewed as immediately stale.
58 *
59 * If the channel is not marked busy, or has finished processing all its
60 * commands, this value should be zero.
61 *
62 * See mxs_dma_append() for details on how DMA command blocks must be configured
63 * to maintain the expected behavior of the semaphore's value.
64 */
mxs_dma_read_semaphore(int channel)65 static int mxs_dma_read_semaphore(int channel)
66 {
67 struct mxs_apbh_regs *apbh_regs =
68 (struct mxs_apbh_regs *)MXS_APBH_BASE;
69 uint32_t tmp;
70 int ret;
71
72 ret = mxs_dma_validate_chan(channel);
73 if (ret)
74 return ret;
75
76 tmp = readl(&apbh_regs->ch[channel].hw_apbh_ch_sema);
77
78 tmp &= APBH_CHn_SEMA_PHORE_MASK;
79 tmp >>= APBH_CHn_SEMA_PHORE_OFFSET;
80
81 return tmp;
82 }
83
84 #ifndef CONFIG_SYS_DCACHE_OFF
mxs_dma_flush_desc(struct mxs_dma_desc * desc)85 void mxs_dma_flush_desc(struct mxs_dma_desc *desc)
86 {
87 uint32_t addr;
88 uint32_t size;
89
90 addr = (uint32_t)desc;
91 size = roundup(sizeof(struct mxs_dma_desc), MXS_DMA_ALIGNMENT);
92
93 flush_dcache_range(addr, addr + size);
94 }
95 #else
mxs_dma_flush_desc(struct mxs_dma_desc * desc)96 inline void mxs_dma_flush_desc(struct mxs_dma_desc *desc) {}
97 #endif
98
99 /*
100 * Enable a DMA channel.
101 *
102 * If the given channel has any DMA descriptors on its active list, this
103 * function causes the DMA hardware to begin processing them.
104 *
105 * This function marks the DMA channel as "busy," whether or not there are any
106 * descriptors to process.
107 */
mxs_dma_enable(int channel)108 static int mxs_dma_enable(int channel)
109 {
110 struct mxs_apbh_regs *apbh_regs =
111 (struct mxs_apbh_regs *)MXS_APBH_BASE;
112 unsigned int sem;
113 struct mxs_dma_chan *pchan;
114 struct mxs_dma_desc *pdesc;
115 int ret;
116
117 ret = mxs_dma_validate_chan(channel);
118 if (ret)
119 return ret;
120
121 pchan = mxs_dma_channels + channel;
122
123 if (pchan->pending_num == 0) {
124 pchan->flags |= MXS_DMA_FLAGS_BUSY;
125 return 0;
126 }
127
128 pdesc = list_first_entry(&pchan->active, struct mxs_dma_desc, node);
129 if (pdesc == NULL)
130 return -EFAULT;
131
132 if (pchan->flags & MXS_DMA_FLAGS_BUSY) {
133 if (!(pdesc->cmd.data & MXS_DMA_DESC_CHAIN))
134 return 0;
135
136 sem = mxs_dma_read_semaphore(channel);
137 if (sem == 0)
138 return 0;
139
140 if (sem == 1) {
141 pdesc = list_entry(pdesc->node.next,
142 struct mxs_dma_desc, node);
143 writel(mxs_dma_cmd_address(pdesc),
144 &apbh_regs->ch[channel].hw_apbh_ch_nxtcmdar);
145 }
146 writel(pchan->pending_num,
147 &apbh_regs->ch[channel].hw_apbh_ch_sema);
148 pchan->active_num += pchan->pending_num;
149 pchan->pending_num = 0;
150 } else {
151 pchan->active_num += pchan->pending_num;
152 pchan->pending_num = 0;
153 writel(mxs_dma_cmd_address(pdesc),
154 &apbh_regs->ch[channel].hw_apbh_ch_nxtcmdar);
155 writel(pchan->active_num,
156 &apbh_regs->ch[channel].hw_apbh_ch_sema);
157 writel(1 << (channel + APBH_CTRL0_CLKGATE_CHANNEL_OFFSET),
158 &apbh_regs->hw_apbh_ctrl0_clr);
159 }
160
161 pchan->flags |= MXS_DMA_FLAGS_BUSY;
162 return 0;
163 }
164
165 /*
166 * Disable a DMA channel.
167 *
168 * This function shuts down a DMA channel and marks it as "not busy." Any
169 * descriptors on the active list are immediately moved to the head of the
170 * "done" list, whether or not they have actually been processed by the
171 * hardware. The "ready" flags of these descriptors are NOT cleared, so they
172 * still appear to be active.
173 *
174 * This function immediately shuts down a DMA channel's hardware, aborting any
175 * I/O that may be in progress, potentially leaving I/O hardware in an undefined
176 * state. It is unwise to call this function if there is ANY chance the hardware
177 * is still processing a command.
178 */
mxs_dma_disable(int channel)179 static int mxs_dma_disable(int channel)
180 {
181 struct mxs_dma_chan *pchan;
182 struct mxs_apbh_regs *apbh_regs =
183 (struct mxs_apbh_regs *)MXS_APBH_BASE;
184 int ret;
185
186 ret = mxs_dma_validate_chan(channel);
187 if (ret)
188 return ret;
189
190 pchan = mxs_dma_channels + channel;
191
192 if (!(pchan->flags & MXS_DMA_FLAGS_BUSY))
193 return -EINVAL;
194
195 writel(1 << (channel + APBH_CTRL0_CLKGATE_CHANNEL_OFFSET),
196 &apbh_regs->hw_apbh_ctrl0_set);
197
198 pchan->flags &= ~MXS_DMA_FLAGS_BUSY;
199 pchan->active_num = 0;
200 pchan->pending_num = 0;
201 list_splice_init(&pchan->active, &pchan->done);
202
203 return 0;
204 }
205
206 /*
207 * Resets the DMA channel hardware.
208 */
mxs_dma_reset(int channel)209 static int mxs_dma_reset(int channel)
210 {
211 struct mxs_apbh_regs *apbh_regs =
212 (struct mxs_apbh_regs *)MXS_APBH_BASE;
213 int ret;
214 #if defined(CONFIG_MX23)
215 uint32_t setreg = (uint32_t)(&apbh_regs->hw_apbh_ctrl0_set);
216 uint32_t offset = APBH_CTRL0_RESET_CHANNEL_OFFSET;
217 #elif (defined(CONFIG_MX28) || defined(CONFIG_MX6) || defined(CONFIG_MX7))
218 uint32_t setreg = (uint32_t)(&apbh_regs->hw_apbh_channel_ctrl_set);
219 uint32_t offset = APBH_CHANNEL_CTRL_RESET_CHANNEL_OFFSET;
220 #endif
221
222 ret = mxs_dma_validate_chan(channel);
223 if (ret)
224 return ret;
225
226 writel(1 << (channel + offset), setreg);
227
228 return 0;
229 }
230
231 /*
232 * Enable or disable DMA interrupt.
233 *
234 * This function enables the given DMA channel to interrupt the CPU.
235 */
mxs_dma_enable_irq(int channel,int enable)236 static int mxs_dma_enable_irq(int channel, int enable)
237 {
238 struct mxs_apbh_regs *apbh_regs =
239 (struct mxs_apbh_regs *)MXS_APBH_BASE;
240 int ret;
241
242 ret = mxs_dma_validate_chan(channel);
243 if (ret)
244 return ret;
245
246 if (enable)
247 writel(1 << (channel + APBH_CTRL1_CH_CMDCMPLT_IRQ_EN_OFFSET),
248 &apbh_regs->hw_apbh_ctrl1_set);
249 else
250 writel(1 << (channel + APBH_CTRL1_CH_CMDCMPLT_IRQ_EN_OFFSET),
251 &apbh_regs->hw_apbh_ctrl1_clr);
252
253 return 0;
254 }
255
256 /*
257 * Clear DMA interrupt.
258 *
259 * The software that is using the DMA channel must register to receive its
260 * interrupts and, when they arrive, must call this function to clear them.
261 */
mxs_dma_ack_irq(int channel)262 static int mxs_dma_ack_irq(int channel)
263 {
264 struct mxs_apbh_regs *apbh_regs =
265 (struct mxs_apbh_regs *)MXS_APBH_BASE;
266 int ret;
267
268 ret = mxs_dma_validate_chan(channel);
269 if (ret)
270 return ret;
271
272 writel(1 << channel, &apbh_regs->hw_apbh_ctrl1_clr);
273 writel(1 << channel, &apbh_regs->hw_apbh_ctrl2_clr);
274
275 return 0;
276 }
277
278 /*
279 * Request to reserve a DMA channel
280 */
mxs_dma_request(int channel)281 static int mxs_dma_request(int channel)
282 {
283 struct mxs_dma_chan *pchan;
284
285 if ((channel < 0) || (channel >= MXS_MAX_DMA_CHANNELS))
286 return -EINVAL;
287
288 pchan = mxs_dma_channels + channel;
289 if ((pchan->flags & MXS_DMA_FLAGS_VALID) != MXS_DMA_FLAGS_VALID)
290 return -ENODEV;
291
292 if (pchan->flags & MXS_DMA_FLAGS_ALLOCATED)
293 return -EBUSY;
294
295 pchan->flags |= MXS_DMA_FLAGS_ALLOCATED;
296 pchan->active_num = 0;
297 pchan->pending_num = 0;
298
299 INIT_LIST_HEAD(&pchan->active);
300 INIT_LIST_HEAD(&pchan->done);
301
302 return 0;
303 }
304
305 /*
306 * Release a DMA channel.
307 *
308 * This function releases a DMA channel from its current owner.
309 *
310 * The channel will NOT be released if it's marked "busy" (see
311 * mxs_dma_enable()).
312 */
mxs_dma_release(int channel)313 int mxs_dma_release(int channel)
314 {
315 struct mxs_dma_chan *pchan;
316 int ret;
317
318 ret = mxs_dma_validate_chan(channel);
319 if (ret)
320 return ret;
321
322 pchan = mxs_dma_channels + channel;
323
324 if (pchan->flags & MXS_DMA_FLAGS_BUSY)
325 return -EBUSY;
326
327 pchan->dev = 0;
328 pchan->active_num = 0;
329 pchan->pending_num = 0;
330 pchan->flags &= ~MXS_DMA_FLAGS_ALLOCATED;
331
332 return 0;
333 }
334
335 /*
336 * Allocate DMA descriptor
337 */
mxs_dma_desc_alloc(void)338 struct mxs_dma_desc *mxs_dma_desc_alloc(void)
339 {
340 struct mxs_dma_desc *pdesc;
341 uint32_t size;
342
343 size = roundup(sizeof(struct mxs_dma_desc), MXS_DMA_ALIGNMENT);
344 pdesc = memalign(MXS_DMA_ALIGNMENT, size);
345
346 if (pdesc == NULL)
347 return NULL;
348
349 memset(pdesc, 0, sizeof(*pdesc));
350 pdesc->address = (dma_addr_t)pdesc;
351
352 return pdesc;
353 };
354
355 /*
356 * Free DMA descriptor
357 */
mxs_dma_desc_free(struct mxs_dma_desc * pdesc)358 void mxs_dma_desc_free(struct mxs_dma_desc *pdesc)
359 {
360 if (pdesc == NULL)
361 return;
362
363 free(pdesc);
364 }
365
366 /*
367 * Add a DMA descriptor to a channel.
368 *
369 * If the descriptor list for this channel is not empty, this function sets the
370 * CHAIN bit and the NEXTCMD_ADDR fields in the last descriptor's DMA command so
371 * it will chain to the new descriptor's command.
372 *
373 * Then, this function marks the new descriptor as "ready," adds it to the end
374 * of the active descriptor list, and increments the count of pending
375 * descriptors.
376 *
377 * The MXS platform DMA software imposes some rules on DMA commands to maintain
378 * important invariants. These rules are NOT checked, but they must be carefully
379 * applied by software that uses MXS DMA channels.
380 *
381 * Invariant:
382 * The DMA channel's hardware semaphore must reflect the number of DMA
383 * commands the hardware will process, but has not yet finished.
384 *
385 * Explanation:
386 * A DMA channel begins processing commands when its hardware semaphore is
387 * written with a value greater than zero, and it stops processing commands
388 * when the semaphore returns to zero.
389 *
390 * When a channel finishes a DMA command, it will decrement its semaphore if
391 * the DECREMENT_SEMAPHORE bit is set in that command's flags bits.
392 *
393 * In principle, it's not necessary for the DECREMENT_SEMAPHORE to be set,
394 * unless it suits the purposes of the software. For example, one could
395 * construct a series of five DMA commands, with the DECREMENT_SEMAPHORE
396 * bit set only in the last one. Then, setting the DMA channel's hardware
397 * semaphore to one would cause the entire series of five commands to be
398 * processed. However, this example would violate the invariant given above.
399 *
400 * Rule:
401 * ALL DMA commands MUST have the DECREMENT_SEMAPHORE bit set so that the DMA
402 * channel's hardware semaphore will be decremented EVERY time a command is
403 * processed.
404 */
mxs_dma_desc_append(int channel,struct mxs_dma_desc * pdesc)405 int mxs_dma_desc_append(int channel, struct mxs_dma_desc *pdesc)
406 {
407 struct mxs_dma_chan *pchan;
408 struct mxs_dma_desc *last;
409 int ret;
410
411 ret = mxs_dma_validate_chan(channel);
412 if (ret)
413 return ret;
414
415 pchan = mxs_dma_channels + channel;
416
417 pdesc->cmd.next = mxs_dma_cmd_address(pdesc);
418 pdesc->flags |= MXS_DMA_DESC_FIRST | MXS_DMA_DESC_LAST;
419
420 if (!list_empty(&pchan->active)) {
421 last = list_entry(pchan->active.prev, struct mxs_dma_desc,
422 node);
423
424 pdesc->flags &= ~MXS_DMA_DESC_FIRST;
425 last->flags &= ~MXS_DMA_DESC_LAST;
426
427 last->cmd.next = mxs_dma_cmd_address(pdesc);
428 last->cmd.data |= MXS_DMA_DESC_CHAIN;
429
430 mxs_dma_flush_desc(last);
431 }
432 pdesc->flags |= MXS_DMA_DESC_READY;
433 if (pdesc->flags & MXS_DMA_DESC_FIRST)
434 pchan->pending_num++;
435 list_add_tail(&pdesc->node, &pchan->active);
436
437 mxs_dma_flush_desc(pdesc);
438
439 return ret;
440 }
441
442 /*
443 * Clean up processed DMA descriptors.
444 *
445 * This function removes processed DMA descriptors from the "active" list. Pass
446 * in a non-NULL list head to get the descriptors moved to your list. Pass NULL
447 * to get the descriptors moved to the channel's "done" list. Descriptors on
448 * the "done" list can be retrieved with mxs_dma_get_finished().
449 *
450 * This function marks the DMA channel as "not busy" if no unprocessed
451 * descriptors remain on the "active" list.
452 */
mxs_dma_finish(int channel,struct list_head * head)453 static int mxs_dma_finish(int channel, struct list_head *head)
454 {
455 int sem;
456 struct mxs_dma_chan *pchan;
457 struct list_head *p, *q;
458 struct mxs_dma_desc *pdesc;
459 int ret;
460
461 ret = mxs_dma_validate_chan(channel);
462 if (ret)
463 return ret;
464
465 pchan = mxs_dma_channels + channel;
466
467 sem = mxs_dma_read_semaphore(channel);
468 if (sem < 0)
469 return sem;
470
471 if (sem == pchan->active_num)
472 return 0;
473
474 list_for_each_safe(p, q, &pchan->active) {
475 if ((pchan->active_num) <= sem)
476 break;
477
478 pdesc = list_entry(p, struct mxs_dma_desc, node);
479 pdesc->flags &= ~MXS_DMA_DESC_READY;
480
481 if (head)
482 list_move_tail(p, head);
483 else
484 list_move_tail(p, &pchan->done);
485
486 if (pdesc->flags & MXS_DMA_DESC_LAST)
487 pchan->active_num--;
488 }
489
490 if (sem == 0)
491 pchan->flags &= ~MXS_DMA_FLAGS_BUSY;
492
493 return 0;
494 }
495
496 /*
497 * Wait for DMA channel to complete
498 */
mxs_dma_wait_complete(uint32_t timeout,unsigned int chan)499 static int mxs_dma_wait_complete(uint32_t timeout, unsigned int chan)
500 {
501 struct mxs_apbh_regs *apbh_regs =
502 (struct mxs_apbh_regs *)MXS_APBH_BASE;
503 int ret;
504
505 ret = mxs_dma_validate_chan(chan);
506 if (ret)
507 return ret;
508
509 if (mxs_wait_mask_set(&apbh_regs->hw_apbh_ctrl1_reg,
510 1 << chan, timeout)) {
511 ret = -ETIMEDOUT;
512 mxs_dma_reset(chan);
513 }
514
515 return ret;
516 }
517
518 /*
519 * Execute the DMA channel
520 */
mxs_dma_go(int chan)521 int mxs_dma_go(int chan)
522 {
523 uint32_t timeout = 10000000;
524 int ret;
525
526 LIST_HEAD(tmp_desc_list);
527
528 mxs_dma_enable_irq(chan, 1);
529 mxs_dma_enable(chan);
530
531 /* Wait for DMA to finish. */
532 ret = mxs_dma_wait_complete(timeout, chan);
533
534 /* Clear out the descriptors we just ran. */
535 mxs_dma_finish(chan, &tmp_desc_list);
536
537 /* Shut the DMA channel down. */
538 mxs_dma_ack_irq(chan);
539 mxs_dma_reset(chan);
540 mxs_dma_enable_irq(chan, 0);
541 mxs_dma_disable(chan);
542
543 return ret;
544 }
545
546 /*
547 * Execute a continuously running circular DMA descriptor.
548 * NOTE: This is not intended for general use, but rather
549 * for the LCD driver in Smart-LCD mode. It allows
550 * continuous triggering of the RUN bit there.
551 */
mxs_dma_circ_start(int chan,struct mxs_dma_desc * pdesc)552 void mxs_dma_circ_start(int chan, struct mxs_dma_desc *pdesc)
553 {
554 struct mxs_apbh_regs *apbh_regs =
555 (struct mxs_apbh_regs *)MXS_APBH_BASE;
556
557 mxs_dma_flush_desc(pdesc);
558
559 mxs_dma_enable_irq(chan, 1);
560
561 writel(mxs_dma_cmd_address(pdesc),
562 &apbh_regs->ch[chan].hw_apbh_ch_nxtcmdar);
563 writel(1, &apbh_regs->ch[chan].hw_apbh_ch_sema);
564 writel(1 << (chan + APBH_CTRL0_CLKGATE_CHANNEL_OFFSET),
565 &apbh_regs->hw_apbh_ctrl0_clr);
566 }
567
568 /*
569 * Initialize the DMA hardware
570 */
mxs_dma_init(void)571 void mxs_dma_init(void)
572 {
573 struct mxs_apbh_regs *apbh_regs =
574 (struct mxs_apbh_regs *)MXS_APBH_BASE;
575
576 mxs_reset_block(&apbh_regs->hw_apbh_ctrl0_reg);
577
578 #ifdef CONFIG_APBH_DMA_BURST8
579 writel(APBH_CTRL0_AHB_BURST8_EN,
580 &apbh_regs->hw_apbh_ctrl0_set);
581 #else
582 writel(APBH_CTRL0_AHB_BURST8_EN,
583 &apbh_regs->hw_apbh_ctrl0_clr);
584 #endif
585
586 #ifdef CONFIG_APBH_DMA_BURST
587 writel(APBH_CTRL0_APB_BURST_EN,
588 &apbh_regs->hw_apbh_ctrl0_set);
589 #else
590 writel(APBH_CTRL0_APB_BURST_EN,
591 &apbh_regs->hw_apbh_ctrl0_clr);
592 #endif
593 }
594
mxs_dma_init_channel(int channel)595 int mxs_dma_init_channel(int channel)
596 {
597 struct mxs_dma_chan *pchan;
598 int ret;
599
600 pchan = mxs_dma_channels + channel;
601 pchan->flags = MXS_DMA_FLAGS_VALID;
602
603 ret = mxs_dma_request(channel);
604
605 if (ret) {
606 printf("MXS DMA: Can't acquire DMA channel %i\n",
607 channel);
608 return ret;
609 }
610
611 mxs_dma_reset(channel);
612 mxs_dma_ack_irq(channel);
613
614 return 0;
615 }
616