1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Freescale i.MX28 APBH DMA driver 4 * 5 * Copyright (C) 2011 Marek Vasut <marek.vasut@gmail.com> 6 * on behalf of DENX Software Engineering GmbH 7 * 8 * Based on code from LTIB: 9 * Copyright (C) 2010 Freescale Semiconductor, Inc. All Rights Reserved. 10 */ 11 12 #include <linux/list.h> 13 14 #include <common.h> 15 #include <malloc.h> 16 #include <linux/errno.h> 17 #include <asm/io.h> 18 #include <asm/arch/clock.h> 19 #include <asm/arch/imx-regs.h> 20 #include <asm/arch/sys_proto.h> 21 #include <asm/mach-imx/dma.h> 22 #include <asm/mach-imx/regs-apbh.h> 23 24 static struct mxs_dma_chan mxs_dma_channels[MXS_MAX_DMA_CHANNELS]; 25 26 /* 27 * Test is the DMA channel is valid channel 28 */ 29 int mxs_dma_validate_chan(int channel) 30 { 31 struct mxs_dma_chan *pchan; 32 33 if ((channel < 0) || (channel >= MXS_MAX_DMA_CHANNELS)) 34 return -EINVAL; 35 36 pchan = mxs_dma_channels + channel; 37 if (!(pchan->flags & MXS_DMA_FLAGS_ALLOCATED)) 38 return -EINVAL; 39 40 return 0; 41 } 42 43 /* 44 * Return the address of the command within a descriptor. 45 */ 46 static unsigned int mxs_dma_cmd_address(struct mxs_dma_desc *desc) 47 { 48 return desc->address + offsetof(struct mxs_dma_desc, cmd); 49 } 50 51 /* 52 * Read a DMA channel's hardware semaphore. 53 * 54 * As used by the MXS platform's DMA software, the DMA channel's hardware 55 * semaphore reflects the number of DMA commands the hardware will process, but 56 * has not yet finished. This is a volatile value read directly from hardware, 57 * so it must be be viewed as immediately stale. 58 * 59 * If the channel is not marked busy, or has finished processing all its 60 * commands, this value should be zero. 61 * 62 * See mxs_dma_append() for details on how DMA command blocks must be configured 63 * to maintain the expected behavior of the semaphore's value. 64 */ 65 static int mxs_dma_read_semaphore(int channel) 66 { 67 struct mxs_apbh_regs *apbh_regs = 68 (struct mxs_apbh_regs *)MXS_APBH_BASE; 69 uint32_t tmp; 70 int ret; 71 72 ret = mxs_dma_validate_chan(channel); 73 if (ret) 74 return ret; 75 76 tmp = readl(&apbh_regs->ch[channel].hw_apbh_ch_sema); 77 78 tmp &= APBH_CHn_SEMA_PHORE_MASK; 79 tmp >>= APBH_CHn_SEMA_PHORE_OFFSET; 80 81 return tmp; 82 } 83 84 #ifndef CONFIG_SYS_DCACHE_OFF 85 void mxs_dma_flush_desc(struct mxs_dma_desc *desc) 86 { 87 uint32_t addr; 88 uint32_t size; 89 90 addr = (uint32_t)desc; 91 size = roundup(sizeof(struct mxs_dma_desc), MXS_DMA_ALIGNMENT); 92 93 flush_dcache_range(addr, addr + size); 94 } 95 #else 96 inline void mxs_dma_flush_desc(struct mxs_dma_desc *desc) {} 97 #endif 98 99 /* 100 * Enable a DMA channel. 101 * 102 * If the given channel has any DMA descriptors on its active list, this 103 * function causes the DMA hardware to begin processing them. 104 * 105 * This function marks the DMA channel as "busy," whether or not there are any 106 * descriptors to process. 107 */ 108 static int mxs_dma_enable(int channel) 109 { 110 struct mxs_apbh_regs *apbh_regs = 111 (struct mxs_apbh_regs *)MXS_APBH_BASE; 112 unsigned int sem; 113 struct mxs_dma_chan *pchan; 114 struct mxs_dma_desc *pdesc; 115 int ret; 116 117 ret = mxs_dma_validate_chan(channel); 118 if (ret) 119 return ret; 120 121 pchan = mxs_dma_channels + channel; 122 123 if (pchan->pending_num == 0) { 124 pchan->flags |= MXS_DMA_FLAGS_BUSY; 125 return 0; 126 } 127 128 pdesc = list_first_entry(&pchan->active, struct mxs_dma_desc, node); 129 if (pdesc == NULL) 130 return -EFAULT; 131 132 if (pchan->flags & MXS_DMA_FLAGS_BUSY) { 133 if (!(pdesc->cmd.data & MXS_DMA_DESC_CHAIN)) 134 return 0; 135 136 sem = mxs_dma_read_semaphore(channel); 137 if (sem == 0) 138 return 0; 139 140 if (sem == 1) { 141 pdesc = list_entry(pdesc->node.next, 142 struct mxs_dma_desc, node); 143 writel(mxs_dma_cmd_address(pdesc), 144 &apbh_regs->ch[channel].hw_apbh_ch_nxtcmdar); 145 } 146 writel(pchan->pending_num, 147 &apbh_regs->ch[channel].hw_apbh_ch_sema); 148 pchan->active_num += pchan->pending_num; 149 pchan->pending_num = 0; 150 } else { 151 pchan->active_num += pchan->pending_num; 152 pchan->pending_num = 0; 153 writel(mxs_dma_cmd_address(pdesc), 154 &apbh_regs->ch[channel].hw_apbh_ch_nxtcmdar); 155 writel(pchan->active_num, 156 &apbh_regs->ch[channel].hw_apbh_ch_sema); 157 writel(1 << (channel + APBH_CTRL0_CLKGATE_CHANNEL_OFFSET), 158 &apbh_regs->hw_apbh_ctrl0_clr); 159 } 160 161 pchan->flags |= MXS_DMA_FLAGS_BUSY; 162 return 0; 163 } 164 165 /* 166 * Disable a DMA channel. 167 * 168 * This function shuts down a DMA channel and marks it as "not busy." Any 169 * descriptors on the active list are immediately moved to the head of the 170 * "done" list, whether or not they have actually been processed by the 171 * hardware. The "ready" flags of these descriptors are NOT cleared, so they 172 * still appear to be active. 173 * 174 * This function immediately shuts down a DMA channel's hardware, aborting any 175 * I/O that may be in progress, potentially leaving I/O hardware in an undefined 176 * state. It is unwise to call this function if there is ANY chance the hardware 177 * is still processing a command. 178 */ 179 static int mxs_dma_disable(int channel) 180 { 181 struct mxs_dma_chan *pchan; 182 struct mxs_apbh_regs *apbh_regs = 183 (struct mxs_apbh_regs *)MXS_APBH_BASE; 184 int ret; 185 186 ret = mxs_dma_validate_chan(channel); 187 if (ret) 188 return ret; 189 190 pchan = mxs_dma_channels + channel; 191 192 if (!(pchan->flags & MXS_DMA_FLAGS_BUSY)) 193 return -EINVAL; 194 195 writel(1 << (channel + APBH_CTRL0_CLKGATE_CHANNEL_OFFSET), 196 &apbh_regs->hw_apbh_ctrl0_set); 197 198 pchan->flags &= ~MXS_DMA_FLAGS_BUSY; 199 pchan->active_num = 0; 200 pchan->pending_num = 0; 201 list_splice_init(&pchan->active, &pchan->done); 202 203 return 0; 204 } 205 206 /* 207 * Resets the DMA channel hardware. 208 */ 209 static int mxs_dma_reset(int channel) 210 { 211 struct mxs_apbh_regs *apbh_regs = 212 (struct mxs_apbh_regs *)MXS_APBH_BASE; 213 int ret; 214 #if defined(CONFIG_MX23) 215 uint32_t setreg = (uint32_t)(&apbh_regs->hw_apbh_ctrl0_set); 216 uint32_t offset = APBH_CTRL0_RESET_CHANNEL_OFFSET; 217 #elif (defined(CONFIG_MX28) || defined(CONFIG_MX6) || defined(CONFIG_MX7)) 218 uint32_t setreg = (uint32_t)(&apbh_regs->hw_apbh_channel_ctrl_set); 219 uint32_t offset = APBH_CHANNEL_CTRL_RESET_CHANNEL_OFFSET; 220 #endif 221 222 ret = mxs_dma_validate_chan(channel); 223 if (ret) 224 return ret; 225 226 writel(1 << (channel + offset), setreg); 227 228 return 0; 229 } 230 231 /* 232 * Enable or disable DMA interrupt. 233 * 234 * This function enables the given DMA channel to interrupt the CPU. 235 */ 236 static int mxs_dma_enable_irq(int channel, int enable) 237 { 238 struct mxs_apbh_regs *apbh_regs = 239 (struct mxs_apbh_regs *)MXS_APBH_BASE; 240 int ret; 241 242 ret = mxs_dma_validate_chan(channel); 243 if (ret) 244 return ret; 245 246 if (enable) 247 writel(1 << (channel + APBH_CTRL1_CH_CMDCMPLT_IRQ_EN_OFFSET), 248 &apbh_regs->hw_apbh_ctrl1_set); 249 else 250 writel(1 << (channel + APBH_CTRL1_CH_CMDCMPLT_IRQ_EN_OFFSET), 251 &apbh_regs->hw_apbh_ctrl1_clr); 252 253 return 0; 254 } 255 256 /* 257 * Clear DMA interrupt. 258 * 259 * The software that is using the DMA channel must register to receive its 260 * interrupts and, when they arrive, must call this function to clear them. 261 */ 262 static int mxs_dma_ack_irq(int channel) 263 { 264 struct mxs_apbh_regs *apbh_regs = 265 (struct mxs_apbh_regs *)MXS_APBH_BASE; 266 int ret; 267 268 ret = mxs_dma_validate_chan(channel); 269 if (ret) 270 return ret; 271 272 writel(1 << channel, &apbh_regs->hw_apbh_ctrl1_clr); 273 writel(1 << channel, &apbh_regs->hw_apbh_ctrl2_clr); 274 275 return 0; 276 } 277 278 /* 279 * Request to reserve a DMA channel 280 */ 281 static int mxs_dma_request(int channel) 282 { 283 struct mxs_dma_chan *pchan; 284 285 if ((channel < 0) || (channel >= MXS_MAX_DMA_CHANNELS)) 286 return -EINVAL; 287 288 pchan = mxs_dma_channels + channel; 289 if ((pchan->flags & MXS_DMA_FLAGS_VALID) != MXS_DMA_FLAGS_VALID) 290 return -ENODEV; 291 292 if (pchan->flags & MXS_DMA_FLAGS_ALLOCATED) 293 return -EBUSY; 294 295 pchan->flags |= MXS_DMA_FLAGS_ALLOCATED; 296 pchan->active_num = 0; 297 pchan->pending_num = 0; 298 299 INIT_LIST_HEAD(&pchan->active); 300 INIT_LIST_HEAD(&pchan->done); 301 302 return 0; 303 } 304 305 /* 306 * Release a DMA channel. 307 * 308 * This function releases a DMA channel from its current owner. 309 * 310 * The channel will NOT be released if it's marked "busy" (see 311 * mxs_dma_enable()). 312 */ 313 int mxs_dma_release(int channel) 314 { 315 struct mxs_dma_chan *pchan; 316 int ret; 317 318 ret = mxs_dma_validate_chan(channel); 319 if (ret) 320 return ret; 321 322 pchan = mxs_dma_channels + channel; 323 324 if (pchan->flags & MXS_DMA_FLAGS_BUSY) 325 return -EBUSY; 326 327 pchan->dev = 0; 328 pchan->active_num = 0; 329 pchan->pending_num = 0; 330 pchan->flags &= ~MXS_DMA_FLAGS_ALLOCATED; 331 332 return 0; 333 } 334 335 /* 336 * Allocate DMA descriptor 337 */ 338 struct mxs_dma_desc *mxs_dma_desc_alloc(void) 339 { 340 struct mxs_dma_desc *pdesc; 341 uint32_t size; 342 343 size = roundup(sizeof(struct mxs_dma_desc), MXS_DMA_ALIGNMENT); 344 pdesc = memalign(MXS_DMA_ALIGNMENT, size); 345 346 if (pdesc == NULL) 347 return NULL; 348 349 memset(pdesc, 0, sizeof(*pdesc)); 350 pdesc->address = (dma_addr_t)pdesc; 351 352 return pdesc; 353 }; 354 355 /* 356 * Free DMA descriptor 357 */ 358 void mxs_dma_desc_free(struct mxs_dma_desc *pdesc) 359 { 360 if (pdesc == NULL) 361 return; 362 363 free(pdesc); 364 } 365 366 /* 367 * Add a DMA descriptor to a channel. 368 * 369 * If the descriptor list for this channel is not empty, this function sets the 370 * CHAIN bit and the NEXTCMD_ADDR fields in the last descriptor's DMA command so 371 * it will chain to the new descriptor's command. 372 * 373 * Then, this function marks the new descriptor as "ready," adds it to the end 374 * of the active descriptor list, and increments the count of pending 375 * descriptors. 376 * 377 * The MXS platform DMA software imposes some rules on DMA commands to maintain 378 * important invariants. These rules are NOT checked, but they must be carefully 379 * applied by software that uses MXS DMA channels. 380 * 381 * Invariant: 382 * The DMA channel's hardware semaphore must reflect the number of DMA 383 * commands the hardware will process, but has not yet finished. 384 * 385 * Explanation: 386 * A DMA channel begins processing commands when its hardware semaphore is 387 * written with a value greater than zero, and it stops processing commands 388 * when the semaphore returns to zero. 389 * 390 * When a channel finishes a DMA command, it will decrement its semaphore if 391 * the DECREMENT_SEMAPHORE bit is set in that command's flags bits. 392 * 393 * In principle, it's not necessary for the DECREMENT_SEMAPHORE to be set, 394 * unless it suits the purposes of the software. For example, one could 395 * construct a series of five DMA commands, with the DECREMENT_SEMAPHORE 396 * bit set only in the last one. Then, setting the DMA channel's hardware 397 * semaphore to one would cause the entire series of five commands to be 398 * processed. However, this example would violate the invariant given above. 399 * 400 * Rule: 401 * ALL DMA commands MUST have the DECREMENT_SEMAPHORE bit set so that the DMA 402 * channel's hardware semaphore will be decremented EVERY time a command is 403 * processed. 404 */ 405 int mxs_dma_desc_append(int channel, struct mxs_dma_desc *pdesc) 406 { 407 struct mxs_dma_chan *pchan; 408 struct mxs_dma_desc *last; 409 int ret; 410 411 ret = mxs_dma_validate_chan(channel); 412 if (ret) 413 return ret; 414 415 pchan = mxs_dma_channels + channel; 416 417 pdesc->cmd.next = mxs_dma_cmd_address(pdesc); 418 pdesc->flags |= MXS_DMA_DESC_FIRST | MXS_DMA_DESC_LAST; 419 420 if (!list_empty(&pchan->active)) { 421 last = list_entry(pchan->active.prev, struct mxs_dma_desc, 422 node); 423 424 pdesc->flags &= ~MXS_DMA_DESC_FIRST; 425 last->flags &= ~MXS_DMA_DESC_LAST; 426 427 last->cmd.next = mxs_dma_cmd_address(pdesc); 428 last->cmd.data |= MXS_DMA_DESC_CHAIN; 429 430 mxs_dma_flush_desc(last); 431 } 432 pdesc->flags |= MXS_DMA_DESC_READY; 433 if (pdesc->flags & MXS_DMA_DESC_FIRST) 434 pchan->pending_num++; 435 list_add_tail(&pdesc->node, &pchan->active); 436 437 mxs_dma_flush_desc(pdesc); 438 439 return ret; 440 } 441 442 /* 443 * Clean up processed DMA descriptors. 444 * 445 * This function removes processed DMA descriptors from the "active" list. Pass 446 * in a non-NULL list head to get the descriptors moved to your list. Pass NULL 447 * to get the descriptors moved to the channel's "done" list. Descriptors on 448 * the "done" list can be retrieved with mxs_dma_get_finished(). 449 * 450 * This function marks the DMA channel as "not busy" if no unprocessed 451 * descriptors remain on the "active" list. 452 */ 453 static int mxs_dma_finish(int channel, struct list_head *head) 454 { 455 int sem; 456 struct mxs_dma_chan *pchan; 457 struct list_head *p, *q; 458 struct mxs_dma_desc *pdesc; 459 int ret; 460 461 ret = mxs_dma_validate_chan(channel); 462 if (ret) 463 return ret; 464 465 pchan = mxs_dma_channels + channel; 466 467 sem = mxs_dma_read_semaphore(channel); 468 if (sem < 0) 469 return sem; 470 471 if (sem == pchan->active_num) 472 return 0; 473 474 list_for_each_safe(p, q, &pchan->active) { 475 if ((pchan->active_num) <= sem) 476 break; 477 478 pdesc = list_entry(p, struct mxs_dma_desc, node); 479 pdesc->flags &= ~MXS_DMA_DESC_READY; 480 481 if (head) 482 list_move_tail(p, head); 483 else 484 list_move_tail(p, &pchan->done); 485 486 if (pdesc->flags & MXS_DMA_DESC_LAST) 487 pchan->active_num--; 488 } 489 490 if (sem == 0) 491 pchan->flags &= ~MXS_DMA_FLAGS_BUSY; 492 493 return 0; 494 } 495 496 /* 497 * Wait for DMA channel to complete 498 */ 499 static int mxs_dma_wait_complete(uint32_t timeout, unsigned int chan) 500 { 501 struct mxs_apbh_regs *apbh_regs = 502 (struct mxs_apbh_regs *)MXS_APBH_BASE; 503 int ret; 504 505 ret = mxs_dma_validate_chan(chan); 506 if (ret) 507 return ret; 508 509 if (mxs_wait_mask_set(&apbh_regs->hw_apbh_ctrl1_reg, 510 1 << chan, timeout)) { 511 ret = -ETIMEDOUT; 512 mxs_dma_reset(chan); 513 } 514 515 return ret; 516 } 517 518 /* 519 * Execute the DMA channel 520 */ 521 int mxs_dma_go(int chan) 522 { 523 uint32_t timeout = 10000000; 524 int ret; 525 526 LIST_HEAD(tmp_desc_list); 527 528 mxs_dma_enable_irq(chan, 1); 529 mxs_dma_enable(chan); 530 531 /* Wait for DMA to finish. */ 532 ret = mxs_dma_wait_complete(timeout, chan); 533 534 /* Clear out the descriptors we just ran. */ 535 mxs_dma_finish(chan, &tmp_desc_list); 536 537 /* Shut the DMA channel down. */ 538 mxs_dma_ack_irq(chan); 539 mxs_dma_reset(chan); 540 mxs_dma_enable_irq(chan, 0); 541 mxs_dma_disable(chan); 542 543 return ret; 544 } 545 546 /* 547 * Execute a continuously running circular DMA descriptor. 548 * NOTE: This is not intended for general use, but rather 549 * for the LCD driver in Smart-LCD mode. It allows 550 * continuous triggering of the RUN bit there. 551 */ 552 void mxs_dma_circ_start(int chan, struct mxs_dma_desc *pdesc) 553 { 554 struct mxs_apbh_regs *apbh_regs = 555 (struct mxs_apbh_regs *)MXS_APBH_BASE; 556 557 mxs_dma_flush_desc(pdesc); 558 559 mxs_dma_enable_irq(chan, 1); 560 561 writel(mxs_dma_cmd_address(pdesc), 562 &apbh_regs->ch[chan].hw_apbh_ch_nxtcmdar); 563 writel(1, &apbh_regs->ch[chan].hw_apbh_ch_sema); 564 writel(1 << (chan + APBH_CTRL0_CLKGATE_CHANNEL_OFFSET), 565 &apbh_regs->hw_apbh_ctrl0_clr); 566 } 567 568 /* 569 * Initialize the DMA hardware 570 */ 571 void mxs_dma_init(void) 572 { 573 struct mxs_apbh_regs *apbh_regs = 574 (struct mxs_apbh_regs *)MXS_APBH_BASE; 575 576 mxs_reset_block(&apbh_regs->hw_apbh_ctrl0_reg); 577 578 #ifdef CONFIG_APBH_DMA_BURST8 579 writel(APBH_CTRL0_AHB_BURST8_EN, 580 &apbh_regs->hw_apbh_ctrl0_set); 581 #else 582 writel(APBH_CTRL0_AHB_BURST8_EN, 583 &apbh_regs->hw_apbh_ctrl0_clr); 584 #endif 585 586 #ifdef CONFIG_APBH_DMA_BURST 587 writel(APBH_CTRL0_APB_BURST_EN, 588 &apbh_regs->hw_apbh_ctrl0_set); 589 #else 590 writel(APBH_CTRL0_APB_BURST_EN, 591 &apbh_regs->hw_apbh_ctrl0_clr); 592 #endif 593 } 594 595 int mxs_dma_init_channel(int channel) 596 { 597 struct mxs_dma_chan *pchan; 598 int ret; 599 600 pchan = mxs_dma_channels + channel; 601 pchan->flags = MXS_DMA_FLAGS_VALID; 602 603 ret = mxs_dma_request(channel); 604 605 if (ret) { 606 printf("MXS DMA: Can't acquire DMA channel %i\n", 607 channel); 608 return ret; 609 } 610 611 mxs_dma_reset(channel); 612 mxs_dma_ack_irq(channel); 613 614 return 0; 615 } 616