1 /* 2 * Freescale MPC85xx, MPC83xx DMA Engine support 3 * 4 * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved. 5 * 6 * Author: 7 * Zhang Wei <wei.zhang@freescale.com>, Jul 2007 8 * Ebony Zhu <ebony.zhu@freescale.com>, May 2007 9 * 10 * Description: 11 * DMA engine driver for Freescale MPC8540 DMA controller, which is 12 * also fit for MPC8560, MPC8555, MPC8548, MPC8641, and etc. 13 * The support for MPC8349 DMA contorller is also added. 14 * 15 * This is free software; you can redistribute it and/or modify 16 * it under the terms of the GNU General Public License as published by 17 * the Free Software Foundation; either version 2 of the License, or 18 * (at your option) any later version. 19 * 20 */ 21 22 #include <linux/init.h> 23 #include <linux/module.h> 24 #include <linux/pci.h> 25 #include <linux/interrupt.h> 26 #include <linux/dmaengine.h> 27 #include <linux/delay.h> 28 #include <linux/dma-mapping.h> 29 #include <linux/dmapool.h> 30 #include <linux/of_platform.h> 31 32 #include "fsldma.h" 33 34 static void dma_init(struct fsl_dma_chan *fsl_chan) 35 { 36 /* Reset the channel */ 37 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, 0, 32); 38 39 switch (fsl_chan->feature & FSL_DMA_IP_MASK) { 40 case FSL_DMA_IP_85XX: 41 /* Set the channel to below modes: 42 * EIE - Error interrupt enable 43 * EOSIE - End of segments interrupt enable (basic mode) 44 * EOLNIE - End of links interrupt enable 45 */ 46 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, FSL_DMA_MR_EIE 47 | FSL_DMA_MR_EOLNIE | FSL_DMA_MR_EOSIE, 32); 48 break; 49 case FSL_DMA_IP_83XX: 50 /* Set the channel to below modes: 51 * EOTIE - End-of-transfer interrupt enable 52 */ 53 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, FSL_DMA_MR_EOTIE, 54 32); 55 break; 56 } 57 58 } 59 60 static void set_sr(struct fsl_dma_chan *fsl_chan, u32 val) 61 { 62 DMA_OUT(fsl_chan, &fsl_chan->reg_base->sr, val, 32); 63 } 64 65 static u32 get_sr(struct fsl_dma_chan *fsl_chan) 66 { 67 return DMA_IN(fsl_chan, &fsl_chan->reg_base->sr, 32); 68 } 69 70 static void set_desc_cnt(struct fsl_dma_chan *fsl_chan, 71 struct fsl_dma_ld_hw *hw, u32 count) 72 { 73 hw->count = CPU_TO_DMA(fsl_chan, count, 32); 74 } 75 76 static void set_desc_src(struct fsl_dma_chan *fsl_chan, 77 struct fsl_dma_ld_hw *hw, dma_addr_t src) 78 { 79 u64 snoop_bits; 80 81 snoop_bits = ((fsl_chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) 82 ? ((u64)FSL_DMA_SATR_SREADTYPE_SNOOP_READ << 32) : 0; 83 hw->src_addr = CPU_TO_DMA(fsl_chan, snoop_bits | src, 64); 84 } 85 86 static void set_desc_dest(struct fsl_dma_chan *fsl_chan, 87 struct fsl_dma_ld_hw *hw, dma_addr_t dest) 88 { 89 u64 snoop_bits; 90 91 snoop_bits = ((fsl_chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) 92 ? ((u64)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE << 32) : 0; 93 hw->dst_addr = CPU_TO_DMA(fsl_chan, snoop_bits | dest, 64); 94 } 95 96 static void set_desc_next(struct fsl_dma_chan *fsl_chan, 97 struct fsl_dma_ld_hw *hw, dma_addr_t next) 98 { 99 u64 snoop_bits; 100 101 snoop_bits = ((fsl_chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX) 102 ? FSL_DMA_SNEN : 0; 103 hw->next_ln_addr = CPU_TO_DMA(fsl_chan, snoop_bits | next, 64); 104 } 105 106 static void set_cdar(struct fsl_dma_chan *fsl_chan, dma_addr_t addr) 107 { 108 DMA_OUT(fsl_chan, &fsl_chan->reg_base->cdar, addr | FSL_DMA_SNEN, 64); 109 } 110 111 static dma_addr_t get_cdar(struct fsl_dma_chan *fsl_chan) 112 { 113 return DMA_IN(fsl_chan, &fsl_chan->reg_base->cdar, 64) & ~FSL_DMA_SNEN; 114 } 115 116 static void set_ndar(struct fsl_dma_chan *fsl_chan, dma_addr_t addr) 117 { 118 DMA_OUT(fsl_chan, &fsl_chan->reg_base->ndar, addr, 64); 119 } 120 121 static dma_addr_t get_ndar(struct fsl_dma_chan *fsl_chan) 122 { 123 return DMA_IN(fsl_chan, &fsl_chan->reg_base->ndar, 64); 124 } 125 126 static u32 get_bcr(struct fsl_dma_chan *fsl_chan) 127 { 128 return DMA_IN(fsl_chan, &fsl_chan->reg_base->bcr, 32); 129 } 130 131 static int dma_is_idle(struct fsl_dma_chan *fsl_chan) 132 { 133 u32 sr = get_sr(fsl_chan); 134 return (!(sr & FSL_DMA_SR_CB)) || (sr & FSL_DMA_SR_CH); 135 } 136 137 static void dma_start(struct fsl_dma_chan *fsl_chan) 138 { 139 u32 mr_set = 0;; 140 141 if (fsl_chan->feature & FSL_DMA_CHAN_PAUSE_EXT) { 142 DMA_OUT(fsl_chan, &fsl_chan->reg_base->bcr, 0, 32); 143 mr_set |= FSL_DMA_MR_EMP_EN; 144 } else 145 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, 146 DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) 147 & ~FSL_DMA_MR_EMP_EN, 32); 148 149 if (fsl_chan->feature & FSL_DMA_CHAN_START_EXT) 150 mr_set |= FSL_DMA_MR_EMS_EN; 151 else 152 mr_set |= FSL_DMA_MR_CS; 153 154 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, 155 DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) 156 | mr_set, 32); 157 } 158 159 static void dma_halt(struct fsl_dma_chan *fsl_chan) 160 { 161 int i; 162 163 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, 164 DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) | FSL_DMA_MR_CA, 165 32); 166 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, 167 DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) & ~(FSL_DMA_MR_CS 168 | FSL_DMA_MR_EMS_EN | FSL_DMA_MR_CA), 32); 169 170 for (i = 0; i < 100; i++) { 171 if (dma_is_idle(fsl_chan)) 172 break; 173 udelay(10); 174 } 175 if (i >= 100 && !dma_is_idle(fsl_chan)) 176 dev_err(fsl_chan->dev, "DMA halt timeout!\n"); 177 } 178 179 static void set_ld_eol(struct fsl_dma_chan *fsl_chan, 180 struct fsl_desc_sw *desc) 181 { 182 desc->hw.next_ln_addr = CPU_TO_DMA(fsl_chan, 183 DMA_TO_CPU(fsl_chan, desc->hw.next_ln_addr, 64) | FSL_DMA_EOL, 184 64); 185 } 186 187 static void append_ld_queue(struct fsl_dma_chan *fsl_chan, 188 struct fsl_desc_sw *new_desc) 189 { 190 struct fsl_desc_sw *queue_tail = to_fsl_desc(fsl_chan->ld_queue.prev); 191 192 if (list_empty(&fsl_chan->ld_queue)) 193 return; 194 195 /* Link to the new descriptor physical address and 196 * Enable End-of-segment interrupt for 197 * the last link descriptor. 198 * (the previous node's next link descriptor) 199 * 200 * For FSL_DMA_IP_83xx, the snoop enable bit need be set. 201 */ 202 queue_tail->hw.next_ln_addr = CPU_TO_DMA(fsl_chan, 203 new_desc->async_tx.phys | FSL_DMA_EOSIE | 204 (((fsl_chan->feature & FSL_DMA_IP_MASK) 205 == FSL_DMA_IP_83XX) ? FSL_DMA_SNEN : 0), 64); 206 } 207 208 /** 209 * fsl_chan_set_src_loop_size - Set source address hold transfer size 210 * @fsl_chan : Freescale DMA channel 211 * @size : Address loop size, 0 for disable loop 212 * 213 * The set source address hold transfer size. The source 214 * address hold or loop transfer size is when the DMA transfer 215 * data from source address (SA), if the loop size is 4, the DMA will 216 * read data from SA, SA + 1, SA + 2, SA + 3, then loop back to SA, 217 * SA + 1 ... and so on. 218 */ 219 static void fsl_chan_set_src_loop_size(struct fsl_dma_chan *fsl_chan, int size) 220 { 221 switch (size) { 222 case 0: 223 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, 224 DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) & 225 (~FSL_DMA_MR_SAHE), 32); 226 break; 227 case 1: 228 case 2: 229 case 4: 230 case 8: 231 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, 232 DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) | 233 FSL_DMA_MR_SAHE | (__ilog2(size) << 14), 234 32); 235 break; 236 } 237 } 238 239 /** 240 * fsl_chan_set_dest_loop_size - Set destination address hold transfer size 241 * @fsl_chan : Freescale DMA channel 242 * @size : Address loop size, 0 for disable loop 243 * 244 * The set destination address hold transfer size. The destination 245 * address hold or loop transfer size is when the DMA transfer 246 * data to destination address (TA), if the loop size is 4, the DMA will 247 * write data to TA, TA + 1, TA + 2, TA + 3, then loop back to TA, 248 * TA + 1 ... and so on. 249 */ 250 static void fsl_chan_set_dest_loop_size(struct fsl_dma_chan *fsl_chan, int size) 251 { 252 switch (size) { 253 case 0: 254 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, 255 DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) & 256 (~FSL_DMA_MR_DAHE), 32); 257 break; 258 case 1: 259 case 2: 260 case 4: 261 case 8: 262 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, 263 DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) | 264 FSL_DMA_MR_DAHE | (__ilog2(size) << 16), 265 32); 266 break; 267 } 268 } 269 270 /** 271 * fsl_chan_toggle_ext_pause - Toggle channel external pause status 272 * @fsl_chan : Freescale DMA channel 273 * @size : Pause control size, 0 for disable external pause control. 274 * The maximum is 1024. 275 * 276 * The Freescale DMA channel can be controlled by the external 277 * signal DREQ#. The pause control size is how many bytes are allowed 278 * to transfer before pausing the channel, after which a new assertion 279 * of DREQ# resumes channel operation. 280 */ 281 static void fsl_chan_toggle_ext_pause(struct fsl_dma_chan *fsl_chan, int size) 282 { 283 if (size > 1024) 284 return; 285 286 if (size) { 287 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, 288 DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) 289 | ((__ilog2(size) << 24) & 0x0f000000), 290 32); 291 fsl_chan->feature |= FSL_DMA_CHAN_PAUSE_EXT; 292 } else 293 fsl_chan->feature &= ~FSL_DMA_CHAN_PAUSE_EXT; 294 } 295 296 /** 297 * fsl_chan_toggle_ext_start - Toggle channel external start status 298 * @fsl_chan : Freescale DMA channel 299 * @enable : 0 is disabled, 1 is enabled. 300 * 301 * If enable the external start, the channel can be started by an 302 * external DMA start pin. So the dma_start() does not start the 303 * transfer immediately. The DMA channel will wait for the 304 * control pin asserted. 305 */ 306 static void fsl_chan_toggle_ext_start(struct fsl_dma_chan *fsl_chan, int enable) 307 { 308 if (enable) 309 fsl_chan->feature |= FSL_DMA_CHAN_START_EXT; 310 else 311 fsl_chan->feature &= ~FSL_DMA_CHAN_START_EXT; 312 } 313 314 static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx) 315 { 316 struct fsl_desc_sw *desc = tx_to_fsl_desc(tx); 317 struct fsl_dma_chan *fsl_chan = to_fsl_chan(tx->chan); 318 unsigned long flags; 319 dma_cookie_t cookie; 320 321 /* cookie increment and adding to ld_queue must be atomic */ 322 spin_lock_irqsave(&fsl_chan->desc_lock, flags); 323 324 cookie = fsl_chan->common.cookie; 325 cookie++; 326 if (cookie < 0) 327 cookie = 1; 328 desc->async_tx.cookie = cookie; 329 fsl_chan->common.cookie = desc->async_tx.cookie; 330 331 append_ld_queue(fsl_chan, desc); 332 list_splice_init(&desc->async_tx.tx_list, fsl_chan->ld_queue.prev); 333 334 spin_unlock_irqrestore(&fsl_chan->desc_lock, flags); 335 336 return cookie; 337 } 338 339 /** 340 * fsl_dma_alloc_descriptor - Allocate descriptor from channel's DMA pool. 341 * @fsl_chan : Freescale DMA channel 342 * 343 * Return - The descriptor allocated. NULL for failed. 344 */ 345 static struct fsl_desc_sw *fsl_dma_alloc_descriptor( 346 struct fsl_dma_chan *fsl_chan) 347 { 348 dma_addr_t pdesc; 349 struct fsl_desc_sw *desc_sw; 350 351 desc_sw = dma_pool_alloc(fsl_chan->desc_pool, GFP_ATOMIC, &pdesc); 352 if (desc_sw) { 353 memset(desc_sw, 0, sizeof(struct fsl_desc_sw)); 354 dma_async_tx_descriptor_init(&desc_sw->async_tx, 355 &fsl_chan->common); 356 desc_sw->async_tx.tx_submit = fsl_dma_tx_submit; 357 desc_sw->async_tx.phys = pdesc; 358 } 359 360 return desc_sw; 361 } 362 363 364 /** 365 * fsl_dma_alloc_chan_resources - Allocate resources for DMA channel. 366 * @fsl_chan : Freescale DMA channel 367 * 368 * This function will create a dma pool for descriptor allocation. 369 * 370 * Return - The number of descriptors allocated. 371 */ 372 static int fsl_dma_alloc_chan_resources(struct dma_chan *chan) 373 { 374 struct fsl_dma_chan *fsl_chan = to_fsl_chan(chan); 375 376 /* Has this channel already been allocated? */ 377 if (fsl_chan->desc_pool) 378 return 1; 379 380 /* We need the descriptor to be aligned to 32bytes 381 * for meeting FSL DMA specification requirement. 382 */ 383 fsl_chan->desc_pool = dma_pool_create("fsl_dma_engine_desc_pool", 384 fsl_chan->dev, sizeof(struct fsl_desc_sw), 385 32, 0); 386 if (!fsl_chan->desc_pool) { 387 dev_err(fsl_chan->dev, "No memory for channel %d " 388 "descriptor dma pool.\n", fsl_chan->id); 389 return 0; 390 } 391 392 return 1; 393 } 394 395 /** 396 * fsl_dma_free_chan_resources - Free all resources of the channel. 397 * @fsl_chan : Freescale DMA channel 398 */ 399 static void fsl_dma_free_chan_resources(struct dma_chan *chan) 400 { 401 struct fsl_dma_chan *fsl_chan = to_fsl_chan(chan); 402 struct fsl_desc_sw *desc, *_desc; 403 unsigned long flags; 404 405 dev_dbg(fsl_chan->dev, "Free all channel resources.\n"); 406 spin_lock_irqsave(&fsl_chan->desc_lock, flags); 407 list_for_each_entry_safe(desc, _desc, &fsl_chan->ld_queue, node) { 408 #ifdef FSL_DMA_LD_DEBUG 409 dev_dbg(fsl_chan->dev, 410 "LD %p will be released.\n", desc); 411 #endif 412 list_del(&desc->node); 413 /* free link descriptor */ 414 dma_pool_free(fsl_chan->desc_pool, desc, desc->async_tx.phys); 415 } 416 spin_unlock_irqrestore(&fsl_chan->desc_lock, flags); 417 dma_pool_destroy(fsl_chan->desc_pool); 418 419 fsl_chan->desc_pool = NULL; 420 } 421 422 static struct dma_async_tx_descriptor * 423 fsl_dma_prep_interrupt(struct dma_chan *chan, unsigned long flags) 424 { 425 struct fsl_dma_chan *fsl_chan; 426 struct fsl_desc_sw *new; 427 428 if (!chan) 429 return NULL; 430 431 fsl_chan = to_fsl_chan(chan); 432 433 new = fsl_dma_alloc_descriptor(fsl_chan); 434 if (!new) { 435 dev_err(fsl_chan->dev, "No free memory for link descriptor\n"); 436 return NULL; 437 } 438 439 new->async_tx.cookie = -EBUSY; 440 new->async_tx.flags = flags; 441 442 /* Insert the link descriptor to the LD ring */ 443 list_add_tail(&new->node, &new->async_tx.tx_list); 444 445 /* Set End-of-link to the last link descriptor of new list*/ 446 set_ld_eol(fsl_chan, new); 447 448 return &new->async_tx; 449 } 450 451 static struct dma_async_tx_descriptor *fsl_dma_prep_memcpy( 452 struct dma_chan *chan, dma_addr_t dma_dest, dma_addr_t dma_src, 453 size_t len, unsigned long flags) 454 { 455 struct fsl_dma_chan *fsl_chan; 456 struct fsl_desc_sw *first = NULL, *prev = NULL, *new; 457 size_t copy; 458 LIST_HEAD(link_chain); 459 460 if (!chan) 461 return NULL; 462 463 if (!len) 464 return NULL; 465 466 fsl_chan = to_fsl_chan(chan); 467 468 do { 469 470 /* Allocate the link descriptor from DMA pool */ 471 new = fsl_dma_alloc_descriptor(fsl_chan); 472 if (!new) { 473 dev_err(fsl_chan->dev, 474 "No free memory for link descriptor\n"); 475 return NULL; 476 } 477 #ifdef FSL_DMA_LD_DEBUG 478 dev_dbg(fsl_chan->dev, "new link desc alloc %p\n", new); 479 #endif 480 481 copy = min(len, (size_t)FSL_DMA_BCR_MAX_CNT); 482 483 set_desc_cnt(fsl_chan, &new->hw, copy); 484 set_desc_src(fsl_chan, &new->hw, dma_src); 485 set_desc_dest(fsl_chan, &new->hw, dma_dest); 486 487 if (!first) 488 first = new; 489 else 490 set_desc_next(fsl_chan, &prev->hw, new->async_tx.phys); 491 492 new->async_tx.cookie = 0; 493 async_tx_ack(&new->async_tx); 494 495 prev = new; 496 len -= copy; 497 dma_src += copy; 498 dma_dest += copy; 499 500 /* Insert the link descriptor to the LD ring */ 501 list_add_tail(&new->node, &first->async_tx.tx_list); 502 } while (len); 503 504 new->async_tx.flags = flags; /* client is in control of this ack */ 505 new->async_tx.cookie = -EBUSY; 506 507 /* Set End-of-link to the last link descriptor of new list*/ 508 set_ld_eol(fsl_chan, new); 509 510 return first ? &first->async_tx : NULL; 511 } 512 513 /** 514 * fsl_dma_update_completed_cookie - Update the completed cookie. 515 * @fsl_chan : Freescale DMA channel 516 */ 517 static void fsl_dma_update_completed_cookie(struct fsl_dma_chan *fsl_chan) 518 { 519 struct fsl_desc_sw *cur_desc, *desc; 520 dma_addr_t ld_phy; 521 522 ld_phy = get_cdar(fsl_chan) & FSL_DMA_NLDA_MASK; 523 524 if (ld_phy) { 525 cur_desc = NULL; 526 list_for_each_entry(desc, &fsl_chan->ld_queue, node) 527 if (desc->async_tx.phys == ld_phy) { 528 cur_desc = desc; 529 break; 530 } 531 532 if (cur_desc && cur_desc->async_tx.cookie) { 533 if (dma_is_idle(fsl_chan)) 534 fsl_chan->completed_cookie = 535 cur_desc->async_tx.cookie; 536 else 537 fsl_chan->completed_cookie = 538 cur_desc->async_tx.cookie - 1; 539 } 540 } 541 } 542 543 /** 544 * fsl_chan_ld_cleanup - Clean up link descriptors 545 * @fsl_chan : Freescale DMA channel 546 * 547 * This function clean up the ld_queue of DMA channel. 548 * If 'in_intr' is set, the function will move the link descriptor to 549 * the recycle list. Otherwise, free it directly. 550 */ 551 static void fsl_chan_ld_cleanup(struct fsl_dma_chan *fsl_chan) 552 { 553 struct fsl_desc_sw *desc, *_desc; 554 unsigned long flags; 555 556 spin_lock_irqsave(&fsl_chan->desc_lock, flags); 557 558 dev_dbg(fsl_chan->dev, "chan completed_cookie = %d\n", 559 fsl_chan->completed_cookie); 560 list_for_each_entry_safe(desc, _desc, &fsl_chan->ld_queue, node) { 561 dma_async_tx_callback callback; 562 void *callback_param; 563 564 if (dma_async_is_complete(desc->async_tx.cookie, 565 fsl_chan->completed_cookie, fsl_chan->common.cookie) 566 == DMA_IN_PROGRESS) 567 break; 568 569 callback = desc->async_tx.callback; 570 callback_param = desc->async_tx.callback_param; 571 572 /* Remove from ld_queue list */ 573 list_del(&desc->node); 574 575 dev_dbg(fsl_chan->dev, "link descriptor %p will be recycle.\n", 576 desc); 577 dma_pool_free(fsl_chan->desc_pool, desc, desc->async_tx.phys); 578 579 /* Run the link descriptor callback function */ 580 if (callback) { 581 spin_unlock_irqrestore(&fsl_chan->desc_lock, flags); 582 dev_dbg(fsl_chan->dev, "link descriptor %p callback\n", 583 desc); 584 callback(callback_param); 585 spin_lock_irqsave(&fsl_chan->desc_lock, flags); 586 } 587 } 588 spin_unlock_irqrestore(&fsl_chan->desc_lock, flags); 589 } 590 591 /** 592 * fsl_chan_xfer_ld_queue - Transfer link descriptors in channel ld_queue. 593 * @fsl_chan : Freescale DMA channel 594 */ 595 static void fsl_chan_xfer_ld_queue(struct fsl_dma_chan *fsl_chan) 596 { 597 struct list_head *ld_node; 598 dma_addr_t next_dest_addr; 599 unsigned long flags; 600 601 if (!dma_is_idle(fsl_chan)) 602 return; 603 604 dma_halt(fsl_chan); 605 606 /* If there are some link descriptors 607 * not transfered in queue. We need to start it. 608 */ 609 spin_lock_irqsave(&fsl_chan->desc_lock, flags); 610 611 /* Find the first un-transfer desciptor */ 612 for (ld_node = fsl_chan->ld_queue.next; 613 (ld_node != &fsl_chan->ld_queue) 614 && (dma_async_is_complete( 615 to_fsl_desc(ld_node)->async_tx.cookie, 616 fsl_chan->completed_cookie, 617 fsl_chan->common.cookie) == DMA_SUCCESS); 618 ld_node = ld_node->next); 619 620 spin_unlock_irqrestore(&fsl_chan->desc_lock, flags); 621 622 if (ld_node != &fsl_chan->ld_queue) { 623 /* Get the ld start address from ld_queue */ 624 next_dest_addr = to_fsl_desc(ld_node)->async_tx.phys; 625 dev_dbg(fsl_chan->dev, "xfer LDs staring from %p\n", 626 (void *)next_dest_addr); 627 set_cdar(fsl_chan, next_dest_addr); 628 dma_start(fsl_chan); 629 } else { 630 set_cdar(fsl_chan, 0); 631 set_ndar(fsl_chan, 0); 632 } 633 } 634 635 /** 636 * fsl_dma_memcpy_issue_pending - Issue the DMA start command 637 * @fsl_chan : Freescale DMA channel 638 */ 639 static void fsl_dma_memcpy_issue_pending(struct dma_chan *chan) 640 { 641 struct fsl_dma_chan *fsl_chan = to_fsl_chan(chan); 642 643 #ifdef FSL_DMA_LD_DEBUG 644 struct fsl_desc_sw *ld; 645 unsigned long flags; 646 647 spin_lock_irqsave(&fsl_chan->desc_lock, flags); 648 if (list_empty(&fsl_chan->ld_queue)) { 649 spin_unlock_irqrestore(&fsl_chan->desc_lock, flags); 650 return; 651 } 652 653 dev_dbg(fsl_chan->dev, "--memcpy issue--\n"); 654 list_for_each_entry(ld, &fsl_chan->ld_queue, node) { 655 int i; 656 dev_dbg(fsl_chan->dev, "Ch %d, LD %08x\n", 657 fsl_chan->id, ld->async_tx.phys); 658 for (i = 0; i < 8; i++) 659 dev_dbg(fsl_chan->dev, "LD offset %d: %08x\n", 660 i, *(((u32 *)&ld->hw) + i)); 661 } 662 dev_dbg(fsl_chan->dev, "----------------\n"); 663 spin_unlock_irqrestore(&fsl_chan->desc_lock, flags); 664 #endif 665 666 fsl_chan_xfer_ld_queue(fsl_chan); 667 } 668 669 /** 670 * fsl_dma_is_complete - Determine the DMA status 671 * @fsl_chan : Freescale DMA channel 672 */ 673 static enum dma_status fsl_dma_is_complete(struct dma_chan *chan, 674 dma_cookie_t cookie, 675 dma_cookie_t *done, 676 dma_cookie_t *used) 677 { 678 struct fsl_dma_chan *fsl_chan = to_fsl_chan(chan); 679 dma_cookie_t last_used; 680 dma_cookie_t last_complete; 681 682 fsl_chan_ld_cleanup(fsl_chan); 683 684 last_used = chan->cookie; 685 last_complete = fsl_chan->completed_cookie; 686 687 if (done) 688 *done = last_complete; 689 690 if (used) 691 *used = last_used; 692 693 return dma_async_is_complete(cookie, last_complete, last_used); 694 } 695 696 static irqreturn_t fsl_dma_chan_do_interrupt(int irq, void *data) 697 { 698 struct fsl_dma_chan *fsl_chan = (struct fsl_dma_chan *)data; 699 u32 stat; 700 int update_cookie = 0; 701 int xfer_ld_q = 0; 702 703 stat = get_sr(fsl_chan); 704 dev_dbg(fsl_chan->dev, "event: channel %d, stat = 0x%x\n", 705 fsl_chan->id, stat); 706 set_sr(fsl_chan, stat); /* Clear the event register */ 707 708 stat &= ~(FSL_DMA_SR_CB | FSL_DMA_SR_CH); 709 if (!stat) 710 return IRQ_NONE; 711 712 if (stat & FSL_DMA_SR_TE) 713 dev_err(fsl_chan->dev, "Transfer Error!\n"); 714 715 /* Programming Error 716 * The DMA_INTERRUPT async_tx is a NULL transfer, which will 717 * triger a PE interrupt. 718 */ 719 if (stat & FSL_DMA_SR_PE) { 720 dev_dbg(fsl_chan->dev, "event: Programming Error INT\n"); 721 if (get_bcr(fsl_chan) == 0) { 722 /* BCR register is 0, this is a DMA_INTERRUPT async_tx. 723 * Now, update the completed cookie, and continue the 724 * next uncompleted transfer. 725 */ 726 update_cookie = 1; 727 xfer_ld_q = 1; 728 } 729 stat &= ~FSL_DMA_SR_PE; 730 } 731 732 /* If the link descriptor segment transfer finishes, 733 * we will recycle the used descriptor. 734 */ 735 if (stat & FSL_DMA_SR_EOSI) { 736 dev_dbg(fsl_chan->dev, "event: End-of-segments INT\n"); 737 dev_dbg(fsl_chan->dev, "event: clndar %p, nlndar %p\n", 738 (void *)get_cdar(fsl_chan), (void *)get_ndar(fsl_chan)); 739 stat &= ~FSL_DMA_SR_EOSI; 740 update_cookie = 1; 741 } 742 743 /* For MPC8349, EOCDI event need to update cookie 744 * and start the next transfer if it exist. 745 */ 746 if (stat & FSL_DMA_SR_EOCDI) { 747 dev_dbg(fsl_chan->dev, "event: End-of-Chain link INT\n"); 748 stat &= ~FSL_DMA_SR_EOCDI; 749 update_cookie = 1; 750 xfer_ld_q = 1; 751 } 752 753 /* If it current transfer is the end-of-transfer, 754 * we should clear the Channel Start bit for 755 * prepare next transfer. 756 */ 757 if (stat & FSL_DMA_SR_EOLNI) { 758 dev_dbg(fsl_chan->dev, "event: End-of-link INT\n"); 759 stat &= ~FSL_DMA_SR_EOLNI; 760 xfer_ld_q = 1; 761 } 762 763 if (update_cookie) 764 fsl_dma_update_completed_cookie(fsl_chan); 765 if (xfer_ld_q) 766 fsl_chan_xfer_ld_queue(fsl_chan); 767 if (stat) 768 dev_dbg(fsl_chan->dev, "event: unhandled sr 0x%02x\n", 769 stat); 770 771 dev_dbg(fsl_chan->dev, "event: Exit\n"); 772 tasklet_schedule(&fsl_chan->tasklet); 773 return IRQ_HANDLED; 774 } 775 776 static irqreturn_t fsl_dma_do_interrupt(int irq, void *data) 777 { 778 struct fsl_dma_device *fdev = (struct fsl_dma_device *)data; 779 u32 gsr; 780 int ch_nr; 781 782 gsr = (fdev->feature & FSL_DMA_BIG_ENDIAN) ? in_be32(fdev->reg_base) 783 : in_le32(fdev->reg_base); 784 ch_nr = (32 - ffs(gsr)) / 8; 785 786 return fdev->chan[ch_nr] ? fsl_dma_chan_do_interrupt(irq, 787 fdev->chan[ch_nr]) : IRQ_NONE; 788 } 789 790 static void dma_do_tasklet(unsigned long data) 791 { 792 struct fsl_dma_chan *fsl_chan = (struct fsl_dma_chan *)data; 793 fsl_chan_ld_cleanup(fsl_chan); 794 } 795 796 static int __devinit fsl_dma_chan_probe(struct fsl_dma_device *fdev, 797 struct device_node *node, u32 feature, const char *compatible) 798 { 799 struct fsl_dma_chan *new_fsl_chan; 800 int err; 801 802 /* alloc channel */ 803 new_fsl_chan = kzalloc(sizeof(struct fsl_dma_chan), GFP_KERNEL); 804 if (!new_fsl_chan) { 805 dev_err(fdev->dev, "No free memory for allocating " 806 "dma channels!\n"); 807 return -ENOMEM; 808 } 809 810 /* get dma channel register base */ 811 err = of_address_to_resource(node, 0, &new_fsl_chan->reg); 812 if (err) { 813 dev_err(fdev->dev, "Can't get %s property 'reg'\n", 814 node->full_name); 815 goto err_no_reg; 816 } 817 818 new_fsl_chan->feature = feature; 819 820 if (!fdev->feature) 821 fdev->feature = new_fsl_chan->feature; 822 823 /* If the DMA device's feature is different than its channels', 824 * report the bug. 825 */ 826 WARN_ON(fdev->feature != new_fsl_chan->feature); 827 828 new_fsl_chan->dev = fdev->dev; 829 new_fsl_chan->reg_base = ioremap(new_fsl_chan->reg.start, 830 new_fsl_chan->reg.end - new_fsl_chan->reg.start + 1); 831 832 new_fsl_chan->id = ((new_fsl_chan->reg.start - 0x100) & 0xfff) >> 7; 833 if (new_fsl_chan->id > FSL_DMA_MAX_CHANS_PER_DEVICE) { 834 dev_err(fdev->dev, "There is no %d channel!\n", 835 new_fsl_chan->id); 836 err = -EINVAL; 837 goto err_no_chan; 838 } 839 fdev->chan[new_fsl_chan->id] = new_fsl_chan; 840 tasklet_init(&new_fsl_chan->tasklet, dma_do_tasklet, 841 (unsigned long)new_fsl_chan); 842 843 /* Init the channel */ 844 dma_init(new_fsl_chan); 845 846 /* Clear cdar registers */ 847 set_cdar(new_fsl_chan, 0); 848 849 switch (new_fsl_chan->feature & FSL_DMA_IP_MASK) { 850 case FSL_DMA_IP_85XX: 851 new_fsl_chan->toggle_ext_start = fsl_chan_toggle_ext_start; 852 new_fsl_chan->toggle_ext_pause = fsl_chan_toggle_ext_pause; 853 case FSL_DMA_IP_83XX: 854 new_fsl_chan->set_src_loop_size = fsl_chan_set_src_loop_size; 855 new_fsl_chan->set_dest_loop_size = fsl_chan_set_dest_loop_size; 856 } 857 858 spin_lock_init(&new_fsl_chan->desc_lock); 859 INIT_LIST_HEAD(&new_fsl_chan->ld_queue); 860 861 new_fsl_chan->common.device = &fdev->common; 862 863 /* Add the channel to DMA device channel list */ 864 list_add_tail(&new_fsl_chan->common.device_node, 865 &fdev->common.channels); 866 fdev->common.chancnt++; 867 868 new_fsl_chan->irq = irq_of_parse_and_map(node, 0); 869 if (new_fsl_chan->irq != NO_IRQ) { 870 err = request_irq(new_fsl_chan->irq, 871 &fsl_dma_chan_do_interrupt, IRQF_SHARED, 872 "fsldma-channel", new_fsl_chan); 873 if (err) { 874 dev_err(fdev->dev, "DMA channel %s request_irq error " 875 "with return %d\n", node->full_name, err); 876 goto err_no_irq; 877 } 878 } 879 880 dev_info(fdev->dev, "#%d (%s), irq %d\n", new_fsl_chan->id, 881 compatible, 882 new_fsl_chan->irq != NO_IRQ ? new_fsl_chan->irq : fdev->irq); 883 884 return 0; 885 886 err_no_irq: 887 list_del(&new_fsl_chan->common.device_node); 888 err_no_chan: 889 iounmap(new_fsl_chan->reg_base); 890 err_no_reg: 891 kfree(new_fsl_chan); 892 return err; 893 } 894 895 static void fsl_dma_chan_remove(struct fsl_dma_chan *fchan) 896 { 897 if (fchan->irq != NO_IRQ) 898 free_irq(fchan->irq, fchan); 899 list_del(&fchan->common.device_node); 900 iounmap(fchan->reg_base); 901 kfree(fchan); 902 } 903 904 static int __devinit of_fsl_dma_probe(struct of_device *dev, 905 const struct of_device_id *match) 906 { 907 int err; 908 struct fsl_dma_device *fdev; 909 struct device_node *child; 910 911 fdev = kzalloc(sizeof(struct fsl_dma_device), GFP_KERNEL); 912 if (!fdev) { 913 dev_err(&dev->dev, "No enough memory for 'priv'\n"); 914 return -ENOMEM; 915 } 916 fdev->dev = &dev->dev; 917 INIT_LIST_HEAD(&fdev->common.channels); 918 919 /* get DMA controller register base */ 920 err = of_address_to_resource(dev->node, 0, &fdev->reg); 921 if (err) { 922 dev_err(&dev->dev, "Can't get %s property 'reg'\n", 923 dev->node->full_name); 924 goto err_no_reg; 925 } 926 927 dev_info(&dev->dev, "Probe the Freescale DMA driver for %s " 928 "controller at %p...\n", 929 match->compatible, (void *)fdev->reg.start); 930 fdev->reg_base = ioremap(fdev->reg.start, fdev->reg.end 931 - fdev->reg.start + 1); 932 933 dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask); 934 dma_cap_set(DMA_INTERRUPT, fdev->common.cap_mask); 935 fdev->common.device_alloc_chan_resources = fsl_dma_alloc_chan_resources; 936 fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources; 937 fdev->common.device_prep_dma_interrupt = fsl_dma_prep_interrupt; 938 fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy; 939 fdev->common.device_is_tx_complete = fsl_dma_is_complete; 940 fdev->common.device_issue_pending = fsl_dma_memcpy_issue_pending; 941 fdev->common.dev = &dev->dev; 942 943 fdev->irq = irq_of_parse_and_map(dev->node, 0); 944 if (fdev->irq != NO_IRQ) { 945 err = request_irq(fdev->irq, &fsl_dma_do_interrupt, IRQF_SHARED, 946 "fsldma-device", fdev); 947 if (err) { 948 dev_err(&dev->dev, "DMA device request_irq error " 949 "with return %d\n", err); 950 goto err; 951 } 952 } 953 954 dev_set_drvdata(&(dev->dev), fdev); 955 956 /* We cannot use of_platform_bus_probe() because there is no 957 * of_platform_bus_remove. Instead, we manually instantiate every DMA 958 * channel object. 959 */ 960 for_each_child_of_node(dev->node, child) { 961 if (of_device_is_compatible(child, "fsl,eloplus-dma-channel")) 962 fsl_dma_chan_probe(fdev, child, 963 FSL_DMA_IP_85XX | FSL_DMA_BIG_ENDIAN, 964 "fsl,eloplus-dma-channel"); 965 if (of_device_is_compatible(child, "fsl,elo-dma-channel")) 966 fsl_dma_chan_probe(fdev, child, 967 FSL_DMA_IP_83XX | FSL_DMA_LITTLE_ENDIAN, 968 "fsl,elo-dma-channel"); 969 } 970 971 dma_async_device_register(&fdev->common); 972 return 0; 973 974 err: 975 iounmap(fdev->reg_base); 976 err_no_reg: 977 kfree(fdev); 978 return err; 979 } 980 981 static int of_fsl_dma_remove(struct of_device *of_dev) 982 { 983 struct fsl_dma_device *fdev; 984 unsigned int i; 985 986 fdev = dev_get_drvdata(&of_dev->dev); 987 988 dma_async_device_unregister(&fdev->common); 989 990 for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) 991 if (fdev->chan[i]) 992 fsl_dma_chan_remove(fdev->chan[i]); 993 994 if (fdev->irq != NO_IRQ) 995 free_irq(fdev->irq, fdev); 996 997 iounmap(fdev->reg_base); 998 999 kfree(fdev); 1000 dev_set_drvdata(&of_dev->dev, NULL); 1001 1002 return 0; 1003 } 1004 1005 static struct of_device_id of_fsl_dma_ids[] = { 1006 { .compatible = "fsl,eloplus-dma", }, 1007 { .compatible = "fsl,elo-dma", }, 1008 {} 1009 }; 1010 1011 static struct of_platform_driver of_fsl_dma_driver = { 1012 .name = "fsl-elo-dma", 1013 .match_table = of_fsl_dma_ids, 1014 .probe = of_fsl_dma_probe, 1015 .remove = of_fsl_dma_remove, 1016 }; 1017 1018 static __init int of_fsl_dma_init(void) 1019 { 1020 int ret; 1021 1022 pr_info("Freescale Elo / Elo Plus DMA driver\n"); 1023 1024 ret = of_register_platform_driver(&of_fsl_dma_driver); 1025 if (ret) 1026 pr_err("fsldma: failed to register platform driver\n"); 1027 1028 return ret; 1029 } 1030 1031 static void __exit of_fsl_dma_exit(void) 1032 { 1033 of_unregister_platform_driver(&of_fsl_dma_driver); 1034 } 1035 1036 subsys_initcall(of_fsl_dma_init); 1037 module_exit(of_fsl_dma_exit); 1038 1039 MODULE_DESCRIPTION("Freescale Elo / Elo Plus DMA driver"); 1040 MODULE_LICENSE("GPL"); 1041