1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. 4 * Synopsys DesignWare eDMA core driver 5 * 6 * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> 7 */ 8 9 #include <linux/module.h> 10 #include <linux/device.h> 11 #include <linux/kernel.h> 12 #include <linux/pm_runtime.h> 13 #include <linux/dmaengine.h> 14 #include <linux/err.h> 15 #include <linux/interrupt.h> 16 #include <linux/dma/edma.h> 17 #include <linux/pci.h> 18 19 #include "dw-edma-core.h" 20 #include "dw-edma-v0-core.h" 21 #include "../dmaengine.h" 22 #include "../virt-dma.h" 23 24 static inline 25 struct device *dchan2dev(struct dma_chan *dchan) 26 { 27 return &dchan->dev->device; 28 } 29 30 static inline 31 struct device *chan2dev(struct dw_edma_chan *chan) 32 { 33 return &chan->vc.chan.dev->device; 34 } 35 36 static inline 37 struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd) 38 { 39 return container_of(vd, struct dw_edma_desc, vd); 40 } 41 42 static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk) 43 { 44 struct dw_edma_burst *burst; 45 46 burst = kzalloc(sizeof(*burst), GFP_NOWAIT); 47 if (unlikely(!burst)) 48 return NULL; 49 50 INIT_LIST_HEAD(&burst->list); 51 if (chunk->burst) { 52 /* Create and add new element into the linked list */ 53 chunk->bursts_alloc++; 54 list_add_tail(&burst->list, &chunk->burst->list); 55 } else { 56 /* List head */ 57 chunk->bursts_alloc = 0; 58 chunk->burst = burst; 59 } 60 61 return burst; 62 } 63 64 static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc) 65 { 66 struct dw_edma_chan *chan = desc->chan; 67 struct dw_edma *dw = chan->chip->dw; 68 struct dw_edma_chunk *chunk; 69 70 chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT); 71 if (unlikely(!chunk)) 72 return NULL; 73 74 INIT_LIST_HEAD(&chunk->list); 75 chunk->chan = chan; 76 /* Toggling change bit (CB) in each chunk, this is a mechanism to 77 * inform the eDMA HW block that this is a new linked list ready 78 * to be consumed. 79 * - Odd chunks originate CB equal to 0 80 * - Even chunks originate CB equal to 1 81 */ 82 chunk->cb = !(desc->chunks_alloc % 2); 83 chunk->ll_region.paddr = dw->ll_region.paddr + chan->ll_off; 84 chunk->ll_region.vaddr = dw->ll_region.vaddr + chan->ll_off; 85 86 if (desc->chunk) { 87 /* Create and add new element into the linked list */ 88 desc->chunks_alloc++; 89 list_add_tail(&chunk->list, &desc->chunk->list); 90 if (!dw_edma_alloc_burst(chunk)) { 91 kfree(chunk); 92 return NULL; 93 } 94 } else { 95 /* List head */ 96 chunk->burst = NULL; 97 desc->chunks_alloc = 0; 98 desc->chunk = chunk; 99 } 100 101 return chunk; 102 } 103 104 static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan) 105 { 106 struct dw_edma_desc *desc; 107 108 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 109 if (unlikely(!desc)) 110 return NULL; 111 112 desc->chan = chan; 113 if (!dw_edma_alloc_chunk(desc)) { 114 kfree(desc); 115 return NULL; 116 } 117 118 return desc; 119 } 120 121 static void dw_edma_free_burst(struct dw_edma_chunk *chunk) 122 { 123 struct dw_edma_burst *child, *_next; 124 125 /* Remove all the list elements */ 126 list_for_each_entry_safe(child, _next, &chunk->burst->list, list) { 127 list_del(&child->list); 128 kfree(child); 129 chunk->bursts_alloc--; 130 } 131 132 /* Remove the list head */ 133 kfree(child); 134 chunk->burst = NULL; 135 } 136 137 static void dw_edma_free_chunk(struct dw_edma_desc *desc) 138 { 139 struct dw_edma_chunk *child, *_next; 140 141 if (!desc->chunk) 142 return; 143 144 /* Remove all the list elements */ 145 list_for_each_entry_safe(child, _next, &desc->chunk->list, list) { 146 dw_edma_free_burst(child); 147 list_del(&child->list); 148 kfree(child); 149 desc->chunks_alloc--; 150 } 151 152 /* Remove the list head */ 153 kfree(child); 154 desc->chunk = NULL; 155 } 156 157 static void dw_edma_free_desc(struct dw_edma_desc *desc) 158 { 159 dw_edma_free_chunk(desc); 160 kfree(desc); 161 } 162 163 static void vchan_free_desc(struct virt_dma_desc *vdesc) 164 { 165 dw_edma_free_desc(vd2dw_edma_desc(vdesc)); 166 } 167 168 static void dw_edma_start_transfer(struct dw_edma_chan *chan) 169 { 170 struct dw_edma_chunk *child; 171 struct dw_edma_desc *desc; 172 struct virt_dma_desc *vd; 173 174 vd = vchan_next_desc(&chan->vc); 175 if (!vd) 176 return; 177 178 desc = vd2dw_edma_desc(vd); 179 if (!desc) 180 return; 181 182 child = list_first_entry_or_null(&desc->chunk->list, 183 struct dw_edma_chunk, list); 184 if (!child) 185 return; 186 187 dw_edma_v0_core_start(child, !desc->xfer_sz); 188 desc->xfer_sz += child->ll_region.sz; 189 dw_edma_free_burst(child); 190 list_del(&child->list); 191 kfree(child); 192 desc->chunks_alloc--; 193 } 194 195 static int dw_edma_device_config(struct dma_chan *dchan, 196 struct dma_slave_config *config) 197 { 198 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 199 200 memcpy(&chan->config, config, sizeof(*config)); 201 chan->configured = true; 202 203 return 0; 204 } 205 206 static int dw_edma_device_pause(struct dma_chan *dchan) 207 { 208 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 209 int err = 0; 210 211 if (!chan->configured) 212 err = -EPERM; 213 else if (chan->status != EDMA_ST_BUSY) 214 err = -EPERM; 215 else if (chan->request != EDMA_REQ_NONE) 216 err = -EPERM; 217 else 218 chan->request = EDMA_REQ_PAUSE; 219 220 return err; 221 } 222 223 static int dw_edma_device_resume(struct dma_chan *dchan) 224 { 225 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 226 int err = 0; 227 228 if (!chan->configured) { 229 err = -EPERM; 230 } else if (chan->status != EDMA_ST_PAUSE) { 231 err = -EPERM; 232 } else if (chan->request != EDMA_REQ_NONE) { 233 err = -EPERM; 234 } else { 235 chan->status = EDMA_ST_BUSY; 236 dw_edma_start_transfer(chan); 237 } 238 239 return err; 240 } 241 242 static int dw_edma_device_terminate_all(struct dma_chan *dchan) 243 { 244 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 245 int err = 0; 246 LIST_HEAD(head); 247 248 if (!chan->configured) { 249 /* Do nothing */ 250 } else if (chan->status == EDMA_ST_PAUSE) { 251 chan->status = EDMA_ST_IDLE; 252 chan->configured = false; 253 } else if (chan->status == EDMA_ST_IDLE) { 254 chan->configured = false; 255 } else if (dw_edma_v0_core_ch_status(chan) == DMA_COMPLETE) { 256 /* 257 * The channel is in a false BUSY state, probably didn't 258 * receive or lost an interrupt 259 */ 260 chan->status = EDMA_ST_IDLE; 261 chan->configured = false; 262 } else if (chan->request > EDMA_REQ_PAUSE) { 263 err = -EPERM; 264 } else { 265 chan->request = EDMA_REQ_STOP; 266 } 267 268 return err; 269 } 270 271 static void dw_edma_device_issue_pending(struct dma_chan *dchan) 272 { 273 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 274 unsigned long flags; 275 276 spin_lock_irqsave(&chan->vc.lock, flags); 277 if (chan->configured && chan->request == EDMA_REQ_NONE && 278 chan->status == EDMA_ST_IDLE && vchan_issue_pending(&chan->vc)) { 279 chan->status = EDMA_ST_BUSY; 280 dw_edma_start_transfer(chan); 281 } 282 spin_unlock_irqrestore(&chan->vc.lock, flags); 283 } 284 285 static enum dma_status 286 dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, 287 struct dma_tx_state *txstate) 288 { 289 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 290 struct dw_edma_desc *desc; 291 struct virt_dma_desc *vd; 292 unsigned long flags; 293 enum dma_status ret; 294 u32 residue = 0; 295 296 ret = dma_cookie_status(dchan, cookie, txstate); 297 if (ret == DMA_COMPLETE) 298 return ret; 299 300 if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE) 301 ret = DMA_PAUSED; 302 303 if (!txstate) 304 goto ret_residue; 305 306 spin_lock_irqsave(&chan->vc.lock, flags); 307 vd = vchan_find_desc(&chan->vc, cookie); 308 if (vd) { 309 desc = vd2dw_edma_desc(vd); 310 if (desc) 311 residue = desc->alloc_sz - desc->xfer_sz; 312 } 313 spin_unlock_irqrestore(&chan->vc.lock, flags); 314 315 ret_residue: 316 dma_set_residue(txstate, residue); 317 318 return ret; 319 } 320 321 static struct dma_async_tx_descriptor * 322 dw_edma_device_transfer(struct dw_edma_transfer *xfer) 323 { 324 struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan); 325 enum dma_transfer_direction direction = xfer->direction; 326 phys_addr_t src_addr, dst_addr; 327 struct scatterlist *sg = NULL; 328 struct dw_edma_chunk *chunk; 329 struct dw_edma_burst *burst; 330 struct dw_edma_desc *desc; 331 u32 cnt; 332 int i; 333 334 if ((direction == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_WRITE) || 335 (direction == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_READ)) 336 return NULL; 337 338 if (xfer->cyclic) { 339 if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt) 340 return NULL; 341 } else { 342 if (xfer->xfer.sg.len < 1) 343 return NULL; 344 } 345 346 if (!chan->configured) 347 return NULL; 348 349 desc = dw_edma_alloc_desc(chan); 350 if (unlikely(!desc)) 351 goto err_alloc; 352 353 chunk = dw_edma_alloc_chunk(desc); 354 if (unlikely(!chunk)) 355 goto err_alloc; 356 357 src_addr = chan->config.src_addr; 358 dst_addr = chan->config.dst_addr; 359 360 if (xfer->cyclic) { 361 cnt = xfer->xfer.cyclic.cnt; 362 } else { 363 cnt = xfer->xfer.sg.len; 364 sg = xfer->xfer.sg.sgl; 365 } 366 367 for (i = 0; i < cnt; i++) { 368 if (!xfer->cyclic && !sg) 369 break; 370 371 if (chunk->bursts_alloc == chan->ll_max) { 372 chunk = dw_edma_alloc_chunk(desc); 373 if (unlikely(!chunk)) 374 goto err_alloc; 375 } 376 377 burst = dw_edma_alloc_burst(chunk); 378 if (unlikely(!burst)) 379 goto err_alloc; 380 381 if (xfer->cyclic) 382 burst->sz = xfer->xfer.cyclic.len; 383 else 384 burst->sz = sg_dma_len(sg); 385 386 chunk->ll_region.sz += burst->sz; 387 desc->alloc_sz += burst->sz; 388 389 if (direction == DMA_DEV_TO_MEM) { 390 burst->sar = src_addr; 391 if (xfer->cyclic) { 392 burst->dar = xfer->xfer.cyclic.paddr; 393 } else { 394 burst->dar = sg_dma_address(sg); 395 /* Unlike the typical assumption by other 396 * drivers/IPs the peripheral memory isn't 397 * a FIFO memory, in this case, it's a 398 * linear memory and that why the source 399 * and destination addresses are increased 400 * by the same portion (data length) 401 */ 402 src_addr += sg_dma_len(sg); 403 } 404 } else { 405 burst->dar = dst_addr; 406 if (xfer->cyclic) { 407 burst->sar = xfer->xfer.cyclic.paddr; 408 } else { 409 burst->sar = sg_dma_address(sg); 410 /* Unlike the typical assumption by other 411 * drivers/IPs the peripheral memory isn't 412 * a FIFO memory, in this case, it's a 413 * linear memory and that why the source 414 * and destination addresses are increased 415 * by the same portion (data length) 416 */ 417 dst_addr += sg_dma_len(sg); 418 } 419 } 420 421 if (!xfer->cyclic) 422 sg = sg_next(sg); 423 } 424 425 return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags); 426 427 err_alloc: 428 if (desc) 429 dw_edma_free_desc(desc); 430 431 return NULL; 432 } 433 434 static struct dma_async_tx_descriptor * 435 dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, 436 unsigned int len, 437 enum dma_transfer_direction direction, 438 unsigned long flags, void *context) 439 { 440 struct dw_edma_transfer xfer; 441 442 xfer.dchan = dchan; 443 xfer.direction = direction; 444 xfer.xfer.sg.sgl = sgl; 445 xfer.xfer.sg.len = len; 446 xfer.flags = flags; 447 xfer.cyclic = false; 448 449 return dw_edma_device_transfer(&xfer); 450 } 451 452 static struct dma_async_tx_descriptor * 453 dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr, 454 size_t len, size_t count, 455 enum dma_transfer_direction direction, 456 unsigned long flags) 457 { 458 struct dw_edma_transfer xfer; 459 460 xfer.dchan = dchan; 461 xfer.direction = direction; 462 xfer.xfer.cyclic.paddr = paddr; 463 xfer.xfer.cyclic.len = len; 464 xfer.xfer.cyclic.cnt = count; 465 xfer.flags = flags; 466 xfer.cyclic = true; 467 468 return dw_edma_device_transfer(&xfer); 469 } 470 471 static void dw_edma_done_interrupt(struct dw_edma_chan *chan) 472 { 473 struct dw_edma_desc *desc; 474 struct virt_dma_desc *vd; 475 unsigned long flags; 476 477 dw_edma_v0_core_clear_done_int(chan); 478 479 spin_lock_irqsave(&chan->vc.lock, flags); 480 vd = vchan_next_desc(&chan->vc); 481 if (vd) { 482 switch (chan->request) { 483 case EDMA_REQ_NONE: 484 desc = vd2dw_edma_desc(vd); 485 if (desc->chunks_alloc) { 486 chan->status = EDMA_ST_BUSY; 487 dw_edma_start_transfer(chan); 488 } else { 489 list_del(&vd->node); 490 vchan_cookie_complete(vd); 491 chan->status = EDMA_ST_IDLE; 492 } 493 break; 494 495 case EDMA_REQ_STOP: 496 list_del(&vd->node); 497 vchan_cookie_complete(vd); 498 chan->request = EDMA_REQ_NONE; 499 chan->status = EDMA_ST_IDLE; 500 break; 501 502 case EDMA_REQ_PAUSE: 503 chan->request = EDMA_REQ_NONE; 504 chan->status = EDMA_ST_PAUSE; 505 break; 506 507 default: 508 break; 509 } 510 } 511 spin_unlock_irqrestore(&chan->vc.lock, flags); 512 } 513 514 static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) 515 { 516 struct virt_dma_desc *vd; 517 unsigned long flags; 518 519 dw_edma_v0_core_clear_abort_int(chan); 520 521 spin_lock_irqsave(&chan->vc.lock, flags); 522 vd = vchan_next_desc(&chan->vc); 523 if (vd) { 524 list_del(&vd->node); 525 vchan_cookie_complete(vd); 526 } 527 spin_unlock_irqrestore(&chan->vc.lock, flags); 528 chan->request = EDMA_REQ_NONE; 529 chan->status = EDMA_ST_IDLE; 530 } 531 532 static irqreturn_t dw_edma_interrupt(int irq, void *data, bool write) 533 { 534 struct dw_edma_irq *dw_irq = data; 535 struct dw_edma *dw = dw_irq->dw; 536 unsigned long total, pos, val; 537 unsigned long off; 538 u32 mask; 539 540 if (write) { 541 total = dw->wr_ch_cnt; 542 off = 0; 543 mask = dw_irq->wr_mask; 544 } else { 545 total = dw->rd_ch_cnt; 546 off = dw->wr_ch_cnt; 547 mask = dw_irq->rd_mask; 548 } 549 550 val = dw_edma_v0_core_status_done_int(dw, write ? 551 EDMA_DIR_WRITE : 552 EDMA_DIR_READ); 553 val &= mask; 554 for_each_set_bit(pos, &val, total) { 555 struct dw_edma_chan *chan = &dw->chan[pos + off]; 556 557 dw_edma_done_interrupt(chan); 558 } 559 560 val = dw_edma_v0_core_status_abort_int(dw, write ? 561 EDMA_DIR_WRITE : 562 EDMA_DIR_READ); 563 val &= mask; 564 for_each_set_bit(pos, &val, total) { 565 struct dw_edma_chan *chan = &dw->chan[pos + off]; 566 567 dw_edma_abort_interrupt(chan); 568 } 569 570 return IRQ_HANDLED; 571 } 572 573 static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) 574 { 575 return dw_edma_interrupt(irq, data, true); 576 } 577 578 static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data) 579 { 580 return dw_edma_interrupt(irq, data, false); 581 } 582 583 static irqreturn_t dw_edma_interrupt_common(int irq, void *data) 584 { 585 dw_edma_interrupt(irq, data, true); 586 dw_edma_interrupt(irq, data, false); 587 588 return IRQ_HANDLED; 589 } 590 591 static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) 592 { 593 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 594 595 if (chan->status != EDMA_ST_IDLE) 596 return -EBUSY; 597 598 pm_runtime_get(chan->chip->dev); 599 600 return 0; 601 } 602 603 static void dw_edma_free_chan_resources(struct dma_chan *dchan) 604 { 605 unsigned long timeout = jiffies + msecs_to_jiffies(5000); 606 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 607 int ret; 608 609 while (time_before(jiffies, timeout)) { 610 ret = dw_edma_device_terminate_all(dchan); 611 if (!ret) 612 break; 613 614 if (time_after_eq(jiffies, timeout)) 615 return; 616 617 cpu_relax(); 618 } 619 620 pm_runtime_put(chan->chip->dev); 621 } 622 623 static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write, 624 u32 wr_alloc, u32 rd_alloc) 625 { 626 struct dw_edma_region *dt_region; 627 struct device *dev = chip->dev; 628 struct dw_edma *dw = chip->dw; 629 struct dw_edma_chan *chan; 630 size_t ll_chunk, dt_chunk; 631 struct dw_edma_irq *irq; 632 struct dma_device *dma; 633 u32 i, j, cnt, ch_cnt; 634 u32 alloc, off_alloc; 635 int err = 0; 636 u32 pos; 637 638 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; 639 ll_chunk = dw->ll_region.sz; 640 dt_chunk = dw->dt_region.sz; 641 642 /* Calculate linked list chunk for each channel */ 643 ll_chunk /= roundup_pow_of_two(ch_cnt); 644 645 /* Calculate linked list chunk for each channel */ 646 dt_chunk /= roundup_pow_of_two(ch_cnt); 647 648 if (write) { 649 i = 0; 650 cnt = dw->wr_ch_cnt; 651 dma = &dw->wr_edma; 652 alloc = wr_alloc; 653 off_alloc = 0; 654 } else { 655 i = dw->wr_ch_cnt; 656 cnt = dw->rd_ch_cnt; 657 dma = &dw->rd_edma; 658 alloc = rd_alloc; 659 off_alloc = wr_alloc; 660 } 661 662 INIT_LIST_HEAD(&dma->channels); 663 for (j = 0; (alloc || dw->nr_irqs == 1) && j < cnt; j++, i++) { 664 chan = &dw->chan[i]; 665 666 dt_region = devm_kzalloc(dev, sizeof(*dt_region), GFP_KERNEL); 667 if (!dt_region) 668 return -ENOMEM; 669 670 chan->vc.chan.private = dt_region; 671 672 chan->chip = chip; 673 chan->id = j; 674 chan->dir = write ? EDMA_DIR_WRITE : EDMA_DIR_READ; 675 chan->configured = false; 676 chan->request = EDMA_REQ_NONE; 677 chan->status = EDMA_ST_IDLE; 678 679 chan->ll_off = (ll_chunk * i); 680 chan->ll_max = (ll_chunk / EDMA_LL_SZ) - 1; 681 682 chan->dt_off = (dt_chunk * i); 683 684 dev_vdbg(dev, "L. List:\tChannel %s[%u] off=0x%.8lx, max_cnt=%u\n", 685 write ? "write" : "read", j, 686 chan->ll_off, chan->ll_max); 687 688 if (dw->nr_irqs == 1) 689 pos = 0; 690 else 691 pos = off_alloc + (j % alloc); 692 693 irq = &dw->irq[pos]; 694 695 if (write) 696 irq->wr_mask |= BIT(j); 697 else 698 irq->rd_mask |= BIT(j); 699 700 irq->dw = dw; 701 memcpy(&chan->msi, &irq->msi, sizeof(chan->msi)); 702 703 dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n", 704 write ? "write" : "read", j, 705 chan->msi.address_hi, chan->msi.address_lo, 706 chan->msi.data); 707 708 chan->vc.desc_free = vchan_free_desc; 709 vchan_init(&chan->vc, dma); 710 711 dt_region->paddr = dw->dt_region.paddr + chan->dt_off; 712 dt_region->vaddr = dw->dt_region.vaddr + chan->dt_off; 713 dt_region->sz = dt_chunk; 714 715 dev_vdbg(dev, "Data:\tChannel %s[%u] off=0x%.8lx\n", 716 write ? "write" : "read", j, chan->dt_off); 717 718 dw_edma_v0_core_device_config(chan); 719 } 720 721 /* Set DMA channel capabilities */ 722 dma_cap_zero(dma->cap_mask); 723 dma_cap_set(DMA_SLAVE, dma->cap_mask); 724 dma_cap_set(DMA_CYCLIC, dma->cap_mask); 725 dma_cap_set(DMA_PRIVATE, dma->cap_mask); 726 dma->directions = BIT(write ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV); 727 dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 728 dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 729 dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 730 dma->chancnt = cnt; 731 732 /* Set DMA channel callbacks */ 733 dma->dev = chip->dev; 734 dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources; 735 dma->device_free_chan_resources = dw_edma_free_chan_resources; 736 dma->device_config = dw_edma_device_config; 737 dma->device_pause = dw_edma_device_pause; 738 dma->device_resume = dw_edma_device_resume; 739 dma->device_terminate_all = dw_edma_device_terminate_all; 740 dma->device_issue_pending = dw_edma_device_issue_pending; 741 dma->device_tx_status = dw_edma_device_tx_status; 742 dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg; 743 dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic; 744 745 dma_set_max_seg_size(dma->dev, U32_MAX); 746 747 /* Register DMA device */ 748 err = dma_async_device_register(dma); 749 750 return err; 751 } 752 753 static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt) 754 { 755 if (*nr_irqs && *alloc < cnt) { 756 (*alloc)++; 757 (*nr_irqs)--; 758 } 759 } 760 761 static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt) 762 { 763 while (*mask * alloc < cnt) 764 (*mask)++; 765 } 766 767 static int dw_edma_irq_request(struct dw_edma_chip *chip, 768 u32 *wr_alloc, u32 *rd_alloc) 769 { 770 struct device *dev = chip->dev; 771 struct dw_edma *dw = chip->dw; 772 u32 wr_mask = 1; 773 u32 rd_mask = 1; 774 int i, err = 0; 775 u32 ch_cnt; 776 777 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; 778 779 if (dw->nr_irqs < 1) 780 return -EINVAL; 781 782 if (dw->nr_irqs == 1) { 783 /* Common IRQ shared among all channels */ 784 err = request_irq(pci_irq_vector(to_pci_dev(dev), 0), 785 dw_edma_interrupt_common, 786 IRQF_SHARED, dw->name, &dw->irq[0]); 787 if (err) { 788 dw->nr_irqs = 0; 789 return err; 790 } 791 792 get_cached_msi_msg(pci_irq_vector(to_pci_dev(dev), 0), 793 &dw->irq[0].msi); 794 } else { 795 /* Distribute IRQs equally among all channels */ 796 int tmp = dw->nr_irqs; 797 798 while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) { 799 dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt); 800 dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt); 801 } 802 803 dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt); 804 dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt); 805 806 for (i = 0; i < (*wr_alloc + *rd_alloc); i++) { 807 err = request_irq(pci_irq_vector(to_pci_dev(dev), i), 808 i < *wr_alloc ? 809 dw_edma_interrupt_write : 810 dw_edma_interrupt_read, 811 IRQF_SHARED, dw->name, 812 &dw->irq[i]); 813 if (err) { 814 dw->nr_irqs = i; 815 return err; 816 } 817 818 get_cached_msi_msg(pci_irq_vector(to_pci_dev(dev), i), 819 &dw->irq[i].msi); 820 } 821 822 dw->nr_irqs = i; 823 } 824 825 return err; 826 } 827 828 int dw_edma_probe(struct dw_edma_chip *chip) 829 { 830 struct device *dev = chip->dev; 831 struct dw_edma *dw = chip->dw; 832 u32 wr_alloc = 0; 833 u32 rd_alloc = 0; 834 int i, err; 835 836 raw_spin_lock_init(&dw->lock); 837 838 /* Find out how many write channels are supported by hardware */ 839 dw->wr_ch_cnt = dw_edma_v0_core_ch_count(dw, EDMA_DIR_WRITE); 840 if (!dw->wr_ch_cnt) 841 return -EINVAL; 842 843 /* Find out how many read channels are supported by hardware */ 844 dw->rd_ch_cnt = dw_edma_v0_core_ch_count(dw, EDMA_DIR_READ); 845 if (!dw->rd_ch_cnt) 846 return -EINVAL; 847 848 dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n", 849 dw->wr_ch_cnt, dw->rd_ch_cnt); 850 851 /* Allocate channels */ 852 dw->chan = devm_kcalloc(dev, dw->wr_ch_cnt + dw->rd_ch_cnt, 853 sizeof(*dw->chan), GFP_KERNEL); 854 if (!dw->chan) 855 return -ENOMEM; 856 857 snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%d", chip->id); 858 859 /* Disable eDMA, only to establish the ideal initial conditions */ 860 dw_edma_v0_core_off(dw); 861 862 /* Request IRQs */ 863 err = dw_edma_irq_request(chip, &wr_alloc, &rd_alloc); 864 if (err) 865 return err; 866 867 /* Setup write channels */ 868 err = dw_edma_channel_setup(chip, true, wr_alloc, rd_alloc); 869 if (err) 870 goto err_irq_free; 871 872 /* Setup read channels */ 873 err = dw_edma_channel_setup(chip, false, wr_alloc, rd_alloc); 874 if (err) 875 goto err_irq_free; 876 877 /* Power management */ 878 pm_runtime_enable(dev); 879 880 /* Turn debugfs on */ 881 dw_edma_v0_core_debugfs_on(chip); 882 883 return 0; 884 885 err_irq_free: 886 for (i = (dw->nr_irqs - 1); i >= 0; i--) 887 free_irq(pci_irq_vector(to_pci_dev(dev), i), &dw->irq[i]); 888 889 dw->nr_irqs = 0; 890 891 return err; 892 } 893 EXPORT_SYMBOL_GPL(dw_edma_probe); 894 895 int dw_edma_remove(struct dw_edma_chip *chip) 896 { 897 struct dw_edma_chan *chan, *_chan; 898 struct device *dev = chip->dev; 899 struct dw_edma *dw = chip->dw; 900 int i; 901 902 /* Disable eDMA */ 903 dw_edma_v0_core_off(dw); 904 905 /* Free irqs */ 906 for (i = (dw->nr_irqs - 1); i >= 0; i--) 907 free_irq(pci_irq_vector(to_pci_dev(dev), i), &dw->irq[i]); 908 909 /* Power management */ 910 pm_runtime_disable(dev); 911 912 list_for_each_entry_safe(chan, _chan, &dw->wr_edma.channels, 913 vc.chan.device_node) { 914 list_del(&chan->vc.chan.device_node); 915 tasklet_kill(&chan->vc.task); 916 } 917 918 list_for_each_entry_safe(chan, _chan, &dw->rd_edma.channels, 919 vc.chan.device_node) { 920 list_del(&chan->vc.chan.device_node); 921 tasklet_kill(&chan->vc.task); 922 } 923 924 /* Deregister eDMA device */ 925 dma_async_device_unregister(&dw->wr_edma); 926 dma_async_device_unregister(&dw->rd_edma); 927 928 /* Turn debugfs off */ 929 dw_edma_v0_core_debugfs_off(); 930 931 return 0; 932 } 933 EXPORT_SYMBOL_GPL(dw_edma_remove); 934 935 MODULE_LICENSE("GPL v2"); 936 MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver"); 937 MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>"); 938