1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. 4 * Synopsys DesignWare eDMA core driver 5 * 6 * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> 7 */ 8 9 #include <linux/module.h> 10 #include <linux/device.h> 11 #include <linux/kernel.h> 12 #include <linux/dmaengine.h> 13 #include <linux/err.h> 14 #include <linux/interrupt.h> 15 #include <linux/irq.h> 16 #include <linux/dma/edma.h> 17 #include <linux/dma-mapping.h> 18 19 #include "dw-edma-core.h" 20 #include "dw-edma-v0-core.h" 21 #include "dw-hdma-v0-core.h" 22 #include "../dmaengine.h" 23 #include "../virt-dma.h" 24 25 static inline 26 struct device *dchan2dev(struct dma_chan *dchan) 27 { 28 return &dchan->dev->device; 29 } 30 31 static inline 32 struct device *chan2dev(struct dw_edma_chan *chan) 33 { 34 return &chan->vc.chan.dev->device; 35 } 36 37 static inline 38 struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd) 39 { 40 return container_of(vd, struct dw_edma_desc, vd); 41 } 42 43 static inline 44 u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr) 45 { 46 struct dw_edma_chip *chip = chan->dw->chip; 47 48 if (chip->ops->pci_address) 49 return chip->ops->pci_address(chip->dev, cpu_addr); 50 51 return cpu_addr; 52 } 53 54 static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk) 55 { 56 struct dw_edma_burst *burst; 57 58 burst = kzalloc(sizeof(*burst), GFP_NOWAIT); 59 if (unlikely(!burst)) 60 return NULL; 61 62 INIT_LIST_HEAD(&burst->list); 63 if (chunk->burst) { 64 /* Create and add new element into the linked list */ 65 chunk->bursts_alloc++; 66 list_add_tail(&burst->list, &chunk->burst->list); 67 } else { 68 /* List head */ 69 chunk->bursts_alloc = 0; 70 chunk->burst = burst; 71 } 72 73 return burst; 74 } 75 76 static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc) 77 { 78 struct dw_edma_chip *chip = desc->chan->dw->chip; 79 struct dw_edma_chan *chan = desc->chan; 80 struct dw_edma_chunk *chunk; 81 82 chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT); 83 if (unlikely(!chunk)) 84 return NULL; 85 86 INIT_LIST_HEAD(&chunk->list); 87 chunk->chan = chan; 88 /* Toggling change bit (CB) in each chunk, this is a mechanism to 89 * inform the eDMA HW block that this is a new linked list ready 90 * to be consumed. 91 * - Odd chunks originate CB equal to 0 92 * - Even chunks originate CB equal to 1 93 */ 94 chunk->cb = !(desc->chunks_alloc % 2); 95 if (chan->dir == EDMA_DIR_WRITE) { 96 chunk->ll_region.paddr = chip->ll_region_wr[chan->id].paddr; 97 chunk->ll_region.vaddr = chip->ll_region_wr[chan->id].vaddr; 98 } else { 99 chunk->ll_region.paddr = chip->ll_region_rd[chan->id].paddr; 100 chunk->ll_region.vaddr = chip->ll_region_rd[chan->id].vaddr; 101 } 102 103 if (desc->chunk) { 104 /* Create and add new element into the linked list */ 105 if (!dw_edma_alloc_burst(chunk)) { 106 kfree(chunk); 107 return NULL; 108 } 109 desc->chunks_alloc++; 110 list_add_tail(&chunk->list, &desc->chunk->list); 111 } else { 112 /* List head */ 113 chunk->burst = NULL; 114 desc->chunks_alloc = 0; 115 desc->chunk = chunk; 116 } 117 118 return chunk; 119 } 120 121 static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan) 122 { 123 struct dw_edma_desc *desc; 124 125 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 126 if (unlikely(!desc)) 127 return NULL; 128 129 desc->chan = chan; 130 if (!dw_edma_alloc_chunk(desc)) { 131 kfree(desc); 132 return NULL; 133 } 134 135 return desc; 136 } 137 138 static void dw_edma_free_burst(struct dw_edma_chunk *chunk) 139 { 140 struct dw_edma_burst *child, *_next; 141 142 /* Remove all the list elements */ 143 list_for_each_entry_safe(child, _next, &chunk->burst->list, list) { 144 list_del(&child->list); 145 kfree(child); 146 chunk->bursts_alloc--; 147 } 148 149 /* Remove the list head */ 150 kfree(child); 151 chunk->burst = NULL; 152 } 153 154 static void dw_edma_free_chunk(struct dw_edma_desc *desc) 155 { 156 struct dw_edma_chunk *child, *_next; 157 158 if (!desc->chunk) 159 return; 160 161 /* Remove all the list elements */ 162 list_for_each_entry_safe(child, _next, &desc->chunk->list, list) { 163 dw_edma_free_burst(child); 164 list_del(&child->list); 165 kfree(child); 166 desc->chunks_alloc--; 167 } 168 169 /* Remove the list head */ 170 kfree(child); 171 desc->chunk = NULL; 172 } 173 174 static void dw_edma_free_desc(struct dw_edma_desc *desc) 175 { 176 dw_edma_free_chunk(desc); 177 kfree(desc); 178 } 179 180 static void vchan_free_desc(struct virt_dma_desc *vdesc) 181 { 182 dw_edma_free_desc(vd2dw_edma_desc(vdesc)); 183 } 184 185 static int dw_edma_start_transfer(struct dw_edma_chan *chan) 186 { 187 struct dw_edma *dw = chan->dw; 188 struct dw_edma_chunk *child; 189 struct dw_edma_desc *desc; 190 struct virt_dma_desc *vd; 191 192 vd = vchan_next_desc(&chan->vc); 193 if (!vd) 194 return 0; 195 196 desc = vd2dw_edma_desc(vd); 197 if (!desc) 198 return 0; 199 200 child = list_first_entry_or_null(&desc->chunk->list, 201 struct dw_edma_chunk, list); 202 if (!child) 203 return 0; 204 205 dw_edma_core_start(dw, child, !desc->xfer_sz); 206 desc->xfer_sz += child->ll_region.sz; 207 dw_edma_free_burst(child); 208 list_del(&child->list); 209 kfree(child); 210 desc->chunks_alloc--; 211 212 return 1; 213 } 214 215 static void dw_edma_device_caps(struct dma_chan *dchan, 216 struct dma_slave_caps *caps) 217 { 218 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 219 220 if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { 221 if (chan->dir == EDMA_DIR_READ) 222 caps->directions = BIT(DMA_DEV_TO_MEM); 223 else 224 caps->directions = BIT(DMA_MEM_TO_DEV); 225 } else { 226 if (chan->dir == EDMA_DIR_WRITE) 227 caps->directions = BIT(DMA_DEV_TO_MEM); 228 else 229 caps->directions = BIT(DMA_MEM_TO_DEV); 230 } 231 } 232 233 static int dw_edma_device_config(struct dma_chan *dchan, 234 struct dma_slave_config *config) 235 { 236 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 237 238 memcpy(&chan->config, config, sizeof(*config)); 239 chan->configured = true; 240 241 return 0; 242 } 243 244 static int dw_edma_device_pause(struct dma_chan *dchan) 245 { 246 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 247 int err = 0; 248 249 if (!chan->configured) 250 err = -EPERM; 251 else if (chan->status != EDMA_ST_BUSY) 252 err = -EPERM; 253 else if (chan->request != EDMA_REQ_NONE) 254 err = -EPERM; 255 else 256 chan->request = EDMA_REQ_PAUSE; 257 258 return err; 259 } 260 261 static int dw_edma_device_resume(struct dma_chan *dchan) 262 { 263 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 264 int err = 0; 265 266 if (!chan->configured) { 267 err = -EPERM; 268 } else if (chan->status != EDMA_ST_PAUSE) { 269 err = -EPERM; 270 } else if (chan->request != EDMA_REQ_NONE) { 271 err = -EPERM; 272 } else { 273 chan->status = EDMA_ST_BUSY; 274 dw_edma_start_transfer(chan); 275 } 276 277 return err; 278 } 279 280 static int dw_edma_device_terminate_all(struct dma_chan *dchan) 281 { 282 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 283 int err = 0; 284 285 if (!chan->configured) { 286 /* Do nothing */ 287 } else if (chan->status == EDMA_ST_PAUSE) { 288 chan->status = EDMA_ST_IDLE; 289 chan->configured = false; 290 } else if (chan->status == EDMA_ST_IDLE) { 291 chan->configured = false; 292 } else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) { 293 /* 294 * The channel is in a false BUSY state, probably didn't 295 * receive or lost an interrupt 296 */ 297 chan->status = EDMA_ST_IDLE; 298 chan->configured = false; 299 } else if (chan->request > EDMA_REQ_PAUSE) { 300 err = -EPERM; 301 } else { 302 chan->request = EDMA_REQ_STOP; 303 } 304 305 return err; 306 } 307 308 static void dw_edma_device_issue_pending(struct dma_chan *dchan) 309 { 310 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 311 unsigned long flags; 312 313 if (!chan->configured) 314 return; 315 316 spin_lock_irqsave(&chan->vc.lock, flags); 317 if (vchan_issue_pending(&chan->vc) && chan->request == EDMA_REQ_NONE && 318 chan->status == EDMA_ST_IDLE) { 319 chan->status = EDMA_ST_BUSY; 320 dw_edma_start_transfer(chan); 321 } 322 spin_unlock_irqrestore(&chan->vc.lock, flags); 323 } 324 325 static enum dma_status 326 dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, 327 struct dma_tx_state *txstate) 328 { 329 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 330 struct dw_edma_desc *desc; 331 struct virt_dma_desc *vd; 332 unsigned long flags; 333 enum dma_status ret; 334 u32 residue = 0; 335 336 ret = dma_cookie_status(dchan, cookie, txstate); 337 if (ret == DMA_COMPLETE) 338 return ret; 339 340 if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE) 341 ret = DMA_PAUSED; 342 343 if (!txstate) 344 goto ret_residue; 345 346 spin_lock_irqsave(&chan->vc.lock, flags); 347 vd = vchan_find_desc(&chan->vc, cookie); 348 if (vd) { 349 desc = vd2dw_edma_desc(vd); 350 if (desc) 351 residue = desc->alloc_sz - desc->xfer_sz; 352 } 353 spin_unlock_irqrestore(&chan->vc.lock, flags); 354 355 ret_residue: 356 dma_set_residue(txstate, residue); 357 358 return ret; 359 } 360 361 static struct dma_async_tx_descriptor * 362 dw_edma_device_transfer(struct dw_edma_transfer *xfer) 363 { 364 struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan); 365 enum dma_transfer_direction dir = xfer->direction; 366 struct scatterlist *sg = NULL; 367 struct dw_edma_chunk *chunk; 368 struct dw_edma_burst *burst; 369 struct dw_edma_desc *desc; 370 u64 src_addr, dst_addr; 371 size_t fsz = 0; 372 u32 cnt = 0; 373 int i; 374 375 if (!chan->configured) 376 return NULL; 377 378 /* 379 * Local Root Port/End-point Remote End-point 380 * +-----------------------+ PCIe bus +----------------------+ 381 * | | +-+ | | 382 * | DEV_TO_MEM Rx Ch <----+ +---+ Tx Ch DEV_TO_MEM | 383 * | | | | | | 384 * | MEM_TO_DEV Tx Ch +----+ +---> Rx Ch MEM_TO_DEV | 385 * | | +-+ | | 386 * +-----------------------+ +----------------------+ 387 * 388 * 1. Normal logic: 389 * If eDMA is embedded into the DW PCIe RP/EP and controlled from the 390 * CPU/Application side, the Rx channel (EDMA_DIR_READ) will be used 391 * for the device read operations (DEV_TO_MEM) and the Tx channel 392 * (EDMA_DIR_WRITE) - for the write operations (MEM_TO_DEV). 393 * 394 * 2. Inverted logic: 395 * If eDMA is embedded into a Remote PCIe EP and is controlled by the 396 * MWr/MRd TLPs sent from the CPU's PCIe host controller, the Tx 397 * channel (EDMA_DIR_WRITE) will be used for the device read operations 398 * (DEV_TO_MEM) and the Rx channel (EDMA_DIR_READ) - for the write 399 * operations (MEM_TO_DEV). 400 * 401 * It is the client driver responsibility to choose a proper channel 402 * for the DMA transfers. 403 */ 404 if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { 405 if ((chan->dir == EDMA_DIR_READ && dir != DMA_DEV_TO_MEM) || 406 (chan->dir == EDMA_DIR_WRITE && dir != DMA_MEM_TO_DEV)) 407 return NULL; 408 } else { 409 if ((chan->dir == EDMA_DIR_WRITE && dir != DMA_DEV_TO_MEM) || 410 (chan->dir == EDMA_DIR_READ && dir != DMA_MEM_TO_DEV)) 411 return NULL; 412 } 413 414 if (xfer->type == EDMA_XFER_CYCLIC) { 415 if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt) 416 return NULL; 417 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 418 if (xfer->xfer.sg.len < 1) 419 return NULL; 420 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 421 if (!xfer->xfer.il->numf || xfer->xfer.il->frame_size < 1) 422 return NULL; 423 if (!xfer->xfer.il->src_inc || !xfer->xfer.il->dst_inc) 424 return NULL; 425 } else { 426 return NULL; 427 } 428 429 desc = dw_edma_alloc_desc(chan); 430 if (unlikely(!desc)) 431 goto err_alloc; 432 433 chunk = dw_edma_alloc_chunk(desc); 434 if (unlikely(!chunk)) 435 goto err_alloc; 436 437 if (xfer->type == EDMA_XFER_INTERLEAVED) { 438 src_addr = xfer->xfer.il->src_start; 439 dst_addr = xfer->xfer.il->dst_start; 440 } else { 441 src_addr = chan->config.src_addr; 442 dst_addr = chan->config.dst_addr; 443 } 444 445 if (dir == DMA_DEV_TO_MEM) 446 src_addr = dw_edma_get_pci_address(chan, (phys_addr_t)src_addr); 447 else 448 dst_addr = dw_edma_get_pci_address(chan, (phys_addr_t)dst_addr); 449 450 if (xfer->type == EDMA_XFER_CYCLIC) { 451 cnt = xfer->xfer.cyclic.cnt; 452 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 453 cnt = xfer->xfer.sg.len; 454 sg = xfer->xfer.sg.sgl; 455 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 456 cnt = xfer->xfer.il->numf * xfer->xfer.il->frame_size; 457 fsz = xfer->xfer.il->frame_size; 458 } 459 460 for (i = 0; i < cnt; i++) { 461 if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg) 462 break; 463 464 if (chunk->bursts_alloc == chan->ll_max) { 465 chunk = dw_edma_alloc_chunk(desc); 466 if (unlikely(!chunk)) 467 goto err_alloc; 468 } 469 470 burst = dw_edma_alloc_burst(chunk); 471 if (unlikely(!burst)) 472 goto err_alloc; 473 474 if (xfer->type == EDMA_XFER_CYCLIC) 475 burst->sz = xfer->xfer.cyclic.len; 476 else if (xfer->type == EDMA_XFER_SCATTER_GATHER) 477 burst->sz = sg_dma_len(sg); 478 else if (xfer->type == EDMA_XFER_INTERLEAVED) 479 burst->sz = xfer->xfer.il->sgl[i % fsz].size; 480 481 chunk->ll_region.sz += burst->sz; 482 desc->alloc_sz += burst->sz; 483 484 if (dir == DMA_DEV_TO_MEM) { 485 burst->sar = src_addr; 486 if (xfer->type == EDMA_XFER_CYCLIC) { 487 burst->dar = xfer->xfer.cyclic.paddr; 488 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 489 src_addr += sg_dma_len(sg); 490 burst->dar = sg_dma_address(sg); 491 /* Unlike the typical assumption by other 492 * drivers/IPs the peripheral memory isn't 493 * a FIFO memory, in this case, it's a 494 * linear memory and that why the source 495 * and destination addresses are increased 496 * by the same portion (data length) 497 */ 498 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 499 burst->dar = dst_addr; 500 } 501 } else { 502 burst->dar = dst_addr; 503 if (xfer->type == EDMA_XFER_CYCLIC) { 504 burst->sar = xfer->xfer.cyclic.paddr; 505 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 506 dst_addr += sg_dma_len(sg); 507 burst->sar = sg_dma_address(sg); 508 /* Unlike the typical assumption by other 509 * drivers/IPs the peripheral memory isn't 510 * a FIFO memory, in this case, it's a 511 * linear memory and that why the source 512 * and destination addresses are increased 513 * by the same portion (data length) 514 */ 515 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 516 burst->sar = src_addr; 517 } 518 } 519 520 if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 521 sg = sg_next(sg); 522 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 523 struct dma_interleaved_template *il = xfer->xfer.il; 524 struct data_chunk *dc = &il->sgl[i % fsz]; 525 526 src_addr += burst->sz; 527 if (il->src_sgl) 528 src_addr += dmaengine_get_src_icg(il, dc); 529 530 dst_addr += burst->sz; 531 if (il->dst_sgl) 532 dst_addr += dmaengine_get_dst_icg(il, dc); 533 } 534 } 535 536 return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags); 537 538 err_alloc: 539 if (desc) 540 dw_edma_free_desc(desc); 541 542 return NULL; 543 } 544 545 static struct dma_async_tx_descriptor * 546 dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, 547 unsigned int len, 548 enum dma_transfer_direction direction, 549 unsigned long flags, void *context) 550 { 551 struct dw_edma_transfer xfer; 552 553 xfer.dchan = dchan; 554 xfer.direction = direction; 555 xfer.xfer.sg.sgl = sgl; 556 xfer.xfer.sg.len = len; 557 xfer.flags = flags; 558 xfer.type = EDMA_XFER_SCATTER_GATHER; 559 560 return dw_edma_device_transfer(&xfer); 561 } 562 563 static struct dma_async_tx_descriptor * 564 dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr, 565 size_t len, size_t count, 566 enum dma_transfer_direction direction, 567 unsigned long flags) 568 { 569 struct dw_edma_transfer xfer; 570 571 xfer.dchan = dchan; 572 xfer.direction = direction; 573 xfer.xfer.cyclic.paddr = paddr; 574 xfer.xfer.cyclic.len = len; 575 xfer.xfer.cyclic.cnt = count; 576 xfer.flags = flags; 577 xfer.type = EDMA_XFER_CYCLIC; 578 579 return dw_edma_device_transfer(&xfer); 580 } 581 582 static struct dma_async_tx_descriptor * 583 dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan, 584 struct dma_interleaved_template *ilt, 585 unsigned long flags) 586 { 587 struct dw_edma_transfer xfer; 588 589 xfer.dchan = dchan; 590 xfer.direction = ilt->dir; 591 xfer.xfer.il = ilt; 592 xfer.flags = flags; 593 xfer.type = EDMA_XFER_INTERLEAVED; 594 595 return dw_edma_device_transfer(&xfer); 596 } 597 598 static void dw_edma_done_interrupt(struct dw_edma_chan *chan) 599 { 600 struct dw_edma_desc *desc; 601 struct virt_dma_desc *vd; 602 unsigned long flags; 603 604 spin_lock_irqsave(&chan->vc.lock, flags); 605 vd = vchan_next_desc(&chan->vc); 606 if (vd) { 607 switch (chan->request) { 608 case EDMA_REQ_NONE: 609 desc = vd2dw_edma_desc(vd); 610 if (!desc->chunks_alloc) { 611 list_del(&vd->node); 612 vchan_cookie_complete(vd); 613 } 614 615 /* Continue transferring if there are remaining chunks or issued requests. 616 */ 617 chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE; 618 break; 619 620 case EDMA_REQ_STOP: 621 list_del(&vd->node); 622 vchan_cookie_complete(vd); 623 chan->request = EDMA_REQ_NONE; 624 chan->status = EDMA_ST_IDLE; 625 break; 626 627 case EDMA_REQ_PAUSE: 628 chan->request = EDMA_REQ_NONE; 629 chan->status = EDMA_ST_PAUSE; 630 break; 631 632 default: 633 break; 634 } 635 } 636 spin_unlock_irqrestore(&chan->vc.lock, flags); 637 } 638 639 static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) 640 { 641 struct virt_dma_desc *vd; 642 unsigned long flags; 643 644 spin_lock_irqsave(&chan->vc.lock, flags); 645 vd = vchan_next_desc(&chan->vc); 646 if (vd) { 647 list_del(&vd->node); 648 vchan_cookie_complete(vd); 649 } 650 spin_unlock_irqrestore(&chan->vc.lock, flags); 651 chan->request = EDMA_REQ_NONE; 652 chan->status = EDMA_ST_IDLE; 653 } 654 655 static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) 656 { 657 struct dw_edma_irq *dw_irq = data; 658 659 return dw_edma_core_handle_int(dw_irq, EDMA_DIR_WRITE, 660 dw_edma_done_interrupt, 661 dw_edma_abort_interrupt); 662 } 663 664 static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data) 665 { 666 struct dw_edma_irq *dw_irq = data; 667 668 return dw_edma_core_handle_int(dw_irq, EDMA_DIR_READ, 669 dw_edma_done_interrupt, 670 dw_edma_abort_interrupt); 671 } 672 673 static irqreturn_t dw_edma_interrupt_common(int irq, void *data) 674 { 675 irqreturn_t ret = IRQ_NONE; 676 677 ret |= dw_edma_interrupt_write(irq, data); 678 ret |= dw_edma_interrupt_read(irq, data); 679 680 return ret; 681 } 682 683 static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) 684 { 685 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 686 687 if (chan->status != EDMA_ST_IDLE) 688 return -EBUSY; 689 690 return 0; 691 } 692 693 static void dw_edma_free_chan_resources(struct dma_chan *dchan) 694 { 695 unsigned long timeout = jiffies + msecs_to_jiffies(5000); 696 int ret; 697 698 while (time_before(jiffies, timeout)) { 699 ret = dw_edma_device_terminate_all(dchan); 700 if (!ret) 701 break; 702 703 if (time_after_eq(jiffies, timeout)) 704 return; 705 706 cpu_relax(); 707 } 708 } 709 710 static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) 711 { 712 struct dw_edma_chip *chip = dw->chip; 713 struct device *dev = chip->dev; 714 struct dw_edma_chan *chan; 715 struct dw_edma_irq *irq; 716 struct dma_device *dma; 717 u32 i, ch_cnt; 718 u32 pos; 719 720 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; 721 dma = &dw->dma; 722 723 INIT_LIST_HEAD(&dma->channels); 724 725 for (i = 0; i < ch_cnt; i++) { 726 chan = &dw->chan[i]; 727 728 chan->dw = dw; 729 730 if (i < dw->wr_ch_cnt) { 731 chan->id = i; 732 chan->dir = EDMA_DIR_WRITE; 733 } else { 734 chan->id = i - dw->wr_ch_cnt; 735 chan->dir = EDMA_DIR_READ; 736 } 737 738 chan->configured = false; 739 chan->request = EDMA_REQ_NONE; 740 chan->status = EDMA_ST_IDLE; 741 742 if (chan->dir == EDMA_DIR_WRITE) 743 chan->ll_max = (chip->ll_region_wr[chan->id].sz / EDMA_LL_SZ); 744 else 745 chan->ll_max = (chip->ll_region_rd[chan->id].sz / EDMA_LL_SZ); 746 chan->ll_max -= 1; 747 748 dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n", 749 chan->dir == EDMA_DIR_WRITE ? "write" : "read", 750 chan->id, chan->ll_max); 751 752 if (dw->nr_irqs == 1) 753 pos = 0; 754 else if (chan->dir == EDMA_DIR_WRITE) 755 pos = chan->id % wr_alloc; 756 else 757 pos = wr_alloc + chan->id % rd_alloc; 758 759 irq = &dw->irq[pos]; 760 761 if (chan->dir == EDMA_DIR_WRITE) 762 irq->wr_mask |= BIT(chan->id); 763 else 764 irq->rd_mask |= BIT(chan->id); 765 766 irq->dw = dw; 767 memcpy(&chan->msi, &irq->msi, sizeof(chan->msi)); 768 769 dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n", 770 chan->dir == EDMA_DIR_WRITE ? "write" : "read", chan->id, 771 chan->msi.address_hi, chan->msi.address_lo, 772 chan->msi.data); 773 774 chan->vc.desc_free = vchan_free_desc; 775 chan->vc.chan.private = chan->dir == EDMA_DIR_WRITE ? 776 &dw->chip->dt_region_wr[chan->id] : 777 &dw->chip->dt_region_rd[chan->id]; 778 779 vchan_init(&chan->vc, dma); 780 781 dw_edma_core_ch_config(chan); 782 } 783 784 /* Set DMA channel capabilities */ 785 dma_cap_zero(dma->cap_mask); 786 dma_cap_set(DMA_SLAVE, dma->cap_mask); 787 dma_cap_set(DMA_CYCLIC, dma->cap_mask); 788 dma_cap_set(DMA_PRIVATE, dma->cap_mask); 789 dma_cap_set(DMA_INTERLEAVE, dma->cap_mask); 790 dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 791 dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 792 dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 793 dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 794 795 /* Set DMA channel callbacks */ 796 dma->dev = chip->dev; 797 dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources; 798 dma->device_free_chan_resources = dw_edma_free_chan_resources; 799 dma->device_caps = dw_edma_device_caps; 800 dma->device_config = dw_edma_device_config; 801 dma->device_pause = dw_edma_device_pause; 802 dma->device_resume = dw_edma_device_resume; 803 dma->device_terminate_all = dw_edma_device_terminate_all; 804 dma->device_issue_pending = dw_edma_device_issue_pending; 805 dma->device_tx_status = dw_edma_device_tx_status; 806 dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg; 807 dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic; 808 dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma; 809 810 dma_set_max_seg_size(dma->dev, U32_MAX); 811 812 /* Register DMA device */ 813 return dma_async_device_register(dma); 814 } 815 816 static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt) 817 { 818 if (*nr_irqs && *alloc < cnt) { 819 (*alloc)++; 820 (*nr_irqs)--; 821 } 822 } 823 824 static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt) 825 { 826 while (*mask * alloc < cnt) 827 (*mask)++; 828 } 829 830 static int dw_edma_irq_request(struct dw_edma *dw, 831 u32 *wr_alloc, u32 *rd_alloc) 832 { 833 struct dw_edma_chip *chip = dw->chip; 834 struct device *dev = dw->chip->dev; 835 u32 wr_mask = 1; 836 u32 rd_mask = 1; 837 int i, err = 0; 838 u32 ch_cnt; 839 int irq; 840 841 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; 842 843 if (chip->nr_irqs < 1 || !chip->ops->irq_vector) 844 return -EINVAL; 845 846 dw->irq = devm_kcalloc(dev, chip->nr_irqs, sizeof(*dw->irq), GFP_KERNEL); 847 if (!dw->irq) 848 return -ENOMEM; 849 850 if (chip->nr_irqs == 1) { 851 /* Common IRQ shared among all channels */ 852 irq = chip->ops->irq_vector(dev, 0); 853 err = request_irq(irq, dw_edma_interrupt_common, 854 IRQF_SHARED, dw->name, &dw->irq[0]); 855 if (err) { 856 dw->nr_irqs = 0; 857 return err; 858 } 859 860 if (irq_get_msi_desc(irq)) 861 get_cached_msi_msg(irq, &dw->irq[0].msi); 862 863 dw->nr_irqs = 1; 864 } else { 865 /* Distribute IRQs equally among all channels */ 866 int tmp = chip->nr_irqs; 867 868 while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) { 869 dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt); 870 dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt); 871 } 872 873 dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt); 874 dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt); 875 876 for (i = 0; i < (*wr_alloc + *rd_alloc); i++) { 877 irq = chip->ops->irq_vector(dev, i); 878 err = request_irq(irq, 879 i < *wr_alloc ? 880 dw_edma_interrupt_write : 881 dw_edma_interrupt_read, 882 IRQF_SHARED, dw->name, 883 &dw->irq[i]); 884 if (err) 885 goto err_irq_free; 886 887 if (irq_get_msi_desc(irq)) 888 get_cached_msi_msg(irq, &dw->irq[i].msi); 889 } 890 891 dw->nr_irqs = i; 892 } 893 894 return 0; 895 896 err_irq_free: 897 for (i--; i >= 0; i--) { 898 irq = chip->ops->irq_vector(dev, i); 899 free_irq(irq, &dw->irq[i]); 900 } 901 902 return err; 903 } 904 905 int dw_edma_probe(struct dw_edma_chip *chip) 906 { 907 struct device *dev; 908 struct dw_edma *dw; 909 u32 wr_alloc = 0; 910 u32 rd_alloc = 0; 911 int i, err; 912 913 if (!chip) 914 return -EINVAL; 915 916 dev = chip->dev; 917 if (!dev || !chip->ops) 918 return -EINVAL; 919 920 dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL); 921 if (!dw) 922 return -ENOMEM; 923 924 dw->chip = chip; 925 926 if (dw->chip->mf == EDMA_MF_HDMA_NATIVE) 927 dw_hdma_v0_core_register(dw); 928 else 929 dw_edma_v0_core_register(dw); 930 931 raw_spin_lock_init(&dw->lock); 932 933 dw->wr_ch_cnt = min_t(u16, chip->ll_wr_cnt, 934 dw_edma_core_ch_count(dw, EDMA_DIR_WRITE)); 935 dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH); 936 937 dw->rd_ch_cnt = min_t(u16, chip->ll_rd_cnt, 938 dw_edma_core_ch_count(dw, EDMA_DIR_READ)); 939 dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH); 940 941 if (!dw->wr_ch_cnt && !dw->rd_ch_cnt) 942 return -EINVAL; 943 944 dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n", 945 dw->wr_ch_cnt, dw->rd_ch_cnt); 946 947 /* Allocate channels */ 948 dw->chan = devm_kcalloc(dev, dw->wr_ch_cnt + dw->rd_ch_cnt, 949 sizeof(*dw->chan), GFP_KERNEL); 950 if (!dw->chan) 951 return -ENOMEM; 952 953 snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%s", 954 dev_name(chip->dev)); 955 956 /* Disable eDMA, only to establish the ideal initial conditions */ 957 dw_edma_core_off(dw); 958 959 /* Request IRQs */ 960 err = dw_edma_irq_request(dw, &wr_alloc, &rd_alloc); 961 if (err) 962 return err; 963 964 /* Setup write/read channels */ 965 err = dw_edma_channel_setup(dw, wr_alloc, rd_alloc); 966 if (err) 967 goto err_irq_free; 968 969 /* Turn debugfs on */ 970 dw_edma_core_debugfs_on(dw); 971 972 chip->dw = dw; 973 974 return 0; 975 976 err_irq_free: 977 for (i = (dw->nr_irqs - 1); i >= 0; i--) 978 free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); 979 980 return err; 981 } 982 EXPORT_SYMBOL_GPL(dw_edma_probe); 983 984 int dw_edma_remove(struct dw_edma_chip *chip) 985 { 986 struct dw_edma_chan *chan, *_chan; 987 struct device *dev = chip->dev; 988 struct dw_edma *dw = chip->dw; 989 int i; 990 991 /* Skip removal if no private data found */ 992 if (!dw) 993 return -ENODEV; 994 995 /* Disable eDMA */ 996 dw_edma_core_off(dw); 997 998 /* Free irqs */ 999 for (i = (dw->nr_irqs - 1); i >= 0; i--) 1000 free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); 1001 1002 /* Deregister eDMA device */ 1003 dma_async_device_unregister(&dw->dma); 1004 list_for_each_entry_safe(chan, _chan, &dw->dma.channels, 1005 vc.chan.device_node) { 1006 tasklet_kill(&chan->vc.task); 1007 list_del(&chan->vc.chan.device_node); 1008 } 1009 1010 return 0; 1011 } 1012 EXPORT_SYMBOL_GPL(dw_edma_remove); 1013 1014 MODULE_LICENSE("GPL v2"); 1015 MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver"); 1016 MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>"); 1017