1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. 4 * Synopsys DesignWare eDMA core driver 5 * 6 * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> 7 */ 8 9 #include <linux/module.h> 10 #include <linux/device.h> 11 #include <linux/kernel.h> 12 #include <linux/dmaengine.h> 13 #include <linux/err.h> 14 #include <linux/interrupt.h> 15 #include <linux/irq.h> 16 #include <linux/dma/edma.h> 17 #include <linux/dma-mapping.h> 18 19 #include "dw-edma-core.h" 20 #include "dw-edma-v0-core.h" 21 #include "../dmaengine.h" 22 #include "../virt-dma.h" 23 24 static inline 25 struct device *dchan2dev(struct dma_chan *dchan) 26 { 27 return &dchan->dev->device; 28 } 29 30 static inline 31 struct device *chan2dev(struct dw_edma_chan *chan) 32 { 33 return &chan->vc.chan.dev->device; 34 } 35 36 static inline 37 struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd) 38 { 39 return container_of(vd, struct dw_edma_desc, vd); 40 } 41 42 static inline 43 u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr) 44 { 45 struct dw_edma_chip *chip = chan->dw->chip; 46 47 if (chip->ops->pci_address) 48 return chip->ops->pci_address(chip->dev, cpu_addr); 49 50 return cpu_addr; 51 } 52 53 static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk) 54 { 55 struct dw_edma_burst *burst; 56 57 burst = kzalloc(sizeof(*burst), GFP_NOWAIT); 58 if (unlikely(!burst)) 59 return NULL; 60 61 INIT_LIST_HEAD(&burst->list); 62 if (chunk->burst) { 63 /* Create and add new element into the linked list */ 64 chunk->bursts_alloc++; 65 list_add_tail(&burst->list, &chunk->burst->list); 66 } else { 67 /* List head */ 68 chunk->bursts_alloc = 0; 69 chunk->burst = burst; 70 } 71 72 return burst; 73 } 74 75 static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc) 76 { 77 struct dw_edma_chip *chip = desc->chan->dw->chip; 78 struct dw_edma_chan *chan = desc->chan; 79 struct dw_edma_chunk *chunk; 80 81 chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT); 82 if (unlikely(!chunk)) 83 return NULL; 84 85 INIT_LIST_HEAD(&chunk->list); 86 chunk->chan = chan; 87 /* Toggling change bit (CB) in each chunk, this is a mechanism to 88 * inform the eDMA HW block that this is a new linked list ready 89 * to be consumed. 90 * - Odd chunks originate CB equal to 0 91 * - Even chunks originate CB equal to 1 92 */ 93 chunk->cb = !(desc->chunks_alloc % 2); 94 if (chan->dir == EDMA_DIR_WRITE) { 95 chunk->ll_region.paddr = chip->ll_region_wr[chan->id].paddr; 96 chunk->ll_region.vaddr = chip->ll_region_wr[chan->id].vaddr; 97 } else { 98 chunk->ll_region.paddr = chip->ll_region_rd[chan->id].paddr; 99 chunk->ll_region.vaddr = chip->ll_region_rd[chan->id].vaddr; 100 } 101 102 if (desc->chunk) { 103 /* Create and add new element into the linked list */ 104 if (!dw_edma_alloc_burst(chunk)) { 105 kfree(chunk); 106 return NULL; 107 } 108 desc->chunks_alloc++; 109 list_add_tail(&chunk->list, &desc->chunk->list); 110 } else { 111 /* List head */ 112 chunk->burst = NULL; 113 desc->chunks_alloc = 0; 114 desc->chunk = chunk; 115 } 116 117 return chunk; 118 } 119 120 static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan) 121 { 122 struct dw_edma_desc *desc; 123 124 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 125 if (unlikely(!desc)) 126 return NULL; 127 128 desc->chan = chan; 129 if (!dw_edma_alloc_chunk(desc)) { 130 kfree(desc); 131 return NULL; 132 } 133 134 return desc; 135 } 136 137 static void dw_edma_free_burst(struct dw_edma_chunk *chunk) 138 { 139 struct dw_edma_burst *child, *_next; 140 141 /* Remove all the list elements */ 142 list_for_each_entry_safe(child, _next, &chunk->burst->list, list) { 143 list_del(&child->list); 144 kfree(child); 145 chunk->bursts_alloc--; 146 } 147 148 /* Remove the list head */ 149 kfree(child); 150 chunk->burst = NULL; 151 } 152 153 static void dw_edma_free_chunk(struct dw_edma_desc *desc) 154 { 155 struct dw_edma_chunk *child, *_next; 156 157 if (!desc->chunk) 158 return; 159 160 /* Remove all the list elements */ 161 list_for_each_entry_safe(child, _next, &desc->chunk->list, list) { 162 dw_edma_free_burst(child); 163 list_del(&child->list); 164 kfree(child); 165 desc->chunks_alloc--; 166 } 167 168 /* Remove the list head */ 169 kfree(child); 170 desc->chunk = NULL; 171 } 172 173 static void dw_edma_free_desc(struct dw_edma_desc *desc) 174 { 175 dw_edma_free_chunk(desc); 176 kfree(desc); 177 } 178 179 static void vchan_free_desc(struct virt_dma_desc *vdesc) 180 { 181 dw_edma_free_desc(vd2dw_edma_desc(vdesc)); 182 } 183 184 static int dw_edma_start_transfer(struct dw_edma_chan *chan) 185 { 186 struct dw_edma_chunk *child; 187 struct dw_edma_desc *desc; 188 struct virt_dma_desc *vd; 189 190 vd = vchan_next_desc(&chan->vc); 191 if (!vd) 192 return 0; 193 194 desc = vd2dw_edma_desc(vd); 195 if (!desc) 196 return 0; 197 198 child = list_first_entry_or_null(&desc->chunk->list, 199 struct dw_edma_chunk, list); 200 if (!child) 201 return 0; 202 203 dw_edma_v0_core_start(child, !desc->xfer_sz); 204 desc->xfer_sz += child->ll_region.sz; 205 dw_edma_free_burst(child); 206 list_del(&child->list); 207 kfree(child); 208 desc->chunks_alloc--; 209 210 return 1; 211 } 212 213 static void dw_edma_device_caps(struct dma_chan *dchan, 214 struct dma_slave_caps *caps) 215 { 216 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 217 218 if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { 219 if (chan->dir == EDMA_DIR_READ) 220 caps->directions = BIT(DMA_DEV_TO_MEM); 221 else 222 caps->directions = BIT(DMA_MEM_TO_DEV); 223 } else { 224 if (chan->dir == EDMA_DIR_WRITE) 225 caps->directions = BIT(DMA_DEV_TO_MEM); 226 else 227 caps->directions = BIT(DMA_MEM_TO_DEV); 228 } 229 } 230 231 static int dw_edma_device_config(struct dma_chan *dchan, 232 struct dma_slave_config *config) 233 { 234 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 235 236 memcpy(&chan->config, config, sizeof(*config)); 237 chan->configured = true; 238 239 return 0; 240 } 241 242 static int dw_edma_device_pause(struct dma_chan *dchan) 243 { 244 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 245 int err = 0; 246 247 if (!chan->configured) 248 err = -EPERM; 249 else if (chan->status != EDMA_ST_BUSY) 250 err = -EPERM; 251 else if (chan->request != EDMA_REQ_NONE) 252 err = -EPERM; 253 else 254 chan->request = EDMA_REQ_PAUSE; 255 256 return err; 257 } 258 259 static int dw_edma_device_resume(struct dma_chan *dchan) 260 { 261 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 262 int err = 0; 263 264 if (!chan->configured) { 265 err = -EPERM; 266 } else if (chan->status != EDMA_ST_PAUSE) { 267 err = -EPERM; 268 } else if (chan->request != EDMA_REQ_NONE) { 269 err = -EPERM; 270 } else { 271 chan->status = EDMA_ST_BUSY; 272 dw_edma_start_transfer(chan); 273 } 274 275 return err; 276 } 277 278 static int dw_edma_device_terminate_all(struct dma_chan *dchan) 279 { 280 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 281 int err = 0; 282 283 if (!chan->configured) { 284 /* Do nothing */ 285 } else if (chan->status == EDMA_ST_PAUSE) { 286 chan->status = EDMA_ST_IDLE; 287 chan->configured = false; 288 } else if (chan->status == EDMA_ST_IDLE) { 289 chan->configured = false; 290 } else if (dw_edma_v0_core_ch_status(chan) == DMA_COMPLETE) { 291 /* 292 * The channel is in a false BUSY state, probably didn't 293 * receive or lost an interrupt 294 */ 295 chan->status = EDMA_ST_IDLE; 296 chan->configured = false; 297 } else if (chan->request > EDMA_REQ_PAUSE) { 298 err = -EPERM; 299 } else { 300 chan->request = EDMA_REQ_STOP; 301 } 302 303 return err; 304 } 305 306 static void dw_edma_device_issue_pending(struct dma_chan *dchan) 307 { 308 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 309 unsigned long flags; 310 311 if (!chan->configured) 312 return; 313 314 spin_lock_irqsave(&chan->vc.lock, flags); 315 if (vchan_issue_pending(&chan->vc) && chan->request == EDMA_REQ_NONE && 316 chan->status == EDMA_ST_IDLE) { 317 chan->status = EDMA_ST_BUSY; 318 dw_edma_start_transfer(chan); 319 } 320 spin_unlock_irqrestore(&chan->vc.lock, flags); 321 } 322 323 static enum dma_status 324 dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, 325 struct dma_tx_state *txstate) 326 { 327 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 328 struct dw_edma_desc *desc; 329 struct virt_dma_desc *vd; 330 unsigned long flags; 331 enum dma_status ret; 332 u32 residue = 0; 333 334 ret = dma_cookie_status(dchan, cookie, txstate); 335 if (ret == DMA_COMPLETE) 336 return ret; 337 338 if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE) 339 ret = DMA_PAUSED; 340 341 if (!txstate) 342 goto ret_residue; 343 344 spin_lock_irqsave(&chan->vc.lock, flags); 345 vd = vchan_find_desc(&chan->vc, cookie); 346 if (vd) { 347 desc = vd2dw_edma_desc(vd); 348 if (desc) 349 residue = desc->alloc_sz - desc->xfer_sz; 350 } 351 spin_unlock_irqrestore(&chan->vc.lock, flags); 352 353 ret_residue: 354 dma_set_residue(txstate, residue); 355 356 return ret; 357 } 358 359 static struct dma_async_tx_descriptor * 360 dw_edma_device_transfer(struct dw_edma_transfer *xfer) 361 { 362 struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan); 363 enum dma_transfer_direction dir = xfer->direction; 364 struct scatterlist *sg = NULL; 365 struct dw_edma_chunk *chunk; 366 struct dw_edma_burst *burst; 367 struct dw_edma_desc *desc; 368 u64 src_addr, dst_addr; 369 size_t fsz = 0; 370 u32 cnt = 0; 371 int i; 372 373 if (!chan->configured) 374 return NULL; 375 376 /* 377 * Local Root Port/End-point Remote End-point 378 * +-----------------------+ PCIe bus +----------------------+ 379 * | | +-+ | | 380 * | DEV_TO_MEM Rx Ch <----+ +---+ Tx Ch DEV_TO_MEM | 381 * | | | | | | 382 * | MEM_TO_DEV Tx Ch +----+ +---> Rx Ch MEM_TO_DEV | 383 * | | +-+ | | 384 * +-----------------------+ +----------------------+ 385 * 386 * 1. Normal logic: 387 * If eDMA is embedded into the DW PCIe RP/EP and controlled from the 388 * CPU/Application side, the Rx channel (EDMA_DIR_READ) will be used 389 * for the device read operations (DEV_TO_MEM) and the Tx channel 390 * (EDMA_DIR_WRITE) - for the write operations (MEM_TO_DEV). 391 * 392 * 2. Inverted logic: 393 * If eDMA is embedded into a Remote PCIe EP and is controlled by the 394 * MWr/MRd TLPs sent from the CPU's PCIe host controller, the Tx 395 * channel (EDMA_DIR_WRITE) will be used for the device read operations 396 * (DEV_TO_MEM) and the Rx channel (EDMA_DIR_READ) - for the write 397 * operations (MEM_TO_DEV). 398 * 399 * It is the client driver responsibility to choose a proper channel 400 * for the DMA transfers. 401 */ 402 if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { 403 if ((chan->dir == EDMA_DIR_READ && dir != DMA_DEV_TO_MEM) || 404 (chan->dir == EDMA_DIR_WRITE && dir != DMA_MEM_TO_DEV)) 405 return NULL; 406 } else { 407 if ((chan->dir == EDMA_DIR_WRITE && dir != DMA_DEV_TO_MEM) || 408 (chan->dir == EDMA_DIR_READ && dir != DMA_MEM_TO_DEV)) 409 return NULL; 410 } 411 412 if (xfer->type == EDMA_XFER_CYCLIC) { 413 if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt) 414 return NULL; 415 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 416 if (xfer->xfer.sg.len < 1) 417 return NULL; 418 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 419 if (!xfer->xfer.il->numf || xfer->xfer.il->frame_size < 1) 420 return NULL; 421 if (!xfer->xfer.il->src_inc || !xfer->xfer.il->dst_inc) 422 return NULL; 423 } else { 424 return NULL; 425 } 426 427 desc = dw_edma_alloc_desc(chan); 428 if (unlikely(!desc)) 429 goto err_alloc; 430 431 chunk = dw_edma_alloc_chunk(desc); 432 if (unlikely(!chunk)) 433 goto err_alloc; 434 435 if (xfer->type == EDMA_XFER_INTERLEAVED) { 436 src_addr = xfer->xfer.il->src_start; 437 dst_addr = xfer->xfer.il->dst_start; 438 } else { 439 src_addr = chan->config.src_addr; 440 dst_addr = chan->config.dst_addr; 441 } 442 443 if (dir == DMA_DEV_TO_MEM) 444 src_addr = dw_edma_get_pci_address(chan, (phys_addr_t)src_addr); 445 else 446 dst_addr = dw_edma_get_pci_address(chan, (phys_addr_t)dst_addr); 447 448 if (xfer->type == EDMA_XFER_CYCLIC) { 449 cnt = xfer->xfer.cyclic.cnt; 450 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 451 cnt = xfer->xfer.sg.len; 452 sg = xfer->xfer.sg.sgl; 453 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 454 cnt = xfer->xfer.il->numf * xfer->xfer.il->frame_size; 455 fsz = xfer->xfer.il->frame_size; 456 } 457 458 for (i = 0; i < cnt; i++) { 459 if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg) 460 break; 461 462 if (chunk->bursts_alloc == chan->ll_max) { 463 chunk = dw_edma_alloc_chunk(desc); 464 if (unlikely(!chunk)) 465 goto err_alloc; 466 } 467 468 burst = dw_edma_alloc_burst(chunk); 469 if (unlikely(!burst)) 470 goto err_alloc; 471 472 if (xfer->type == EDMA_XFER_CYCLIC) 473 burst->sz = xfer->xfer.cyclic.len; 474 else if (xfer->type == EDMA_XFER_SCATTER_GATHER) 475 burst->sz = sg_dma_len(sg); 476 else if (xfer->type == EDMA_XFER_INTERLEAVED) 477 burst->sz = xfer->xfer.il->sgl[i % fsz].size; 478 479 chunk->ll_region.sz += burst->sz; 480 desc->alloc_sz += burst->sz; 481 482 if (dir == DMA_DEV_TO_MEM) { 483 burst->sar = src_addr; 484 if (xfer->type == EDMA_XFER_CYCLIC) { 485 burst->dar = xfer->xfer.cyclic.paddr; 486 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 487 src_addr += sg_dma_len(sg); 488 burst->dar = sg_dma_address(sg); 489 /* Unlike the typical assumption by other 490 * drivers/IPs the peripheral memory isn't 491 * a FIFO memory, in this case, it's a 492 * linear memory and that why the source 493 * and destination addresses are increased 494 * by the same portion (data length) 495 */ 496 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 497 burst->dar = dst_addr; 498 } 499 } else { 500 burst->dar = dst_addr; 501 if (xfer->type == EDMA_XFER_CYCLIC) { 502 burst->sar = xfer->xfer.cyclic.paddr; 503 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 504 dst_addr += sg_dma_len(sg); 505 burst->sar = sg_dma_address(sg); 506 /* Unlike the typical assumption by other 507 * drivers/IPs the peripheral memory isn't 508 * a FIFO memory, in this case, it's a 509 * linear memory and that why the source 510 * and destination addresses are increased 511 * by the same portion (data length) 512 */ 513 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 514 burst->sar = src_addr; 515 } 516 } 517 518 if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 519 sg = sg_next(sg); 520 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 521 struct dma_interleaved_template *il = xfer->xfer.il; 522 struct data_chunk *dc = &il->sgl[i % fsz]; 523 524 src_addr += burst->sz; 525 if (il->src_sgl) 526 src_addr += dmaengine_get_src_icg(il, dc); 527 528 dst_addr += burst->sz; 529 if (il->dst_sgl) 530 dst_addr += dmaengine_get_dst_icg(il, dc); 531 } 532 } 533 534 return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags); 535 536 err_alloc: 537 if (desc) 538 dw_edma_free_desc(desc); 539 540 return NULL; 541 } 542 543 static struct dma_async_tx_descriptor * 544 dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, 545 unsigned int len, 546 enum dma_transfer_direction direction, 547 unsigned long flags, void *context) 548 { 549 struct dw_edma_transfer xfer; 550 551 xfer.dchan = dchan; 552 xfer.direction = direction; 553 xfer.xfer.sg.sgl = sgl; 554 xfer.xfer.sg.len = len; 555 xfer.flags = flags; 556 xfer.type = EDMA_XFER_SCATTER_GATHER; 557 558 return dw_edma_device_transfer(&xfer); 559 } 560 561 static struct dma_async_tx_descriptor * 562 dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr, 563 size_t len, size_t count, 564 enum dma_transfer_direction direction, 565 unsigned long flags) 566 { 567 struct dw_edma_transfer xfer; 568 569 xfer.dchan = dchan; 570 xfer.direction = direction; 571 xfer.xfer.cyclic.paddr = paddr; 572 xfer.xfer.cyclic.len = len; 573 xfer.xfer.cyclic.cnt = count; 574 xfer.flags = flags; 575 xfer.type = EDMA_XFER_CYCLIC; 576 577 return dw_edma_device_transfer(&xfer); 578 } 579 580 static struct dma_async_tx_descriptor * 581 dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan, 582 struct dma_interleaved_template *ilt, 583 unsigned long flags) 584 { 585 struct dw_edma_transfer xfer; 586 587 xfer.dchan = dchan; 588 xfer.direction = ilt->dir; 589 xfer.xfer.il = ilt; 590 xfer.flags = flags; 591 xfer.type = EDMA_XFER_INTERLEAVED; 592 593 return dw_edma_device_transfer(&xfer); 594 } 595 596 static void dw_edma_done_interrupt(struct dw_edma_chan *chan) 597 { 598 struct dw_edma_desc *desc; 599 struct virt_dma_desc *vd; 600 unsigned long flags; 601 602 dw_edma_v0_core_clear_done_int(chan); 603 604 spin_lock_irqsave(&chan->vc.lock, flags); 605 vd = vchan_next_desc(&chan->vc); 606 if (vd) { 607 switch (chan->request) { 608 case EDMA_REQ_NONE: 609 desc = vd2dw_edma_desc(vd); 610 if (!desc->chunks_alloc) { 611 list_del(&vd->node); 612 vchan_cookie_complete(vd); 613 } 614 615 /* Continue transferring if there are remaining chunks or issued requests. 616 */ 617 chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE; 618 break; 619 620 case EDMA_REQ_STOP: 621 list_del(&vd->node); 622 vchan_cookie_complete(vd); 623 chan->request = EDMA_REQ_NONE; 624 chan->status = EDMA_ST_IDLE; 625 break; 626 627 case EDMA_REQ_PAUSE: 628 chan->request = EDMA_REQ_NONE; 629 chan->status = EDMA_ST_PAUSE; 630 break; 631 632 default: 633 break; 634 } 635 } 636 spin_unlock_irqrestore(&chan->vc.lock, flags); 637 } 638 639 static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) 640 { 641 struct virt_dma_desc *vd; 642 unsigned long flags; 643 644 dw_edma_v0_core_clear_abort_int(chan); 645 646 spin_lock_irqsave(&chan->vc.lock, flags); 647 vd = vchan_next_desc(&chan->vc); 648 if (vd) { 649 list_del(&vd->node); 650 vchan_cookie_complete(vd); 651 } 652 spin_unlock_irqrestore(&chan->vc.lock, flags); 653 chan->request = EDMA_REQ_NONE; 654 chan->status = EDMA_ST_IDLE; 655 } 656 657 static irqreturn_t dw_edma_interrupt(int irq, void *data, bool write) 658 { 659 struct dw_edma_irq *dw_irq = data; 660 struct dw_edma *dw = dw_irq->dw; 661 unsigned long total, pos, val; 662 unsigned long off; 663 u32 mask; 664 665 if (write) { 666 total = dw->wr_ch_cnt; 667 off = 0; 668 mask = dw_irq->wr_mask; 669 } else { 670 total = dw->rd_ch_cnt; 671 off = dw->wr_ch_cnt; 672 mask = dw_irq->rd_mask; 673 } 674 675 val = dw_edma_v0_core_status_done_int(dw, write ? 676 EDMA_DIR_WRITE : 677 EDMA_DIR_READ); 678 val &= mask; 679 for_each_set_bit(pos, &val, total) { 680 struct dw_edma_chan *chan = &dw->chan[pos + off]; 681 682 dw_edma_done_interrupt(chan); 683 } 684 685 val = dw_edma_v0_core_status_abort_int(dw, write ? 686 EDMA_DIR_WRITE : 687 EDMA_DIR_READ); 688 val &= mask; 689 for_each_set_bit(pos, &val, total) { 690 struct dw_edma_chan *chan = &dw->chan[pos + off]; 691 692 dw_edma_abort_interrupt(chan); 693 } 694 695 return IRQ_HANDLED; 696 } 697 698 static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) 699 { 700 return dw_edma_interrupt(irq, data, true); 701 } 702 703 static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data) 704 { 705 return dw_edma_interrupt(irq, data, false); 706 } 707 708 static irqreturn_t dw_edma_interrupt_common(int irq, void *data) 709 { 710 dw_edma_interrupt(irq, data, true); 711 dw_edma_interrupt(irq, data, false); 712 713 return IRQ_HANDLED; 714 } 715 716 static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) 717 { 718 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 719 720 if (chan->status != EDMA_ST_IDLE) 721 return -EBUSY; 722 723 return 0; 724 } 725 726 static void dw_edma_free_chan_resources(struct dma_chan *dchan) 727 { 728 unsigned long timeout = jiffies + msecs_to_jiffies(5000); 729 int ret; 730 731 while (time_before(jiffies, timeout)) { 732 ret = dw_edma_device_terminate_all(dchan); 733 if (!ret) 734 break; 735 736 if (time_after_eq(jiffies, timeout)) 737 return; 738 739 cpu_relax(); 740 } 741 } 742 743 static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) 744 { 745 struct dw_edma_chip *chip = dw->chip; 746 struct device *dev = chip->dev; 747 struct dw_edma_chan *chan; 748 struct dw_edma_irq *irq; 749 struct dma_device *dma; 750 u32 i, ch_cnt; 751 u32 pos; 752 753 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; 754 dma = &dw->dma; 755 756 INIT_LIST_HEAD(&dma->channels); 757 758 for (i = 0; i < ch_cnt; i++) { 759 chan = &dw->chan[i]; 760 761 chan->dw = dw; 762 763 if (i < dw->wr_ch_cnt) { 764 chan->id = i; 765 chan->dir = EDMA_DIR_WRITE; 766 } else { 767 chan->id = i - dw->wr_ch_cnt; 768 chan->dir = EDMA_DIR_READ; 769 } 770 771 chan->configured = false; 772 chan->request = EDMA_REQ_NONE; 773 chan->status = EDMA_ST_IDLE; 774 775 if (chan->dir == EDMA_DIR_WRITE) 776 chan->ll_max = (chip->ll_region_wr[chan->id].sz / EDMA_LL_SZ); 777 else 778 chan->ll_max = (chip->ll_region_rd[chan->id].sz / EDMA_LL_SZ); 779 chan->ll_max -= 1; 780 781 dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n", 782 chan->dir == EDMA_DIR_WRITE ? "write" : "read", 783 chan->id, chan->ll_max); 784 785 if (dw->nr_irqs == 1) 786 pos = 0; 787 else if (chan->dir == EDMA_DIR_WRITE) 788 pos = chan->id % wr_alloc; 789 else 790 pos = wr_alloc + chan->id % rd_alloc; 791 792 irq = &dw->irq[pos]; 793 794 if (chan->dir == EDMA_DIR_WRITE) 795 irq->wr_mask |= BIT(chan->id); 796 else 797 irq->rd_mask |= BIT(chan->id); 798 799 irq->dw = dw; 800 memcpy(&chan->msi, &irq->msi, sizeof(chan->msi)); 801 802 dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n", 803 chan->dir == EDMA_DIR_WRITE ? "write" : "read", chan->id, 804 chan->msi.address_hi, chan->msi.address_lo, 805 chan->msi.data); 806 807 chan->vc.desc_free = vchan_free_desc; 808 chan->vc.chan.private = chan->dir == EDMA_DIR_WRITE ? 809 &dw->chip->dt_region_wr[chan->id] : 810 &dw->chip->dt_region_rd[chan->id]; 811 812 vchan_init(&chan->vc, dma); 813 814 dw_edma_v0_core_device_config(chan); 815 } 816 817 /* Set DMA channel capabilities */ 818 dma_cap_zero(dma->cap_mask); 819 dma_cap_set(DMA_SLAVE, dma->cap_mask); 820 dma_cap_set(DMA_CYCLIC, dma->cap_mask); 821 dma_cap_set(DMA_PRIVATE, dma->cap_mask); 822 dma_cap_set(DMA_INTERLEAVE, dma->cap_mask); 823 dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 824 dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 825 dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 826 dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 827 828 /* Set DMA channel callbacks */ 829 dma->dev = chip->dev; 830 dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources; 831 dma->device_free_chan_resources = dw_edma_free_chan_resources; 832 dma->device_caps = dw_edma_device_caps; 833 dma->device_config = dw_edma_device_config; 834 dma->device_pause = dw_edma_device_pause; 835 dma->device_resume = dw_edma_device_resume; 836 dma->device_terminate_all = dw_edma_device_terminate_all; 837 dma->device_issue_pending = dw_edma_device_issue_pending; 838 dma->device_tx_status = dw_edma_device_tx_status; 839 dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg; 840 dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic; 841 dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma; 842 843 dma_set_max_seg_size(dma->dev, U32_MAX); 844 845 /* Register DMA device */ 846 return dma_async_device_register(dma); 847 } 848 849 static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt) 850 { 851 if (*nr_irqs && *alloc < cnt) { 852 (*alloc)++; 853 (*nr_irqs)--; 854 } 855 } 856 857 static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt) 858 { 859 while (*mask * alloc < cnt) 860 (*mask)++; 861 } 862 863 static int dw_edma_irq_request(struct dw_edma *dw, 864 u32 *wr_alloc, u32 *rd_alloc) 865 { 866 struct dw_edma_chip *chip = dw->chip; 867 struct device *dev = dw->chip->dev; 868 u32 wr_mask = 1; 869 u32 rd_mask = 1; 870 int i, err = 0; 871 u32 ch_cnt; 872 int irq; 873 874 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; 875 876 if (chip->nr_irqs < 1 || !chip->ops->irq_vector) 877 return -EINVAL; 878 879 dw->irq = devm_kcalloc(dev, chip->nr_irqs, sizeof(*dw->irq), GFP_KERNEL); 880 if (!dw->irq) 881 return -ENOMEM; 882 883 if (chip->nr_irqs == 1) { 884 /* Common IRQ shared among all channels */ 885 irq = chip->ops->irq_vector(dev, 0); 886 err = request_irq(irq, dw_edma_interrupt_common, 887 IRQF_SHARED, dw->name, &dw->irq[0]); 888 if (err) { 889 dw->nr_irqs = 0; 890 return err; 891 } 892 893 if (irq_get_msi_desc(irq)) 894 get_cached_msi_msg(irq, &dw->irq[0].msi); 895 896 dw->nr_irqs = 1; 897 } else { 898 /* Distribute IRQs equally among all channels */ 899 int tmp = chip->nr_irqs; 900 901 while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) { 902 dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt); 903 dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt); 904 } 905 906 dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt); 907 dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt); 908 909 for (i = 0; i < (*wr_alloc + *rd_alloc); i++) { 910 irq = chip->ops->irq_vector(dev, i); 911 err = request_irq(irq, 912 i < *wr_alloc ? 913 dw_edma_interrupt_write : 914 dw_edma_interrupt_read, 915 IRQF_SHARED, dw->name, 916 &dw->irq[i]); 917 if (err) 918 goto err_irq_free; 919 920 if (irq_get_msi_desc(irq)) 921 get_cached_msi_msg(irq, &dw->irq[i].msi); 922 } 923 924 dw->nr_irqs = i; 925 } 926 927 return 0; 928 929 err_irq_free: 930 for (i--; i >= 0; i--) { 931 irq = chip->ops->irq_vector(dev, i); 932 free_irq(irq, &dw->irq[i]); 933 } 934 935 return err; 936 } 937 938 int dw_edma_probe(struct dw_edma_chip *chip) 939 { 940 struct device *dev; 941 struct dw_edma *dw; 942 u32 wr_alloc = 0; 943 u32 rd_alloc = 0; 944 int i, err; 945 946 if (!chip) 947 return -EINVAL; 948 949 dev = chip->dev; 950 if (!dev || !chip->ops) 951 return -EINVAL; 952 953 dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL); 954 if (!dw) 955 return -ENOMEM; 956 957 dw->chip = chip; 958 959 raw_spin_lock_init(&dw->lock); 960 961 dw->wr_ch_cnt = min_t(u16, chip->ll_wr_cnt, 962 dw_edma_v0_core_ch_count(dw, EDMA_DIR_WRITE)); 963 dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH); 964 965 dw->rd_ch_cnt = min_t(u16, chip->ll_rd_cnt, 966 dw_edma_v0_core_ch_count(dw, EDMA_DIR_READ)); 967 dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH); 968 969 if (!dw->wr_ch_cnt && !dw->rd_ch_cnt) 970 return -EINVAL; 971 972 dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n", 973 dw->wr_ch_cnt, dw->rd_ch_cnt); 974 975 /* Allocate channels */ 976 dw->chan = devm_kcalloc(dev, dw->wr_ch_cnt + dw->rd_ch_cnt, 977 sizeof(*dw->chan), GFP_KERNEL); 978 if (!dw->chan) 979 return -ENOMEM; 980 981 snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%s", 982 dev_name(chip->dev)); 983 984 /* Disable eDMA, only to establish the ideal initial conditions */ 985 dw_edma_v0_core_off(dw); 986 987 /* Request IRQs */ 988 err = dw_edma_irq_request(dw, &wr_alloc, &rd_alloc); 989 if (err) 990 return err; 991 992 /* Setup write/read channels */ 993 err = dw_edma_channel_setup(dw, wr_alloc, rd_alloc); 994 if (err) 995 goto err_irq_free; 996 997 /* Turn debugfs on */ 998 dw_edma_v0_core_debugfs_on(dw); 999 1000 chip->dw = dw; 1001 1002 return 0; 1003 1004 err_irq_free: 1005 for (i = (dw->nr_irqs - 1); i >= 0; i--) 1006 free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); 1007 1008 return err; 1009 } 1010 EXPORT_SYMBOL_GPL(dw_edma_probe); 1011 1012 int dw_edma_remove(struct dw_edma_chip *chip) 1013 { 1014 struct dw_edma_chan *chan, *_chan; 1015 struct device *dev = chip->dev; 1016 struct dw_edma *dw = chip->dw; 1017 int i; 1018 1019 /* Skip removal if no private data found */ 1020 if (!dw) 1021 return -ENODEV; 1022 1023 /* Disable eDMA */ 1024 dw_edma_v0_core_off(dw); 1025 1026 /* Free irqs */ 1027 for (i = (dw->nr_irqs - 1); i >= 0; i--) 1028 free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); 1029 1030 /* Deregister eDMA device */ 1031 dma_async_device_unregister(&dw->dma); 1032 list_for_each_entry_safe(chan, _chan, &dw->dma.channels, 1033 vc.chan.device_node) { 1034 tasklet_kill(&chan->vc.task); 1035 list_del(&chan->vc.chan.device_node); 1036 } 1037 1038 return 0; 1039 } 1040 EXPORT_SYMBOL_GPL(dw_edma_remove); 1041 1042 MODULE_LICENSE("GPL v2"); 1043 MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver"); 1044 MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>"); 1045