1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. 4 * Synopsys DesignWare eDMA core driver 5 * 6 * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> 7 */ 8 9 #include <linux/module.h> 10 #include <linux/device.h> 11 #include <linux/kernel.h> 12 #include <linux/dmaengine.h> 13 #include <linux/err.h> 14 #include <linux/interrupt.h> 15 #include <linux/irq.h> 16 #include <linux/dma/edma.h> 17 #include <linux/dma-mapping.h> 18 19 #include "dw-edma-core.h" 20 #include "dw-edma-v0-core.h" 21 #include "../dmaengine.h" 22 #include "../virt-dma.h" 23 24 static inline 25 struct device *dchan2dev(struct dma_chan *dchan) 26 { 27 return &dchan->dev->device; 28 } 29 30 static inline 31 struct device *chan2dev(struct dw_edma_chan *chan) 32 { 33 return &chan->vc.chan.dev->device; 34 } 35 36 static inline 37 struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd) 38 { 39 return container_of(vd, struct dw_edma_desc, vd); 40 } 41 42 static inline 43 u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr) 44 { 45 struct dw_edma_chip *chip = chan->dw->chip; 46 47 if (chip->ops->pci_address) 48 return chip->ops->pci_address(chip->dev, cpu_addr); 49 50 return cpu_addr; 51 } 52 53 static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk) 54 { 55 struct dw_edma_burst *burst; 56 57 burst = kzalloc(sizeof(*burst), GFP_NOWAIT); 58 if (unlikely(!burst)) 59 return NULL; 60 61 INIT_LIST_HEAD(&burst->list); 62 if (chunk->burst) { 63 /* Create and add new element into the linked list */ 64 chunk->bursts_alloc++; 65 list_add_tail(&burst->list, &chunk->burst->list); 66 } else { 67 /* List head */ 68 chunk->bursts_alloc = 0; 69 chunk->burst = burst; 70 } 71 72 return burst; 73 } 74 75 static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc) 76 { 77 struct dw_edma_chip *chip = desc->chan->dw->chip; 78 struct dw_edma_chan *chan = desc->chan; 79 struct dw_edma_chunk *chunk; 80 81 chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT); 82 if (unlikely(!chunk)) 83 return NULL; 84 85 INIT_LIST_HEAD(&chunk->list); 86 chunk->chan = chan; 87 /* Toggling change bit (CB) in each chunk, this is a mechanism to 88 * inform the eDMA HW block that this is a new linked list ready 89 * to be consumed. 90 * - Odd chunks originate CB equal to 0 91 * - Even chunks originate CB equal to 1 92 */ 93 chunk->cb = !(desc->chunks_alloc % 2); 94 if (chan->dir == EDMA_DIR_WRITE) { 95 chunk->ll_region.paddr = chip->ll_region_wr[chan->id].paddr; 96 chunk->ll_region.vaddr = chip->ll_region_wr[chan->id].vaddr; 97 } else { 98 chunk->ll_region.paddr = chip->ll_region_rd[chan->id].paddr; 99 chunk->ll_region.vaddr = chip->ll_region_rd[chan->id].vaddr; 100 } 101 102 if (desc->chunk) { 103 /* Create and add new element into the linked list */ 104 if (!dw_edma_alloc_burst(chunk)) { 105 kfree(chunk); 106 return NULL; 107 } 108 desc->chunks_alloc++; 109 list_add_tail(&chunk->list, &desc->chunk->list); 110 } else { 111 /* List head */ 112 chunk->burst = NULL; 113 desc->chunks_alloc = 0; 114 desc->chunk = chunk; 115 } 116 117 return chunk; 118 } 119 120 static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan) 121 { 122 struct dw_edma_desc *desc; 123 124 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 125 if (unlikely(!desc)) 126 return NULL; 127 128 desc->chan = chan; 129 if (!dw_edma_alloc_chunk(desc)) { 130 kfree(desc); 131 return NULL; 132 } 133 134 return desc; 135 } 136 137 static void dw_edma_free_burst(struct dw_edma_chunk *chunk) 138 { 139 struct dw_edma_burst *child, *_next; 140 141 /* Remove all the list elements */ 142 list_for_each_entry_safe(child, _next, &chunk->burst->list, list) { 143 list_del(&child->list); 144 kfree(child); 145 chunk->bursts_alloc--; 146 } 147 148 /* Remove the list head */ 149 kfree(child); 150 chunk->burst = NULL; 151 } 152 153 static void dw_edma_free_chunk(struct dw_edma_desc *desc) 154 { 155 struct dw_edma_chunk *child, *_next; 156 157 if (!desc->chunk) 158 return; 159 160 /* Remove all the list elements */ 161 list_for_each_entry_safe(child, _next, &desc->chunk->list, list) { 162 dw_edma_free_burst(child); 163 list_del(&child->list); 164 kfree(child); 165 desc->chunks_alloc--; 166 } 167 168 /* Remove the list head */ 169 kfree(child); 170 desc->chunk = NULL; 171 } 172 173 static void dw_edma_free_desc(struct dw_edma_desc *desc) 174 { 175 dw_edma_free_chunk(desc); 176 kfree(desc); 177 } 178 179 static void vchan_free_desc(struct virt_dma_desc *vdesc) 180 { 181 dw_edma_free_desc(vd2dw_edma_desc(vdesc)); 182 } 183 184 static void dw_edma_start_transfer(struct dw_edma_chan *chan) 185 { 186 struct dw_edma_chunk *child; 187 struct dw_edma_desc *desc; 188 struct virt_dma_desc *vd; 189 190 vd = vchan_next_desc(&chan->vc); 191 if (!vd) 192 return; 193 194 desc = vd2dw_edma_desc(vd); 195 if (!desc) 196 return; 197 198 child = list_first_entry_or_null(&desc->chunk->list, 199 struct dw_edma_chunk, list); 200 if (!child) 201 return; 202 203 dw_edma_v0_core_start(child, !desc->xfer_sz); 204 desc->xfer_sz += child->ll_region.sz; 205 dw_edma_free_burst(child); 206 list_del(&child->list); 207 kfree(child); 208 desc->chunks_alloc--; 209 } 210 211 static void dw_edma_device_caps(struct dma_chan *dchan, 212 struct dma_slave_caps *caps) 213 { 214 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 215 216 if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { 217 if (chan->dir == EDMA_DIR_READ) 218 caps->directions = BIT(DMA_DEV_TO_MEM); 219 else 220 caps->directions = BIT(DMA_MEM_TO_DEV); 221 } else { 222 if (chan->dir == EDMA_DIR_WRITE) 223 caps->directions = BIT(DMA_DEV_TO_MEM); 224 else 225 caps->directions = BIT(DMA_MEM_TO_DEV); 226 } 227 } 228 229 static int dw_edma_device_config(struct dma_chan *dchan, 230 struct dma_slave_config *config) 231 { 232 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 233 234 memcpy(&chan->config, config, sizeof(*config)); 235 chan->configured = true; 236 237 return 0; 238 } 239 240 static int dw_edma_device_pause(struct dma_chan *dchan) 241 { 242 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 243 int err = 0; 244 245 if (!chan->configured) 246 err = -EPERM; 247 else if (chan->status != EDMA_ST_BUSY) 248 err = -EPERM; 249 else if (chan->request != EDMA_REQ_NONE) 250 err = -EPERM; 251 else 252 chan->request = EDMA_REQ_PAUSE; 253 254 return err; 255 } 256 257 static int dw_edma_device_resume(struct dma_chan *dchan) 258 { 259 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 260 int err = 0; 261 262 if (!chan->configured) { 263 err = -EPERM; 264 } else if (chan->status != EDMA_ST_PAUSE) { 265 err = -EPERM; 266 } else if (chan->request != EDMA_REQ_NONE) { 267 err = -EPERM; 268 } else { 269 chan->status = EDMA_ST_BUSY; 270 dw_edma_start_transfer(chan); 271 } 272 273 return err; 274 } 275 276 static int dw_edma_device_terminate_all(struct dma_chan *dchan) 277 { 278 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 279 int err = 0; 280 281 if (!chan->configured) { 282 /* Do nothing */ 283 } else if (chan->status == EDMA_ST_PAUSE) { 284 chan->status = EDMA_ST_IDLE; 285 chan->configured = false; 286 } else if (chan->status == EDMA_ST_IDLE) { 287 chan->configured = false; 288 } else if (dw_edma_v0_core_ch_status(chan) == DMA_COMPLETE) { 289 /* 290 * The channel is in a false BUSY state, probably didn't 291 * receive or lost an interrupt 292 */ 293 chan->status = EDMA_ST_IDLE; 294 chan->configured = false; 295 } else if (chan->request > EDMA_REQ_PAUSE) { 296 err = -EPERM; 297 } else { 298 chan->request = EDMA_REQ_STOP; 299 } 300 301 return err; 302 } 303 304 static void dw_edma_device_issue_pending(struct dma_chan *dchan) 305 { 306 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 307 unsigned long flags; 308 309 spin_lock_irqsave(&chan->vc.lock, flags); 310 if (chan->configured && chan->request == EDMA_REQ_NONE && 311 chan->status == EDMA_ST_IDLE && vchan_issue_pending(&chan->vc)) { 312 chan->status = EDMA_ST_BUSY; 313 dw_edma_start_transfer(chan); 314 } 315 spin_unlock_irqrestore(&chan->vc.lock, flags); 316 } 317 318 static enum dma_status 319 dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, 320 struct dma_tx_state *txstate) 321 { 322 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 323 struct dw_edma_desc *desc; 324 struct virt_dma_desc *vd; 325 unsigned long flags; 326 enum dma_status ret; 327 u32 residue = 0; 328 329 ret = dma_cookie_status(dchan, cookie, txstate); 330 if (ret == DMA_COMPLETE) 331 return ret; 332 333 if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE) 334 ret = DMA_PAUSED; 335 336 if (!txstate) 337 goto ret_residue; 338 339 spin_lock_irqsave(&chan->vc.lock, flags); 340 vd = vchan_find_desc(&chan->vc, cookie); 341 if (vd) { 342 desc = vd2dw_edma_desc(vd); 343 if (desc) 344 residue = desc->alloc_sz - desc->xfer_sz; 345 } 346 spin_unlock_irqrestore(&chan->vc.lock, flags); 347 348 ret_residue: 349 dma_set_residue(txstate, residue); 350 351 return ret; 352 } 353 354 static struct dma_async_tx_descriptor * 355 dw_edma_device_transfer(struct dw_edma_transfer *xfer) 356 { 357 struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan); 358 enum dma_transfer_direction dir = xfer->direction; 359 struct scatterlist *sg = NULL; 360 struct dw_edma_chunk *chunk; 361 struct dw_edma_burst *burst; 362 struct dw_edma_desc *desc; 363 u64 src_addr, dst_addr; 364 size_t fsz = 0; 365 u32 cnt = 0; 366 int i; 367 368 if (!chan->configured) 369 return NULL; 370 371 /* 372 * Local Root Port/End-point Remote End-point 373 * +-----------------------+ PCIe bus +----------------------+ 374 * | | +-+ | | 375 * | DEV_TO_MEM Rx Ch <----+ +---+ Tx Ch DEV_TO_MEM | 376 * | | | | | | 377 * | MEM_TO_DEV Tx Ch +----+ +---> Rx Ch MEM_TO_DEV | 378 * | | +-+ | | 379 * +-----------------------+ +----------------------+ 380 * 381 * 1. Normal logic: 382 * If eDMA is embedded into the DW PCIe RP/EP and controlled from the 383 * CPU/Application side, the Rx channel (EDMA_DIR_READ) will be used 384 * for the device read operations (DEV_TO_MEM) and the Tx channel 385 * (EDMA_DIR_WRITE) - for the write operations (MEM_TO_DEV). 386 * 387 * 2. Inverted logic: 388 * If eDMA is embedded into a Remote PCIe EP and is controlled by the 389 * MWr/MRd TLPs sent from the CPU's PCIe host controller, the Tx 390 * channel (EDMA_DIR_WRITE) will be used for the device read operations 391 * (DEV_TO_MEM) and the Rx channel (EDMA_DIR_READ) - for the write 392 * operations (MEM_TO_DEV). 393 * 394 * It is the client driver responsibility to choose a proper channel 395 * for the DMA transfers. 396 */ 397 if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { 398 if ((chan->dir == EDMA_DIR_READ && dir != DMA_DEV_TO_MEM) || 399 (chan->dir == EDMA_DIR_WRITE && dir != DMA_MEM_TO_DEV)) 400 return NULL; 401 } else { 402 if ((chan->dir == EDMA_DIR_WRITE && dir != DMA_DEV_TO_MEM) || 403 (chan->dir == EDMA_DIR_READ && dir != DMA_MEM_TO_DEV)) 404 return NULL; 405 } 406 407 if (xfer->type == EDMA_XFER_CYCLIC) { 408 if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt) 409 return NULL; 410 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 411 if (xfer->xfer.sg.len < 1) 412 return NULL; 413 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 414 if (!xfer->xfer.il->numf || xfer->xfer.il->frame_size < 1) 415 return NULL; 416 if (!xfer->xfer.il->src_inc || !xfer->xfer.il->dst_inc) 417 return NULL; 418 } else { 419 return NULL; 420 } 421 422 desc = dw_edma_alloc_desc(chan); 423 if (unlikely(!desc)) 424 goto err_alloc; 425 426 chunk = dw_edma_alloc_chunk(desc); 427 if (unlikely(!chunk)) 428 goto err_alloc; 429 430 if (xfer->type == EDMA_XFER_INTERLEAVED) { 431 src_addr = xfer->xfer.il->src_start; 432 dst_addr = xfer->xfer.il->dst_start; 433 } else { 434 src_addr = chan->config.src_addr; 435 dst_addr = chan->config.dst_addr; 436 } 437 438 if (dir == DMA_DEV_TO_MEM) 439 src_addr = dw_edma_get_pci_address(chan, (phys_addr_t)src_addr); 440 else 441 dst_addr = dw_edma_get_pci_address(chan, (phys_addr_t)dst_addr); 442 443 if (xfer->type == EDMA_XFER_CYCLIC) { 444 cnt = xfer->xfer.cyclic.cnt; 445 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 446 cnt = xfer->xfer.sg.len; 447 sg = xfer->xfer.sg.sgl; 448 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 449 cnt = xfer->xfer.il->numf * xfer->xfer.il->frame_size; 450 fsz = xfer->xfer.il->frame_size; 451 } 452 453 for (i = 0; i < cnt; i++) { 454 if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg) 455 break; 456 457 if (chunk->bursts_alloc == chan->ll_max) { 458 chunk = dw_edma_alloc_chunk(desc); 459 if (unlikely(!chunk)) 460 goto err_alloc; 461 } 462 463 burst = dw_edma_alloc_burst(chunk); 464 if (unlikely(!burst)) 465 goto err_alloc; 466 467 if (xfer->type == EDMA_XFER_CYCLIC) 468 burst->sz = xfer->xfer.cyclic.len; 469 else if (xfer->type == EDMA_XFER_SCATTER_GATHER) 470 burst->sz = sg_dma_len(sg); 471 else if (xfer->type == EDMA_XFER_INTERLEAVED) 472 burst->sz = xfer->xfer.il->sgl[i % fsz].size; 473 474 chunk->ll_region.sz += burst->sz; 475 desc->alloc_sz += burst->sz; 476 477 if (dir == DMA_DEV_TO_MEM) { 478 burst->sar = src_addr; 479 if (xfer->type == EDMA_XFER_CYCLIC) { 480 burst->dar = xfer->xfer.cyclic.paddr; 481 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 482 src_addr += sg_dma_len(sg); 483 burst->dar = sg_dma_address(sg); 484 /* Unlike the typical assumption by other 485 * drivers/IPs the peripheral memory isn't 486 * a FIFO memory, in this case, it's a 487 * linear memory and that why the source 488 * and destination addresses are increased 489 * by the same portion (data length) 490 */ 491 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 492 burst->dar = dst_addr; 493 } 494 } else { 495 burst->dar = dst_addr; 496 if (xfer->type == EDMA_XFER_CYCLIC) { 497 burst->sar = xfer->xfer.cyclic.paddr; 498 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 499 dst_addr += sg_dma_len(sg); 500 burst->sar = sg_dma_address(sg); 501 /* Unlike the typical assumption by other 502 * drivers/IPs the peripheral memory isn't 503 * a FIFO memory, in this case, it's a 504 * linear memory and that why the source 505 * and destination addresses are increased 506 * by the same portion (data length) 507 */ 508 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 509 burst->sar = src_addr; 510 } 511 } 512 513 if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 514 sg = sg_next(sg); 515 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 516 struct dma_interleaved_template *il = xfer->xfer.il; 517 struct data_chunk *dc = &il->sgl[i % fsz]; 518 519 src_addr += burst->sz; 520 if (il->src_sgl) 521 src_addr += dmaengine_get_src_icg(il, dc); 522 523 dst_addr += burst->sz; 524 if (il->dst_sgl) 525 dst_addr += dmaengine_get_dst_icg(il, dc); 526 } 527 } 528 529 return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags); 530 531 err_alloc: 532 if (desc) 533 dw_edma_free_desc(desc); 534 535 return NULL; 536 } 537 538 static struct dma_async_tx_descriptor * 539 dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, 540 unsigned int len, 541 enum dma_transfer_direction direction, 542 unsigned long flags, void *context) 543 { 544 struct dw_edma_transfer xfer; 545 546 xfer.dchan = dchan; 547 xfer.direction = direction; 548 xfer.xfer.sg.sgl = sgl; 549 xfer.xfer.sg.len = len; 550 xfer.flags = flags; 551 xfer.type = EDMA_XFER_SCATTER_GATHER; 552 553 return dw_edma_device_transfer(&xfer); 554 } 555 556 static struct dma_async_tx_descriptor * 557 dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr, 558 size_t len, size_t count, 559 enum dma_transfer_direction direction, 560 unsigned long flags) 561 { 562 struct dw_edma_transfer xfer; 563 564 xfer.dchan = dchan; 565 xfer.direction = direction; 566 xfer.xfer.cyclic.paddr = paddr; 567 xfer.xfer.cyclic.len = len; 568 xfer.xfer.cyclic.cnt = count; 569 xfer.flags = flags; 570 xfer.type = EDMA_XFER_CYCLIC; 571 572 return dw_edma_device_transfer(&xfer); 573 } 574 575 static struct dma_async_tx_descriptor * 576 dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan, 577 struct dma_interleaved_template *ilt, 578 unsigned long flags) 579 { 580 struct dw_edma_transfer xfer; 581 582 xfer.dchan = dchan; 583 xfer.direction = ilt->dir; 584 xfer.xfer.il = ilt; 585 xfer.flags = flags; 586 xfer.type = EDMA_XFER_INTERLEAVED; 587 588 return dw_edma_device_transfer(&xfer); 589 } 590 591 static void dw_edma_done_interrupt(struct dw_edma_chan *chan) 592 { 593 struct dw_edma_desc *desc; 594 struct virt_dma_desc *vd; 595 unsigned long flags; 596 597 dw_edma_v0_core_clear_done_int(chan); 598 599 spin_lock_irqsave(&chan->vc.lock, flags); 600 vd = vchan_next_desc(&chan->vc); 601 if (vd) { 602 switch (chan->request) { 603 case EDMA_REQ_NONE: 604 desc = vd2dw_edma_desc(vd); 605 if (desc->chunks_alloc) { 606 chan->status = EDMA_ST_BUSY; 607 dw_edma_start_transfer(chan); 608 } else { 609 list_del(&vd->node); 610 vchan_cookie_complete(vd); 611 chan->status = EDMA_ST_IDLE; 612 } 613 break; 614 615 case EDMA_REQ_STOP: 616 list_del(&vd->node); 617 vchan_cookie_complete(vd); 618 chan->request = EDMA_REQ_NONE; 619 chan->status = EDMA_ST_IDLE; 620 break; 621 622 case EDMA_REQ_PAUSE: 623 chan->request = EDMA_REQ_NONE; 624 chan->status = EDMA_ST_PAUSE; 625 break; 626 627 default: 628 break; 629 } 630 } 631 spin_unlock_irqrestore(&chan->vc.lock, flags); 632 } 633 634 static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) 635 { 636 struct virt_dma_desc *vd; 637 unsigned long flags; 638 639 dw_edma_v0_core_clear_abort_int(chan); 640 641 spin_lock_irqsave(&chan->vc.lock, flags); 642 vd = vchan_next_desc(&chan->vc); 643 if (vd) { 644 list_del(&vd->node); 645 vchan_cookie_complete(vd); 646 } 647 spin_unlock_irqrestore(&chan->vc.lock, flags); 648 chan->request = EDMA_REQ_NONE; 649 chan->status = EDMA_ST_IDLE; 650 } 651 652 static irqreturn_t dw_edma_interrupt(int irq, void *data, bool write) 653 { 654 struct dw_edma_irq *dw_irq = data; 655 struct dw_edma *dw = dw_irq->dw; 656 unsigned long total, pos, val; 657 unsigned long off; 658 u32 mask; 659 660 if (write) { 661 total = dw->wr_ch_cnt; 662 off = 0; 663 mask = dw_irq->wr_mask; 664 } else { 665 total = dw->rd_ch_cnt; 666 off = dw->wr_ch_cnt; 667 mask = dw_irq->rd_mask; 668 } 669 670 val = dw_edma_v0_core_status_done_int(dw, write ? 671 EDMA_DIR_WRITE : 672 EDMA_DIR_READ); 673 val &= mask; 674 for_each_set_bit(pos, &val, total) { 675 struct dw_edma_chan *chan = &dw->chan[pos + off]; 676 677 dw_edma_done_interrupt(chan); 678 } 679 680 val = dw_edma_v0_core_status_abort_int(dw, write ? 681 EDMA_DIR_WRITE : 682 EDMA_DIR_READ); 683 val &= mask; 684 for_each_set_bit(pos, &val, total) { 685 struct dw_edma_chan *chan = &dw->chan[pos + off]; 686 687 dw_edma_abort_interrupt(chan); 688 } 689 690 return IRQ_HANDLED; 691 } 692 693 static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) 694 { 695 return dw_edma_interrupt(irq, data, true); 696 } 697 698 static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data) 699 { 700 return dw_edma_interrupt(irq, data, false); 701 } 702 703 static irqreturn_t dw_edma_interrupt_common(int irq, void *data) 704 { 705 dw_edma_interrupt(irq, data, true); 706 dw_edma_interrupt(irq, data, false); 707 708 return IRQ_HANDLED; 709 } 710 711 static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) 712 { 713 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 714 715 if (chan->status != EDMA_ST_IDLE) 716 return -EBUSY; 717 718 return 0; 719 } 720 721 static void dw_edma_free_chan_resources(struct dma_chan *dchan) 722 { 723 unsigned long timeout = jiffies + msecs_to_jiffies(5000); 724 int ret; 725 726 while (time_before(jiffies, timeout)) { 727 ret = dw_edma_device_terminate_all(dchan); 728 if (!ret) 729 break; 730 731 if (time_after_eq(jiffies, timeout)) 732 return; 733 734 cpu_relax(); 735 } 736 } 737 738 static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) 739 { 740 struct dw_edma_chip *chip = dw->chip; 741 struct device *dev = chip->dev; 742 struct dw_edma_chan *chan; 743 struct dw_edma_irq *irq; 744 struct dma_device *dma; 745 u32 i, ch_cnt; 746 u32 pos; 747 748 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; 749 dma = &dw->dma; 750 751 INIT_LIST_HEAD(&dma->channels); 752 753 for (i = 0; i < ch_cnt; i++) { 754 chan = &dw->chan[i]; 755 756 chan->dw = dw; 757 758 if (i < dw->wr_ch_cnt) { 759 chan->id = i; 760 chan->dir = EDMA_DIR_WRITE; 761 } else { 762 chan->id = i - dw->wr_ch_cnt; 763 chan->dir = EDMA_DIR_READ; 764 } 765 766 chan->configured = false; 767 chan->request = EDMA_REQ_NONE; 768 chan->status = EDMA_ST_IDLE; 769 770 if (chan->dir == EDMA_DIR_WRITE) 771 chan->ll_max = (chip->ll_region_wr[chan->id].sz / EDMA_LL_SZ); 772 else 773 chan->ll_max = (chip->ll_region_rd[chan->id].sz / EDMA_LL_SZ); 774 chan->ll_max -= 1; 775 776 dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n", 777 chan->dir == EDMA_DIR_WRITE ? "write" : "read", 778 chan->id, chan->ll_max); 779 780 if (dw->nr_irqs == 1) 781 pos = 0; 782 else if (chan->dir == EDMA_DIR_WRITE) 783 pos = chan->id % wr_alloc; 784 else 785 pos = wr_alloc + chan->id % rd_alloc; 786 787 irq = &dw->irq[pos]; 788 789 if (chan->dir == EDMA_DIR_WRITE) 790 irq->wr_mask |= BIT(chan->id); 791 else 792 irq->rd_mask |= BIT(chan->id); 793 794 irq->dw = dw; 795 memcpy(&chan->msi, &irq->msi, sizeof(chan->msi)); 796 797 dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n", 798 chan->dir == EDMA_DIR_WRITE ? "write" : "read", chan->id, 799 chan->msi.address_hi, chan->msi.address_lo, 800 chan->msi.data); 801 802 chan->vc.desc_free = vchan_free_desc; 803 chan->vc.chan.private = chan->dir == EDMA_DIR_WRITE ? 804 &dw->chip->dt_region_wr[chan->id] : 805 &dw->chip->dt_region_rd[chan->id]; 806 807 vchan_init(&chan->vc, dma); 808 809 dw_edma_v0_core_device_config(chan); 810 } 811 812 /* Set DMA channel capabilities */ 813 dma_cap_zero(dma->cap_mask); 814 dma_cap_set(DMA_SLAVE, dma->cap_mask); 815 dma_cap_set(DMA_CYCLIC, dma->cap_mask); 816 dma_cap_set(DMA_PRIVATE, dma->cap_mask); 817 dma_cap_set(DMA_INTERLEAVE, dma->cap_mask); 818 dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 819 dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 820 dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 821 dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 822 823 /* Set DMA channel callbacks */ 824 dma->dev = chip->dev; 825 dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources; 826 dma->device_free_chan_resources = dw_edma_free_chan_resources; 827 dma->device_caps = dw_edma_device_caps; 828 dma->device_config = dw_edma_device_config; 829 dma->device_pause = dw_edma_device_pause; 830 dma->device_resume = dw_edma_device_resume; 831 dma->device_terminate_all = dw_edma_device_terminate_all; 832 dma->device_issue_pending = dw_edma_device_issue_pending; 833 dma->device_tx_status = dw_edma_device_tx_status; 834 dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg; 835 dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic; 836 dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma; 837 838 dma_set_max_seg_size(dma->dev, U32_MAX); 839 840 /* Register DMA device */ 841 return dma_async_device_register(dma); 842 } 843 844 static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt) 845 { 846 if (*nr_irqs && *alloc < cnt) { 847 (*alloc)++; 848 (*nr_irqs)--; 849 } 850 } 851 852 static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt) 853 { 854 while (*mask * alloc < cnt) 855 (*mask)++; 856 } 857 858 static int dw_edma_irq_request(struct dw_edma *dw, 859 u32 *wr_alloc, u32 *rd_alloc) 860 { 861 struct dw_edma_chip *chip = dw->chip; 862 struct device *dev = dw->chip->dev; 863 u32 wr_mask = 1; 864 u32 rd_mask = 1; 865 int i, err = 0; 866 u32 ch_cnt; 867 int irq; 868 869 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; 870 871 if (chip->nr_irqs < 1 || !chip->ops->irq_vector) 872 return -EINVAL; 873 874 dw->irq = devm_kcalloc(dev, chip->nr_irqs, sizeof(*dw->irq), GFP_KERNEL); 875 if (!dw->irq) 876 return -ENOMEM; 877 878 if (chip->nr_irqs == 1) { 879 /* Common IRQ shared among all channels */ 880 irq = chip->ops->irq_vector(dev, 0); 881 err = request_irq(irq, dw_edma_interrupt_common, 882 IRQF_SHARED, dw->name, &dw->irq[0]); 883 if (err) { 884 dw->nr_irqs = 0; 885 return err; 886 } 887 888 if (irq_get_msi_desc(irq)) 889 get_cached_msi_msg(irq, &dw->irq[0].msi); 890 891 dw->nr_irqs = 1; 892 } else { 893 /* Distribute IRQs equally among all channels */ 894 int tmp = chip->nr_irqs; 895 896 while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) { 897 dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt); 898 dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt); 899 } 900 901 dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt); 902 dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt); 903 904 for (i = 0; i < (*wr_alloc + *rd_alloc); i++) { 905 irq = chip->ops->irq_vector(dev, i); 906 err = request_irq(irq, 907 i < *wr_alloc ? 908 dw_edma_interrupt_write : 909 dw_edma_interrupt_read, 910 IRQF_SHARED, dw->name, 911 &dw->irq[i]); 912 if (err) 913 goto err_irq_free; 914 915 if (irq_get_msi_desc(irq)) 916 get_cached_msi_msg(irq, &dw->irq[i].msi); 917 } 918 919 dw->nr_irqs = i; 920 } 921 922 return 0; 923 924 err_irq_free: 925 for (i--; i >= 0; i--) { 926 irq = chip->ops->irq_vector(dev, i); 927 free_irq(irq, &dw->irq[i]); 928 } 929 930 return err; 931 } 932 933 int dw_edma_probe(struct dw_edma_chip *chip) 934 { 935 struct device *dev; 936 struct dw_edma *dw; 937 u32 wr_alloc = 0; 938 u32 rd_alloc = 0; 939 int i, err; 940 941 if (!chip) 942 return -EINVAL; 943 944 dev = chip->dev; 945 if (!dev || !chip->ops) 946 return -EINVAL; 947 948 dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL); 949 if (!dw) 950 return -ENOMEM; 951 952 dw->chip = chip; 953 954 raw_spin_lock_init(&dw->lock); 955 956 dw->wr_ch_cnt = min_t(u16, chip->ll_wr_cnt, 957 dw_edma_v0_core_ch_count(dw, EDMA_DIR_WRITE)); 958 dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH); 959 960 dw->rd_ch_cnt = min_t(u16, chip->ll_rd_cnt, 961 dw_edma_v0_core_ch_count(dw, EDMA_DIR_READ)); 962 dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH); 963 964 if (!dw->wr_ch_cnt && !dw->rd_ch_cnt) 965 return -EINVAL; 966 967 dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n", 968 dw->wr_ch_cnt, dw->rd_ch_cnt); 969 970 /* Allocate channels */ 971 dw->chan = devm_kcalloc(dev, dw->wr_ch_cnt + dw->rd_ch_cnt, 972 sizeof(*dw->chan), GFP_KERNEL); 973 if (!dw->chan) 974 return -ENOMEM; 975 976 snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%s", 977 dev_name(chip->dev)); 978 979 /* Disable eDMA, only to establish the ideal initial conditions */ 980 dw_edma_v0_core_off(dw); 981 982 /* Request IRQs */ 983 err = dw_edma_irq_request(dw, &wr_alloc, &rd_alloc); 984 if (err) 985 return err; 986 987 /* Setup write/read channels */ 988 err = dw_edma_channel_setup(dw, wr_alloc, rd_alloc); 989 if (err) 990 goto err_irq_free; 991 992 /* Turn debugfs on */ 993 dw_edma_v0_core_debugfs_on(dw); 994 995 chip->dw = dw; 996 997 return 0; 998 999 err_irq_free: 1000 for (i = (dw->nr_irqs - 1); i >= 0; i--) 1001 free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); 1002 1003 return err; 1004 } 1005 EXPORT_SYMBOL_GPL(dw_edma_probe); 1006 1007 int dw_edma_remove(struct dw_edma_chip *chip) 1008 { 1009 struct dw_edma_chan *chan, *_chan; 1010 struct device *dev = chip->dev; 1011 struct dw_edma *dw = chip->dw; 1012 int i; 1013 1014 /* Skip removal if no private data found */ 1015 if (!dw) 1016 return -ENODEV; 1017 1018 /* Disable eDMA */ 1019 dw_edma_v0_core_off(dw); 1020 1021 /* Free irqs */ 1022 for (i = (dw->nr_irqs - 1); i >= 0; i--) 1023 free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); 1024 1025 /* Deregister eDMA device */ 1026 dma_async_device_unregister(&dw->dma); 1027 list_for_each_entry_safe(chan, _chan, &dw->dma.channels, 1028 vc.chan.device_node) { 1029 tasklet_kill(&chan->vc.task); 1030 list_del(&chan->vc.chan.device_node); 1031 } 1032 1033 return 0; 1034 } 1035 EXPORT_SYMBOL_GPL(dw_edma_remove); 1036 1037 MODULE_LICENSE("GPL v2"); 1038 MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver"); 1039 MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>"); 1040