1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2018-2019 MediaTek Inc. 3 4 /* 5 * Driver for MediaTek Command-Queue DMA Controller 6 * 7 * Author: Shun-Chih Yu <shun-chih.yu@mediatek.com> 8 * 9 */ 10 11 #include <linux/bitops.h> 12 #include <linux/clk.h> 13 #include <linux/dmaengine.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/err.h> 16 #include <linux/iopoll.h> 17 #include <linux/interrupt.h> 18 #include <linux/list.h> 19 #include <linux/module.h> 20 #include <linux/of.h> 21 #include <linux/of_device.h> 22 #include <linux/of_dma.h> 23 #include <linux/platform_device.h> 24 #include <linux/pm_runtime.h> 25 #include <linux/refcount.h> 26 #include <linux/slab.h> 27 28 #include "../virt-dma.h" 29 30 #define MTK_CQDMA_USEC_POLL 10 31 #define MTK_CQDMA_TIMEOUT_POLL 1000 32 #define MTK_CQDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) 33 #define MTK_CQDMA_ALIGN_SIZE 1 34 35 /* The default number of virtual channel */ 36 #define MTK_CQDMA_NR_VCHANS 32 37 38 /* The default number of physical channel */ 39 #define MTK_CQDMA_NR_PCHANS 3 40 41 /* Registers for underlying dma manipulation */ 42 #define MTK_CQDMA_INT_FLAG 0x0 43 #define MTK_CQDMA_INT_EN 0x4 44 #define MTK_CQDMA_EN 0x8 45 #define MTK_CQDMA_RESET 0xc 46 #define MTK_CQDMA_FLUSH 0x14 47 #define MTK_CQDMA_SRC 0x1c 48 #define MTK_CQDMA_DST 0x20 49 #define MTK_CQDMA_LEN1 0x24 50 #define MTK_CQDMA_LEN2 0x28 51 #define MTK_CQDMA_SRC2 0x60 52 #define MTK_CQDMA_DST2 0x64 53 54 /* Registers setting */ 55 #define MTK_CQDMA_EN_BIT BIT(0) 56 #define MTK_CQDMA_INT_FLAG_BIT BIT(0) 57 #define MTK_CQDMA_INT_EN_BIT BIT(0) 58 #define MTK_CQDMA_FLUSH_BIT BIT(0) 59 60 #define MTK_CQDMA_WARM_RST_BIT BIT(0) 61 #define MTK_CQDMA_HARD_RST_BIT BIT(1) 62 63 #define MTK_CQDMA_MAX_LEN GENMASK(27, 0) 64 #define MTK_CQDMA_ADDR_LIMIT GENMASK(31, 0) 65 #define MTK_CQDMA_ADDR2_SHFIT (32) 66 67 /** 68 * struct mtk_cqdma_vdesc - The struct holding info describing virtual 69 * descriptor (CVD) 70 * @vd: An instance for struct virt_dma_desc 71 * @len: The total data size device wants to move 72 * @residue: The remaining data size device will move 73 * @dest: The destination address device wants to move to 74 * @src: The source address device wants to move from 75 * @ch: The pointer to the corresponding dma channel 76 * @node: The lise_head struct to build link-list for VDs 77 * @parent: The pointer to the parent CVD 78 */ 79 struct mtk_cqdma_vdesc { 80 struct virt_dma_desc vd; 81 size_t len; 82 size_t residue; 83 dma_addr_t dest; 84 dma_addr_t src; 85 struct dma_chan *ch; 86 87 struct list_head node; 88 struct mtk_cqdma_vdesc *parent; 89 }; 90 91 /** 92 * struct mtk_cqdma_pchan - The struct holding info describing physical 93 * channel (PC) 94 * @queue: Queue for the PDs issued to this PC 95 * @base: The mapped register I/O base of this PC 96 * @irq: The IRQ that this PC are using 97 * @refcnt: Track how many VCs are using this PC 98 * @tasklet: Tasklet for this PC 99 * @lock: Lock protect agaisting multiple VCs access PC 100 */ 101 struct mtk_cqdma_pchan { 102 struct list_head queue; 103 void __iomem *base; 104 u32 irq; 105 106 refcount_t refcnt; 107 108 struct tasklet_struct tasklet; 109 110 /* lock to protect PC */ 111 spinlock_t lock; 112 }; 113 114 /** 115 * struct mtk_cqdma_vchan - The struct holding info describing virtual 116 * channel (VC) 117 * @vc: An instance for struct virt_dma_chan 118 * @pc: The pointer to the underlying PC 119 * @issue_completion: The wait for all issued descriptors completited 120 * @issue_synchronize: Bool indicating channel synchronization starts 121 */ 122 struct mtk_cqdma_vchan { 123 struct virt_dma_chan vc; 124 struct mtk_cqdma_pchan *pc; 125 struct completion issue_completion; 126 bool issue_synchronize; 127 }; 128 129 /** 130 * struct mtk_cqdma_device - The struct holding info describing CQDMA 131 * device 132 * @ddev: An instance for struct dma_device 133 * @clk: The clock that device internal is using 134 * @dma_requests: The number of VCs the device supports to 135 * @dma_channels: The number of PCs the device supports to 136 * @vc: The pointer to all available VCs 137 * @pc: The pointer to all the underlying PCs 138 */ 139 struct mtk_cqdma_device { 140 struct dma_device ddev; 141 struct clk *clk; 142 143 u32 dma_requests; 144 u32 dma_channels; 145 struct mtk_cqdma_vchan *vc; 146 struct mtk_cqdma_pchan **pc; 147 }; 148 149 static struct mtk_cqdma_device *to_cqdma_dev(struct dma_chan *chan) 150 { 151 return container_of(chan->device, struct mtk_cqdma_device, ddev); 152 } 153 154 static struct mtk_cqdma_vchan *to_cqdma_vchan(struct dma_chan *chan) 155 { 156 return container_of(chan, struct mtk_cqdma_vchan, vc.chan); 157 } 158 159 static struct mtk_cqdma_vdesc *to_cqdma_vdesc(struct virt_dma_desc *vd) 160 { 161 return container_of(vd, struct mtk_cqdma_vdesc, vd); 162 } 163 164 static struct device *cqdma2dev(struct mtk_cqdma_device *cqdma) 165 { 166 return cqdma->ddev.dev; 167 } 168 169 static u32 mtk_dma_read(struct mtk_cqdma_pchan *pc, u32 reg) 170 { 171 return readl(pc->base + reg); 172 } 173 174 static void mtk_dma_write(struct mtk_cqdma_pchan *pc, u32 reg, u32 val) 175 { 176 writel_relaxed(val, pc->base + reg); 177 } 178 179 static void mtk_dma_rmw(struct mtk_cqdma_pchan *pc, u32 reg, 180 u32 mask, u32 set) 181 { 182 u32 val; 183 184 val = mtk_dma_read(pc, reg); 185 val &= ~mask; 186 val |= set; 187 mtk_dma_write(pc, reg, val); 188 } 189 190 static void mtk_dma_set(struct mtk_cqdma_pchan *pc, u32 reg, u32 val) 191 { 192 mtk_dma_rmw(pc, reg, 0, val); 193 } 194 195 static void mtk_dma_clr(struct mtk_cqdma_pchan *pc, u32 reg, u32 val) 196 { 197 mtk_dma_rmw(pc, reg, val, 0); 198 } 199 200 static void mtk_cqdma_vdesc_free(struct virt_dma_desc *vd) 201 { 202 kfree(to_cqdma_vdesc(vd)); 203 } 204 205 static int mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan *pc, bool atomic) 206 { 207 u32 status = 0; 208 209 if (!atomic) 210 return readl_poll_timeout(pc->base + MTK_CQDMA_EN, 211 status, 212 !(status & MTK_CQDMA_EN_BIT), 213 MTK_CQDMA_USEC_POLL, 214 MTK_CQDMA_TIMEOUT_POLL); 215 216 return readl_poll_timeout_atomic(pc->base + MTK_CQDMA_EN, 217 status, 218 !(status & MTK_CQDMA_EN_BIT), 219 MTK_CQDMA_USEC_POLL, 220 MTK_CQDMA_TIMEOUT_POLL); 221 } 222 223 static int mtk_cqdma_hard_reset(struct mtk_cqdma_pchan *pc) 224 { 225 mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT); 226 mtk_dma_clr(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT); 227 228 return mtk_cqdma_poll_engine_done(pc, true); 229 } 230 231 static void mtk_cqdma_start(struct mtk_cqdma_pchan *pc, 232 struct mtk_cqdma_vdesc *cvd) 233 { 234 /* wait for the previous transaction done */ 235 if (mtk_cqdma_poll_engine_done(pc, true) < 0) 236 dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma wait transaction timeout\n"); 237 238 /* warm reset the dma engine for the new transaction */ 239 mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_WARM_RST_BIT); 240 if (mtk_cqdma_poll_engine_done(pc, true) < 0) 241 dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma warm reset timeout\n"); 242 243 /* setup the source */ 244 mtk_dma_set(pc, MTK_CQDMA_SRC, cvd->src & MTK_CQDMA_ADDR_LIMIT); 245 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 246 mtk_dma_set(pc, MTK_CQDMA_SRC2, cvd->src >> MTK_CQDMA_ADDR2_SHFIT); 247 #else 248 mtk_dma_set(pc, MTK_CQDMA_SRC2, 0); 249 #endif 250 251 /* setup the destination */ 252 mtk_dma_set(pc, MTK_CQDMA_DST, cvd->dest & MTK_CQDMA_ADDR_LIMIT); 253 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 254 mtk_dma_set(pc, MTK_CQDMA_DST2, cvd->dest >> MTK_CQDMA_ADDR2_SHFIT); 255 #else 256 mtk_dma_set(pc, MTK_CQDMA_DST2, 0); 257 #endif 258 259 /* setup the length */ 260 mtk_dma_set(pc, MTK_CQDMA_LEN1, cvd->len); 261 262 /* start dma engine */ 263 mtk_dma_set(pc, MTK_CQDMA_EN, MTK_CQDMA_EN_BIT); 264 } 265 266 static void mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan *cvc) 267 { 268 struct virt_dma_desc *vd, *vd2; 269 struct mtk_cqdma_pchan *pc = cvc->pc; 270 struct mtk_cqdma_vdesc *cvd; 271 bool trigger_engine = false; 272 273 lockdep_assert_held(&cvc->vc.lock); 274 lockdep_assert_held(&pc->lock); 275 276 list_for_each_entry_safe(vd, vd2, &cvc->vc.desc_issued, node) { 277 /* need to trigger dma engine if PC's queue is empty */ 278 if (list_empty(&pc->queue)) 279 trigger_engine = true; 280 281 cvd = to_cqdma_vdesc(vd); 282 283 /* add VD into PC's queue */ 284 list_add_tail(&cvd->node, &pc->queue); 285 286 /* start the dma engine */ 287 if (trigger_engine) 288 mtk_cqdma_start(pc, cvd); 289 290 /* remove VD from list desc_issued */ 291 list_del(&vd->node); 292 } 293 } 294 295 /* 296 * return true if this VC is active, 297 * meaning that there are VDs under processing by the PC 298 */ 299 static bool mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan *cvc) 300 { 301 struct mtk_cqdma_vdesc *cvd; 302 303 list_for_each_entry(cvd, &cvc->pc->queue, node) 304 if (cvc == to_cqdma_vchan(cvd->ch)) 305 return true; 306 307 return false; 308 } 309 310 /* 311 * return the pointer of the CVD that is just consumed by the PC 312 */ 313 static struct mtk_cqdma_vdesc 314 *mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan *pc) 315 { 316 struct mtk_cqdma_vchan *cvc; 317 struct mtk_cqdma_vdesc *cvd, *ret = NULL; 318 319 /* consume a CVD from PC's queue */ 320 cvd = list_first_entry_or_null(&pc->queue, 321 struct mtk_cqdma_vdesc, node); 322 if (unlikely(!cvd || !cvd->parent)) 323 return NULL; 324 325 cvc = to_cqdma_vchan(cvd->ch); 326 ret = cvd; 327 328 /* update residue of the parent CVD */ 329 cvd->parent->residue -= cvd->len; 330 331 /* delete CVD from PC's queue */ 332 list_del(&cvd->node); 333 334 spin_lock(&cvc->vc.lock); 335 336 /* check whether all the child CVDs completed */ 337 if (!cvd->parent->residue) { 338 /* add the parent VD into list desc_completed */ 339 vchan_cookie_complete(&cvd->parent->vd); 340 341 /* setup completion if this VC is under synchronization */ 342 if (cvc->issue_synchronize && !mtk_cqdma_is_vchan_active(cvc)) { 343 complete(&cvc->issue_completion); 344 cvc->issue_synchronize = false; 345 } 346 } 347 348 spin_unlock(&cvc->vc.lock); 349 350 /* start transaction for next CVD in the queue */ 351 cvd = list_first_entry_or_null(&pc->queue, 352 struct mtk_cqdma_vdesc, node); 353 if (cvd) 354 mtk_cqdma_start(pc, cvd); 355 356 return ret; 357 } 358 359 static void mtk_cqdma_tasklet_cb(struct tasklet_struct *t) 360 { 361 struct mtk_cqdma_pchan *pc = from_tasklet(pc, t, tasklet); 362 struct mtk_cqdma_vdesc *cvd = NULL; 363 unsigned long flags; 364 365 spin_lock_irqsave(&pc->lock, flags); 366 /* consume the queue */ 367 cvd = mtk_cqdma_consume_work_queue(pc); 368 spin_unlock_irqrestore(&pc->lock, flags); 369 370 /* submit the next CVD */ 371 if (cvd) { 372 dma_run_dependencies(&cvd->vd.tx); 373 374 /* 375 * free child CVD after completion. 376 * the parent CVD would be freed with desc_free by user. 377 */ 378 if (cvd->parent != cvd) 379 kfree(cvd); 380 } 381 382 /* re-enable interrupt before leaving tasklet */ 383 enable_irq(pc->irq); 384 } 385 386 static irqreturn_t mtk_cqdma_irq(int irq, void *devid) 387 { 388 struct mtk_cqdma_device *cqdma = devid; 389 irqreturn_t ret = IRQ_NONE; 390 bool schedule_tasklet = false; 391 u32 i; 392 393 /* clear interrupt flags for each PC */ 394 for (i = 0; i < cqdma->dma_channels; ++i, schedule_tasklet = false) { 395 spin_lock(&cqdma->pc[i]->lock); 396 if (mtk_dma_read(cqdma->pc[i], 397 MTK_CQDMA_INT_FLAG) & MTK_CQDMA_INT_FLAG_BIT) { 398 /* clear interrupt */ 399 mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_FLAG, 400 MTK_CQDMA_INT_FLAG_BIT); 401 402 schedule_tasklet = true; 403 ret = IRQ_HANDLED; 404 } 405 spin_unlock(&cqdma->pc[i]->lock); 406 407 if (schedule_tasklet) { 408 /* disable interrupt */ 409 disable_irq_nosync(cqdma->pc[i]->irq); 410 411 /* schedule the tasklet to handle the transactions */ 412 tasklet_schedule(&cqdma->pc[i]->tasklet); 413 } 414 } 415 416 return ret; 417 } 418 419 static struct virt_dma_desc *mtk_cqdma_find_active_desc(struct dma_chan *c, 420 dma_cookie_t cookie) 421 { 422 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); 423 struct virt_dma_desc *vd; 424 unsigned long flags; 425 426 spin_lock_irqsave(&cvc->pc->lock, flags); 427 list_for_each_entry(vd, &cvc->pc->queue, node) 428 if (vd->tx.cookie == cookie) { 429 spin_unlock_irqrestore(&cvc->pc->lock, flags); 430 return vd; 431 } 432 spin_unlock_irqrestore(&cvc->pc->lock, flags); 433 434 list_for_each_entry(vd, &cvc->vc.desc_issued, node) 435 if (vd->tx.cookie == cookie) 436 return vd; 437 438 return NULL; 439 } 440 441 static enum dma_status mtk_cqdma_tx_status(struct dma_chan *c, 442 dma_cookie_t cookie, 443 struct dma_tx_state *txstate) 444 { 445 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); 446 struct mtk_cqdma_vdesc *cvd; 447 struct virt_dma_desc *vd; 448 enum dma_status ret; 449 unsigned long flags; 450 size_t bytes = 0; 451 452 ret = dma_cookie_status(c, cookie, txstate); 453 if (ret == DMA_COMPLETE || !txstate) 454 return ret; 455 456 spin_lock_irqsave(&cvc->vc.lock, flags); 457 vd = mtk_cqdma_find_active_desc(c, cookie); 458 spin_unlock_irqrestore(&cvc->vc.lock, flags); 459 460 if (vd) { 461 cvd = to_cqdma_vdesc(vd); 462 bytes = cvd->residue; 463 } 464 465 dma_set_residue(txstate, bytes); 466 467 return ret; 468 } 469 470 static void mtk_cqdma_issue_pending(struct dma_chan *c) 471 { 472 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); 473 unsigned long pc_flags; 474 unsigned long vc_flags; 475 476 /* acquire PC's lock before VS's lock for lock dependency in tasklet */ 477 spin_lock_irqsave(&cvc->pc->lock, pc_flags); 478 spin_lock_irqsave(&cvc->vc.lock, vc_flags); 479 480 if (vchan_issue_pending(&cvc->vc)) 481 mtk_cqdma_issue_vchan_pending(cvc); 482 483 spin_unlock_irqrestore(&cvc->vc.lock, vc_flags); 484 spin_unlock_irqrestore(&cvc->pc->lock, pc_flags); 485 } 486 487 static struct dma_async_tx_descriptor * 488 mtk_cqdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, 489 dma_addr_t src, size_t len, unsigned long flags) 490 { 491 struct mtk_cqdma_vdesc **cvd; 492 struct dma_async_tx_descriptor *tx = NULL, *prev_tx = NULL; 493 size_t i, tlen, nr_vd; 494 495 /* 496 * In the case that trsanction length is larger than the 497 * DMA engine supports, a single memcpy transaction needs 498 * to be separated into several DMA transactions. 499 * Each DMA transaction would be described by a CVD, 500 * and the first one is referred as the parent CVD, 501 * while the others are child CVDs. 502 * The parent CVD's tx descriptor is the only tx descriptor 503 * returned to the DMA user, and it should not be completed 504 * until all the child CVDs completed. 505 */ 506 nr_vd = DIV_ROUND_UP(len, MTK_CQDMA_MAX_LEN); 507 cvd = kcalloc(nr_vd, sizeof(*cvd), GFP_NOWAIT); 508 if (!cvd) 509 return NULL; 510 511 for (i = 0; i < nr_vd; ++i) { 512 cvd[i] = kzalloc(sizeof(*cvd[i]), GFP_NOWAIT); 513 if (!cvd[i]) { 514 for (; i > 0; --i) 515 kfree(cvd[i - 1]); 516 return NULL; 517 } 518 519 /* setup dma channel */ 520 cvd[i]->ch = c; 521 522 /* setup sourece, destination, and length */ 523 tlen = (len > MTK_CQDMA_MAX_LEN) ? MTK_CQDMA_MAX_LEN : len; 524 cvd[i]->len = tlen; 525 cvd[i]->src = src; 526 cvd[i]->dest = dest; 527 528 /* setup tx descriptor */ 529 tx = vchan_tx_prep(to_virt_chan(c), &cvd[i]->vd, flags); 530 tx->next = NULL; 531 532 if (!i) { 533 cvd[0]->residue = len; 534 } else { 535 prev_tx->next = tx; 536 cvd[i]->residue = tlen; 537 } 538 539 cvd[i]->parent = cvd[0]; 540 541 /* update the src, dest, len, prev_tx for the next CVD */ 542 src += tlen; 543 dest += tlen; 544 len -= tlen; 545 prev_tx = tx; 546 } 547 548 return &cvd[0]->vd.tx; 549 } 550 551 static void mtk_cqdma_free_inactive_desc(struct dma_chan *c) 552 { 553 struct virt_dma_chan *vc = to_virt_chan(c); 554 unsigned long flags; 555 LIST_HEAD(head); 556 557 /* 558 * set desc_allocated, desc_submitted, 559 * and desc_issued as the candicates to be freed 560 */ 561 spin_lock_irqsave(&vc->lock, flags); 562 list_splice_tail_init(&vc->desc_allocated, &head); 563 list_splice_tail_init(&vc->desc_submitted, &head); 564 list_splice_tail_init(&vc->desc_issued, &head); 565 spin_unlock_irqrestore(&vc->lock, flags); 566 567 /* free descriptor lists */ 568 vchan_dma_desc_free_list(vc, &head); 569 } 570 571 static void mtk_cqdma_free_active_desc(struct dma_chan *c) 572 { 573 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); 574 bool sync_needed = false; 575 unsigned long pc_flags; 576 unsigned long vc_flags; 577 578 /* acquire PC's lock first due to lock dependency in dma ISR */ 579 spin_lock_irqsave(&cvc->pc->lock, pc_flags); 580 spin_lock_irqsave(&cvc->vc.lock, vc_flags); 581 582 /* synchronization is required if this VC is active */ 583 if (mtk_cqdma_is_vchan_active(cvc)) { 584 cvc->issue_synchronize = true; 585 sync_needed = true; 586 } 587 588 spin_unlock_irqrestore(&cvc->vc.lock, vc_flags); 589 spin_unlock_irqrestore(&cvc->pc->lock, pc_flags); 590 591 /* waiting for the completion of this VC */ 592 if (sync_needed) 593 wait_for_completion(&cvc->issue_completion); 594 595 /* free all descriptors in list desc_completed */ 596 vchan_synchronize(&cvc->vc); 597 598 WARN_ONCE(!list_empty(&cvc->vc.desc_completed), 599 "Desc pending still in list desc_completed\n"); 600 } 601 602 static int mtk_cqdma_terminate_all(struct dma_chan *c) 603 { 604 /* free descriptors not processed yet by hardware */ 605 mtk_cqdma_free_inactive_desc(c); 606 607 /* free descriptors being processed by hardware */ 608 mtk_cqdma_free_active_desc(c); 609 610 return 0; 611 } 612 613 static int mtk_cqdma_alloc_chan_resources(struct dma_chan *c) 614 { 615 struct mtk_cqdma_device *cqdma = to_cqdma_dev(c); 616 struct mtk_cqdma_vchan *vc = to_cqdma_vchan(c); 617 struct mtk_cqdma_pchan *pc = NULL; 618 u32 i, min_refcnt = U32_MAX, refcnt; 619 unsigned long flags; 620 621 /* allocate PC with the minimun refcount */ 622 for (i = 0; i < cqdma->dma_channels; ++i) { 623 refcnt = refcount_read(&cqdma->pc[i]->refcnt); 624 if (refcnt < min_refcnt) { 625 pc = cqdma->pc[i]; 626 min_refcnt = refcnt; 627 } 628 } 629 630 if (!pc) 631 return -ENOSPC; 632 633 spin_lock_irqsave(&pc->lock, flags); 634 635 if (!refcount_read(&pc->refcnt)) { 636 /* allocate PC when the refcount is zero */ 637 mtk_cqdma_hard_reset(pc); 638 639 /* enable interrupt for this PC */ 640 mtk_dma_set(pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT); 641 642 /* 643 * refcount_inc would complain increment on 0; use-after-free. 644 * Thus, we need to explicitly set it as 1 initially. 645 */ 646 refcount_set(&pc->refcnt, 1); 647 } else { 648 refcount_inc(&pc->refcnt); 649 } 650 651 spin_unlock_irqrestore(&pc->lock, flags); 652 653 vc->pc = pc; 654 655 return 0; 656 } 657 658 static void mtk_cqdma_free_chan_resources(struct dma_chan *c) 659 { 660 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); 661 unsigned long flags; 662 663 /* free all descriptors in all lists on the VC */ 664 mtk_cqdma_terminate_all(c); 665 666 spin_lock_irqsave(&cvc->pc->lock, flags); 667 668 /* PC is not freed until there is no VC mapped to it */ 669 if (refcount_dec_and_test(&cvc->pc->refcnt)) { 670 /* start the flush operation and stop the engine */ 671 mtk_dma_set(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT); 672 673 /* wait for the completion of flush operation */ 674 if (mtk_cqdma_poll_engine_done(cvc->pc, true) < 0) 675 dev_err(cqdma2dev(to_cqdma_dev(c)), "cqdma flush timeout\n"); 676 677 /* clear the flush bit and interrupt flag */ 678 mtk_dma_clr(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT); 679 mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_FLAG, 680 MTK_CQDMA_INT_FLAG_BIT); 681 682 /* disable interrupt for this PC */ 683 mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT); 684 } 685 686 spin_unlock_irqrestore(&cvc->pc->lock, flags); 687 } 688 689 static int mtk_cqdma_hw_init(struct mtk_cqdma_device *cqdma) 690 { 691 unsigned long flags; 692 int err; 693 u32 i; 694 695 pm_runtime_enable(cqdma2dev(cqdma)); 696 pm_runtime_get_sync(cqdma2dev(cqdma)); 697 698 err = clk_prepare_enable(cqdma->clk); 699 700 if (err) { 701 pm_runtime_put_sync(cqdma2dev(cqdma)); 702 pm_runtime_disable(cqdma2dev(cqdma)); 703 return err; 704 } 705 706 /* reset all PCs */ 707 for (i = 0; i < cqdma->dma_channels; ++i) { 708 spin_lock_irqsave(&cqdma->pc[i]->lock, flags); 709 if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0) { 710 dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n"); 711 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags); 712 713 clk_disable_unprepare(cqdma->clk); 714 pm_runtime_put_sync(cqdma2dev(cqdma)); 715 pm_runtime_disable(cqdma2dev(cqdma)); 716 return -EINVAL; 717 } 718 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags); 719 } 720 721 return 0; 722 } 723 724 static void mtk_cqdma_hw_deinit(struct mtk_cqdma_device *cqdma) 725 { 726 unsigned long flags; 727 u32 i; 728 729 /* reset all PCs */ 730 for (i = 0; i < cqdma->dma_channels; ++i) { 731 spin_lock_irqsave(&cqdma->pc[i]->lock, flags); 732 if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0) 733 dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n"); 734 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags); 735 } 736 737 clk_disable_unprepare(cqdma->clk); 738 739 pm_runtime_put_sync(cqdma2dev(cqdma)); 740 pm_runtime_disable(cqdma2dev(cqdma)); 741 } 742 743 static const struct of_device_id mtk_cqdma_match[] = { 744 { .compatible = "mediatek,mt6765-cqdma" }, 745 { /* sentinel */ } 746 }; 747 MODULE_DEVICE_TABLE(of, mtk_cqdma_match); 748 749 static int mtk_cqdma_probe(struct platform_device *pdev) 750 { 751 struct mtk_cqdma_device *cqdma; 752 struct mtk_cqdma_vchan *vc; 753 struct dma_device *dd; 754 int err; 755 u32 i; 756 757 cqdma = devm_kzalloc(&pdev->dev, sizeof(*cqdma), GFP_KERNEL); 758 if (!cqdma) 759 return -ENOMEM; 760 761 dd = &cqdma->ddev; 762 763 cqdma->clk = devm_clk_get(&pdev->dev, "cqdma"); 764 if (IS_ERR(cqdma->clk)) { 765 dev_err(&pdev->dev, "No clock for %s\n", 766 dev_name(&pdev->dev)); 767 return PTR_ERR(cqdma->clk); 768 } 769 770 dma_cap_set(DMA_MEMCPY, dd->cap_mask); 771 772 dd->copy_align = MTK_CQDMA_ALIGN_SIZE; 773 dd->device_alloc_chan_resources = mtk_cqdma_alloc_chan_resources; 774 dd->device_free_chan_resources = mtk_cqdma_free_chan_resources; 775 dd->device_tx_status = mtk_cqdma_tx_status; 776 dd->device_issue_pending = mtk_cqdma_issue_pending; 777 dd->device_prep_dma_memcpy = mtk_cqdma_prep_dma_memcpy; 778 dd->device_terminate_all = mtk_cqdma_terminate_all; 779 dd->src_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS; 780 dd->dst_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS; 781 dd->directions = BIT(DMA_MEM_TO_MEM); 782 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; 783 dd->dev = &pdev->dev; 784 INIT_LIST_HEAD(&dd->channels); 785 786 if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node, 787 "dma-requests", 788 &cqdma->dma_requests)) { 789 dev_info(&pdev->dev, 790 "Using %u as missing dma-requests property\n", 791 MTK_CQDMA_NR_VCHANS); 792 793 cqdma->dma_requests = MTK_CQDMA_NR_VCHANS; 794 } 795 796 if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node, 797 "dma-channels", 798 &cqdma->dma_channels)) { 799 dev_info(&pdev->dev, 800 "Using %u as missing dma-channels property\n", 801 MTK_CQDMA_NR_PCHANS); 802 803 cqdma->dma_channels = MTK_CQDMA_NR_PCHANS; 804 } 805 806 cqdma->pc = devm_kcalloc(&pdev->dev, cqdma->dma_channels, 807 sizeof(*cqdma->pc), GFP_KERNEL); 808 if (!cqdma->pc) 809 return -ENOMEM; 810 811 /* initialization for PCs */ 812 for (i = 0; i < cqdma->dma_channels; ++i) { 813 cqdma->pc[i] = devm_kcalloc(&pdev->dev, 1, 814 sizeof(**cqdma->pc), GFP_KERNEL); 815 if (!cqdma->pc[i]) 816 return -ENOMEM; 817 818 INIT_LIST_HEAD(&cqdma->pc[i]->queue); 819 spin_lock_init(&cqdma->pc[i]->lock); 820 refcount_set(&cqdma->pc[i]->refcnt, 0); 821 cqdma->pc[i]->base = devm_platform_ioremap_resource(pdev, i); 822 if (IS_ERR(cqdma->pc[i]->base)) 823 return PTR_ERR(cqdma->pc[i]->base); 824 825 /* allocate IRQ resource */ 826 err = platform_get_irq(pdev, i); 827 if (err < 0) 828 return err; 829 cqdma->pc[i]->irq = err; 830 831 err = devm_request_irq(&pdev->dev, cqdma->pc[i]->irq, 832 mtk_cqdma_irq, 0, dev_name(&pdev->dev), 833 cqdma); 834 if (err) { 835 dev_err(&pdev->dev, 836 "request_irq failed with err %d\n", err); 837 return -EINVAL; 838 } 839 } 840 841 /* allocate resource for VCs */ 842 cqdma->vc = devm_kcalloc(&pdev->dev, cqdma->dma_requests, 843 sizeof(*cqdma->vc), GFP_KERNEL); 844 if (!cqdma->vc) 845 return -ENOMEM; 846 847 for (i = 0; i < cqdma->dma_requests; i++) { 848 vc = &cqdma->vc[i]; 849 vc->vc.desc_free = mtk_cqdma_vdesc_free; 850 vchan_init(&vc->vc, dd); 851 init_completion(&vc->issue_completion); 852 } 853 854 err = dma_async_device_register(dd); 855 if (err) 856 return err; 857 858 err = of_dma_controller_register(pdev->dev.of_node, 859 of_dma_xlate_by_chan_id, cqdma); 860 if (err) { 861 dev_err(&pdev->dev, 862 "MediaTek CQDMA OF registration failed %d\n", err); 863 goto err_unregister; 864 } 865 866 err = mtk_cqdma_hw_init(cqdma); 867 if (err) { 868 dev_err(&pdev->dev, 869 "MediaTek CQDMA HW initialization failed %d\n", err); 870 goto err_unregister; 871 } 872 873 platform_set_drvdata(pdev, cqdma); 874 875 /* initialize tasklet for each PC */ 876 for (i = 0; i < cqdma->dma_channels; ++i) 877 tasklet_setup(&cqdma->pc[i]->tasklet, mtk_cqdma_tasklet_cb); 878 879 dev_info(&pdev->dev, "MediaTek CQDMA driver registered\n"); 880 881 return 0; 882 883 err_unregister: 884 dma_async_device_unregister(dd); 885 886 return err; 887 } 888 889 static int mtk_cqdma_remove(struct platform_device *pdev) 890 { 891 struct mtk_cqdma_device *cqdma = platform_get_drvdata(pdev); 892 struct mtk_cqdma_vchan *vc; 893 unsigned long flags; 894 int i; 895 896 /* kill VC task */ 897 for (i = 0; i < cqdma->dma_requests; i++) { 898 vc = &cqdma->vc[i]; 899 900 list_del(&vc->vc.chan.device_node); 901 tasklet_kill(&vc->vc.task); 902 } 903 904 /* disable interrupt */ 905 for (i = 0; i < cqdma->dma_channels; i++) { 906 spin_lock_irqsave(&cqdma->pc[i]->lock, flags); 907 mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_EN, 908 MTK_CQDMA_INT_EN_BIT); 909 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags); 910 911 /* Waits for any pending IRQ handlers to complete */ 912 synchronize_irq(cqdma->pc[i]->irq); 913 914 tasklet_kill(&cqdma->pc[i]->tasklet); 915 } 916 917 /* disable hardware */ 918 mtk_cqdma_hw_deinit(cqdma); 919 920 dma_async_device_unregister(&cqdma->ddev); 921 of_dma_controller_free(pdev->dev.of_node); 922 923 return 0; 924 } 925 926 static struct platform_driver mtk_cqdma_driver = { 927 .probe = mtk_cqdma_probe, 928 .remove = mtk_cqdma_remove, 929 .driver = { 930 .name = KBUILD_MODNAME, 931 .of_match_table = mtk_cqdma_match, 932 }, 933 }; 934 module_platform_driver(mtk_cqdma_driver); 935 936 MODULE_DESCRIPTION("MediaTek CQDMA Controller Driver"); 937 MODULE_AUTHOR("Shun-Chih Yu <shun-chih.yu@mediatek.com>"); 938 MODULE_LICENSE("GPL v2"); 939