1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * SiFive FU540 Platform DMA driver 4 * Copyright (C) 2019 SiFive 5 * 6 * Based partially on: 7 * - drivers/dma/fsl-edma.c 8 * - drivers/dma/dw-edma/ 9 * - drivers/dma/pxa-dma.c 10 * 11 * See the following sources for further documentation: 12 * - Chapter 12 "Platform DMA Engine (PDMA)" of 13 * SiFive FU540-C000 v1.0 14 * https://static.dev.sifive.com/FU540-C000-v1.0.pdf 15 */ 16 #include <linux/module.h> 17 #include <linux/device.h> 18 #include <linux/kernel.h> 19 #include <linux/platform_device.h> 20 #include <linux/mod_devicetable.h> 21 #include <linux/dma-mapping.h> 22 #include <linux/of.h> 23 #include <linux/slab.h> 24 25 #include "sf-pdma.h" 26 27 #ifndef readq 28 static inline unsigned long long readq(void __iomem *addr) 29 { 30 return readl(addr) | (((unsigned long long)readl(addr + 4)) << 32LL); 31 } 32 #endif 33 34 #ifndef writeq 35 static inline void writeq(unsigned long long v, void __iomem *addr) 36 { 37 writel(lower_32_bits(v), addr); 38 writel(upper_32_bits(v), addr + 4); 39 } 40 #endif 41 42 static inline struct sf_pdma_chan *to_sf_pdma_chan(struct dma_chan *dchan) 43 { 44 return container_of(dchan, struct sf_pdma_chan, vchan.chan); 45 } 46 47 static inline struct sf_pdma_desc *to_sf_pdma_desc(struct virt_dma_desc *vd) 48 { 49 return container_of(vd, struct sf_pdma_desc, vdesc); 50 } 51 52 static struct sf_pdma_desc *sf_pdma_alloc_desc(struct sf_pdma_chan *chan) 53 { 54 struct sf_pdma_desc *desc; 55 56 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 57 if (!desc) 58 return NULL; 59 60 desc->chan = chan; 61 62 return desc; 63 } 64 65 static void sf_pdma_fill_desc(struct sf_pdma_desc *desc, 66 u64 dst, u64 src, u64 size) 67 { 68 desc->xfer_type = PDMA_FULL_SPEED; 69 desc->xfer_size = size; 70 desc->dst_addr = dst; 71 desc->src_addr = src; 72 } 73 74 static void sf_pdma_disclaim_chan(struct sf_pdma_chan *chan) 75 { 76 struct pdma_regs *regs = &chan->regs; 77 78 writel(PDMA_CLEAR_CTRL, regs->ctrl); 79 } 80 81 static struct dma_async_tx_descriptor * 82 sf_pdma_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dest, dma_addr_t src, 83 size_t len, unsigned long flags) 84 { 85 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 86 struct sf_pdma_desc *desc; 87 unsigned long iflags; 88 89 if (chan && (!len || !dest || !src)) { 90 dev_err(chan->pdma->dma_dev.dev, 91 "Please check dma len, dest, src!\n"); 92 return NULL; 93 } 94 95 desc = sf_pdma_alloc_desc(chan); 96 if (!desc) 97 return NULL; 98 99 desc->in_use = true; 100 desc->dirn = DMA_MEM_TO_MEM; 101 desc->async_tx = vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); 102 103 spin_lock_irqsave(&chan->vchan.lock, iflags); 104 sf_pdma_fill_desc(desc, dest, src, len); 105 spin_unlock_irqrestore(&chan->vchan.lock, iflags); 106 107 return desc->async_tx; 108 } 109 110 static int sf_pdma_slave_config(struct dma_chan *dchan, 111 struct dma_slave_config *cfg) 112 { 113 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 114 115 memcpy(&chan->cfg, cfg, sizeof(*cfg)); 116 117 return 0; 118 } 119 120 static int sf_pdma_alloc_chan_resources(struct dma_chan *dchan) 121 { 122 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 123 struct pdma_regs *regs = &chan->regs; 124 125 dma_cookie_init(dchan); 126 writel(PDMA_CLAIM_MASK, regs->ctrl); 127 128 return 0; 129 } 130 131 static void sf_pdma_disable_request(struct sf_pdma_chan *chan) 132 { 133 struct pdma_regs *regs = &chan->regs; 134 135 writel(readl(regs->ctrl) & ~PDMA_RUN_MASK, regs->ctrl); 136 } 137 138 static void sf_pdma_free_chan_resources(struct dma_chan *dchan) 139 { 140 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 141 unsigned long flags; 142 LIST_HEAD(head); 143 144 spin_lock_irqsave(&chan->vchan.lock, flags); 145 sf_pdma_disable_request(chan); 146 kfree(chan->desc); 147 chan->desc = NULL; 148 vchan_get_all_descriptors(&chan->vchan, &head); 149 sf_pdma_disclaim_chan(chan); 150 spin_unlock_irqrestore(&chan->vchan.lock, flags); 151 vchan_dma_desc_free_list(&chan->vchan, &head); 152 } 153 154 static size_t sf_pdma_desc_residue(struct sf_pdma_chan *chan, 155 dma_cookie_t cookie) 156 { 157 struct virt_dma_desc *vd = NULL; 158 struct pdma_regs *regs = &chan->regs; 159 unsigned long flags; 160 u64 residue = 0; 161 struct sf_pdma_desc *desc; 162 struct dma_async_tx_descriptor *tx = NULL; 163 164 spin_lock_irqsave(&chan->vchan.lock, flags); 165 166 list_for_each_entry(vd, &chan->vchan.desc_submitted, node) 167 if (vd->tx.cookie == cookie) 168 tx = &vd->tx; 169 170 if (!tx) 171 goto out; 172 173 if (cookie == tx->chan->completed_cookie) 174 goto out; 175 176 if (cookie == tx->cookie) { 177 residue = readq(regs->residue); 178 } else { 179 vd = vchan_find_desc(&chan->vchan, cookie); 180 if (!vd) 181 goto out; 182 183 desc = to_sf_pdma_desc(vd); 184 residue = desc->xfer_size; 185 } 186 187 out: 188 spin_unlock_irqrestore(&chan->vchan.lock, flags); 189 return residue; 190 } 191 192 static enum dma_status 193 sf_pdma_tx_status(struct dma_chan *dchan, 194 dma_cookie_t cookie, 195 struct dma_tx_state *txstate) 196 { 197 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 198 enum dma_status status; 199 200 status = dma_cookie_status(dchan, cookie, txstate); 201 202 if (txstate && status != DMA_ERROR) 203 dma_set_residue(txstate, sf_pdma_desc_residue(chan, cookie)); 204 205 return status; 206 } 207 208 static int sf_pdma_terminate_all(struct dma_chan *dchan) 209 { 210 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 211 unsigned long flags; 212 LIST_HEAD(head); 213 214 spin_lock_irqsave(&chan->vchan.lock, flags); 215 sf_pdma_disable_request(chan); 216 kfree(chan->desc); 217 chan->desc = NULL; 218 chan->xfer_err = false; 219 vchan_get_all_descriptors(&chan->vchan, &head); 220 spin_unlock_irqrestore(&chan->vchan.lock, flags); 221 vchan_dma_desc_free_list(&chan->vchan, &head); 222 223 return 0; 224 } 225 226 static void sf_pdma_enable_request(struct sf_pdma_chan *chan) 227 { 228 struct pdma_regs *regs = &chan->regs; 229 u32 v; 230 231 v = PDMA_CLAIM_MASK | 232 PDMA_ENABLE_DONE_INT_MASK | 233 PDMA_ENABLE_ERR_INT_MASK | 234 PDMA_RUN_MASK; 235 236 writel(v, regs->ctrl); 237 } 238 239 static struct sf_pdma_desc *sf_pdma_get_first_pending_desc(struct sf_pdma_chan *chan) 240 { 241 struct virt_dma_chan *vchan = &chan->vchan; 242 struct virt_dma_desc *vdesc; 243 244 if (list_empty(&vchan->desc_issued)) 245 return NULL; 246 247 vdesc = list_first_entry(&vchan->desc_issued, struct virt_dma_desc, node); 248 249 return container_of(vdesc, struct sf_pdma_desc, vdesc); 250 } 251 252 static void sf_pdma_xfer_desc(struct sf_pdma_chan *chan) 253 { 254 struct sf_pdma_desc *desc = chan->desc; 255 struct pdma_regs *regs = &chan->regs; 256 257 if (!desc) { 258 dev_err(chan->pdma->dma_dev.dev, "NULL desc.\n"); 259 return; 260 } 261 262 writel(desc->xfer_type, regs->xfer_type); 263 writeq(desc->xfer_size, regs->xfer_size); 264 writeq(desc->dst_addr, regs->dst_addr); 265 writeq(desc->src_addr, regs->src_addr); 266 267 chan->desc = desc; 268 chan->status = DMA_IN_PROGRESS; 269 sf_pdma_enable_request(chan); 270 } 271 272 static void sf_pdma_issue_pending(struct dma_chan *dchan) 273 { 274 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 275 unsigned long flags; 276 277 spin_lock_irqsave(&chan->vchan.lock, flags); 278 279 if (!chan->desc && vchan_issue_pending(&chan->vchan)) { 280 /* vchan_issue_pending has made a check that desc in not NULL */ 281 chan->desc = sf_pdma_get_first_pending_desc(chan); 282 sf_pdma_xfer_desc(chan); 283 } 284 285 spin_unlock_irqrestore(&chan->vchan.lock, flags); 286 } 287 288 static void sf_pdma_free_desc(struct virt_dma_desc *vdesc) 289 { 290 struct sf_pdma_desc *desc; 291 292 desc = to_sf_pdma_desc(vdesc); 293 desc->in_use = false; 294 } 295 296 static void sf_pdma_donebh_tasklet(struct tasklet_struct *t) 297 { 298 struct sf_pdma_chan *chan = from_tasklet(chan, t, done_tasklet); 299 unsigned long flags; 300 301 spin_lock_irqsave(&chan->lock, flags); 302 if (chan->xfer_err) { 303 chan->retries = MAX_RETRY; 304 chan->status = DMA_COMPLETE; 305 chan->xfer_err = false; 306 } 307 spin_unlock_irqrestore(&chan->lock, flags); 308 309 spin_lock_irqsave(&chan->vchan.lock, flags); 310 list_del(&chan->desc->vdesc.node); 311 vchan_cookie_complete(&chan->desc->vdesc); 312 313 chan->desc = sf_pdma_get_first_pending_desc(chan); 314 if (chan->desc) 315 sf_pdma_xfer_desc(chan); 316 317 spin_unlock_irqrestore(&chan->vchan.lock, flags); 318 } 319 320 static void sf_pdma_errbh_tasklet(struct tasklet_struct *t) 321 { 322 struct sf_pdma_chan *chan = from_tasklet(chan, t, err_tasklet); 323 struct sf_pdma_desc *desc = chan->desc; 324 unsigned long flags; 325 326 spin_lock_irqsave(&chan->lock, flags); 327 if (chan->retries <= 0) { 328 /* fail to recover */ 329 spin_unlock_irqrestore(&chan->lock, flags); 330 dmaengine_desc_get_callback_invoke(desc->async_tx, NULL); 331 } else { 332 /* retry */ 333 chan->retries--; 334 chan->xfer_err = true; 335 chan->status = DMA_ERROR; 336 337 sf_pdma_enable_request(chan); 338 spin_unlock_irqrestore(&chan->lock, flags); 339 } 340 } 341 342 static irqreturn_t sf_pdma_done_isr(int irq, void *dev_id) 343 { 344 struct sf_pdma_chan *chan = dev_id; 345 struct pdma_regs *regs = &chan->regs; 346 u64 residue; 347 348 spin_lock(&chan->vchan.lock); 349 writel((readl(regs->ctrl)) & ~PDMA_DONE_STATUS_MASK, regs->ctrl); 350 residue = readq(regs->residue); 351 352 if (!residue) { 353 tasklet_hi_schedule(&chan->done_tasklet); 354 } else { 355 /* submit next trascatioin if possible */ 356 struct sf_pdma_desc *desc = chan->desc; 357 358 desc->src_addr += desc->xfer_size - residue; 359 desc->dst_addr += desc->xfer_size - residue; 360 desc->xfer_size = residue; 361 362 sf_pdma_xfer_desc(chan); 363 } 364 365 spin_unlock(&chan->vchan.lock); 366 367 return IRQ_HANDLED; 368 } 369 370 static irqreturn_t sf_pdma_err_isr(int irq, void *dev_id) 371 { 372 struct sf_pdma_chan *chan = dev_id; 373 struct pdma_regs *regs = &chan->regs; 374 375 spin_lock(&chan->lock); 376 writel((readl(regs->ctrl)) & ~PDMA_ERR_STATUS_MASK, regs->ctrl); 377 spin_unlock(&chan->lock); 378 379 tasklet_schedule(&chan->err_tasklet); 380 381 return IRQ_HANDLED; 382 } 383 384 /** 385 * sf_pdma_irq_init() - Init PDMA IRQ Handlers 386 * @pdev: pointer of platform_device 387 * @pdma: pointer of PDMA engine. Caller should check NULL 388 * 389 * Initialize DONE and ERROR interrupt handler for 4 channels. Caller should 390 * make sure the pointer passed in are non-NULL. This function should be called 391 * only one time during the device probe. 392 * 393 * Context: Any context. 394 * 395 * Return: 396 * * 0 - OK to init all IRQ handlers 397 * * -EINVAL - Fail to request IRQ 398 */ 399 static int sf_pdma_irq_init(struct platform_device *pdev, struct sf_pdma *pdma) 400 { 401 int irq, r, i; 402 struct sf_pdma_chan *chan; 403 404 for (i = 0; i < pdma->n_chans; i++) { 405 chan = &pdma->chans[i]; 406 407 irq = platform_get_irq(pdev, i * 2); 408 if (irq < 0) { 409 dev_err(&pdev->dev, "ch(%d) Can't get done irq.\n", i); 410 return -EINVAL; 411 } 412 413 r = devm_request_irq(&pdev->dev, irq, sf_pdma_done_isr, 0, 414 dev_name(&pdev->dev), (void *)chan); 415 if (r) { 416 dev_err(&pdev->dev, "Fail to attach done ISR: %d\n", r); 417 return -EINVAL; 418 } 419 420 chan->txirq = irq; 421 422 irq = platform_get_irq(pdev, (i * 2) + 1); 423 if (irq < 0) { 424 dev_err(&pdev->dev, "ch(%d) Can't get err irq.\n", i); 425 return -EINVAL; 426 } 427 428 r = devm_request_irq(&pdev->dev, irq, sf_pdma_err_isr, 0, 429 dev_name(&pdev->dev), (void *)chan); 430 if (r) { 431 dev_err(&pdev->dev, "Fail to attach err ISR: %d\n", r); 432 return -EINVAL; 433 } 434 435 chan->errirq = irq; 436 } 437 438 return 0; 439 } 440 441 /** 442 * sf_pdma_setup_chans() - Init settings of each channel 443 * @pdma: pointer of PDMA engine. Caller should check NULL 444 * 445 * Initialize all data structure and register base. Caller should make sure 446 * the pointer passed in are non-NULL. This function should be called only 447 * one time during the device probe. 448 * 449 * Context: Any context. 450 * 451 * Return: none 452 */ 453 static void sf_pdma_setup_chans(struct sf_pdma *pdma) 454 { 455 int i; 456 struct sf_pdma_chan *chan; 457 458 INIT_LIST_HEAD(&pdma->dma_dev.channels); 459 460 for (i = 0; i < pdma->n_chans; i++) { 461 chan = &pdma->chans[i]; 462 463 chan->regs.ctrl = 464 SF_PDMA_REG_BASE(i) + PDMA_CTRL; 465 chan->regs.xfer_type = 466 SF_PDMA_REG_BASE(i) + PDMA_XFER_TYPE; 467 chan->regs.xfer_size = 468 SF_PDMA_REG_BASE(i) + PDMA_XFER_SIZE; 469 chan->regs.dst_addr = 470 SF_PDMA_REG_BASE(i) + PDMA_DST_ADDR; 471 chan->regs.src_addr = 472 SF_PDMA_REG_BASE(i) + PDMA_SRC_ADDR; 473 chan->regs.act_type = 474 SF_PDMA_REG_BASE(i) + PDMA_ACT_TYPE; 475 chan->regs.residue = 476 SF_PDMA_REG_BASE(i) + PDMA_REMAINING_BYTE; 477 chan->regs.cur_dst_addr = 478 SF_PDMA_REG_BASE(i) + PDMA_CUR_DST_ADDR; 479 chan->regs.cur_src_addr = 480 SF_PDMA_REG_BASE(i) + PDMA_CUR_SRC_ADDR; 481 482 chan->pdma = pdma; 483 chan->pm_state = RUNNING; 484 chan->slave_id = i; 485 chan->xfer_err = false; 486 spin_lock_init(&chan->lock); 487 488 chan->vchan.desc_free = sf_pdma_free_desc; 489 vchan_init(&chan->vchan, &pdma->dma_dev); 490 491 writel(PDMA_CLEAR_CTRL, chan->regs.ctrl); 492 493 tasklet_setup(&chan->done_tasklet, sf_pdma_donebh_tasklet); 494 tasklet_setup(&chan->err_tasklet, sf_pdma_errbh_tasklet); 495 } 496 } 497 498 static int sf_pdma_probe(struct platform_device *pdev) 499 { 500 struct sf_pdma *pdma; 501 struct resource *res; 502 int ret, n_chans; 503 const enum dma_slave_buswidth widths = 504 DMA_SLAVE_BUSWIDTH_1_BYTE | DMA_SLAVE_BUSWIDTH_2_BYTES | 505 DMA_SLAVE_BUSWIDTH_4_BYTES | DMA_SLAVE_BUSWIDTH_8_BYTES | 506 DMA_SLAVE_BUSWIDTH_16_BYTES | DMA_SLAVE_BUSWIDTH_32_BYTES | 507 DMA_SLAVE_BUSWIDTH_64_BYTES; 508 509 ret = of_property_read_u32(pdev->dev.of_node, "dma-channels", &n_chans); 510 if (ret) { 511 /* backwards-compatibility for no dma-channels property */ 512 dev_dbg(&pdev->dev, "set number of channels to default value: 4\n"); 513 n_chans = PDMA_MAX_NR_CH; 514 } else if (n_chans > PDMA_MAX_NR_CH) { 515 dev_err(&pdev->dev, "the number of channels exceeds the maximum\n"); 516 return -EINVAL; 517 } 518 519 pdma = devm_kzalloc(&pdev->dev, struct_size(pdma, chans, n_chans), 520 GFP_KERNEL); 521 if (!pdma) 522 return -ENOMEM; 523 524 pdma->n_chans = n_chans; 525 526 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 527 pdma->membase = devm_ioremap_resource(&pdev->dev, res); 528 if (IS_ERR(pdma->membase)) 529 return PTR_ERR(pdma->membase); 530 531 ret = sf_pdma_irq_init(pdev, pdma); 532 if (ret) 533 return ret; 534 535 sf_pdma_setup_chans(pdma); 536 537 pdma->dma_dev.dev = &pdev->dev; 538 539 /* Setup capability */ 540 dma_cap_set(DMA_MEMCPY, pdma->dma_dev.cap_mask); 541 pdma->dma_dev.copy_align = 2; 542 pdma->dma_dev.src_addr_widths = widths; 543 pdma->dma_dev.dst_addr_widths = widths; 544 pdma->dma_dev.directions = BIT(DMA_MEM_TO_MEM); 545 pdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 546 pdma->dma_dev.descriptor_reuse = true; 547 548 /* Setup DMA APIs */ 549 pdma->dma_dev.device_alloc_chan_resources = 550 sf_pdma_alloc_chan_resources; 551 pdma->dma_dev.device_free_chan_resources = 552 sf_pdma_free_chan_resources; 553 pdma->dma_dev.device_tx_status = sf_pdma_tx_status; 554 pdma->dma_dev.device_prep_dma_memcpy = sf_pdma_prep_dma_memcpy; 555 pdma->dma_dev.device_config = sf_pdma_slave_config; 556 pdma->dma_dev.device_terminate_all = sf_pdma_terminate_all; 557 pdma->dma_dev.device_issue_pending = sf_pdma_issue_pending; 558 559 platform_set_drvdata(pdev, pdma); 560 561 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 562 if (ret) 563 dev_warn(&pdev->dev, 564 "Failed to set DMA mask. Fall back to default.\n"); 565 566 ret = dma_async_device_register(&pdma->dma_dev); 567 if (ret) { 568 dev_err(&pdev->dev, 569 "Can't register SiFive Platform DMA. (%d)\n", ret); 570 return ret; 571 } 572 573 return 0; 574 } 575 576 static int sf_pdma_remove(struct platform_device *pdev) 577 { 578 struct sf_pdma *pdma = platform_get_drvdata(pdev); 579 struct sf_pdma_chan *ch; 580 int i; 581 582 for (i = 0; i < pdma->n_chans; i++) { 583 ch = &pdma->chans[i]; 584 585 devm_free_irq(&pdev->dev, ch->txirq, ch); 586 devm_free_irq(&pdev->dev, ch->errirq, ch); 587 list_del(&ch->vchan.chan.device_node); 588 tasklet_kill(&ch->vchan.task); 589 tasklet_kill(&ch->done_tasklet); 590 tasklet_kill(&ch->err_tasklet); 591 } 592 593 dma_async_device_unregister(&pdma->dma_dev); 594 595 return 0; 596 } 597 598 static const struct of_device_id sf_pdma_dt_ids[] = { 599 { .compatible = "sifive,fu540-c000-pdma" }, 600 { .compatible = "sifive,pdma0" }, 601 {}, 602 }; 603 MODULE_DEVICE_TABLE(of, sf_pdma_dt_ids); 604 605 static struct platform_driver sf_pdma_driver = { 606 .probe = sf_pdma_probe, 607 .remove = sf_pdma_remove, 608 .driver = { 609 .name = "sf-pdma", 610 .of_match_table = sf_pdma_dt_ids, 611 }, 612 }; 613 614 static int __init sf_pdma_init(void) 615 { 616 return platform_driver_register(&sf_pdma_driver); 617 } 618 619 static void __exit sf_pdma_exit(void) 620 { 621 platform_driver_unregister(&sf_pdma_driver); 622 } 623 624 /* do early init */ 625 subsys_initcall(sf_pdma_init); 626 module_exit(sf_pdma_exit); 627 628 MODULE_LICENSE("GPL v2"); 629 MODULE_DESCRIPTION("SiFive Platform DMA driver"); 630 MODULE_AUTHOR("Green Wan <green.wan@sifive.com>"); 631