1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * SiFive FU540 Platform DMA driver 4 * Copyright (C) 2019 SiFive 5 * 6 * Based partially on: 7 * - drivers/dma/fsl-edma.c 8 * - drivers/dma/dw-edma/ 9 * - drivers/dma/pxa-dma.c 10 * 11 * See the following sources for further documentation: 12 * - Chapter 12 "Platform DMA Engine (PDMA)" of 13 * SiFive FU540-C000 v1.0 14 * https://static.dev.sifive.com/FU540-C000-v1.0.pdf 15 */ 16 #include <linux/module.h> 17 #include <linux/device.h> 18 #include <linux/kernel.h> 19 #include <linux/platform_device.h> 20 #include <linux/mod_devicetable.h> 21 #include <linux/dma-mapping.h> 22 #include <linux/of.h> 23 #include <linux/slab.h> 24 25 #include "sf-pdma.h" 26 27 #ifndef readq 28 static inline unsigned long long readq(void __iomem *addr) 29 { 30 return readl(addr) | (((unsigned long long)readl(addr + 4)) << 32LL); 31 } 32 #endif 33 34 #ifndef writeq 35 static inline void writeq(unsigned long long v, void __iomem *addr) 36 { 37 writel(lower_32_bits(v), addr); 38 writel(upper_32_bits(v), addr + 4); 39 } 40 #endif 41 42 static inline struct sf_pdma_chan *to_sf_pdma_chan(struct dma_chan *dchan) 43 { 44 return container_of(dchan, struct sf_pdma_chan, vchan.chan); 45 } 46 47 static inline struct sf_pdma_desc *to_sf_pdma_desc(struct virt_dma_desc *vd) 48 { 49 return container_of(vd, struct sf_pdma_desc, vdesc); 50 } 51 52 static struct sf_pdma_desc *sf_pdma_alloc_desc(struct sf_pdma_chan *chan) 53 { 54 struct sf_pdma_desc *desc; 55 56 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 57 if (!desc) 58 return NULL; 59 60 desc->chan = chan; 61 62 return desc; 63 } 64 65 static void sf_pdma_fill_desc(struct sf_pdma_desc *desc, 66 u64 dst, u64 src, u64 size) 67 { 68 desc->xfer_type = PDMA_FULL_SPEED; 69 desc->xfer_size = size; 70 desc->dst_addr = dst; 71 desc->src_addr = src; 72 } 73 74 static void sf_pdma_disclaim_chan(struct sf_pdma_chan *chan) 75 { 76 struct pdma_regs *regs = &chan->regs; 77 78 writel(PDMA_CLEAR_CTRL, regs->ctrl); 79 } 80 81 static struct dma_async_tx_descriptor * 82 sf_pdma_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dest, dma_addr_t src, 83 size_t len, unsigned long flags) 84 { 85 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 86 struct sf_pdma_desc *desc; 87 unsigned long iflags; 88 89 if (chan && (!len || !dest || !src)) { 90 dev_err(chan->pdma->dma_dev.dev, 91 "Please check dma len, dest, src!\n"); 92 return NULL; 93 } 94 95 desc = sf_pdma_alloc_desc(chan); 96 if (!desc) 97 return NULL; 98 99 desc->dirn = DMA_MEM_TO_MEM; 100 desc->async_tx = vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); 101 102 spin_lock_irqsave(&chan->vchan.lock, iflags); 103 sf_pdma_fill_desc(desc, dest, src, len); 104 spin_unlock_irqrestore(&chan->vchan.lock, iflags); 105 106 return desc->async_tx; 107 } 108 109 static int sf_pdma_slave_config(struct dma_chan *dchan, 110 struct dma_slave_config *cfg) 111 { 112 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 113 114 memcpy(&chan->cfg, cfg, sizeof(*cfg)); 115 116 return 0; 117 } 118 119 static int sf_pdma_alloc_chan_resources(struct dma_chan *dchan) 120 { 121 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 122 struct pdma_regs *regs = &chan->regs; 123 124 dma_cookie_init(dchan); 125 writel(PDMA_CLAIM_MASK, regs->ctrl); 126 127 return 0; 128 } 129 130 static void sf_pdma_disable_request(struct sf_pdma_chan *chan) 131 { 132 struct pdma_regs *regs = &chan->regs; 133 134 writel(readl(regs->ctrl) & ~PDMA_RUN_MASK, regs->ctrl); 135 } 136 137 static void sf_pdma_free_chan_resources(struct dma_chan *dchan) 138 { 139 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 140 unsigned long flags; 141 LIST_HEAD(head); 142 143 spin_lock_irqsave(&chan->vchan.lock, flags); 144 sf_pdma_disable_request(chan); 145 kfree(chan->desc); 146 chan->desc = NULL; 147 vchan_get_all_descriptors(&chan->vchan, &head); 148 sf_pdma_disclaim_chan(chan); 149 spin_unlock_irqrestore(&chan->vchan.lock, flags); 150 vchan_dma_desc_free_list(&chan->vchan, &head); 151 } 152 153 static size_t sf_pdma_desc_residue(struct sf_pdma_chan *chan, 154 dma_cookie_t cookie) 155 { 156 struct virt_dma_desc *vd = NULL; 157 struct pdma_regs *regs = &chan->regs; 158 unsigned long flags; 159 u64 residue = 0; 160 struct sf_pdma_desc *desc; 161 struct dma_async_tx_descriptor *tx = NULL; 162 163 spin_lock_irqsave(&chan->vchan.lock, flags); 164 165 list_for_each_entry(vd, &chan->vchan.desc_submitted, node) 166 if (vd->tx.cookie == cookie) 167 tx = &vd->tx; 168 169 if (!tx) 170 goto out; 171 172 if (cookie == tx->chan->completed_cookie) 173 goto out; 174 175 if (cookie == tx->cookie) { 176 residue = readq(regs->residue); 177 } else { 178 vd = vchan_find_desc(&chan->vchan, cookie); 179 if (!vd) 180 goto out; 181 182 desc = to_sf_pdma_desc(vd); 183 residue = desc->xfer_size; 184 } 185 186 out: 187 spin_unlock_irqrestore(&chan->vchan.lock, flags); 188 return residue; 189 } 190 191 static enum dma_status 192 sf_pdma_tx_status(struct dma_chan *dchan, 193 dma_cookie_t cookie, 194 struct dma_tx_state *txstate) 195 { 196 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 197 enum dma_status status; 198 199 status = dma_cookie_status(dchan, cookie, txstate); 200 201 if (txstate && status != DMA_ERROR) 202 dma_set_residue(txstate, sf_pdma_desc_residue(chan, cookie)); 203 204 return status; 205 } 206 207 static int sf_pdma_terminate_all(struct dma_chan *dchan) 208 { 209 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 210 unsigned long flags; 211 LIST_HEAD(head); 212 213 spin_lock_irqsave(&chan->vchan.lock, flags); 214 sf_pdma_disable_request(chan); 215 kfree(chan->desc); 216 chan->desc = NULL; 217 chan->xfer_err = false; 218 vchan_get_all_descriptors(&chan->vchan, &head); 219 spin_unlock_irqrestore(&chan->vchan.lock, flags); 220 vchan_dma_desc_free_list(&chan->vchan, &head); 221 222 return 0; 223 } 224 225 static void sf_pdma_enable_request(struct sf_pdma_chan *chan) 226 { 227 struct pdma_regs *regs = &chan->regs; 228 u32 v; 229 230 v = PDMA_CLAIM_MASK | 231 PDMA_ENABLE_DONE_INT_MASK | 232 PDMA_ENABLE_ERR_INT_MASK | 233 PDMA_RUN_MASK; 234 235 writel(v, regs->ctrl); 236 } 237 238 static struct sf_pdma_desc *sf_pdma_get_first_pending_desc(struct sf_pdma_chan *chan) 239 { 240 struct virt_dma_chan *vchan = &chan->vchan; 241 struct virt_dma_desc *vdesc; 242 243 if (list_empty(&vchan->desc_issued)) 244 return NULL; 245 246 vdesc = list_first_entry(&vchan->desc_issued, struct virt_dma_desc, node); 247 248 return container_of(vdesc, struct sf_pdma_desc, vdesc); 249 } 250 251 static void sf_pdma_xfer_desc(struct sf_pdma_chan *chan) 252 { 253 struct sf_pdma_desc *desc = chan->desc; 254 struct pdma_regs *regs = &chan->regs; 255 256 if (!desc) { 257 dev_err(chan->pdma->dma_dev.dev, "NULL desc.\n"); 258 return; 259 } 260 261 writel(desc->xfer_type, regs->xfer_type); 262 writeq(desc->xfer_size, regs->xfer_size); 263 writeq(desc->dst_addr, regs->dst_addr); 264 writeq(desc->src_addr, regs->src_addr); 265 266 chan->desc = desc; 267 chan->status = DMA_IN_PROGRESS; 268 sf_pdma_enable_request(chan); 269 } 270 271 static void sf_pdma_issue_pending(struct dma_chan *dchan) 272 { 273 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 274 unsigned long flags; 275 276 spin_lock_irqsave(&chan->vchan.lock, flags); 277 278 if (!chan->desc && vchan_issue_pending(&chan->vchan)) { 279 /* vchan_issue_pending has made a check that desc in not NULL */ 280 chan->desc = sf_pdma_get_first_pending_desc(chan); 281 sf_pdma_xfer_desc(chan); 282 } 283 284 spin_unlock_irqrestore(&chan->vchan.lock, flags); 285 } 286 287 static void sf_pdma_free_desc(struct virt_dma_desc *vdesc) 288 { 289 struct sf_pdma_desc *desc; 290 291 desc = to_sf_pdma_desc(vdesc); 292 kfree(desc); 293 } 294 295 static void sf_pdma_donebh_tasklet(struct tasklet_struct *t) 296 { 297 struct sf_pdma_chan *chan = from_tasklet(chan, t, done_tasklet); 298 unsigned long flags; 299 300 spin_lock_irqsave(&chan->lock, flags); 301 if (chan->xfer_err) { 302 chan->retries = MAX_RETRY; 303 chan->status = DMA_COMPLETE; 304 chan->xfer_err = false; 305 } 306 spin_unlock_irqrestore(&chan->lock, flags); 307 308 spin_lock_irqsave(&chan->vchan.lock, flags); 309 list_del(&chan->desc->vdesc.node); 310 vchan_cookie_complete(&chan->desc->vdesc); 311 312 chan->desc = sf_pdma_get_first_pending_desc(chan); 313 if (chan->desc) 314 sf_pdma_xfer_desc(chan); 315 316 spin_unlock_irqrestore(&chan->vchan.lock, flags); 317 } 318 319 static void sf_pdma_errbh_tasklet(struct tasklet_struct *t) 320 { 321 struct sf_pdma_chan *chan = from_tasklet(chan, t, err_tasklet); 322 struct sf_pdma_desc *desc = chan->desc; 323 unsigned long flags; 324 325 spin_lock_irqsave(&chan->lock, flags); 326 if (chan->retries <= 0) { 327 /* fail to recover */ 328 spin_unlock_irqrestore(&chan->lock, flags); 329 dmaengine_desc_get_callback_invoke(desc->async_tx, NULL); 330 } else { 331 /* retry */ 332 chan->retries--; 333 chan->xfer_err = true; 334 chan->status = DMA_ERROR; 335 336 sf_pdma_enable_request(chan); 337 spin_unlock_irqrestore(&chan->lock, flags); 338 } 339 } 340 341 static irqreturn_t sf_pdma_done_isr(int irq, void *dev_id) 342 { 343 struct sf_pdma_chan *chan = dev_id; 344 struct pdma_regs *regs = &chan->regs; 345 u64 residue; 346 347 spin_lock(&chan->vchan.lock); 348 writel((readl(regs->ctrl)) & ~PDMA_DONE_STATUS_MASK, regs->ctrl); 349 residue = readq(regs->residue); 350 351 if (!residue) { 352 tasklet_hi_schedule(&chan->done_tasklet); 353 } else { 354 /* submit next trascatioin if possible */ 355 struct sf_pdma_desc *desc = chan->desc; 356 357 desc->src_addr += desc->xfer_size - residue; 358 desc->dst_addr += desc->xfer_size - residue; 359 desc->xfer_size = residue; 360 361 sf_pdma_xfer_desc(chan); 362 } 363 364 spin_unlock(&chan->vchan.lock); 365 366 return IRQ_HANDLED; 367 } 368 369 static irqreturn_t sf_pdma_err_isr(int irq, void *dev_id) 370 { 371 struct sf_pdma_chan *chan = dev_id; 372 struct pdma_regs *regs = &chan->regs; 373 374 spin_lock(&chan->lock); 375 writel((readl(regs->ctrl)) & ~PDMA_ERR_STATUS_MASK, regs->ctrl); 376 spin_unlock(&chan->lock); 377 378 tasklet_schedule(&chan->err_tasklet); 379 380 return IRQ_HANDLED; 381 } 382 383 /** 384 * sf_pdma_irq_init() - Init PDMA IRQ Handlers 385 * @pdev: pointer of platform_device 386 * @pdma: pointer of PDMA engine. Caller should check NULL 387 * 388 * Initialize DONE and ERROR interrupt handler for 4 channels. Caller should 389 * make sure the pointer passed in are non-NULL. This function should be called 390 * only one time during the device probe. 391 * 392 * Context: Any context. 393 * 394 * Return: 395 * * 0 - OK to init all IRQ handlers 396 * * -EINVAL - Fail to request IRQ 397 */ 398 static int sf_pdma_irq_init(struct platform_device *pdev, struct sf_pdma *pdma) 399 { 400 int irq, r, i; 401 struct sf_pdma_chan *chan; 402 403 for (i = 0; i < pdma->n_chans; i++) { 404 chan = &pdma->chans[i]; 405 406 irq = platform_get_irq(pdev, i * 2); 407 if (irq < 0) 408 return -EINVAL; 409 410 r = devm_request_irq(&pdev->dev, irq, sf_pdma_done_isr, 0, 411 dev_name(&pdev->dev), (void *)chan); 412 if (r) { 413 dev_err(&pdev->dev, "Fail to attach done ISR: %d\n", r); 414 return -EINVAL; 415 } 416 417 chan->txirq = irq; 418 419 irq = platform_get_irq(pdev, (i * 2) + 1); 420 if (irq < 0) 421 return -EINVAL; 422 423 r = devm_request_irq(&pdev->dev, irq, sf_pdma_err_isr, 0, 424 dev_name(&pdev->dev), (void *)chan); 425 if (r) { 426 dev_err(&pdev->dev, "Fail to attach err ISR: %d\n", r); 427 return -EINVAL; 428 } 429 430 chan->errirq = irq; 431 } 432 433 return 0; 434 } 435 436 /** 437 * sf_pdma_setup_chans() - Init settings of each channel 438 * @pdma: pointer of PDMA engine. Caller should check NULL 439 * 440 * Initialize all data structure and register base. Caller should make sure 441 * the pointer passed in are non-NULL. This function should be called only 442 * one time during the device probe. 443 * 444 * Context: Any context. 445 * 446 * Return: none 447 */ 448 static void sf_pdma_setup_chans(struct sf_pdma *pdma) 449 { 450 int i; 451 struct sf_pdma_chan *chan; 452 453 INIT_LIST_HEAD(&pdma->dma_dev.channels); 454 455 for (i = 0; i < pdma->n_chans; i++) { 456 chan = &pdma->chans[i]; 457 458 chan->regs.ctrl = 459 SF_PDMA_REG_BASE(i) + PDMA_CTRL; 460 chan->regs.xfer_type = 461 SF_PDMA_REG_BASE(i) + PDMA_XFER_TYPE; 462 chan->regs.xfer_size = 463 SF_PDMA_REG_BASE(i) + PDMA_XFER_SIZE; 464 chan->regs.dst_addr = 465 SF_PDMA_REG_BASE(i) + PDMA_DST_ADDR; 466 chan->regs.src_addr = 467 SF_PDMA_REG_BASE(i) + PDMA_SRC_ADDR; 468 chan->regs.act_type = 469 SF_PDMA_REG_BASE(i) + PDMA_ACT_TYPE; 470 chan->regs.residue = 471 SF_PDMA_REG_BASE(i) + PDMA_REMAINING_BYTE; 472 chan->regs.cur_dst_addr = 473 SF_PDMA_REG_BASE(i) + PDMA_CUR_DST_ADDR; 474 chan->regs.cur_src_addr = 475 SF_PDMA_REG_BASE(i) + PDMA_CUR_SRC_ADDR; 476 477 chan->pdma = pdma; 478 chan->pm_state = RUNNING; 479 chan->slave_id = i; 480 chan->xfer_err = false; 481 spin_lock_init(&chan->lock); 482 483 chan->vchan.desc_free = sf_pdma_free_desc; 484 vchan_init(&chan->vchan, &pdma->dma_dev); 485 486 writel(PDMA_CLEAR_CTRL, chan->regs.ctrl); 487 488 tasklet_setup(&chan->done_tasklet, sf_pdma_donebh_tasklet); 489 tasklet_setup(&chan->err_tasklet, sf_pdma_errbh_tasklet); 490 } 491 } 492 493 static int sf_pdma_probe(struct platform_device *pdev) 494 { 495 struct sf_pdma *pdma; 496 int ret, n_chans; 497 const enum dma_slave_buswidth widths = 498 DMA_SLAVE_BUSWIDTH_1_BYTE | DMA_SLAVE_BUSWIDTH_2_BYTES | 499 DMA_SLAVE_BUSWIDTH_4_BYTES | DMA_SLAVE_BUSWIDTH_8_BYTES | 500 DMA_SLAVE_BUSWIDTH_16_BYTES | DMA_SLAVE_BUSWIDTH_32_BYTES | 501 DMA_SLAVE_BUSWIDTH_64_BYTES; 502 503 ret = of_property_read_u32(pdev->dev.of_node, "dma-channels", &n_chans); 504 if (ret) { 505 /* backwards-compatibility for no dma-channels property */ 506 dev_dbg(&pdev->dev, "set number of channels to default value: 4\n"); 507 n_chans = PDMA_MAX_NR_CH; 508 } else if (n_chans > PDMA_MAX_NR_CH) { 509 dev_err(&pdev->dev, "the number of channels exceeds the maximum\n"); 510 return -EINVAL; 511 } 512 513 pdma = devm_kzalloc(&pdev->dev, struct_size(pdma, chans, n_chans), 514 GFP_KERNEL); 515 if (!pdma) 516 return -ENOMEM; 517 518 pdma->n_chans = n_chans; 519 520 pdma->membase = devm_platform_ioremap_resource(pdev, 0); 521 if (IS_ERR(pdma->membase)) 522 return PTR_ERR(pdma->membase); 523 524 ret = sf_pdma_irq_init(pdev, pdma); 525 if (ret) 526 return ret; 527 528 sf_pdma_setup_chans(pdma); 529 530 pdma->dma_dev.dev = &pdev->dev; 531 532 /* Setup capability */ 533 dma_cap_set(DMA_MEMCPY, pdma->dma_dev.cap_mask); 534 pdma->dma_dev.copy_align = 2; 535 pdma->dma_dev.src_addr_widths = widths; 536 pdma->dma_dev.dst_addr_widths = widths; 537 pdma->dma_dev.directions = BIT(DMA_MEM_TO_MEM); 538 pdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 539 pdma->dma_dev.descriptor_reuse = true; 540 541 /* Setup DMA APIs */ 542 pdma->dma_dev.device_alloc_chan_resources = 543 sf_pdma_alloc_chan_resources; 544 pdma->dma_dev.device_free_chan_resources = 545 sf_pdma_free_chan_resources; 546 pdma->dma_dev.device_tx_status = sf_pdma_tx_status; 547 pdma->dma_dev.device_prep_dma_memcpy = sf_pdma_prep_dma_memcpy; 548 pdma->dma_dev.device_config = sf_pdma_slave_config; 549 pdma->dma_dev.device_terminate_all = sf_pdma_terminate_all; 550 pdma->dma_dev.device_issue_pending = sf_pdma_issue_pending; 551 552 platform_set_drvdata(pdev, pdma); 553 554 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 555 if (ret) 556 dev_warn(&pdev->dev, 557 "Failed to set DMA mask. Fall back to default.\n"); 558 559 ret = dma_async_device_register(&pdma->dma_dev); 560 if (ret) { 561 dev_err(&pdev->dev, 562 "Can't register SiFive Platform DMA. (%d)\n", ret); 563 return ret; 564 } 565 566 return 0; 567 } 568 569 static int sf_pdma_remove(struct platform_device *pdev) 570 { 571 struct sf_pdma *pdma = platform_get_drvdata(pdev); 572 struct sf_pdma_chan *ch; 573 int i; 574 575 for (i = 0; i < pdma->n_chans; i++) { 576 ch = &pdma->chans[i]; 577 578 devm_free_irq(&pdev->dev, ch->txirq, ch); 579 devm_free_irq(&pdev->dev, ch->errirq, ch); 580 list_del(&ch->vchan.chan.device_node); 581 tasklet_kill(&ch->vchan.task); 582 tasklet_kill(&ch->done_tasklet); 583 tasklet_kill(&ch->err_tasklet); 584 } 585 586 dma_async_device_unregister(&pdma->dma_dev); 587 588 return 0; 589 } 590 591 static const struct of_device_id sf_pdma_dt_ids[] = { 592 { .compatible = "sifive,fu540-c000-pdma" }, 593 { .compatible = "sifive,pdma0" }, 594 {}, 595 }; 596 MODULE_DEVICE_TABLE(of, sf_pdma_dt_ids); 597 598 static struct platform_driver sf_pdma_driver = { 599 .probe = sf_pdma_probe, 600 .remove = sf_pdma_remove, 601 .driver = { 602 .name = "sf-pdma", 603 .of_match_table = sf_pdma_dt_ids, 604 }, 605 }; 606 607 static int __init sf_pdma_init(void) 608 { 609 return platform_driver_register(&sf_pdma_driver); 610 } 611 612 static void __exit sf_pdma_exit(void) 613 { 614 platform_driver_unregister(&sf_pdma_driver); 615 } 616 617 /* do early init */ 618 subsys_initcall(sf_pdma_init); 619 module_exit(sf_pdma_exit); 620 621 MODULE_LICENSE("GPL v2"); 622 MODULE_DESCRIPTION("SiFive Platform DMA driver"); 623 MODULE_AUTHOR("Green Wan <green.wan@sifive.com>"); 624