1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * SiFive FU540 Platform DMA driver 4 * Copyright (C) 2019 SiFive 5 * 6 * Based partially on: 7 * - drivers/dma/fsl-edma.c 8 * - drivers/dma/dw-edma/ 9 * - drivers/dma/pxa-dma.c 10 * 11 * See the following sources for further documentation: 12 * - Chapter 12 "Platform DMA Engine (PDMA)" of 13 * SiFive FU540-C000 v1.0 14 * https://static.dev.sifive.com/FU540-C000-v1.0.pdf 15 */ 16 #include <linux/module.h> 17 #include <linux/device.h> 18 #include <linux/kernel.h> 19 #include <linux/platform_device.h> 20 #include <linux/mod_devicetable.h> 21 #include <linux/dma-mapping.h> 22 #include <linux/of.h> 23 #include <linux/slab.h> 24 25 #include "sf-pdma.h" 26 27 #ifndef readq 28 static inline unsigned long long readq(void __iomem *addr) 29 { 30 return readl(addr) | (((unsigned long long)readl(addr + 4)) << 32LL); 31 } 32 #endif 33 34 #ifndef writeq 35 static inline void writeq(unsigned long long v, void __iomem *addr) 36 { 37 writel(lower_32_bits(v), addr); 38 writel(upper_32_bits(v), addr + 4); 39 } 40 #endif 41 42 static inline struct sf_pdma_chan *to_sf_pdma_chan(struct dma_chan *dchan) 43 { 44 return container_of(dchan, struct sf_pdma_chan, vchan.chan); 45 } 46 47 static inline struct sf_pdma_desc *to_sf_pdma_desc(struct virt_dma_desc *vd) 48 { 49 return container_of(vd, struct sf_pdma_desc, vdesc); 50 } 51 52 static struct sf_pdma_desc *sf_pdma_alloc_desc(struct sf_pdma_chan *chan) 53 { 54 struct sf_pdma_desc *desc; 55 unsigned long flags; 56 57 spin_lock_irqsave(&chan->lock, flags); 58 59 if (chan->desc && !chan->desc->in_use) { 60 spin_unlock_irqrestore(&chan->lock, flags); 61 return chan->desc; 62 } 63 64 spin_unlock_irqrestore(&chan->lock, flags); 65 66 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 67 if (!desc) 68 return NULL; 69 70 desc->chan = chan; 71 72 return desc; 73 } 74 75 static void sf_pdma_fill_desc(struct sf_pdma_desc *desc, 76 u64 dst, u64 src, u64 size) 77 { 78 desc->xfer_type = PDMA_FULL_SPEED; 79 desc->xfer_size = size; 80 desc->dst_addr = dst; 81 desc->src_addr = src; 82 } 83 84 static void sf_pdma_disclaim_chan(struct sf_pdma_chan *chan) 85 { 86 struct pdma_regs *regs = &chan->regs; 87 88 writel(PDMA_CLEAR_CTRL, regs->ctrl); 89 } 90 91 static struct dma_async_tx_descriptor * 92 sf_pdma_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dest, dma_addr_t src, 93 size_t len, unsigned long flags) 94 { 95 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 96 struct sf_pdma_desc *desc; 97 98 if (chan && (!len || !dest || !src)) { 99 dev_err(chan->pdma->dma_dev.dev, 100 "Please check dma len, dest, src!\n"); 101 return NULL; 102 } 103 104 desc = sf_pdma_alloc_desc(chan); 105 if (!desc) 106 return NULL; 107 108 desc->in_use = true; 109 desc->dirn = DMA_MEM_TO_MEM; 110 desc->async_tx = vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); 111 112 spin_lock_irqsave(&chan->vchan.lock, flags); 113 chan->desc = desc; 114 sf_pdma_fill_desc(desc, dest, src, len); 115 spin_unlock_irqrestore(&chan->vchan.lock, flags); 116 117 return desc->async_tx; 118 } 119 120 static int sf_pdma_slave_config(struct dma_chan *dchan, 121 struct dma_slave_config *cfg) 122 { 123 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 124 125 memcpy(&chan->cfg, cfg, sizeof(*cfg)); 126 127 return 0; 128 } 129 130 static int sf_pdma_alloc_chan_resources(struct dma_chan *dchan) 131 { 132 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 133 struct pdma_regs *regs = &chan->regs; 134 135 dma_cookie_init(dchan); 136 writel(PDMA_CLAIM_MASK, regs->ctrl); 137 138 return 0; 139 } 140 141 static void sf_pdma_disable_request(struct sf_pdma_chan *chan) 142 { 143 struct pdma_regs *regs = &chan->regs; 144 145 writel(readl(regs->ctrl) & ~PDMA_RUN_MASK, regs->ctrl); 146 } 147 148 static void sf_pdma_free_chan_resources(struct dma_chan *dchan) 149 { 150 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 151 unsigned long flags; 152 LIST_HEAD(head); 153 154 spin_lock_irqsave(&chan->vchan.lock, flags); 155 sf_pdma_disable_request(chan); 156 kfree(chan->desc); 157 chan->desc = NULL; 158 vchan_get_all_descriptors(&chan->vchan, &head); 159 sf_pdma_disclaim_chan(chan); 160 spin_unlock_irqrestore(&chan->vchan.lock, flags); 161 vchan_dma_desc_free_list(&chan->vchan, &head); 162 } 163 164 static size_t sf_pdma_desc_residue(struct sf_pdma_chan *chan, 165 dma_cookie_t cookie) 166 { 167 struct virt_dma_desc *vd = NULL; 168 struct pdma_regs *regs = &chan->regs; 169 unsigned long flags; 170 u64 residue = 0; 171 struct sf_pdma_desc *desc; 172 struct dma_async_tx_descriptor *tx; 173 174 spin_lock_irqsave(&chan->vchan.lock, flags); 175 176 tx = &chan->desc->vdesc.tx; 177 if (cookie == tx->chan->completed_cookie) 178 goto out; 179 180 if (cookie == tx->cookie) { 181 residue = readq(regs->residue); 182 } else { 183 vd = vchan_find_desc(&chan->vchan, cookie); 184 if (!vd) 185 goto out; 186 187 desc = to_sf_pdma_desc(vd); 188 residue = desc->xfer_size; 189 } 190 191 out: 192 spin_unlock_irqrestore(&chan->vchan.lock, flags); 193 return residue; 194 } 195 196 static enum dma_status 197 sf_pdma_tx_status(struct dma_chan *dchan, 198 dma_cookie_t cookie, 199 struct dma_tx_state *txstate) 200 { 201 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 202 enum dma_status status; 203 204 status = dma_cookie_status(dchan, cookie, txstate); 205 206 if (txstate && status != DMA_ERROR) 207 dma_set_residue(txstate, sf_pdma_desc_residue(chan, cookie)); 208 209 return status; 210 } 211 212 static int sf_pdma_terminate_all(struct dma_chan *dchan) 213 { 214 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 215 unsigned long flags; 216 LIST_HEAD(head); 217 218 spin_lock_irqsave(&chan->vchan.lock, flags); 219 sf_pdma_disable_request(chan); 220 kfree(chan->desc); 221 chan->desc = NULL; 222 chan->xfer_err = false; 223 vchan_get_all_descriptors(&chan->vchan, &head); 224 spin_unlock_irqrestore(&chan->vchan.lock, flags); 225 vchan_dma_desc_free_list(&chan->vchan, &head); 226 227 return 0; 228 } 229 230 static void sf_pdma_enable_request(struct sf_pdma_chan *chan) 231 { 232 struct pdma_regs *regs = &chan->regs; 233 u32 v; 234 235 v = PDMA_CLAIM_MASK | 236 PDMA_ENABLE_DONE_INT_MASK | 237 PDMA_ENABLE_ERR_INT_MASK | 238 PDMA_RUN_MASK; 239 240 writel(v, regs->ctrl); 241 } 242 243 static void sf_pdma_xfer_desc(struct sf_pdma_chan *chan) 244 { 245 struct sf_pdma_desc *desc = chan->desc; 246 struct pdma_regs *regs = &chan->regs; 247 248 if (!desc) { 249 dev_err(chan->pdma->dma_dev.dev, "NULL desc.\n"); 250 return; 251 } 252 253 writel(desc->xfer_type, regs->xfer_type); 254 writeq(desc->xfer_size, regs->xfer_size); 255 writeq(desc->dst_addr, regs->dst_addr); 256 writeq(desc->src_addr, regs->src_addr); 257 258 chan->desc = desc; 259 chan->status = DMA_IN_PROGRESS; 260 sf_pdma_enable_request(chan); 261 } 262 263 static void sf_pdma_issue_pending(struct dma_chan *dchan) 264 { 265 struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan); 266 unsigned long flags; 267 268 spin_lock_irqsave(&chan->vchan.lock, flags); 269 270 if (vchan_issue_pending(&chan->vchan) && chan->desc) 271 sf_pdma_xfer_desc(chan); 272 273 spin_unlock_irqrestore(&chan->vchan.lock, flags); 274 } 275 276 static void sf_pdma_free_desc(struct virt_dma_desc *vdesc) 277 { 278 struct sf_pdma_desc *desc; 279 280 desc = to_sf_pdma_desc(vdesc); 281 desc->in_use = false; 282 } 283 284 static void sf_pdma_donebh_tasklet(unsigned long arg) 285 { 286 struct sf_pdma_chan *chan = (struct sf_pdma_chan *)arg; 287 struct sf_pdma_desc *desc = chan->desc; 288 unsigned long flags; 289 290 spin_lock_irqsave(&chan->lock, flags); 291 if (chan->xfer_err) { 292 chan->retries = MAX_RETRY; 293 chan->status = DMA_COMPLETE; 294 chan->xfer_err = false; 295 } 296 spin_unlock_irqrestore(&chan->lock, flags); 297 298 dmaengine_desc_get_callback_invoke(desc->async_tx, NULL); 299 } 300 301 static void sf_pdma_errbh_tasklet(unsigned long arg) 302 { 303 struct sf_pdma_chan *chan = (struct sf_pdma_chan *)arg; 304 struct sf_pdma_desc *desc = chan->desc; 305 unsigned long flags; 306 307 spin_lock_irqsave(&chan->lock, flags); 308 if (chan->retries <= 0) { 309 /* fail to recover */ 310 spin_unlock_irqrestore(&chan->lock, flags); 311 dmaengine_desc_get_callback_invoke(desc->async_tx, NULL); 312 } else { 313 /* retry */ 314 chan->retries--; 315 chan->xfer_err = true; 316 chan->status = DMA_ERROR; 317 318 sf_pdma_enable_request(chan); 319 spin_unlock_irqrestore(&chan->lock, flags); 320 } 321 } 322 323 static irqreturn_t sf_pdma_done_isr(int irq, void *dev_id) 324 { 325 struct sf_pdma_chan *chan = dev_id; 326 struct pdma_regs *regs = &chan->regs; 327 unsigned long flags; 328 u64 residue; 329 330 spin_lock_irqsave(&chan->vchan.lock, flags); 331 writel((readl(regs->ctrl)) & ~PDMA_DONE_STATUS_MASK, regs->ctrl); 332 residue = readq(regs->residue); 333 334 if (!residue) { 335 list_del(&chan->desc->vdesc.node); 336 vchan_cookie_complete(&chan->desc->vdesc); 337 } else { 338 /* submit next trascatioin if possible */ 339 struct sf_pdma_desc *desc = chan->desc; 340 341 desc->src_addr += desc->xfer_size - residue; 342 desc->dst_addr += desc->xfer_size - residue; 343 desc->xfer_size = residue; 344 345 sf_pdma_xfer_desc(chan); 346 } 347 348 spin_unlock_irqrestore(&chan->vchan.lock, flags); 349 350 tasklet_hi_schedule(&chan->done_tasklet); 351 352 return IRQ_HANDLED; 353 } 354 355 static irqreturn_t sf_pdma_err_isr(int irq, void *dev_id) 356 { 357 struct sf_pdma_chan *chan = dev_id; 358 struct pdma_regs *regs = &chan->regs; 359 unsigned long flags; 360 361 spin_lock_irqsave(&chan->lock, flags); 362 writel((readl(regs->ctrl)) & ~PDMA_ERR_STATUS_MASK, regs->ctrl); 363 spin_unlock_irqrestore(&chan->lock, flags); 364 365 tasklet_schedule(&chan->err_tasklet); 366 367 return IRQ_HANDLED; 368 } 369 370 /** 371 * sf_pdma_irq_init() - Init PDMA IRQ Handlers 372 * @pdev: pointer of platform_device 373 * @pdma: pointer of PDMA engine. Caller should check NULL 374 * 375 * Initialize DONE and ERROR interrupt handler for 4 channels. Caller should 376 * make sure the pointer passed in are non-NULL. This function should be called 377 * only one time during the device probe. 378 * 379 * Context: Any context. 380 * 381 * Return: 382 * * 0 - OK to init all IRQ handlers 383 * * -EINVAL - Fail to request IRQ 384 */ 385 static int sf_pdma_irq_init(struct platform_device *pdev, struct sf_pdma *pdma) 386 { 387 int irq, r, i; 388 struct sf_pdma_chan *chan; 389 390 for (i = 0; i < pdma->n_chans; i++) { 391 chan = &pdma->chans[i]; 392 393 irq = platform_get_irq(pdev, i * 2); 394 if (irq < 0) { 395 dev_err(&pdev->dev, "ch(%d) Can't get done irq.\n", i); 396 return -EINVAL; 397 } 398 399 r = devm_request_irq(&pdev->dev, irq, sf_pdma_done_isr, 0, 400 dev_name(&pdev->dev), (void *)chan); 401 if (r) { 402 dev_err(&pdev->dev, "Fail to attach done ISR: %d\n", r); 403 return -EINVAL; 404 } 405 406 chan->txirq = irq; 407 408 irq = platform_get_irq(pdev, (i * 2) + 1); 409 if (irq < 0) { 410 dev_err(&pdev->dev, "ch(%d) Can't get err irq.\n", i); 411 return -EINVAL; 412 } 413 414 r = devm_request_irq(&pdev->dev, irq, sf_pdma_err_isr, 0, 415 dev_name(&pdev->dev), (void *)chan); 416 if (r) { 417 dev_err(&pdev->dev, "Fail to attach err ISR: %d\n", r); 418 return -EINVAL; 419 } 420 421 chan->errirq = irq; 422 } 423 424 return 0; 425 } 426 427 /** 428 * sf_pdma_setup_chans() - Init settings of each channel 429 * @pdma: pointer of PDMA engine. Caller should check NULL 430 * 431 * Initialize all data structure and register base. Caller should make sure 432 * the pointer passed in are non-NULL. This function should be called only 433 * one time during the device probe. 434 * 435 * Context: Any context. 436 * 437 * Return: none 438 */ 439 static void sf_pdma_setup_chans(struct sf_pdma *pdma) 440 { 441 int i; 442 struct sf_pdma_chan *chan; 443 444 INIT_LIST_HEAD(&pdma->dma_dev.channels); 445 446 for (i = 0; i < pdma->n_chans; i++) { 447 chan = &pdma->chans[i]; 448 449 chan->regs.ctrl = 450 SF_PDMA_REG_BASE(i) + PDMA_CTRL; 451 chan->regs.xfer_type = 452 SF_PDMA_REG_BASE(i) + PDMA_XFER_TYPE; 453 chan->regs.xfer_size = 454 SF_PDMA_REG_BASE(i) + PDMA_XFER_SIZE; 455 chan->regs.dst_addr = 456 SF_PDMA_REG_BASE(i) + PDMA_DST_ADDR; 457 chan->regs.src_addr = 458 SF_PDMA_REG_BASE(i) + PDMA_SRC_ADDR; 459 chan->regs.act_type = 460 SF_PDMA_REG_BASE(i) + PDMA_ACT_TYPE; 461 chan->regs.residue = 462 SF_PDMA_REG_BASE(i) + PDMA_REMAINING_BYTE; 463 chan->regs.cur_dst_addr = 464 SF_PDMA_REG_BASE(i) + PDMA_CUR_DST_ADDR; 465 chan->regs.cur_src_addr = 466 SF_PDMA_REG_BASE(i) + PDMA_CUR_SRC_ADDR; 467 468 chan->pdma = pdma; 469 chan->pm_state = RUNNING; 470 chan->slave_id = i; 471 chan->xfer_err = false; 472 spin_lock_init(&chan->lock); 473 474 chan->vchan.desc_free = sf_pdma_free_desc; 475 vchan_init(&chan->vchan, &pdma->dma_dev); 476 477 writel(PDMA_CLEAR_CTRL, chan->regs.ctrl); 478 479 tasklet_init(&chan->done_tasklet, 480 sf_pdma_donebh_tasklet, (unsigned long)chan); 481 tasklet_init(&chan->err_tasklet, 482 sf_pdma_errbh_tasklet, (unsigned long)chan); 483 } 484 } 485 486 static int sf_pdma_probe(struct platform_device *pdev) 487 { 488 struct sf_pdma *pdma; 489 struct sf_pdma_chan *chan; 490 struct resource *res; 491 int len, chans; 492 int ret; 493 const enum dma_slave_buswidth widths = 494 DMA_SLAVE_BUSWIDTH_1_BYTE | DMA_SLAVE_BUSWIDTH_2_BYTES | 495 DMA_SLAVE_BUSWIDTH_4_BYTES | DMA_SLAVE_BUSWIDTH_8_BYTES | 496 DMA_SLAVE_BUSWIDTH_16_BYTES | DMA_SLAVE_BUSWIDTH_32_BYTES | 497 DMA_SLAVE_BUSWIDTH_64_BYTES; 498 499 chans = PDMA_NR_CH; 500 len = sizeof(*pdma) + sizeof(*chan) * chans; 501 pdma = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); 502 if (!pdma) 503 return -ENOMEM; 504 505 pdma->n_chans = chans; 506 507 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 508 pdma->membase = devm_ioremap_resource(&pdev->dev, res); 509 if (IS_ERR(pdma->membase)) 510 return PTR_ERR(pdma->membase); 511 512 ret = sf_pdma_irq_init(pdev, pdma); 513 if (ret) 514 return ret; 515 516 sf_pdma_setup_chans(pdma); 517 518 pdma->dma_dev.dev = &pdev->dev; 519 520 /* Setup capability */ 521 dma_cap_set(DMA_MEMCPY, pdma->dma_dev.cap_mask); 522 pdma->dma_dev.copy_align = 2; 523 pdma->dma_dev.src_addr_widths = widths; 524 pdma->dma_dev.dst_addr_widths = widths; 525 pdma->dma_dev.directions = BIT(DMA_MEM_TO_MEM); 526 pdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 527 pdma->dma_dev.descriptor_reuse = true; 528 529 /* Setup DMA APIs */ 530 pdma->dma_dev.device_alloc_chan_resources = 531 sf_pdma_alloc_chan_resources; 532 pdma->dma_dev.device_free_chan_resources = 533 sf_pdma_free_chan_resources; 534 pdma->dma_dev.device_tx_status = sf_pdma_tx_status; 535 pdma->dma_dev.device_prep_dma_memcpy = sf_pdma_prep_dma_memcpy; 536 pdma->dma_dev.device_config = sf_pdma_slave_config; 537 pdma->dma_dev.device_terminate_all = sf_pdma_terminate_all; 538 pdma->dma_dev.device_issue_pending = sf_pdma_issue_pending; 539 540 platform_set_drvdata(pdev, pdma); 541 542 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 543 if (ret) 544 dev_warn(&pdev->dev, 545 "Failed to set DMA mask. Fall back to default.\n"); 546 547 ret = dma_async_device_register(&pdma->dma_dev); 548 if (ret) { 549 dev_err(&pdev->dev, 550 "Can't register SiFive Platform DMA. (%d)\n", ret); 551 return ret; 552 } 553 554 return 0; 555 } 556 557 static int sf_pdma_remove(struct platform_device *pdev) 558 { 559 struct sf_pdma *pdma = platform_get_drvdata(pdev); 560 struct sf_pdma_chan *ch; 561 int i; 562 563 for (i = 0; i < PDMA_NR_CH; i++) { 564 ch = &pdma->chans[i]; 565 566 devm_free_irq(&pdev->dev, ch->txirq, ch); 567 devm_free_irq(&pdev->dev, ch->errirq, ch); 568 list_del(&ch->vchan.chan.device_node); 569 tasklet_kill(&ch->vchan.task); 570 tasklet_kill(&ch->done_tasklet); 571 tasklet_kill(&ch->err_tasklet); 572 } 573 574 dma_async_device_unregister(&pdma->dma_dev); 575 576 return 0; 577 } 578 579 static const struct of_device_id sf_pdma_dt_ids[] = { 580 { .compatible = "sifive,fu540-c000-pdma" }, 581 {}, 582 }; 583 MODULE_DEVICE_TABLE(of, sf_pdma_dt_ids); 584 585 static struct platform_driver sf_pdma_driver = { 586 .probe = sf_pdma_probe, 587 .remove = sf_pdma_remove, 588 .driver = { 589 .name = "sf-pdma", 590 .of_match_table = of_match_ptr(sf_pdma_dt_ids), 591 }, 592 }; 593 594 static int __init sf_pdma_init(void) 595 { 596 return platform_driver_register(&sf_pdma_driver); 597 } 598 599 static void __exit sf_pdma_exit(void) 600 { 601 platform_driver_unregister(&sf_pdma_driver); 602 } 603 604 /* do early init */ 605 subsys_initcall(sf_pdma_init); 606 module_exit(sf_pdma_exit); 607 608 MODULE_LICENSE("GPL v2"); 609 MODULE_DESCRIPTION("SiFive Platform DMA driver"); 610 MODULE_AUTHOR("Green Wan <green.wan@sifive.com>"); 611