1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Renesas RZ/G2L DMA Controller Driver 4 * 5 * Based on imx-dma.c 6 * 7 * Copyright (C) 2021 Renesas Electronics Corp. 8 * Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de> 9 * Copyright 2012 Javier Martin, Vista Silicon <javier.martin@vista-silicon.com> 10 */ 11 12 #include <linux/dma-mapping.h> 13 #include <linux/dmaengine.h> 14 #include <linux/interrupt.h> 15 #include <linux/iopoll.h> 16 #include <linux/list.h> 17 #include <linux/module.h> 18 #include <linux/of.h> 19 #include <linux/of_dma.h> 20 #include <linux/of_platform.h> 21 #include <linux/platform_device.h> 22 #include <linux/pm_runtime.h> 23 #include <linux/reset.h> 24 #include <linux/slab.h> 25 #include <linux/spinlock.h> 26 27 #include "../dmaengine.h" 28 #include "../virt-dma.h" 29 30 enum rz_dmac_prep_type { 31 RZ_DMAC_DESC_MEMCPY, 32 RZ_DMAC_DESC_SLAVE_SG, 33 }; 34 35 struct rz_lmdesc { 36 u32 header; 37 u32 sa; 38 u32 da; 39 u32 tb; 40 u32 chcfg; 41 u32 chitvl; 42 u32 chext; 43 u32 nxla; 44 }; 45 46 struct rz_dmac_desc { 47 struct virt_dma_desc vd; 48 dma_addr_t src; 49 dma_addr_t dest; 50 size_t len; 51 struct list_head node; 52 enum dma_transfer_direction direction; 53 enum rz_dmac_prep_type type; 54 /* For slave sg */ 55 struct scatterlist *sg; 56 unsigned int sgcount; 57 }; 58 59 #define to_rz_dmac_desc(d) container_of(d, struct rz_dmac_desc, vd) 60 61 struct rz_dmac_chan { 62 struct virt_dma_chan vc; 63 void __iomem *ch_base; 64 void __iomem *ch_cmn_base; 65 unsigned int index; 66 int irq; 67 struct rz_dmac_desc *desc; 68 int descs_allocated; 69 70 dma_addr_t src_per_address; 71 dma_addr_t dst_per_address; 72 73 u32 chcfg; 74 u32 chctrl; 75 int mid_rid; 76 77 struct list_head ld_free; 78 struct list_head ld_queue; 79 struct list_head ld_active; 80 81 struct { 82 struct rz_lmdesc *base; 83 struct rz_lmdesc *head; 84 struct rz_lmdesc *tail; 85 dma_addr_t base_dma; 86 } lmdesc; 87 }; 88 89 #define to_rz_dmac_chan(c) container_of(c, struct rz_dmac_chan, vc.chan) 90 91 struct rz_dmac { 92 struct dma_device engine; 93 struct device *dev; 94 struct reset_control *rstc; 95 void __iomem *base; 96 void __iomem *ext_base; 97 98 unsigned int n_channels; 99 struct rz_dmac_chan *channels; 100 101 DECLARE_BITMAP(modules, 1024); 102 }; 103 104 #define to_rz_dmac(d) container_of(d, struct rz_dmac, engine) 105 106 /* 107 * ----------------------------------------------------------------------------- 108 * Registers 109 */ 110 111 #define CHSTAT 0x0024 112 #define CHCTRL 0x0028 113 #define CHCFG 0x002c 114 #define NXLA 0x0038 115 116 #define DCTRL 0x0000 117 118 #define EACH_CHANNEL_OFFSET 0x0040 119 #define CHANNEL_0_7_OFFSET 0x0000 120 #define CHANNEL_0_7_COMMON_BASE 0x0300 121 #define CHANNEL_8_15_OFFSET 0x0400 122 #define CHANNEL_8_15_COMMON_BASE 0x0700 123 124 #define CHSTAT_ER BIT(4) 125 #define CHSTAT_EN BIT(0) 126 127 #define CHCTRL_CLRINTMSK BIT(17) 128 #define CHCTRL_CLRSUS BIT(9) 129 #define CHCTRL_CLRTC BIT(6) 130 #define CHCTRL_CLREND BIT(5) 131 #define CHCTRL_CLRRQ BIT(4) 132 #define CHCTRL_SWRST BIT(3) 133 #define CHCTRL_STG BIT(2) 134 #define CHCTRL_CLREN BIT(1) 135 #define CHCTRL_SETEN BIT(0) 136 #define CHCTRL_DEFAULT (CHCTRL_CLRINTMSK | CHCTRL_CLRSUS | \ 137 CHCTRL_CLRTC | CHCTRL_CLREND | \ 138 CHCTRL_CLRRQ | CHCTRL_SWRST | \ 139 CHCTRL_CLREN) 140 141 #define CHCFG_DMS BIT(31) 142 #define CHCFG_DEM BIT(24) 143 #define CHCFG_DAD BIT(21) 144 #define CHCFG_SAD BIT(20) 145 #define CHCFG_REQD BIT(3) 146 #define CHCFG_SEL(bits) ((bits) & 0x07) 147 #define CHCFG_MEM_COPY (0x80400008) 148 #define CHCFG_FILL_DDS(a) (((a) << 16) & GENMASK(19, 16)) 149 #define CHCFG_FILL_SDS(a) (((a) << 12) & GENMASK(15, 12)) 150 #define CHCFG_FILL_TM(a) (((a) & BIT(5)) << 22) 151 #define CHCFG_FILL_AM(a) (((a) & GENMASK(4, 2)) << 6) 152 #define CHCFG_FILL_LVL(a) (((a) & BIT(1)) << 5) 153 #define CHCFG_FILL_HIEN(a) (((a) & BIT(0)) << 5) 154 155 #define MID_RID_MASK GENMASK(9, 0) 156 #define CHCFG_MASK GENMASK(15, 10) 157 #define CHCFG_DS_INVALID 0xFF 158 #define DCTRL_LVINT BIT(1) 159 #define DCTRL_PR BIT(0) 160 #define DCTRL_DEFAULT (DCTRL_LVINT | DCTRL_PR) 161 162 /* LINK MODE DESCRIPTOR */ 163 #define HEADER_LV BIT(0) 164 165 #define RZ_DMAC_MAX_CHAN_DESCRIPTORS 16 166 #define RZ_DMAC_MAX_CHANNELS 16 167 #define DMAC_NR_LMDESC 64 168 169 /* 170 * ----------------------------------------------------------------------------- 171 * Device access 172 */ 173 174 static void rz_dmac_writel(struct rz_dmac *dmac, unsigned int val, 175 unsigned int offset) 176 { 177 writel(val, dmac->base + offset); 178 } 179 180 static void rz_dmac_ext_writel(struct rz_dmac *dmac, unsigned int val, 181 unsigned int offset) 182 { 183 writel(val, dmac->ext_base + offset); 184 } 185 186 static u32 rz_dmac_ext_readl(struct rz_dmac *dmac, unsigned int offset) 187 { 188 return readl(dmac->ext_base + offset); 189 } 190 191 static void rz_dmac_ch_writel(struct rz_dmac_chan *channel, unsigned int val, 192 unsigned int offset, int which) 193 { 194 if (which) 195 writel(val, channel->ch_base + offset); 196 else 197 writel(val, channel->ch_cmn_base + offset); 198 } 199 200 static u32 rz_dmac_ch_readl(struct rz_dmac_chan *channel, 201 unsigned int offset, int which) 202 { 203 if (which) 204 return readl(channel->ch_base + offset); 205 else 206 return readl(channel->ch_cmn_base + offset); 207 } 208 209 /* 210 * ----------------------------------------------------------------------------- 211 * Initialization 212 */ 213 214 static void rz_lmdesc_setup(struct rz_dmac_chan *channel, 215 struct rz_lmdesc *lmdesc) 216 { 217 u32 nxla; 218 219 channel->lmdesc.base = lmdesc; 220 channel->lmdesc.head = lmdesc; 221 channel->lmdesc.tail = lmdesc; 222 nxla = channel->lmdesc.base_dma; 223 while (lmdesc < (channel->lmdesc.base + (DMAC_NR_LMDESC - 1))) { 224 lmdesc->header = 0; 225 nxla += sizeof(*lmdesc); 226 lmdesc->nxla = nxla; 227 lmdesc++; 228 } 229 230 lmdesc->header = 0; 231 lmdesc->nxla = channel->lmdesc.base_dma; 232 } 233 234 /* 235 * ----------------------------------------------------------------------------- 236 * Descriptors preparation 237 */ 238 239 static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel) 240 { 241 struct rz_lmdesc *lmdesc = channel->lmdesc.head; 242 243 while (!(lmdesc->header & HEADER_LV)) { 244 lmdesc->header = 0; 245 lmdesc++; 246 if (lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC)) 247 lmdesc = channel->lmdesc.base; 248 } 249 channel->lmdesc.head = lmdesc; 250 } 251 252 static void rz_dmac_enable_hw(struct rz_dmac_chan *channel) 253 { 254 struct dma_chan *chan = &channel->vc.chan; 255 struct rz_dmac *dmac = to_rz_dmac(chan->device); 256 unsigned long flags; 257 u32 nxla; 258 u32 chctrl; 259 u32 chstat; 260 261 dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index); 262 263 local_irq_save(flags); 264 265 rz_dmac_lmdesc_recycle(channel); 266 267 nxla = channel->lmdesc.base_dma + 268 (sizeof(struct rz_lmdesc) * (channel->lmdesc.head - 269 channel->lmdesc.base)); 270 271 chstat = rz_dmac_ch_readl(channel, CHSTAT, 1); 272 if (!(chstat & CHSTAT_EN)) { 273 chctrl = (channel->chctrl | CHCTRL_SETEN); 274 rz_dmac_ch_writel(channel, nxla, NXLA, 1); 275 rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1); 276 rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1); 277 rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1); 278 } 279 280 local_irq_restore(flags); 281 } 282 283 static void rz_dmac_disable_hw(struct rz_dmac_chan *channel) 284 { 285 struct dma_chan *chan = &channel->vc.chan; 286 struct rz_dmac *dmac = to_rz_dmac(chan->device); 287 unsigned long flags; 288 289 dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index); 290 291 local_irq_save(flags); 292 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); 293 local_irq_restore(flags); 294 } 295 296 static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars) 297 { 298 u32 dmars_offset = (nr / 2) * 4; 299 u32 shift = (nr % 2) * 16; 300 u32 dmars32; 301 302 dmars32 = rz_dmac_ext_readl(dmac, dmars_offset); 303 dmars32 &= ~(0xffff << shift); 304 dmars32 |= dmars << shift; 305 306 rz_dmac_ext_writel(dmac, dmars32, dmars_offset); 307 } 308 309 static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel) 310 { 311 struct dma_chan *chan = &channel->vc.chan; 312 struct rz_dmac *dmac = to_rz_dmac(chan->device); 313 struct rz_lmdesc *lmdesc = channel->lmdesc.tail; 314 struct rz_dmac_desc *d = channel->desc; 315 u32 chcfg = CHCFG_MEM_COPY; 316 317 /* prepare descriptor */ 318 lmdesc->sa = d->src; 319 lmdesc->da = d->dest; 320 lmdesc->tb = d->len; 321 lmdesc->chcfg = chcfg; 322 lmdesc->chitvl = 0; 323 lmdesc->chext = 0; 324 lmdesc->header = HEADER_LV; 325 326 rz_dmac_set_dmars_register(dmac, channel->index, 0); 327 328 channel->chcfg = chcfg; 329 channel->chctrl = CHCTRL_STG | CHCTRL_SETEN; 330 } 331 332 static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel) 333 { 334 struct dma_chan *chan = &channel->vc.chan; 335 struct rz_dmac *dmac = to_rz_dmac(chan->device); 336 struct rz_dmac_desc *d = channel->desc; 337 struct scatterlist *sg, *sgl = d->sg; 338 struct rz_lmdesc *lmdesc; 339 unsigned int i, sg_len = d->sgcount; 340 341 channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DEM | CHCFG_DMS; 342 343 if (d->direction == DMA_DEV_TO_MEM) { 344 channel->chcfg |= CHCFG_SAD; 345 channel->chcfg &= ~CHCFG_REQD; 346 } else { 347 channel->chcfg |= CHCFG_DAD | CHCFG_REQD; 348 } 349 350 lmdesc = channel->lmdesc.tail; 351 352 for (i = 0, sg = sgl; i < sg_len; i++, sg = sg_next(sg)) { 353 if (d->direction == DMA_DEV_TO_MEM) { 354 lmdesc->sa = channel->src_per_address; 355 lmdesc->da = sg_dma_address(sg); 356 } else { 357 lmdesc->sa = sg_dma_address(sg); 358 lmdesc->da = channel->dst_per_address; 359 } 360 361 lmdesc->tb = sg_dma_len(sg); 362 lmdesc->chitvl = 0; 363 lmdesc->chext = 0; 364 if (i == (sg_len - 1)) { 365 lmdesc->chcfg = (channel->chcfg & ~CHCFG_DEM); 366 lmdesc->header = HEADER_LV; 367 } else { 368 lmdesc->chcfg = channel->chcfg; 369 lmdesc->header = HEADER_LV; 370 } 371 if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC)) 372 lmdesc = channel->lmdesc.base; 373 } 374 375 channel->lmdesc.tail = lmdesc; 376 377 rz_dmac_set_dmars_register(dmac, channel->index, channel->mid_rid); 378 channel->chctrl = CHCTRL_SETEN; 379 } 380 381 static int rz_dmac_xfer_desc(struct rz_dmac_chan *chan) 382 { 383 struct rz_dmac_desc *d = chan->desc; 384 struct virt_dma_desc *vd; 385 386 vd = vchan_next_desc(&chan->vc); 387 if (!vd) 388 return 0; 389 390 list_del(&vd->node); 391 392 switch (d->type) { 393 case RZ_DMAC_DESC_MEMCPY: 394 rz_dmac_prepare_desc_for_memcpy(chan); 395 break; 396 397 case RZ_DMAC_DESC_SLAVE_SG: 398 rz_dmac_prepare_descs_for_slave_sg(chan); 399 break; 400 401 default: 402 return -EINVAL; 403 } 404 405 rz_dmac_enable_hw(chan); 406 407 return 0; 408 } 409 410 /* 411 * ----------------------------------------------------------------------------- 412 * DMA engine operations 413 */ 414 415 static int rz_dmac_alloc_chan_resources(struct dma_chan *chan) 416 { 417 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 418 419 while (channel->descs_allocated < RZ_DMAC_MAX_CHAN_DESCRIPTORS) { 420 struct rz_dmac_desc *desc; 421 422 desc = kzalloc(sizeof(*desc), GFP_KERNEL); 423 if (!desc) 424 break; 425 426 list_add_tail(&desc->node, &channel->ld_free); 427 channel->descs_allocated++; 428 } 429 430 if (!channel->descs_allocated) 431 return -ENOMEM; 432 433 return channel->descs_allocated; 434 } 435 436 static void rz_dmac_free_chan_resources(struct dma_chan *chan) 437 { 438 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 439 struct rz_dmac *dmac = to_rz_dmac(chan->device); 440 struct rz_lmdesc *lmdesc = channel->lmdesc.base; 441 struct rz_dmac_desc *desc, *_desc; 442 unsigned long flags; 443 unsigned int i; 444 445 spin_lock_irqsave(&channel->vc.lock, flags); 446 447 for (i = 0; i < DMAC_NR_LMDESC; i++) 448 lmdesc[i].header = 0; 449 450 rz_dmac_disable_hw(channel); 451 list_splice_tail_init(&channel->ld_active, &channel->ld_free); 452 list_splice_tail_init(&channel->ld_queue, &channel->ld_free); 453 454 if (channel->mid_rid >= 0) { 455 clear_bit(channel->mid_rid, dmac->modules); 456 channel->mid_rid = -EINVAL; 457 } 458 459 spin_unlock_irqrestore(&channel->vc.lock, flags); 460 461 list_for_each_entry_safe(desc, _desc, &channel->ld_free, node) { 462 kfree(desc); 463 channel->descs_allocated--; 464 } 465 466 INIT_LIST_HEAD(&channel->ld_free); 467 vchan_free_chan_resources(&channel->vc); 468 } 469 470 static struct dma_async_tx_descriptor * 471 rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 472 size_t len, unsigned long flags) 473 { 474 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 475 struct rz_dmac *dmac = to_rz_dmac(chan->device); 476 struct rz_dmac_desc *desc; 477 478 dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n", 479 __func__, channel->index, &src, &dest, len); 480 481 if (list_empty(&channel->ld_free)) 482 return NULL; 483 484 desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); 485 486 desc->type = RZ_DMAC_DESC_MEMCPY; 487 desc->src = src; 488 desc->dest = dest; 489 desc->len = len; 490 desc->direction = DMA_MEM_TO_MEM; 491 492 list_move_tail(channel->ld_free.next, &channel->ld_queue); 493 return vchan_tx_prep(&channel->vc, &desc->vd, flags); 494 } 495 496 static struct dma_async_tx_descriptor * 497 rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 498 unsigned int sg_len, 499 enum dma_transfer_direction direction, 500 unsigned long flags, void *context) 501 { 502 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 503 struct rz_dmac_desc *desc; 504 struct scatterlist *sg; 505 int dma_length = 0; 506 int i = 0; 507 508 if (list_empty(&channel->ld_free)) 509 return NULL; 510 511 desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); 512 513 for_each_sg(sgl, sg, sg_len, i) { 514 dma_length += sg_dma_len(sg); 515 } 516 517 desc->type = RZ_DMAC_DESC_SLAVE_SG; 518 desc->sg = sgl; 519 desc->sgcount = sg_len; 520 desc->len = dma_length; 521 desc->direction = direction; 522 523 if (direction == DMA_DEV_TO_MEM) 524 desc->src = channel->src_per_address; 525 else 526 desc->dest = channel->dst_per_address; 527 528 list_move_tail(channel->ld_free.next, &channel->ld_queue); 529 return vchan_tx_prep(&channel->vc, &desc->vd, flags); 530 } 531 532 static int rz_dmac_terminate_all(struct dma_chan *chan) 533 { 534 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 535 unsigned long flags; 536 LIST_HEAD(head); 537 538 rz_dmac_disable_hw(channel); 539 spin_lock_irqsave(&channel->vc.lock, flags); 540 list_splice_tail_init(&channel->ld_active, &channel->ld_free); 541 list_splice_tail_init(&channel->ld_queue, &channel->ld_free); 542 spin_unlock_irqrestore(&channel->vc.lock, flags); 543 vchan_get_all_descriptors(&channel->vc, &head); 544 vchan_dma_desc_free_list(&channel->vc, &head); 545 546 return 0; 547 } 548 549 static void rz_dmac_issue_pending(struct dma_chan *chan) 550 { 551 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 552 struct rz_dmac *dmac = to_rz_dmac(chan->device); 553 struct rz_dmac_desc *desc; 554 unsigned long flags; 555 556 spin_lock_irqsave(&channel->vc.lock, flags); 557 558 if (!list_empty(&channel->ld_queue)) { 559 desc = list_first_entry(&channel->ld_queue, 560 struct rz_dmac_desc, node); 561 channel->desc = desc; 562 if (vchan_issue_pending(&channel->vc)) { 563 if (rz_dmac_xfer_desc(channel) < 0) 564 dev_warn(dmac->dev, "ch: %d couldn't issue DMA xfer\n", 565 channel->index); 566 else 567 list_move_tail(channel->ld_queue.next, 568 &channel->ld_active); 569 } 570 } 571 572 spin_unlock_irqrestore(&channel->vc.lock, flags); 573 } 574 575 static u8 rz_dmac_ds_to_val_mapping(enum dma_slave_buswidth ds) 576 { 577 u8 i; 578 static const enum dma_slave_buswidth ds_lut[] = { 579 DMA_SLAVE_BUSWIDTH_1_BYTE, 580 DMA_SLAVE_BUSWIDTH_2_BYTES, 581 DMA_SLAVE_BUSWIDTH_4_BYTES, 582 DMA_SLAVE_BUSWIDTH_8_BYTES, 583 DMA_SLAVE_BUSWIDTH_16_BYTES, 584 DMA_SLAVE_BUSWIDTH_32_BYTES, 585 DMA_SLAVE_BUSWIDTH_64_BYTES, 586 DMA_SLAVE_BUSWIDTH_128_BYTES, 587 }; 588 589 for (i = 0; i < ARRAY_SIZE(ds_lut); i++) { 590 if (ds_lut[i] == ds) 591 return i; 592 } 593 594 return CHCFG_DS_INVALID; 595 } 596 597 static int rz_dmac_config(struct dma_chan *chan, 598 struct dma_slave_config *config) 599 { 600 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 601 u32 val; 602 603 channel->src_per_address = config->src_addr; 604 channel->dst_per_address = config->dst_addr; 605 606 val = rz_dmac_ds_to_val_mapping(config->dst_addr_width); 607 if (val == CHCFG_DS_INVALID) 608 return -EINVAL; 609 610 channel->chcfg |= CHCFG_FILL_DDS(val); 611 612 val = rz_dmac_ds_to_val_mapping(config->src_addr_width); 613 if (val == CHCFG_DS_INVALID) 614 return -EINVAL; 615 616 channel->chcfg |= CHCFG_FILL_SDS(val); 617 618 return 0; 619 } 620 621 static void rz_dmac_virt_desc_free(struct virt_dma_desc *vd) 622 { 623 /* 624 * Place holder 625 * Descriptor allocation is done during alloc_chan_resources and 626 * get freed during free_chan_resources. 627 * list is used to manage the descriptors and avoid any memory 628 * allocation/free during DMA read/write. 629 */ 630 } 631 632 static void rz_dmac_device_synchronize(struct dma_chan *chan) 633 { 634 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 635 struct rz_dmac *dmac = to_rz_dmac(chan->device); 636 u32 chstat; 637 int ret; 638 639 ret = read_poll_timeout(rz_dmac_ch_readl, chstat, !(chstat & CHSTAT_EN), 640 100, 100000, false, channel, CHSTAT, 1); 641 if (ret < 0) 642 dev_warn(dmac->dev, "DMA Timeout"); 643 644 rz_dmac_set_dmars_register(dmac, channel->index, 0); 645 } 646 647 /* 648 * ----------------------------------------------------------------------------- 649 * IRQ handling 650 */ 651 652 static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel) 653 { 654 struct dma_chan *chan = &channel->vc.chan; 655 struct rz_dmac *dmac = to_rz_dmac(chan->device); 656 u32 chstat, chctrl; 657 658 chstat = rz_dmac_ch_readl(channel, CHSTAT, 1); 659 if (chstat & CHSTAT_ER) { 660 dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n", 661 channel->index, chstat); 662 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); 663 goto done; 664 } 665 666 chctrl = rz_dmac_ch_readl(channel, CHCTRL, 1); 667 rz_dmac_ch_writel(channel, chctrl | CHCTRL_CLREND, CHCTRL, 1); 668 done: 669 return; 670 } 671 672 static irqreturn_t rz_dmac_irq_handler(int irq, void *dev_id) 673 { 674 struct rz_dmac_chan *channel = dev_id; 675 676 if (channel) { 677 rz_dmac_irq_handle_channel(channel); 678 return IRQ_WAKE_THREAD; 679 } 680 /* handle DMAERR irq */ 681 return IRQ_HANDLED; 682 } 683 684 static irqreturn_t rz_dmac_irq_handler_thread(int irq, void *dev_id) 685 { 686 struct rz_dmac_chan *channel = dev_id; 687 struct rz_dmac_desc *desc = NULL; 688 unsigned long flags; 689 690 spin_lock_irqsave(&channel->vc.lock, flags); 691 692 if (list_empty(&channel->ld_active)) { 693 /* Someone might have called terminate all */ 694 goto out; 695 } 696 697 desc = list_first_entry(&channel->ld_active, struct rz_dmac_desc, node); 698 vchan_cookie_complete(&desc->vd); 699 list_move_tail(channel->ld_active.next, &channel->ld_free); 700 if (!list_empty(&channel->ld_queue)) { 701 desc = list_first_entry(&channel->ld_queue, struct rz_dmac_desc, 702 node); 703 channel->desc = desc; 704 if (rz_dmac_xfer_desc(channel) == 0) 705 list_move_tail(channel->ld_queue.next, &channel->ld_active); 706 } 707 out: 708 spin_unlock_irqrestore(&channel->vc.lock, flags); 709 710 return IRQ_HANDLED; 711 } 712 713 /* 714 * ----------------------------------------------------------------------------- 715 * OF xlate and channel filter 716 */ 717 718 static bool rz_dmac_chan_filter(struct dma_chan *chan, void *arg) 719 { 720 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 721 struct rz_dmac *dmac = to_rz_dmac(chan->device); 722 struct of_phandle_args *dma_spec = arg; 723 u32 ch_cfg; 724 725 channel->mid_rid = dma_spec->args[0] & MID_RID_MASK; 726 ch_cfg = (dma_spec->args[0] & CHCFG_MASK) >> 10; 727 channel->chcfg = CHCFG_FILL_TM(ch_cfg) | CHCFG_FILL_AM(ch_cfg) | 728 CHCFG_FILL_LVL(ch_cfg) | CHCFG_FILL_HIEN(ch_cfg); 729 730 return !test_and_set_bit(channel->mid_rid, dmac->modules); 731 } 732 733 static struct dma_chan *rz_dmac_of_xlate(struct of_phandle_args *dma_spec, 734 struct of_dma *ofdma) 735 { 736 dma_cap_mask_t mask; 737 738 if (dma_spec->args_count != 1) 739 return NULL; 740 741 /* Only slave DMA channels can be allocated via DT */ 742 dma_cap_zero(mask); 743 dma_cap_set(DMA_SLAVE, mask); 744 745 return dma_request_channel(mask, rz_dmac_chan_filter, dma_spec); 746 } 747 748 /* 749 * ----------------------------------------------------------------------------- 750 * Probe and remove 751 */ 752 753 static int rz_dmac_chan_probe(struct rz_dmac *dmac, 754 struct rz_dmac_chan *channel, 755 unsigned int index) 756 { 757 struct platform_device *pdev = to_platform_device(dmac->dev); 758 struct rz_lmdesc *lmdesc; 759 char pdev_irqname[5]; 760 char *irqname; 761 int ret; 762 763 channel->index = index; 764 channel->mid_rid = -EINVAL; 765 766 /* Request the channel interrupt. */ 767 sprintf(pdev_irqname, "ch%u", index); 768 channel->irq = platform_get_irq_byname(pdev, pdev_irqname); 769 if (channel->irq < 0) 770 return channel->irq; 771 772 irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", 773 dev_name(dmac->dev), index); 774 if (!irqname) 775 return -ENOMEM; 776 777 ret = devm_request_threaded_irq(dmac->dev, channel->irq, 778 rz_dmac_irq_handler, 779 rz_dmac_irq_handler_thread, 0, 780 irqname, channel); 781 if (ret) { 782 dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", 783 channel->irq, ret); 784 return ret; 785 } 786 787 /* Set io base address for each channel */ 788 if (index < 8) { 789 channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET + 790 EACH_CHANNEL_OFFSET * index; 791 channel->ch_cmn_base = dmac->base + CHANNEL_0_7_COMMON_BASE; 792 } else { 793 channel->ch_base = dmac->base + CHANNEL_8_15_OFFSET + 794 EACH_CHANNEL_OFFSET * (index - 8); 795 channel->ch_cmn_base = dmac->base + CHANNEL_8_15_COMMON_BASE; 796 } 797 798 /* Allocate descriptors */ 799 lmdesc = dma_alloc_coherent(&pdev->dev, 800 sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, 801 &channel->lmdesc.base_dma, GFP_KERNEL); 802 if (!lmdesc) { 803 dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n"); 804 return -ENOMEM; 805 } 806 rz_lmdesc_setup(channel, lmdesc); 807 808 /* Initialize register for each channel */ 809 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); 810 811 channel->vc.desc_free = rz_dmac_virt_desc_free; 812 vchan_init(&channel->vc, &dmac->engine); 813 INIT_LIST_HEAD(&channel->ld_queue); 814 INIT_LIST_HEAD(&channel->ld_free); 815 INIT_LIST_HEAD(&channel->ld_active); 816 817 return 0; 818 } 819 820 static int rz_dmac_parse_of(struct device *dev, struct rz_dmac *dmac) 821 { 822 struct device_node *np = dev->of_node; 823 int ret; 824 825 ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels); 826 if (ret < 0) { 827 dev_err(dev, "unable to read dma-channels property\n"); 828 return ret; 829 } 830 831 if (!dmac->n_channels || dmac->n_channels > RZ_DMAC_MAX_CHANNELS) { 832 dev_err(dev, "invalid number of channels %u\n", dmac->n_channels); 833 return -EINVAL; 834 } 835 836 return 0; 837 } 838 839 static int rz_dmac_probe(struct platform_device *pdev) 840 { 841 const char *irqname = "error"; 842 struct dma_device *engine; 843 struct rz_dmac *dmac; 844 int channel_num; 845 unsigned int i; 846 int ret; 847 int irq; 848 849 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); 850 if (!dmac) 851 return -ENOMEM; 852 853 dmac->dev = &pdev->dev; 854 platform_set_drvdata(pdev, dmac); 855 856 ret = rz_dmac_parse_of(&pdev->dev, dmac); 857 if (ret < 0) 858 return ret; 859 860 dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels, 861 sizeof(*dmac->channels), GFP_KERNEL); 862 if (!dmac->channels) 863 return -ENOMEM; 864 865 /* Request resources */ 866 dmac->base = devm_platform_ioremap_resource(pdev, 0); 867 if (IS_ERR(dmac->base)) 868 return PTR_ERR(dmac->base); 869 870 dmac->ext_base = devm_platform_ioremap_resource(pdev, 1); 871 if (IS_ERR(dmac->ext_base)) 872 return PTR_ERR(dmac->ext_base); 873 874 /* Register interrupt handler for error */ 875 irq = platform_get_irq_byname(pdev, irqname); 876 if (irq < 0) 877 return irq; 878 879 ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0, 880 irqname, NULL); 881 if (ret) { 882 dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n", 883 irq, ret); 884 return ret; 885 } 886 887 /* Initialize the channels. */ 888 INIT_LIST_HEAD(&dmac->engine.channels); 889 890 dmac->rstc = devm_reset_control_array_get_exclusive(&pdev->dev); 891 if (IS_ERR(dmac->rstc)) 892 return dev_err_probe(&pdev->dev, PTR_ERR(dmac->rstc), 893 "failed to get resets\n"); 894 895 pm_runtime_enable(&pdev->dev); 896 ret = pm_runtime_resume_and_get(&pdev->dev); 897 if (ret < 0) { 898 dev_err(&pdev->dev, "pm_runtime_resume_and_get failed\n"); 899 goto err_pm_disable; 900 } 901 902 ret = reset_control_deassert(dmac->rstc); 903 if (ret) 904 goto err_pm_runtime_put; 905 906 for (i = 0; i < dmac->n_channels; i++) { 907 ret = rz_dmac_chan_probe(dmac, &dmac->channels[i], i); 908 if (ret < 0) 909 goto err; 910 } 911 912 /* Register the DMAC as a DMA provider for DT. */ 913 ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate, 914 NULL); 915 if (ret < 0) 916 goto err; 917 918 /* Register the DMA engine device. */ 919 engine = &dmac->engine; 920 dma_cap_set(DMA_SLAVE, engine->cap_mask); 921 dma_cap_set(DMA_MEMCPY, engine->cap_mask); 922 rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL); 923 rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL); 924 925 engine->dev = &pdev->dev; 926 927 engine->device_alloc_chan_resources = rz_dmac_alloc_chan_resources; 928 engine->device_free_chan_resources = rz_dmac_free_chan_resources; 929 engine->device_tx_status = dma_cookie_status; 930 engine->device_prep_slave_sg = rz_dmac_prep_slave_sg; 931 engine->device_prep_dma_memcpy = rz_dmac_prep_dma_memcpy; 932 engine->device_config = rz_dmac_config; 933 engine->device_terminate_all = rz_dmac_terminate_all; 934 engine->device_issue_pending = rz_dmac_issue_pending; 935 engine->device_synchronize = rz_dmac_device_synchronize; 936 937 engine->copy_align = DMAENGINE_ALIGN_1_BYTE; 938 dma_set_max_seg_size(engine->dev, U32_MAX); 939 940 ret = dma_async_device_register(engine); 941 if (ret < 0) { 942 dev_err(&pdev->dev, "unable to register\n"); 943 goto dma_register_err; 944 } 945 return 0; 946 947 dma_register_err: 948 of_dma_controller_free(pdev->dev.of_node); 949 err: 950 reset_control_assert(dmac->rstc); 951 channel_num = i ? i - 1 : 0; 952 for (i = 0; i < channel_num; i++) { 953 struct rz_dmac_chan *channel = &dmac->channels[i]; 954 955 dma_free_coherent(&pdev->dev, 956 sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, 957 channel->lmdesc.base, 958 channel->lmdesc.base_dma); 959 } 960 961 err_pm_runtime_put: 962 pm_runtime_put(&pdev->dev); 963 err_pm_disable: 964 pm_runtime_disable(&pdev->dev); 965 966 return ret; 967 } 968 969 static int rz_dmac_remove(struct platform_device *pdev) 970 { 971 struct rz_dmac *dmac = platform_get_drvdata(pdev); 972 unsigned int i; 973 974 for (i = 0; i < dmac->n_channels; i++) { 975 struct rz_dmac_chan *channel = &dmac->channels[i]; 976 977 dma_free_coherent(&pdev->dev, 978 sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, 979 channel->lmdesc.base, 980 channel->lmdesc.base_dma); 981 } 982 of_dma_controller_free(pdev->dev.of_node); 983 dma_async_device_unregister(&dmac->engine); 984 reset_control_assert(dmac->rstc); 985 pm_runtime_put(&pdev->dev); 986 pm_runtime_disable(&pdev->dev); 987 988 return 0; 989 } 990 991 static const struct of_device_id of_rz_dmac_match[] = { 992 { .compatible = "renesas,rz-dmac", }, 993 { /* Sentinel */ } 994 }; 995 MODULE_DEVICE_TABLE(of, of_rz_dmac_match); 996 997 static struct platform_driver rz_dmac_driver = { 998 .driver = { 999 .name = "rz-dmac", 1000 .of_match_table = of_rz_dmac_match, 1001 }, 1002 .probe = rz_dmac_probe, 1003 .remove = rz_dmac_remove, 1004 }; 1005 1006 module_platform_driver(rz_dmac_driver); 1007 1008 MODULE_DESCRIPTION("Renesas RZ/G2L DMA Controller Driver"); 1009 MODULE_AUTHOR("Biju Das <biju.das.jz@bp.renesas.com>"); 1010 MODULE_LICENSE("GPL v2"); 1011