1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Renesas RZ/G2L DMA Controller Driver 4 * 5 * Based on imx-dma.c 6 * 7 * Copyright (C) 2021 Renesas Electronics Corp. 8 * Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de> 9 * Copyright 2012 Javier Martin, Vista Silicon <javier.martin@vista-silicon.com> 10 */ 11 12 #include <linux/dma-mapping.h> 13 #include <linux/dmaengine.h> 14 #include <linux/interrupt.h> 15 #include <linux/iopoll.h> 16 #include <linux/list.h> 17 #include <linux/module.h> 18 #include <linux/of.h> 19 #include <linux/of_dma.h> 20 #include <linux/of_platform.h> 21 #include <linux/platform_device.h> 22 #include <linux/pm_runtime.h> 23 #include <linux/slab.h> 24 #include <linux/spinlock.h> 25 26 #include "../dmaengine.h" 27 #include "../virt-dma.h" 28 29 enum rz_dmac_prep_type { 30 RZ_DMAC_DESC_MEMCPY, 31 RZ_DMAC_DESC_SLAVE_SG, 32 }; 33 34 struct rz_lmdesc { 35 u32 header; 36 u32 sa; 37 u32 da; 38 u32 tb; 39 u32 chcfg; 40 u32 chitvl; 41 u32 chext; 42 u32 nxla; 43 }; 44 45 struct rz_dmac_desc { 46 struct virt_dma_desc vd; 47 dma_addr_t src; 48 dma_addr_t dest; 49 size_t len; 50 struct list_head node; 51 enum dma_transfer_direction direction; 52 enum rz_dmac_prep_type type; 53 /* For slave sg */ 54 struct scatterlist *sg; 55 unsigned int sgcount; 56 }; 57 58 #define to_rz_dmac_desc(d) container_of(d, struct rz_dmac_desc, vd) 59 60 struct rz_dmac_chan { 61 struct virt_dma_chan vc; 62 void __iomem *ch_base; 63 void __iomem *ch_cmn_base; 64 unsigned int index; 65 int irq; 66 struct rz_dmac_desc *desc; 67 int descs_allocated; 68 69 enum dma_slave_buswidth src_word_size; 70 enum dma_slave_buswidth dst_word_size; 71 dma_addr_t src_per_address; 72 dma_addr_t dst_per_address; 73 74 u32 chcfg; 75 u32 chctrl; 76 int mid_rid; 77 78 struct list_head ld_free; 79 struct list_head ld_queue; 80 struct list_head ld_active; 81 82 struct { 83 struct rz_lmdesc *base; 84 struct rz_lmdesc *head; 85 struct rz_lmdesc *tail; 86 dma_addr_t base_dma; 87 } lmdesc; 88 }; 89 90 #define to_rz_dmac_chan(c) container_of(c, struct rz_dmac_chan, vc.chan) 91 92 struct rz_dmac { 93 struct dma_device engine; 94 struct device *dev; 95 void __iomem *base; 96 void __iomem *ext_base; 97 98 unsigned int n_channels; 99 struct rz_dmac_chan *channels; 100 101 DECLARE_BITMAP(modules, 1024); 102 }; 103 104 #define to_rz_dmac(d) container_of(d, struct rz_dmac, engine) 105 106 /* 107 * ----------------------------------------------------------------------------- 108 * Registers 109 */ 110 111 #define CHSTAT 0x0024 112 #define CHCTRL 0x0028 113 #define CHCFG 0x002c 114 #define NXLA 0x0038 115 116 #define DCTRL 0x0000 117 118 #define EACH_CHANNEL_OFFSET 0x0040 119 #define CHANNEL_0_7_OFFSET 0x0000 120 #define CHANNEL_0_7_COMMON_BASE 0x0300 121 #define CHANNEL_8_15_OFFSET 0x0400 122 #define CHANNEL_8_15_COMMON_BASE 0x0700 123 124 #define CHSTAT_ER BIT(4) 125 #define CHSTAT_EN BIT(0) 126 127 #define CHCTRL_CLRINTMSK BIT(17) 128 #define CHCTRL_CLRSUS BIT(9) 129 #define CHCTRL_CLRTC BIT(6) 130 #define CHCTRL_CLREND BIT(5) 131 #define CHCTRL_CLRRQ BIT(4) 132 #define CHCTRL_SWRST BIT(3) 133 #define CHCTRL_STG BIT(2) 134 #define CHCTRL_CLREN BIT(1) 135 #define CHCTRL_SETEN BIT(0) 136 #define CHCTRL_DEFAULT (CHCTRL_CLRINTMSK | CHCTRL_CLRSUS | \ 137 CHCTRL_CLRTC | CHCTRL_CLREND | \ 138 CHCTRL_CLRRQ | CHCTRL_SWRST | \ 139 CHCTRL_CLREN) 140 141 #define CHCFG_DMS BIT(31) 142 #define CHCFG_DEM BIT(24) 143 #define CHCFG_DAD BIT(21) 144 #define CHCFG_SAD BIT(20) 145 #define CHCFG_REQD BIT(3) 146 #define CHCFG_SEL(bits) ((bits) & 0x07) 147 #define CHCFG_MEM_COPY (0x80400008) 148 #define CHCFG_FILL_DDS(a) (((a) << 16) & GENMASK(19, 16)) 149 #define CHCFG_FILL_SDS(a) (((a) << 12) & GENMASK(15, 12)) 150 #define CHCFG_FILL_TM(a) (((a) & BIT(5)) << 22) 151 #define CHCFG_FILL_AM(a) (((a) & GENMASK(4, 2)) << 6) 152 #define CHCFG_FILL_LVL(a) (((a) & BIT(1)) << 5) 153 #define CHCFG_FILL_HIEN(a) (((a) & BIT(0)) << 5) 154 155 #define MID_RID_MASK GENMASK(9, 0) 156 #define CHCFG_MASK GENMASK(15, 10) 157 #define CHCFG_DS_INVALID 0xFF 158 #define DCTRL_LVINT BIT(1) 159 #define DCTRL_PR BIT(0) 160 #define DCTRL_DEFAULT (DCTRL_LVINT | DCTRL_PR) 161 162 /* LINK MODE DESCRIPTOR */ 163 #define HEADER_LV BIT(0) 164 165 #define RZ_DMAC_MAX_CHAN_DESCRIPTORS 16 166 #define RZ_DMAC_MAX_CHANNELS 16 167 #define DMAC_NR_LMDESC 64 168 169 /* 170 * ----------------------------------------------------------------------------- 171 * Device access 172 */ 173 174 static void rz_dmac_writel(struct rz_dmac *dmac, unsigned int val, 175 unsigned int offset) 176 { 177 writel(val, dmac->base + offset); 178 } 179 180 static void rz_dmac_ext_writel(struct rz_dmac *dmac, unsigned int val, 181 unsigned int offset) 182 { 183 writel(val, dmac->ext_base + offset); 184 } 185 186 static u32 rz_dmac_ext_readl(struct rz_dmac *dmac, unsigned int offset) 187 { 188 return readl(dmac->ext_base + offset); 189 } 190 191 static void rz_dmac_ch_writel(struct rz_dmac_chan *channel, unsigned int val, 192 unsigned int offset, int which) 193 { 194 if (which) 195 writel(val, channel->ch_base + offset); 196 else 197 writel(val, channel->ch_cmn_base + offset); 198 } 199 200 static u32 rz_dmac_ch_readl(struct rz_dmac_chan *channel, 201 unsigned int offset, int which) 202 { 203 if (which) 204 return readl(channel->ch_base + offset); 205 else 206 return readl(channel->ch_cmn_base + offset); 207 } 208 209 /* 210 * ----------------------------------------------------------------------------- 211 * Initialization 212 */ 213 214 static void rz_lmdesc_setup(struct rz_dmac_chan *channel, 215 struct rz_lmdesc *lmdesc) 216 { 217 u32 nxla; 218 219 channel->lmdesc.base = lmdesc; 220 channel->lmdesc.head = lmdesc; 221 channel->lmdesc.tail = lmdesc; 222 nxla = channel->lmdesc.base_dma; 223 while (lmdesc < (channel->lmdesc.base + (DMAC_NR_LMDESC - 1))) { 224 lmdesc->header = 0; 225 nxla += sizeof(*lmdesc); 226 lmdesc->nxla = nxla; 227 lmdesc++; 228 } 229 230 lmdesc->header = 0; 231 lmdesc->nxla = channel->lmdesc.base_dma; 232 } 233 234 /* 235 * ----------------------------------------------------------------------------- 236 * Descriptors preparation 237 */ 238 239 static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel) 240 { 241 struct rz_lmdesc *lmdesc = channel->lmdesc.head; 242 243 while (!(lmdesc->header & HEADER_LV)) { 244 lmdesc->header = 0; 245 lmdesc++; 246 if (lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC)) 247 lmdesc = channel->lmdesc.base; 248 } 249 channel->lmdesc.head = lmdesc; 250 } 251 252 static void rz_dmac_enable_hw(struct rz_dmac_chan *channel) 253 { 254 struct dma_chan *chan = &channel->vc.chan; 255 struct rz_dmac *dmac = to_rz_dmac(chan->device); 256 unsigned long flags; 257 u32 nxla; 258 u32 chctrl; 259 u32 chstat; 260 261 dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index); 262 263 local_irq_save(flags); 264 265 rz_dmac_lmdesc_recycle(channel); 266 267 nxla = channel->lmdesc.base_dma + 268 (sizeof(struct rz_lmdesc) * (channel->lmdesc.head - 269 channel->lmdesc.base)); 270 271 chstat = rz_dmac_ch_readl(channel, CHSTAT, 1); 272 if (!(chstat & CHSTAT_EN)) { 273 chctrl = (channel->chctrl | CHCTRL_SETEN); 274 rz_dmac_ch_writel(channel, nxla, NXLA, 1); 275 rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1); 276 rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1); 277 rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1); 278 } 279 280 local_irq_restore(flags); 281 } 282 283 static void rz_dmac_disable_hw(struct rz_dmac_chan *channel) 284 { 285 struct dma_chan *chan = &channel->vc.chan; 286 struct rz_dmac *dmac = to_rz_dmac(chan->device); 287 unsigned long flags; 288 289 dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index); 290 291 local_irq_save(flags); 292 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); 293 local_irq_restore(flags); 294 } 295 296 static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars) 297 { 298 u32 dmars_offset = (nr / 2) * 4; 299 u32 shift = (nr % 2) * 16; 300 u32 dmars32; 301 302 dmars32 = rz_dmac_ext_readl(dmac, dmars_offset); 303 dmars32 &= ~(0xffff << shift); 304 dmars32 |= dmars << shift; 305 306 rz_dmac_ext_writel(dmac, dmars32, dmars_offset); 307 } 308 309 static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel) 310 { 311 struct dma_chan *chan = &channel->vc.chan; 312 struct rz_dmac *dmac = to_rz_dmac(chan->device); 313 struct rz_lmdesc *lmdesc = channel->lmdesc.tail; 314 struct rz_dmac_desc *d = channel->desc; 315 u32 chcfg = CHCFG_MEM_COPY; 316 317 /* prepare descriptor */ 318 lmdesc->sa = d->src; 319 lmdesc->da = d->dest; 320 lmdesc->tb = d->len; 321 lmdesc->chcfg = chcfg; 322 lmdesc->chitvl = 0; 323 lmdesc->chext = 0; 324 lmdesc->header = HEADER_LV; 325 326 rz_dmac_set_dmars_register(dmac, channel->index, 0); 327 328 channel->chcfg = chcfg; 329 channel->chctrl = CHCTRL_STG | CHCTRL_SETEN; 330 } 331 332 static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel) 333 { 334 struct dma_chan *chan = &channel->vc.chan; 335 struct rz_dmac *dmac = to_rz_dmac(chan->device); 336 struct rz_dmac_desc *d = channel->desc; 337 struct scatterlist *sg, *sgl = d->sg; 338 struct rz_lmdesc *lmdesc; 339 unsigned int i, sg_len = d->sgcount; 340 341 channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DEM | CHCFG_DMS; 342 343 if (d->direction == DMA_DEV_TO_MEM) { 344 channel->chcfg |= CHCFG_SAD; 345 channel->chcfg &= ~CHCFG_REQD; 346 } else { 347 channel->chcfg |= CHCFG_DAD | CHCFG_REQD; 348 } 349 350 lmdesc = channel->lmdesc.tail; 351 352 for (i = 0, sg = sgl; i < sg_len; i++, sg = sg_next(sg)) { 353 if (d->direction == DMA_DEV_TO_MEM) { 354 lmdesc->sa = channel->src_per_address; 355 lmdesc->da = sg_dma_address(sg); 356 } else { 357 lmdesc->sa = sg_dma_address(sg); 358 lmdesc->da = channel->dst_per_address; 359 } 360 361 lmdesc->tb = sg_dma_len(sg); 362 lmdesc->chitvl = 0; 363 lmdesc->chext = 0; 364 if (i == (sg_len - 1)) { 365 lmdesc->chcfg = (channel->chcfg & ~CHCFG_DEM); 366 lmdesc->header = HEADER_LV; 367 } else { 368 lmdesc->chcfg = channel->chcfg; 369 lmdesc->header = HEADER_LV; 370 } 371 if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC)) 372 lmdesc = channel->lmdesc.base; 373 } 374 375 channel->lmdesc.tail = lmdesc; 376 377 rz_dmac_set_dmars_register(dmac, channel->index, channel->mid_rid); 378 channel->chctrl = CHCTRL_SETEN; 379 } 380 381 static int rz_dmac_xfer_desc(struct rz_dmac_chan *chan) 382 { 383 struct rz_dmac_desc *d = chan->desc; 384 struct virt_dma_desc *vd; 385 386 vd = vchan_next_desc(&chan->vc); 387 if (!vd) 388 return 0; 389 390 list_del(&vd->node); 391 392 switch (d->type) { 393 case RZ_DMAC_DESC_MEMCPY: 394 rz_dmac_prepare_desc_for_memcpy(chan); 395 break; 396 397 case RZ_DMAC_DESC_SLAVE_SG: 398 rz_dmac_prepare_descs_for_slave_sg(chan); 399 break; 400 401 default: 402 return -EINVAL; 403 } 404 405 rz_dmac_enable_hw(chan); 406 407 return 0; 408 } 409 410 /* 411 * ----------------------------------------------------------------------------- 412 * DMA engine operations 413 */ 414 415 static int rz_dmac_alloc_chan_resources(struct dma_chan *chan) 416 { 417 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 418 419 while (channel->descs_allocated < RZ_DMAC_MAX_CHAN_DESCRIPTORS) { 420 struct rz_dmac_desc *desc; 421 422 desc = kzalloc(sizeof(*desc), GFP_KERNEL); 423 if (!desc) 424 break; 425 426 list_add_tail(&desc->node, &channel->ld_free); 427 channel->descs_allocated++; 428 } 429 430 if (!channel->descs_allocated) 431 return -ENOMEM; 432 433 return channel->descs_allocated; 434 } 435 436 static void rz_dmac_free_chan_resources(struct dma_chan *chan) 437 { 438 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 439 struct rz_dmac *dmac = to_rz_dmac(chan->device); 440 struct rz_lmdesc *lmdesc = channel->lmdesc.base; 441 struct rz_dmac_desc *desc, *_desc; 442 unsigned long flags; 443 unsigned int i; 444 445 spin_lock_irqsave(&channel->vc.lock, flags); 446 447 for (i = 0; i < DMAC_NR_LMDESC; i++) 448 lmdesc[i].header = 0; 449 450 rz_dmac_disable_hw(channel); 451 list_splice_tail_init(&channel->ld_active, &channel->ld_free); 452 list_splice_tail_init(&channel->ld_queue, &channel->ld_free); 453 454 if (channel->mid_rid >= 0) { 455 clear_bit(channel->mid_rid, dmac->modules); 456 channel->mid_rid = -EINVAL; 457 } 458 459 spin_unlock_irqrestore(&channel->vc.lock, flags); 460 461 list_for_each_entry_safe(desc, _desc, &channel->ld_free, node) { 462 kfree(desc); 463 channel->descs_allocated--; 464 } 465 466 INIT_LIST_HEAD(&channel->ld_free); 467 vchan_free_chan_resources(&channel->vc); 468 } 469 470 static struct dma_async_tx_descriptor * 471 rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 472 size_t len, unsigned long flags) 473 { 474 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 475 struct rz_dmac *dmac = to_rz_dmac(chan->device); 476 struct rz_dmac_desc *desc; 477 478 dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n", 479 __func__, channel->index, &src, &dest, len); 480 481 if (list_empty(&channel->ld_free)) 482 return NULL; 483 484 desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); 485 486 desc->type = RZ_DMAC_DESC_MEMCPY; 487 desc->src = src; 488 desc->dest = dest; 489 desc->len = len; 490 desc->direction = DMA_MEM_TO_MEM; 491 492 list_move_tail(channel->ld_free.next, &channel->ld_queue); 493 return vchan_tx_prep(&channel->vc, &desc->vd, flags); 494 } 495 496 static struct dma_async_tx_descriptor * 497 rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 498 unsigned int sg_len, 499 enum dma_transfer_direction direction, 500 unsigned long flags, void *context) 501 { 502 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 503 struct rz_dmac_desc *desc; 504 struct scatterlist *sg; 505 int dma_length = 0; 506 int i = 0; 507 508 if (list_empty(&channel->ld_free)) 509 return NULL; 510 511 desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); 512 513 for_each_sg(sgl, sg, sg_len, i) { 514 dma_length += sg_dma_len(sg); 515 } 516 517 desc->type = RZ_DMAC_DESC_SLAVE_SG; 518 desc->sg = sgl; 519 desc->sgcount = sg_len; 520 desc->len = dma_length; 521 desc->direction = direction; 522 523 if (direction == DMA_DEV_TO_MEM) 524 desc->src = channel->src_per_address; 525 else 526 desc->dest = channel->dst_per_address; 527 528 list_move_tail(channel->ld_free.next, &channel->ld_queue); 529 return vchan_tx_prep(&channel->vc, &desc->vd, flags); 530 } 531 532 static int rz_dmac_terminate_all(struct dma_chan *chan) 533 { 534 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 535 unsigned long flags; 536 LIST_HEAD(head); 537 538 rz_dmac_disable_hw(channel); 539 spin_lock_irqsave(&channel->vc.lock, flags); 540 list_splice_tail_init(&channel->ld_active, &channel->ld_free); 541 list_splice_tail_init(&channel->ld_queue, &channel->ld_free); 542 spin_unlock_irqrestore(&channel->vc.lock, flags); 543 vchan_get_all_descriptors(&channel->vc, &head); 544 vchan_dma_desc_free_list(&channel->vc, &head); 545 546 return 0; 547 } 548 549 static void rz_dmac_issue_pending(struct dma_chan *chan) 550 { 551 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 552 struct rz_dmac *dmac = to_rz_dmac(chan->device); 553 struct rz_dmac_desc *desc; 554 unsigned long flags; 555 556 spin_lock_irqsave(&channel->vc.lock, flags); 557 558 if (!list_empty(&channel->ld_queue)) { 559 desc = list_first_entry(&channel->ld_queue, 560 struct rz_dmac_desc, node); 561 channel->desc = desc; 562 if (vchan_issue_pending(&channel->vc)) { 563 if (rz_dmac_xfer_desc(channel) < 0) 564 dev_warn(dmac->dev, "ch: %d couldn't issue DMA xfer\n", 565 channel->index); 566 else 567 list_move_tail(channel->ld_queue.next, 568 &channel->ld_active); 569 } 570 } 571 572 spin_unlock_irqrestore(&channel->vc.lock, flags); 573 } 574 575 static u8 rz_dmac_ds_to_val_mapping(enum dma_slave_buswidth ds) 576 { 577 u8 i; 578 static const enum dma_slave_buswidth ds_lut[] = { 579 DMA_SLAVE_BUSWIDTH_1_BYTE, 580 DMA_SLAVE_BUSWIDTH_2_BYTES, 581 DMA_SLAVE_BUSWIDTH_4_BYTES, 582 DMA_SLAVE_BUSWIDTH_8_BYTES, 583 DMA_SLAVE_BUSWIDTH_16_BYTES, 584 DMA_SLAVE_BUSWIDTH_32_BYTES, 585 DMA_SLAVE_BUSWIDTH_64_BYTES, 586 DMA_SLAVE_BUSWIDTH_128_BYTES, 587 }; 588 589 for (i = 0; i < ARRAY_SIZE(ds_lut); i++) { 590 if (ds_lut[i] == ds) 591 return i; 592 } 593 594 return CHCFG_DS_INVALID; 595 } 596 597 static int rz_dmac_config(struct dma_chan *chan, 598 struct dma_slave_config *config) 599 { 600 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 601 u32 val; 602 603 channel->src_per_address = config->src_addr; 604 channel->src_word_size = config->src_addr_width; 605 channel->dst_per_address = config->dst_addr; 606 channel->dst_word_size = config->dst_addr_width; 607 608 val = rz_dmac_ds_to_val_mapping(config->dst_addr_width); 609 if (val == CHCFG_DS_INVALID) 610 return -EINVAL; 611 612 channel->chcfg |= CHCFG_FILL_DDS(val); 613 614 val = rz_dmac_ds_to_val_mapping(config->src_addr_width); 615 if (val == CHCFG_DS_INVALID) 616 return -EINVAL; 617 618 channel->chcfg |= CHCFG_FILL_SDS(val); 619 620 return 0; 621 } 622 623 static void rz_dmac_virt_desc_free(struct virt_dma_desc *vd) 624 { 625 /* 626 * Place holder 627 * Descriptor allocation is done during alloc_chan_resources and 628 * get freed during free_chan_resources. 629 * list is used to manage the descriptors and avoid any memory 630 * allocation/free during DMA read/write. 631 */ 632 } 633 634 static void rz_dmac_device_synchronize(struct dma_chan *chan) 635 { 636 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 637 struct rz_dmac *dmac = to_rz_dmac(chan->device); 638 u32 chstat; 639 int ret; 640 641 ret = read_poll_timeout(rz_dmac_ch_readl, chstat, !(chstat & CHSTAT_EN), 642 100, 100000, false, channel, CHSTAT, 1); 643 if (ret < 0) 644 dev_warn(dmac->dev, "DMA Timeout"); 645 646 rz_dmac_set_dmars_register(dmac, channel->index, 0); 647 } 648 649 /* 650 * ----------------------------------------------------------------------------- 651 * IRQ handling 652 */ 653 654 static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel) 655 { 656 struct dma_chan *chan = &channel->vc.chan; 657 struct rz_dmac *dmac = to_rz_dmac(chan->device); 658 u32 chstat, chctrl; 659 660 chstat = rz_dmac_ch_readl(channel, CHSTAT, 1); 661 if (chstat & CHSTAT_ER) { 662 dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n", 663 channel->index, chstat); 664 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); 665 goto done; 666 } 667 668 chctrl = rz_dmac_ch_readl(channel, CHCTRL, 1); 669 rz_dmac_ch_writel(channel, chctrl | CHCTRL_CLREND, CHCTRL, 1); 670 done: 671 return; 672 } 673 674 static irqreturn_t rz_dmac_irq_handler(int irq, void *dev_id) 675 { 676 struct rz_dmac_chan *channel = dev_id; 677 678 if (channel) { 679 rz_dmac_irq_handle_channel(channel); 680 return IRQ_WAKE_THREAD; 681 } 682 /* handle DMAERR irq */ 683 return IRQ_HANDLED; 684 } 685 686 static irqreturn_t rz_dmac_irq_handler_thread(int irq, void *dev_id) 687 { 688 struct rz_dmac_chan *channel = dev_id; 689 struct rz_dmac_desc *desc = NULL; 690 unsigned long flags; 691 692 spin_lock_irqsave(&channel->vc.lock, flags); 693 694 if (list_empty(&channel->ld_active)) { 695 /* Someone might have called terminate all */ 696 goto out; 697 } 698 699 desc = list_first_entry(&channel->ld_active, struct rz_dmac_desc, node); 700 vchan_cookie_complete(&desc->vd); 701 list_move_tail(channel->ld_active.next, &channel->ld_free); 702 if (!list_empty(&channel->ld_queue)) { 703 desc = list_first_entry(&channel->ld_queue, struct rz_dmac_desc, 704 node); 705 channel->desc = desc; 706 if (rz_dmac_xfer_desc(channel) == 0) 707 list_move_tail(channel->ld_queue.next, &channel->ld_active); 708 } 709 out: 710 spin_unlock_irqrestore(&channel->vc.lock, flags); 711 712 return IRQ_HANDLED; 713 } 714 715 /* 716 * ----------------------------------------------------------------------------- 717 * OF xlate and channel filter 718 */ 719 720 static bool rz_dmac_chan_filter(struct dma_chan *chan, void *arg) 721 { 722 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 723 struct rz_dmac *dmac = to_rz_dmac(chan->device); 724 struct of_phandle_args *dma_spec = arg; 725 u32 ch_cfg; 726 727 channel->mid_rid = dma_spec->args[0] & MID_RID_MASK; 728 ch_cfg = (dma_spec->args[0] & CHCFG_MASK) >> 10; 729 channel->chcfg = CHCFG_FILL_TM(ch_cfg) | CHCFG_FILL_AM(ch_cfg) | 730 CHCFG_FILL_LVL(ch_cfg) | CHCFG_FILL_HIEN(ch_cfg); 731 732 return !test_and_set_bit(channel->mid_rid, dmac->modules); 733 } 734 735 static struct dma_chan *rz_dmac_of_xlate(struct of_phandle_args *dma_spec, 736 struct of_dma *ofdma) 737 { 738 dma_cap_mask_t mask; 739 740 if (dma_spec->args_count != 1) 741 return NULL; 742 743 /* Only slave DMA channels can be allocated via DT */ 744 dma_cap_zero(mask); 745 dma_cap_set(DMA_SLAVE, mask); 746 747 return dma_request_channel(mask, rz_dmac_chan_filter, dma_spec); 748 } 749 750 /* 751 * ----------------------------------------------------------------------------- 752 * Probe and remove 753 */ 754 755 static int rz_dmac_chan_probe(struct rz_dmac *dmac, 756 struct rz_dmac_chan *channel, 757 unsigned int index) 758 { 759 struct platform_device *pdev = to_platform_device(dmac->dev); 760 struct rz_lmdesc *lmdesc; 761 char pdev_irqname[5]; 762 char *irqname; 763 int ret; 764 765 channel->index = index; 766 channel->mid_rid = -EINVAL; 767 768 /* Request the channel interrupt. */ 769 sprintf(pdev_irqname, "ch%u", index); 770 channel->irq = platform_get_irq_byname(pdev, pdev_irqname); 771 if (channel->irq < 0) 772 return channel->irq; 773 774 irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", 775 dev_name(dmac->dev), index); 776 if (!irqname) 777 return -ENOMEM; 778 779 ret = devm_request_threaded_irq(dmac->dev, channel->irq, 780 rz_dmac_irq_handler, 781 rz_dmac_irq_handler_thread, 0, 782 irqname, channel); 783 if (ret) { 784 dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", 785 channel->irq, ret); 786 return ret; 787 } 788 789 /* Set io base address for each channel */ 790 if (index < 8) { 791 channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET + 792 EACH_CHANNEL_OFFSET * index; 793 channel->ch_cmn_base = dmac->base + CHANNEL_0_7_COMMON_BASE; 794 } else { 795 channel->ch_base = dmac->base + CHANNEL_8_15_OFFSET + 796 EACH_CHANNEL_OFFSET * (index - 8); 797 channel->ch_cmn_base = dmac->base + CHANNEL_8_15_COMMON_BASE; 798 } 799 800 /* Allocate descriptors */ 801 lmdesc = dma_alloc_coherent(&pdev->dev, 802 sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, 803 &channel->lmdesc.base_dma, GFP_KERNEL); 804 if (!lmdesc) { 805 dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n"); 806 return -ENOMEM; 807 } 808 rz_lmdesc_setup(channel, lmdesc); 809 810 /* Initialize register for each channel */ 811 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); 812 813 channel->vc.desc_free = rz_dmac_virt_desc_free; 814 vchan_init(&channel->vc, &dmac->engine); 815 INIT_LIST_HEAD(&channel->ld_queue); 816 INIT_LIST_HEAD(&channel->ld_free); 817 INIT_LIST_HEAD(&channel->ld_active); 818 819 return 0; 820 } 821 822 static int rz_dmac_parse_of(struct device *dev, struct rz_dmac *dmac) 823 { 824 struct device_node *np = dev->of_node; 825 int ret; 826 827 ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels); 828 if (ret < 0) { 829 dev_err(dev, "unable to read dma-channels property\n"); 830 return ret; 831 } 832 833 if (!dmac->n_channels || dmac->n_channels > RZ_DMAC_MAX_CHANNELS) { 834 dev_err(dev, "invalid number of channels %u\n", dmac->n_channels); 835 return -EINVAL; 836 } 837 838 return 0; 839 } 840 841 static int rz_dmac_probe(struct platform_device *pdev) 842 { 843 const char *irqname = "error"; 844 struct dma_device *engine; 845 struct rz_dmac *dmac; 846 int channel_num; 847 unsigned int i; 848 int ret; 849 int irq; 850 851 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); 852 if (!dmac) 853 return -ENOMEM; 854 855 dmac->dev = &pdev->dev; 856 platform_set_drvdata(pdev, dmac); 857 858 ret = rz_dmac_parse_of(&pdev->dev, dmac); 859 if (ret < 0) 860 return ret; 861 862 dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels, 863 sizeof(*dmac->channels), GFP_KERNEL); 864 if (!dmac->channels) 865 return -ENOMEM; 866 867 /* Request resources */ 868 dmac->base = devm_platform_ioremap_resource(pdev, 0); 869 if (IS_ERR(dmac->base)) 870 return PTR_ERR(dmac->base); 871 872 dmac->ext_base = devm_platform_ioremap_resource(pdev, 1); 873 if (IS_ERR(dmac->ext_base)) 874 return PTR_ERR(dmac->ext_base); 875 876 /* Register interrupt handler for error */ 877 irq = platform_get_irq_byname(pdev, irqname); 878 if (irq < 0) 879 return irq; 880 881 ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0, 882 irqname, NULL); 883 if (ret) { 884 dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n", 885 irq, ret); 886 return ret; 887 } 888 889 /* Initialize the channels. */ 890 INIT_LIST_HEAD(&dmac->engine.channels); 891 892 pm_runtime_enable(&pdev->dev); 893 ret = pm_runtime_resume_and_get(&pdev->dev); 894 if (ret < 0) { 895 dev_err(&pdev->dev, "pm_runtime_resume_and_get failed\n"); 896 goto err_pm_disable; 897 } 898 899 for (i = 0; i < dmac->n_channels; i++) { 900 ret = rz_dmac_chan_probe(dmac, &dmac->channels[i], i); 901 if (ret < 0) 902 goto err; 903 } 904 905 /* Register the DMAC as a DMA provider for DT. */ 906 ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate, 907 NULL); 908 if (ret < 0) 909 goto err; 910 911 /* Register the DMA engine device. */ 912 engine = &dmac->engine; 913 dma_cap_set(DMA_SLAVE, engine->cap_mask); 914 dma_cap_set(DMA_MEMCPY, engine->cap_mask); 915 rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL); 916 rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL); 917 918 engine->dev = &pdev->dev; 919 920 engine->device_alloc_chan_resources = rz_dmac_alloc_chan_resources; 921 engine->device_free_chan_resources = rz_dmac_free_chan_resources; 922 engine->device_tx_status = dma_cookie_status; 923 engine->device_prep_slave_sg = rz_dmac_prep_slave_sg; 924 engine->device_prep_dma_memcpy = rz_dmac_prep_dma_memcpy; 925 engine->device_config = rz_dmac_config; 926 engine->device_terminate_all = rz_dmac_terminate_all; 927 engine->device_issue_pending = rz_dmac_issue_pending; 928 engine->device_synchronize = rz_dmac_device_synchronize; 929 930 engine->copy_align = DMAENGINE_ALIGN_1_BYTE; 931 dma_set_max_seg_size(engine->dev, U32_MAX); 932 933 ret = dma_async_device_register(engine); 934 if (ret < 0) { 935 dev_err(&pdev->dev, "unable to register\n"); 936 goto dma_register_err; 937 } 938 return 0; 939 940 dma_register_err: 941 of_dma_controller_free(pdev->dev.of_node); 942 err: 943 channel_num = i ? i - 1 : 0; 944 for (i = 0; i < channel_num; i++) { 945 struct rz_dmac_chan *channel = &dmac->channels[i]; 946 947 dma_free_coherent(&pdev->dev, 948 sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, 949 channel->lmdesc.base, 950 channel->lmdesc.base_dma); 951 } 952 953 pm_runtime_put(&pdev->dev); 954 err_pm_disable: 955 pm_runtime_disable(&pdev->dev); 956 957 return ret; 958 } 959 960 static int rz_dmac_remove(struct platform_device *pdev) 961 { 962 struct rz_dmac *dmac = platform_get_drvdata(pdev); 963 unsigned int i; 964 965 for (i = 0; i < dmac->n_channels; i++) { 966 struct rz_dmac_chan *channel = &dmac->channels[i]; 967 968 dma_free_coherent(&pdev->dev, 969 sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, 970 channel->lmdesc.base, 971 channel->lmdesc.base_dma); 972 } 973 of_dma_controller_free(pdev->dev.of_node); 974 dma_async_device_unregister(&dmac->engine); 975 pm_runtime_put(&pdev->dev); 976 pm_runtime_disable(&pdev->dev); 977 978 return 0; 979 } 980 981 static const struct of_device_id of_rz_dmac_match[] = { 982 { .compatible = "renesas,rz-dmac", }, 983 { /* Sentinel */ } 984 }; 985 MODULE_DEVICE_TABLE(of, of_rz_dmac_match); 986 987 static struct platform_driver rz_dmac_driver = { 988 .driver = { 989 .name = "rz-dmac", 990 .of_match_table = of_rz_dmac_match, 991 }, 992 .probe = rz_dmac_probe, 993 .remove = rz_dmac_remove, 994 }; 995 996 module_platform_driver(rz_dmac_driver); 997 998 MODULE_DESCRIPTION("Renesas RZ/G2L DMA Controller Driver"); 999 MODULE_AUTHOR("Biju Das <biju.das.jz@bp.renesas.com>"); 1000 MODULE_LICENSE("GPL v2"); 1001