1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2013 - 2015 Linaro Ltd. 4 * Copyright (c) 2013 Hisilicon Limited. 5 */ 6 #include <linux/sched.h> 7 #include <linux/device.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/dmapool.h> 10 #include <linux/dmaengine.h> 11 #include <linux/init.h> 12 #include <linux/interrupt.h> 13 #include <linux/kernel.h> 14 #include <linux/module.h> 15 #include <linux/platform_device.h> 16 #include <linux/slab.h> 17 #include <linux/spinlock.h> 18 #include <linux/of_device.h> 19 #include <linux/of.h> 20 #include <linux/clk.h> 21 #include <linux/of_dma.h> 22 23 #include "virt-dma.h" 24 25 #define DRIVER_NAME "k3-dma" 26 #define DMA_MAX_SIZE 0x1ffc 27 #define DMA_CYCLIC_MAX_PERIOD 0x1000 28 #define LLI_BLOCK_SIZE (4 * PAGE_SIZE) 29 30 #define INT_STAT 0x00 31 #define INT_TC1 0x04 32 #define INT_TC2 0x08 33 #define INT_ERR1 0x0c 34 #define INT_ERR2 0x10 35 #define INT_TC1_MASK 0x18 36 #define INT_TC2_MASK 0x1c 37 #define INT_ERR1_MASK 0x20 38 #define INT_ERR2_MASK 0x24 39 #define INT_TC1_RAW 0x600 40 #define INT_TC2_RAW 0x608 41 #define INT_ERR1_RAW 0x610 42 #define INT_ERR2_RAW 0x618 43 #define CH_PRI 0x688 44 #define CH_STAT 0x690 45 #define CX_CUR_CNT 0x704 46 #define CX_LLI 0x800 47 #define CX_CNT1 0x80c 48 #define CX_CNT0 0x810 49 #define CX_SRC 0x814 50 #define CX_DST 0x818 51 #define CX_CFG 0x81c 52 53 #define CX_LLI_CHAIN_EN 0x2 54 #define CX_CFG_EN 0x1 55 #define CX_CFG_NODEIRQ BIT(1) 56 #define CX_CFG_MEM2PER (0x1 << 2) 57 #define CX_CFG_PER2MEM (0x2 << 2) 58 #define CX_CFG_SRCINCR (0x1 << 31) 59 #define CX_CFG_DSTINCR (0x1 << 30) 60 61 struct k3_desc_hw { 62 u32 lli; 63 u32 reserved[3]; 64 u32 count; 65 u32 saddr; 66 u32 daddr; 67 u32 config; 68 } __aligned(32); 69 70 struct k3_dma_desc_sw { 71 struct virt_dma_desc vd; 72 dma_addr_t desc_hw_lli; 73 size_t desc_num; 74 size_t size; 75 struct k3_desc_hw *desc_hw; 76 }; 77 78 struct k3_dma_phy; 79 80 struct k3_dma_chan { 81 u32 ccfg; 82 struct virt_dma_chan vc; 83 struct k3_dma_phy *phy; 84 struct list_head node; 85 dma_addr_t dev_addr; 86 enum dma_status status; 87 bool cyclic; 88 struct dma_slave_config slave_config; 89 }; 90 91 struct k3_dma_phy { 92 u32 idx; 93 void __iomem *base; 94 struct k3_dma_chan *vchan; 95 struct k3_dma_desc_sw *ds_run; 96 struct k3_dma_desc_sw *ds_done; 97 }; 98 99 struct k3_dma_dev { 100 struct dma_device slave; 101 void __iomem *base; 102 struct tasklet_struct task; 103 spinlock_t lock; 104 struct list_head chan_pending; 105 struct k3_dma_phy *phy; 106 struct k3_dma_chan *chans; 107 struct clk *clk; 108 struct dma_pool *pool; 109 u32 dma_channels; 110 u32 dma_requests; 111 u32 dma_channel_mask; 112 unsigned int irq; 113 }; 114 115 116 #define K3_FLAG_NOCLK BIT(1) 117 118 struct k3dma_soc_data { 119 unsigned long flags; 120 }; 121 122 123 #define to_k3_dma(dmadev) container_of(dmadev, struct k3_dma_dev, slave) 124 125 static int k3_dma_config_write(struct dma_chan *chan, 126 enum dma_transfer_direction dir, 127 struct dma_slave_config *cfg); 128 129 static struct k3_dma_chan *to_k3_chan(struct dma_chan *chan) 130 { 131 return container_of(chan, struct k3_dma_chan, vc.chan); 132 } 133 134 static void k3_dma_pause_dma(struct k3_dma_phy *phy, bool on) 135 { 136 u32 val = 0; 137 138 if (on) { 139 val = readl_relaxed(phy->base + CX_CFG); 140 val |= CX_CFG_EN; 141 writel_relaxed(val, phy->base + CX_CFG); 142 } else { 143 val = readl_relaxed(phy->base + CX_CFG); 144 val &= ~CX_CFG_EN; 145 writel_relaxed(val, phy->base + CX_CFG); 146 } 147 } 148 149 static void k3_dma_terminate_chan(struct k3_dma_phy *phy, struct k3_dma_dev *d) 150 { 151 u32 val = 0; 152 153 k3_dma_pause_dma(phy, false); 154 155 val = 0x1 << phy->idx; 156 writel_relaxed(val, d->base + INT_TC1_RAW); 157 writel_relaxed(val, d->base + INT_TC2_RAW); 158 writel_relaxed(val, d->base + INT_ERR1_RAW); 159 writel_relaxed(val, d->base + INT_ERR2_RAW); 160 } 161 162 static void k3_dma_set_desc(struct k3_dma_phy *phy, struct k3_desc_hw *hw) 163 { 164 writel_relaxed(hw->lli, phy->base + CX_LLI); 165 writel_relaxed(hw->count, phy->base + CX_CNT0); 166 writel_relaxed(hw->saddr, phy->base + CX_SRC); 167 writel_relaxed(hw->daddr, phy->base + CX_DST); 168 writel_relaxed(hw->config, phy->base + CX_CFG); 169 } 170 171 static u32 k3_dma_get_curr_cnt(struct k3_dma_dev *d, struct k3_dma_phy *phy) 172 { 173 u32 cnt = 0; 174 175 cnt = readl_relaxed(d->base + CX_CUR_CNT + phy->idx * 0x10); 176 cnt &= 0xffff; 177 return cnt; 178 } 179 180 static u32 k3_dma_get_curr_lli(struct k3_dma_phy *phy) 181 { 182 return readl_relaxed(phy->base + CX_LLI); 183 } 184 185 static u32 k3_dma_get_chan_stat(struct k3_dma_dev *d) 186 { 187 return readl_relaxed(d->base + CH_STAT); 188 } 189 190 static void k3_dma_enable_dma(struct k3_dma_dev *d, bool on) 191 { 192 if (on) { 193 /* set same priority */ 194 writel_relaxed(0x0, d->base + CH_PRI); 195 196 /* unmask irq */ 197 writel_relaxed(0xffff, d->base + INT_TC1_MASK); 198 writel_relaxed(0xffff, d->base + INT_TC2_MASK); 199 writel_relaxed(0xffff, d->base + INT_ERR1_MASK); 200 writel_relaxed(0xffff, d->base + INT_ERR2_MASK); 201 } else { 202 /* mask irq */ 203 writel_relaxed(0x0, d->base + INT_TC1_MASK); 204 writel_relaxed(0x0, d->base + INT_TC2_MASK); 205 writel_relaxed(0x0, d->base + INT_ERR1_MASK); 206 writel_relaxed(0x0, d->base + INT_ERR2_MASK); 207 } 208 } 209 210 static irqreturn_t k3_dma_int_handler(int irq, void *dev_id) 211 { 212 struct k3_dma_dev *d = (struct k3_dma_dev *)dev_id; 213 struct k3_dma_phy *p; 214 struct k3_dma_chan *c; 215 u32 stat = readl_relaxed(d->base + INT_STAT); 216 u32 tc1 = readl_relaxed(d->base + INT_TC1); 217 u32 tc2 = readl_relaxed(d->base + INT_TC2); 218 u32 err1 = readl_relaxed(d->base + INT_ERR1); 219 u32 err2 = readl_relaxed(d->base + INT_ERR2); 220 u32 i, irq_chan = 0; 221 222 while (stat) { 223 i = __ffs(stat); 224 stat &= ~BIT(i); 225 if (likely(tc1 & BIT(i)) || (tc2 & BIT(i))) { 226 unsigned long flags; 227 228 p = &d->phy[i]; 229 c = p->vchan; 230 if (c && (tc1 & BIT(i))) { 231 spin_lock_irqsave(&c->vc.lock, flags); 232 vchan_cookie_complete(&p->ds_run->vd); 233 p->ds_done = p->ds_run; 234 p->ds_run = NULL; 235 spin_unlock_irqrestore(&c->vc.lock, flags); 236 } 237 if (c && (tc2 & BIT(i))) { 238 spin_lock_irqsave(&c->vc.lock, flags); 239 if (p->ds_run != NULL) 240 vchan_cyclic_callback(&p->ds_run->vd); 241 spin_unlock_irqrestore(&c->vc.lock, flags); 242 } 243 irq_chan |= BIT(i); 244 } 245 if (unlikely((err1 & BIT(i)) || (err2 & BIT(i)))) 246 dev_warn(d->slave.dev, "DMA ERR\n"); 247 } 248 249 writel_relaxed(irq_chan, d->base + INT_TC1_RAW); 250 writel_relaxed(irq_chan, d->base + INT_TC2_RAW); 251 writel_relaxed(err1, d->base + INT_ERR1_RAW); 252 writel_relaxed(err2, d->base + INT_ERR2_RAW); 253 254 if (irq_chan) 255 tasklet_schedule(&d->task); 256 257 if (irq_chan || err1 || err2) 258 return IRQ_HANDLED; 259 260 return IRQ_NONE; 261 } 262 263 static int k3_dma_start_txd(struct k3_dma_chan *c) 264 { 265 struct k3_dma_dev *d = to_k3_dma(c->vc.chan.device); 266 struct virt_dma_desc *vd = vchan_next_desc(&c->vc); 267 268 if (!c->phy) 269 return -EAGAIN; 270 271 if (BIT(c->phy->idx) & k3_dma_get_chan_stat(d)) 272 return -EAGAIN; 273 274 if (vd) { 275 struct k3_dma_desc_sw *ds = 276 container_of(vd, struct k3_dma_desc_sw, vd); 277 /* 278 * fetch and remove request from vc->desc_issued 279 * so vc->desc_issued only contains desc pending 280 */ 281 list_del(&ds->vd.node); 282 283 c->phy->ds_run = ds; 284 c->phy->ds_done = NULL; 285 /* start dma */ 286 k3_dma_set_desc(c->phy, &ds->desc_hw[0]); 287 return 0; 288 } 289 c->phy->ds_run = NULL; 290 c->phy->ds_done = NULL; 291 return -EAGAIN; 292 } 293 294 static void k3_dma_tasklet(unsigned long arg) 295 { 296 struct k3_dma_dev *d = (struct k3_dma_dev *)arg; 297 struct k3_dma_phy *p; 298 struct k3_dma_chan *c, *cn; 299 unsigned pch, pch_alloc = 0; 300 301 /* check new dma request of running channel in vc->desc_issued */ 302 list_for_each_entry_safe(c, cn, &d->slave.channels, vc.chan.device_node) { 303 spin_lock_irq(&c->vc.lock); 304 p = c->phy; 305 if (p && p->ds_done) { 306 if (k3_dma_start_txd(c)) { 307 /* No current txd associated with this channel */ 308 dev_dbg(d->slave.dev, "pchan %u: free\n", p->idx); 309 /* Mark this channel free */ 310 c->phy = NULL; 311 p->vchan = NULL; 312 } 313 } 314 spin_unlock_irq(&c->vc.lock); 315 } 316 317 /* check new channel request in d->chan_pending */ 318 spin_lock_irq(&d->lock); 319 for (pch = 0; pch < d->dma_channels; pch++) { 320 if (!(d->dma_channel_mask & (1 << pch))) 321 continue; 322 323 p = &d->phy[pch]; 324 325 if (p->vchan == NULL && !list_empty(&d->chan_pending)) { 326 c = list_first_entry(&d->chan_pending, 327 struct k3_dma_chan, node); 328 /* remove from d->chan_pending */ 329 list_del_init(&c->node); 330 pch_alloc |= 1 << pch; 331 /* Mark this channel allocated */ 332 p->vchan = c; 333 c->phy = p; 334 dev_dbg(d->slave.dev, "pchan %u: alloc vchan %p\n", pch, &c->vc); 335 } 336 } 337 spin_unlock_irq(&d->lock); 338 339 for (pch = 0; pch < d->dma_channels; pch++) { 340 if (!(d->dma_channel_mask & (1 << pch))) 341 continue; 342 343 if (pch_alloc & (1 << pch)) { 344 p = &d->phy[pch]; 345 c = p->vchan; 346 if (c) { 347 spin_lock_irq(&c->vc.lock); 348 k3_dma_start_txd(c); 349 spin_unlock_irq(&c->vc.lock); 350 } 351 } 352 } 353 } 354 355 static void k3_dma_free_chan_resources(struct dma_chan *chan) 356 { 357 struct k3_dma_chan *c = to_k3_chan(chan); 358 struct k3_dma_dev *d = to_k3_dma(chan->device); 359 unsigned long flags; 360 361 spin_lock_irqsave(&d->lock, flags); 362 list_del_init(&c->node); 363 spin_unlock_irqrestore(&d->lock, flags); 364 365 vchan_free_chan_resources(&c->vc); 366 c->ccfg = 0; 367 } 368 369 static enum dma_status k3_dma_tx_status(struct dma_chan *chan, 370 dma_cookie_t cookie, struct dma_tx_state *state) 371 { 372 struct k3_dma_chan *c = to_k3_chan(chan); 373 struct k3_dma_dev *d = to_k3_dma(chan->device); 374 struct k3_dma_phy *p; 375 struct virt_dma_desc *vd; 376 unsigned long flags; 377 enum dma_status ret; 378 size_t bytes = 0; 379 380 ret = dma_cookie_status(&c->vc.chan, cookie, state); 381 if (ret == DMA_COMPLETE) 382 return ret; 383 384 spin_lock_irqsave(&c->vc.lock, flags); 385 p = c->phy; 386 ret = c->status; 387 388 /* 389 * If the cookie is on our issue queue, then the residue is 390 * its total size. 391 */ 392 vd = vchan_find_desc(&c->vc, cookie); 393 if (vd && !c->cyclic) { 394 bytes = container_of(vd, struct k3_dma_desc_sw, vd)->size; 395 } else if ((!p) || (!p->ds_run)) { 396 bytes = 0; 397 } else { 398 struct k3_dma_desc_sw *ds = p->ds_run; 399 u32 clli = 0, index = 0; 400 401 bytes = k3_dma_get_curr_cnt(d, p); 402 clli = k3_dma_get_curr_lli(p); 403 index = ((clli - ds->desc_hw_lli) / 404 sizeof(struct k3_desc_hw)) + 1; 405 for (; index < ds->desc_num; index++) { 406 bytes += ds->desc_hw[index].count; 407 /* end of lli */ 408 if (!ds->desc_hw[index].lli) 409 break; 410 } 411 } 412 spin_unlock_irqrestore(&c->vc.lock, flags); 413 dma_set_residue(state, bytes); 414 return ret; 415 } 416 417 static void k3_dma_issue_pending(struct dma_chan *chan) 418 { 419 struct k3_dma_chan *c = to_k3_chan(chan); 420 struct k3_dma_dev *d = to_k3_dma(chan->device); 421 unsigned long flags; 422 423 spin_lock_irqsave(&c->vc.lock, flags); 424 /* add request to vc->desc_issued */ 425 if (vchan_issue_pending(&c->vc)) { 426 spin_lock(&d->lock); 427 if (!c->phy) { 428 if (list_empty(&c->node)) { 429 /* if new channel, add chan_pending */ 430 list_add_tail(&c->node, &d->chan_pending); 431 /* check in tasklet */ 432 tasklet_schedule(&d->task); 433 dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc); 434 } 435 } 436 spin_unlock(&d->lock); 437 } else 438 dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc); 439 spin_unlock_irqrestore(&c->vc.lock, flags); 440 } 441 442 static void k3_dma_fill_desc(struct k3_dma_desc_sw *ds, dma_addr_t dst, 443 dma_addr_t src, size_t len, u32 num, u32 ccfg) 444 { 445 if (num != ds->desc_num - 1) 446 ds->desc_hw[num].lli = ds->desc_hw_lli + (num + 1) * 447 sizeof(struct k3_desc_hw); 448 449 ds->desc_hw[num].lli |= CX_LLI_CHAIN_EN; 450 ds->desc_hw[num].count = len; 451 ds->desc_hw[num].saddr = src; 452 ds->desc_hw[num].daddr = dst; 453 ds->desc_hw[num].config = ccfg; 454 } 455 456 static struct k3_dma_desc_sw *k3_dma_alloc_desc_resource(int num, 457 struct dma_chan *chan) 458 { 459 struct k3_dma_chan *c = to_k3_chan(chan); 460 struct k3_dma_desc_sw *ds; 461 struct k3_dma_dev *d = to_k3_dma(chan->device); 462 int lli_limit = LLI_BLOCK_SIZE / sizeof(struct k3_desc_hw); 463 464 if (num > lli_limit) { 465 dev_dbg(chan->device->dev, "vch %p: sg num %d exceed max %d\n", 466 &c->vc, num, lli_limit); 467 return NULL; 468 } 469 470 ds = kzalloc(sizeof(*ds), GFP_NOWAIT); 471 if (!ds) 472 return NULL; 473 474 ds->desc_hw = dma_pool_zalloc(d->pool, GFP_NOWAIT, &ds->desc_hw_lli); 475 if (!ds->desc_hw) { 476 dev_dbg(chan->device->dev, "vch %p: dma alloc fail\n", &c->vc); 477 kfree(ds); 478 return NULL; 479 } 480 ds->desc_num = num; 481 return ds; 482 } 483 484 static struct dma_async_tx_descriptor *k3_dma_prep_memcpy( 485 struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, 486 size_t len, unsigned long flags) 487 { 488 struct k3_dma_chan *c = to_k3_chan(chan); 489 struct k3_dma_desc_sw *ds; 490 size_t copy = 0; 491 int num = 0; 492 493 if (!len) 494 return NULL; 495 496 num = DIV_ROUND_UP(len, DMA_MAX_SIZE); 497 498 ds = k3_dma_alloc_desc_resource(num, chan); 499 if (!ds) 500 return NULL; 501 502 c->cyclic = 0; 503 ds->size = len; 504 num = 0; 505 506 if (!c->ccfg) { 507 /* default is memtomem, without calling device_config */ 508 c->ccfg = CX_CFG_SRCINCR | CX_CFG_DSTINCR | CX_CFG_EN; 509 c->ccfg |= (0xf << 20) | (0xf << 24); /* burst = 16 */ 510 c->ccfg |= (0x3 << 12) | (0x3 << 16); /* width = 64 bit */ 511 } 512 513 do { 514 copy = min_t(size_t, len, DMA_MAX_SIZE); 515 k3_dma_fill_desc(ds, dst, src, copy, num++, c->ccfg); 516 517 src += copy; 518 dst += copy; 519 len -= copy; 520 } while (len); 521 522 ds->desc_hw[num-1].lli = 0; /* end of link */ 523 return vchan_tx_prep(&c->vc, &ds->vd, flags); 524 } 525 526 static struct dma_async_tx_descriptor *k3_dma_prep_slave_sg( 527 struct dma_chan *chan, struct scatterlist *sgl, unsigned int sglen, 528 enum dma_transfer_direction dir, unsigned long flags, void *context) 529 { 530 struct k3_dma_chan *c = to_k3_chan(chan); 531 struct k3_dma_desc_sw *ds; 532 size_t len, avail, total = 0; 533 struct scatterlist *sg; 534 dma_addr_t addr, src = 0, dst = 0; 535 int num = sglen, i; 536 537 if (sgl == NULL) 538 return NULL; 539 540 c->cyclic = 0; 541 542 for_each_sg(sgl, sg, sglen, i) { 543 avail = sg_dma_len(sg); 544 if (avail > DMA_MAX_SIZE) 545 num += DIV_ROUND_UP(avail, DMA_MAX_SIZE) - 1; 546 } 547 548 ds = k3_dma_alloc_desc_resource(num, chan); 549 if (!ds) 550 return NULL; 551 num = 0; 552 k3_dma_config_write(chan, dir, &c->slave_config); 553 554 for_each_sg(sgl, sg, sglen, i) { 555 addr = sg_dma_address(sg); 556 avail = sg_dma_len(sg); 557 total += avail; 558 559 do { 560 len = min_t(size_t, avail, DMA_MAX_SIZE); 561 562 if (dir == DMA_MEM_TO_DEV) { 563 src = addr; 564 dst = c->dev_addr; 565 } else if (dir == DMA_DEV_TO_MEM) { 566 src = c->dev_addr; 567 dst = addr; 568 } 569 570 k3_dma_fill_desc(ds, dst, src, len, num++, c->ccfg); 571 572 addr += len; 573 avail -= len; 574 } while (avail); 575 } 576 577 ds->desc_hw[num-1].lli = 0; /* end of link */ 578 ds->size = total; 579 return vchan_tx_prep(&c->vc, &ds->vd, flags); 580 } 581 582 static struct dma_async_tx_descriptor * 583 k3_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, 584 size_t buf_len, size_t period_len, 585 enum dma_transfer_direction dir, 586 unsigned long flags) 587 { 588 struct k3_dma_chan *c = to_k3_chan(chan); 589 struct k3_dma_desc_sw *ds; 590 size_t len, avail, total = 0; 591 dma_addr_t addr, src = 0, dst = 0; 592 int num = 1, since = 0; 593 size_t modulo = DMA_CYCLIC_MAX_PERIOD; 594 u32 en_tc2 = 0; 595 596 dev_dbg(chan->device->dev, "%s: buf %pad, dst %pad, buf len %zu, period_len = %zu, dir %d\n", 597 __func__, &buf_addr, &to_k3_chan(chan)->dev_addr, 598 buf_len, period_len, (int)dir); 599 600 avail = buf_len; 601 if (avail > modulo) 602 num += DIV_ROUND_UP(avail, modulo) - 1; 603 604 ds = k3_dma_alloc_desc_resource(num, chan); 605 if (!ds) 606 return NULL; 607 608 c->cyclic = 1; 609 addr = buf_addr; 610 avail = buf_len; 611 total = avail; 612 num = 0; 613 k3_dma_config_write(chan, dir, &c->slave_config); 614 615 if (period_len < modulo) 616 modulo = period_len; 617 618 do { 619 len = min_t(size_t, avail, modulo); 620 621 if (dir == DMA_MEM_TO_DEV) { 622 src = addr; 623 dst = c->dev_addr; 624 } else if (dir == DMA_DEV_TO_MEM) { 625 src = c->dev_addr; 626 dst = addr; 627 } 628 since += len; 629 if (since >= period_len) { 630 /* descriptor asks for TC2 interrupt on completion */ 631 en_tc2 = CX_CFG_NODEIRQ; 632 since -= period_len; 633 } else 634 en_tc2 = 0; 635 636 k3_dma_fill_desc(ds, dst, src, len, num++, c->ccfg | en_tc2); 637 638 addr += len; 639 avail -= len; 640 } while (avail); 641 642 /* "Cyclic" == end of link points back to start of link */ 643 ds->desc_hw[num - 1].lli |= ds->desc_hw_lli; 644 645 ds->size = total; 646 647 return vchan_tx_prep(&c->vc, &ds->vd, flags); 648 } 649 650 static int k3_dma_config(struct dma_chan *chan, 651 struct dma_slave_config *cfg) 652 { 653 struct k3_dma_chan *c = to_k3_chan(chan); 654 655 memcpy(&c->slave_config, cfg, sizeof(*cfg)); 656 657 return 0; 658 } 659 660 static int k3_dma_config_write(struct dma_chan *chan, 661 enum dma_transfer_direction dir, 662 struct dma_slave_config *cfg) 663 { 664 struct k3_dma_chan *c = to_k3_chan(chan); 665 u32 maxburst = 0, val = 0; 666 enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED; 667 668 if (dir == DMA_DEV_TO_MEM) { 669 c->ccfg = CX_CFG_DSTINCR; 670 c->dev_addr = cfg->src_addr; 671 maxburst = cfg->src_maxburst; 672 width = cfg->src_addr_width; 673 } else if (dir == DMA_MEM_TO_DEV) { 674 c->ccfg = CX_CFG_SRCINCR; 675 c->dev_addr = cfg->dst_addr; 676 maxburst = cfg->dst_maxburst; 677 width = cfg->dst_addr_width; 678 } 679 switch (width) { 680 case DMA_SLAVE_BUSWIDTH_1_BYTE: 681 case DMA_SLAVE_BUSWIDTH_2_BYTES: 682 case DMA_SLAVE_BUSWIDTH_4_BYTES: 683 case DMA_SLAVE_BUSWIDTH_8_BYTES: 684 val = __ffs(width); 685 break; 686 default: 687 val = 3; 688 break; 689 } 690 c->ccfg |= (val << 12) | (val << 16); 691 692 if ((maxburst == 0) || (maxburst > 16)) 693 val = 15; 694 else 695 val = maxburst - 1; 696 c->ccfg |= (val << 20) | (val << 24); 697 c->ccfg |= CX_CFG_MEM2PER | CX_CFG_EN; 698 699 /* specific request line */ 700 c->ccfg |= c->vc.chan.chan_id << 4; 701 702 return 0; 703 } 704 705 static void k3_dma_free_desc(struct virt_dma_desc *vd) 706 { 707 struct k3_dma_desc_sw *ds = 708 container_of(vd, struct k3_dma_desc_sw, vd); 709 struct k3_dma_dev *d = to_k3_dma(vd->tx.chan->device); 710 711 dma_pool_free(d->pool, ds->desc_hw, ds->desc_hw_lli); 712 kfree(ds); 713 } 714 715 static int k3_dma_terminate_all(struct dma_chan *chan) 716 { 717 struct k3_dma_chan *c = to_k3_chan(chan); 718 struct k3_dma_dev *d = to_k3_dma(chan->device); 719 struct k3_dma_phy *p = c->phy; 720 unsigned long flags; 721 LIST_HEAD(head); 722 723 dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc); 724 725 /* Prevent this channel being scheduled */ 726 spin_lock(&d->lock); 727 list_del_init(&c->node); 728 spin_unlock(&d->lock); 729 730 /* Clear the tx descriptor lists */ 731 spin_lock_irqsave(&c->vc.lock, flags); 732 vchan_get_all_descriptors(&c->vc, &head); 733 if (p) { 734 /* vchan is assigned to a pchan - stop the channel */ 735 k3_dma_terminate_chan(p, d); 736 c->phy = NULL; 737 p->vchan = NULL; 738 if (p->ds_run) { 739 vchan_terminate_vdesc(&p->ds_run->vd); 740 p->ds_run = NULL; 741 } 742 p->ds_done = NULL; 743 } 744 spin_unlock_irqrestore(&c->vc.lock, flags); 745 vchan_dma_desc_free_list(&c->vc, &head); 746 747 return 0; 748 } 749 750 static void k3_dma_synchronize(struct dma_chan *chan) 751 { 752 struct k3_dma_chan *c = to_k3_chan(chan); 753 754 vchan_synchronize(&c->vc); 755 } 756 757 static int k3_dma_transfer_pause(struct dma_chan *chan) 758 { 759 struct k3_dma_chan *c = to_k3_chan(chan); 760 struct k3_dma_dev *d = to_k3_dma(chan->device); 761 struct k3_dma_phy *p = c->phy; 762 763 dev_dbg(d->slave.dev, "vchan %p: pause\n", &c->vc); 764 if (c->status == DMA_IN_PROGRESS) { 765 c->status = DMA_PAUSED; 766 if (p) { 767 k3_dma_pause_dma(p, false); 768 } else { 769 spin_lock(&d->lock); 770 list_del_init(&c->node); 771 spin_unlock(&d->lock); 772 } 773 } 774 775 return 0; 776 } 777 778 static int k3_dma_transfer_resume(struct dma_chan *chan) 779 { 780 struct k3_dma_chan *c = to_k3_chan(chan); 781 struct k3_dma_dev *d = to_k3_dma(chan->device); 782 struct k3_dma_phy *p = c->phy; 783 unsigned long flags; 784 785 dev_dbg(d->slave.dev, "vchan %p: resume\n", &c->vc); 786 spin_lock_irqsave(&c->vc.lock, flags); 787 if (c->status == DMA_PAUSED) { 788 c->status = DMA_IN_PROGRESS; 789 if (p) { 790 k3_dma_pause_dma(p, true); 791 } else if (!list_empty(&c->vc.desc_issued)) { 792 spin_lock(&d->lock); 793 list_add_tail(&c->node, &d->chan_pending); 794 spin_unlock(&d->lock); 795 } 796 } 797 spin_unlock_irqrestore(&c->vc.lock, flags); 798 799 return 0; 800 } 801 802 static const struct k3dma_soc_data k3_v1_dma_data = { 803 .flags = 0, 804 }; 805 806 static const struct k3dma_soc_data asp_v1_dma_data = { 807 .flags = K3_FLAG_NOCLK, 808 }; 809 810 static const struct of_device_id k3_pdma_dt_ids[] = { 811 { .compatible = "hisilicon,k3-dma-1.0", 812 .data = &k3_v1_dma_data 813 }, 814 { .compatible = "hisilicon,hisi-pcm-asp-dma-1.0", 815 .data = &asp_v1_dma_data 816 }, 817 {} 818 }; 819 MODULE_DEVICE_TABLE(of, k3_pdma_dt_ids); 820 821 static struct dma_chan *k3_of_dma_simple_xlate(struct of_phandle_args *dma_spec, 822 struct of_dma *ofdma) 823 { 824 struct k3_dma_dev *d = ofdma->of_dma_data; 825 unsigned int request = dma_spec->args[0]; 826 827 if (request >= d->dma_requests) 828 return NULL; 829 830 return dma_get_slave_channel(&(d->chans[request].vc.chan)); 831 } 832 833 static int k3_dma_probe(struct platform_device *op) 834 { 835 const struct k3dma_soc_data *soc_data; 836 struct k3_dma_dev *d; 837 const struct of_device_id *of_id; 838 int i, ret, irq = 0; 839 840 d = devm_kzalloc(&op->dev, sizeof(*d), GFP_KERNEL); 841 if (!d) 842 return -ENOMEM; 843 844 soc_data = device_get_match_data(&op->dev); 845 if (!soc_data) 846 return -EINVAL; 847 848 d->base = devm_platform_ioremap_resource(op, 0); 849 if (IS_ERR(d->base)) 850 return PTR_ERR(d->base); 851 852 of_id = of_match_device(k3_pdma_dt_ids, &op->dev); 853 if (of_id) { 854 of_property_read_u32((&op->dev)->of_node, 855 "dma-channels", &d->dma_channels); 856 of_property_read_u32((&op->dev)->of_node, 857 "dma-requests", &d->dma_requests); 858 ret = of_property_read_u32((&op->dev)->of_node, 859 "dma-channel-mask", &d->dma_channel_mask); 860 if (ret) { 861 dev_warn(&op->dev, 862 "dma-channel-mask doesn't exist, considering all as available.\n"); 863 d->dma_channel_mask = (u32)~0UL; 864 } 865 } 866 867 if (!(soc_data->flags & K3_FLAG_NOCLK)) { 868 d->clk = devm_clk_get(&op->dev, NULL); 869 if (IS_ERR(d->clk)) { 870 dev_err(&op->dev, "no dma clk\n"); 871 return PTR_ERR(d->clk); 872 } 873 } 874 875 irq = platform_get_irq(op, 0); 876 ret = devm_request_irq(&op->dev, irq, 877 k3_dma_int_handler, 0, DRIVER_NAME, d); 878 if (ret) 879 return ret; 880 881 d->irq = irq; 882 883 /* A DMA memory pool for LLIs, align on 32-byte boundary */ 884 d->pool = dmam_pool_create(DRIVER_NAME, &op->dev, 885 LLI_BLOCK_SIZE, 32, 0); 886 if (!d->pool) 887 return -ENOMEM; 888 889 /* init phy channel */ 890 d->phy = devm_kcalloc(&op->dev, 891 d->dma_channels, sizeof(struct k3_dma_phy), GFP_KERNEL); 892 if (d->phy == NULL) 893 return -ENOMEM; 894 895 for (i = 0; i < d->dma_channels; i++) { 896 struct k3_dma_phy *p; 897 898 if (!(d->dma_channel_mask & BIT(i))) 899 continue; 900 901 p = &d->phy[i]; 902 p->idx = i; 903 p->base = d->base + i * 0x40; 904 } 905 906 INIT_LIST_HEAD(&d->slave.channels); 907 dma_cap_set(DMA_SLAVE, d->slave.cap_mask); 908 dma_cap_set(DMA_MEMCPY, d->slave.cap_mask); 909 dma_cap_set(DMA_CYCLIC, d->slave.cap_mask); 910 d->slave.dev = &op->dev; 911 d->slave.device_free_chan_resources = k3_dma_free_chan_resources; 912 d->slave.device_tx_status = k3_dma_tx_status; 913 d->slave.device_prep_dma_memcpy = k3_dma_prep_memcpy; 914 d->slave.device_prep_slave_sg = k3_dma_prep_slave_sg; 915 d->slave.device_prep_dma_cyclic = k3_dma_prep_dma_cyclic; 916 d->slave.device_issue_pending = k3_dma_issue_pending; 917 d->slave.device_config = k3_dma_config; 918 d->slave.device_pause = k3_dma_transfer_pause; 919 d->slave.device_resume = k3_dma_transfer_resume; 920 d->slave.device_terminate_all = k3_dma_terminate_all; 921 d->slave.device_synchronize = k3_dma_synchronize; 922 d->slave.copy_align = DMAENGINE_ALIGN_8_BYTES; 923 924 /* init virtual channel */ 925 d->chans = devm_kcalloc(&op->dev, 926 d->dma_requests, sizeof(struct k3_dma_chan), GFP_KERNEL); 927 if (d->chans == NULL) 928 return -ENOMEM; 929 930 for (i = 0; i < d->dma_requests; i++) { 931 struct k3_dma_chan *c = &d->chans[i]; 932 933 c->status = DMA_IN_PROGRESS; 934 INIT_LIST_HEAD(&c->node); 935 c->vc.desc_free = k3_dma_free_desc; 936 vchan_init(&c->vc, &d->slave); 937 } 938 939 /* Enable clock before accessing registers */ 940 ret = clk_prepare_enable(d->clk); 941 if (ret < 0) { 942 dev_err(&op->dev, "clk_prepare_enable failed: %d\n", ret); 943 return ret; 944 } 945 946 k3_dma_enable_dma(d, true); 947 948 ret = dma_async_device_register(&d->slave); 949 if (ret) 950 goto dma_async_register_fail; 951 952 ret = of_dma_controller_register((&op->dev)->of_node, 953 k3_of_dma_simple_xlate, d); 954 if (ret) 955 goto of_dma_register_fail; 956 957 spin_lock_init(&d->lock); 958 INIT_LIST_HEAD(&d->chan_pending); 959 tasklet_init(&d->task, k3_dma_tasklet, (unsigned long)d); 960 platform_set_drvdata(op, d); 961 dev_info(&op->dev, "initialized\n"); 962 963 return 0; 964 965 of_dma_register_fail: 966 dma_async_device_unregister(&d->slave); 967 dma_async_register_fail: 968 clk_disable_unprepare(d->clk); 969 return ret; 970 } 971 972 static int k3_dma_remove(struct platform_device *op) 973 { 974 struct k3_dma_chan *c, *cn; 975 struct k3_dma_dev *d = platform_get_drvdata(op); 976 977 dma_async_device_unregister(&d->slave); 978 of_dma_controller_free((&op->dev)->of_node); 979 980 devm_free_irq(&op->dev, d->irq, d); 981 982 list_for_each_entry_safe(c, cn, &d->slave.channels, vc.chan.device_node) { 983 list_del(&c->vc.chan.device_node); 984 tasklet_kill(&c->vc.task); 985 } 986 tasklet_kill(&d->task); 987 clk_disable_unprepare(d->clk); 988 return 0; 989 } 990 991 #ifdef CONFIG_PM_SLEEP 992 static int k3_dma_suspend_dev(struct device *dev) 993 { 994 struct k3_dma_dev *d = dev_get_drvdata(dev); 995 u32 stat = 0; 996 997 stat = k3_dma_get_chan_stat(d); 998 if (stat) { 999 dev_warn(d->slave.dev, 1000 "chan %d is running fail to suspend\n", stat); 1001 return -1; 1002 } 1003 k3_dma_enable_dma(d, false); 1004 clk_disable_unprepare(d->clk); 1005 return 0; 1006 } 1007 1008 static int k3_dma_resume_dev(struct device *dev) 1009 { 1010 struct k3_dma_dev *d = dev_get_drvdata(dev); 1011 int ret = 0; 1012 1013 ret = clk_prepare_enable(d->clk); 1014 if (ret < 0) { 1015 dev_err(d->slave.dev, "clk_prepare_enable failed: %d\n", ret); 1016 return ret; 1017 } 1018 k3_dma_enable_dma(d, true); 1019 return 0; 1020 } 1021 #endif 1022 1023 static SIMPLE_DEV_PM_OPS(k3_dma_pmops, k3_dma_suspend_dev, k3_dma_resume_dev); 1024 1025 static struct platform_driver k3_pdma_driver = { 1026 .driver = { 1027 .name = DRIVER_NAME, 1028 .pm = &k3_dma_pmops, 1029 .of_match_table = k3_pdma_dt_ids, 1030 }, 1031 .probe = k3_dma_probe, 1032 .remove = k3_dma_remove, 1033 }; 1034 1035 module_platform_driver(k3_pdma_driver); 1036 1037 MODULE_DESCRIPTION("Hisilicon k3 DMA Driver"); 1038 MODULE_ALIAS("platform:k3dma"); 1039 MODULE_LICENSE("GPL v2"); 1040