1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2013 - 2015 Linaro Ltd. 4 * Copyright (c) 2013 Hisilicon Limited. 5 */ 6 #include <linux/sched.h> 7 #include <linux/device.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/dmapool.h> 10 #include <linux/dmaengine.h> 11 #include <linux/init.h> 12 #include <linux/interrupt.h> 13 #include <linux/kernel.h> 14 #include <linux/module.h> 15 #include <linux/platform_device.h> 16 #include <linux/slab.h> 17 #include <linux/spinlock.h> 18 #include <linux/of_device.h> 19 #include <linux/of.h> 20 #include <linux/clk.h> 21 #include <linux/of_dma.h> 22 23 #include "virt-dma.h" 24 25 #define DRIVER_NAME "k3-dma" 26 #define DMA_MAX_SIZE 0x1ffc 27 #define DMA_CYCLIC_MAX_PERIOD 0x1000 28 #define LLI_BLOCK_SIZE (4 * PAGE_SIZE) 29 30 #define INT_STAT 0x00 31 #define INT_TC1 0x04 32 #define INT_TC2 0x08 33 #define INT_ERR1 0x0c 34 #define INT_ERR2 0x10 35 #define INT_TC1_MASK 0x18 36 #define INT_TC2_MASK 0x1c 37 #define INT_ERR1_MASK 0x20 38 #define INT_ERR2_MASK 0x24 39 #define INT_TC1_RAW 0x600 40 #define INT_TC2_RAW 0x608 41 #define INT_ERR1_RAW 0x610 42 #define INT_ERR2_RAW 0x618 43 #define CH_PRI 0x688 44 #define CH_STAT 0x690 45 #define CX_CUR_CNT 0x704 46 #define CX_LLI 0x800 47 #define CX_CNT1 0x80c 48 #define CX_CNT0 0x810 49 #define CX_SRC 0x814 50 #define CX_DST 0x818 51 #define CX_CFG 0x81c 52 53 #define CX_LLI_CHAIN_EN 0x2 54 #define CX_CFG_EN 0x1 55 #define CX_CFG_NODEIRQ BIT(1) 56 #define CX_CFG_MEM2PER (0x1 << 2) 57 #define CX_CFG_PER2MEM (0x2 << 2) 58 #define CX_CFG_SRCINCR (0x1 << 31) 59 #define CX_CFG_DSTINCR (0x1 << 30) 60 61 struct k3_desc_hw { 62 u32 lli; 63 u32 reserved[3]; 64 u32 count; 65 u32 saddr; 66 u32 daddr; 67 u32 config; 68 } __aligned(32); 69 70 struct k3_dma_desc_sw { 71 struct virt_dma_desc vd; 72 dma_addr_t desc_hw_lli; 73 size_t desc_num; 74 size_t size; 75 struct k3_desc_hw *desc_hw; 76 }; 77 78 struct k3_dma_phy; 79 80 struct k3_dma_chan { 81 u32 ccfg; 82 struct virt_dma_chan vc; 83 struct k3_dma_phy *phy; 84 struct list_head node; 85 dma_addr_t dev_addr; 86 enum dma_status status; 87 bool cyclic; 88 struct dma_slave_config slave_config; 89 }; 90 91 struct k3_dma_phy { 92 u32 idx; 93 void __iomem *base; 94 struct k3_dma_chan *vchan; 95 struct k3_dma_desc_sw *ds_run; 96 struct k3_dma_desc_sw *ds_done; 97 }; 98 99 struct k3_dma_dev { 100 struct dma_device slave; 101 void __iomem *base; 102 struct tasklet_struct task; 103 spinlock_t lock; 104 struct list_head chan_pending; 105 struct k3_dma_phy *phy; 106 struct k3_dma_chan *chans; 107 struct clk *clk; 108 struct dma_pool *pool; 109 u32 dma_channels; 110 u32 dma_requests; 111 u32 dma_channel_mask; 112 unsigned int irq; 113 }; 114 115 116 #define K3_FLAG_NOCLK BIT(1) 117 118 struct k3dma_soc_data { 119 unsigned long flags; 120 }; 121 122 123 #define to_k3_dma(dmadev) container_of(dmadev, struct k3_dma_dev, slave) 124 125 static int k3_dma_config_write(struct dma_chan *chan, 126 enum dma_transfer_direction dir, 127 struct dma_slave_config *cfg); 128 129 static struct k3_dma_chan *to_k3_chan(struct dma_chan *chan) 130 { 131 return container_of(chan, struct k3_dma_chan, vc.chan); 132 } 133 134 static void k3_dma_pause_dma(struct k3_dma_phy *phy, bool on) 135 { 136 u32 val = 0; 137 138 if (on) { 139 val = readl_relaxed(phy->base + CX_CFG); 140 val |= CX_CFG_EN; 141 writel_relaxed(val, phy->base + CX_CFG); 142 } else { 143 val = readl_relaxed(phy->base + CX_CFG); 144 val &= ~CX_CFG_EN; 145 writel_relaxed(val, phy->base + CX_CFG); 146 } 147 } 148 149 static void k3_dma_terminate_chan(struct k3_dma_phy *phy, struct k3_dma_dev *d) 150 { 151 u32 val = 0; 152 153 k3_dma_pause_dma(phy, false); 154 155 val = 0x1 << phy->idx; 156 writel_relaxed(val, d->base + INT_TC1_RAW); 157 writel_relaxed(val, d->base + INT_TC2_RAW); 158 writel_relaxed(val, d->base + INT_ERR1_RAW); 159 writel_relaxed(val, d->base + INT_ERR2_RAW); 160 } 161 162 static void k3_dma_set_desc(struct k3_dma_phy *phy, struct k3_desc_hw *hw) 163 { 164 writel_relaxed(hw->lli, phy->base + CX_LLI); 165 writel_relaxed(hw->count, phy->base + CX_CNT0); 166 writel_relaxed(hw->saddr, phy->base + CX_SRC); 167 writel_relaxed(hw->daddr, phy->base + CX_DST); 168 writel_relaxed(hw->config, phy->base + CX_CFG); 169 } 170 171 static u32 k3_dma_get_curr_cnt(struct k3_dma_dev *d, struct k3_dma_phy *phy) 172 { 173 u32 cnt = 0; 174 175 cnt = readl_relaxed(d->base + CX_CUR_CNT + phy->idx * 0x10); 176 cnt &= 0xffff; 177 return cnt; 178 } 179 180 static u32 k3_dma_get_curr_lli(struct k3_dma_phy *phy) 181 { 182 return readl_relaxed(phy->base + CX_LLI); 183 } 184 185 static u32 k3_dma_get_chan_stat(struct k3_dma_dev *d) 186 { 187 return readl_relaxed(d->base + CH_STAT); 188 } 189 190 static void k3_dma_enable_dma(struct k3_dma_dev *d, bool on) 191 { 192 if (on) { 193 /* set same priority */ 194 writel_relaxed(0x0, d->base + CH_PRI); 195 196 /* unmask irq */ 197 writel_relaxed(0xffff, d->base + INT_TC1_MASK); 198 writel_relaxed(0xffff, d->base + INT_TC2_MASK); 199 writel_relaxed(0xffff, d->base + INT_ERR1_MASK); 200 writel_relaxed(0xffff, d->base + INT_ERR2_MASK); 201 } else { 202 /* mask irq */ 203 writel_relaxed(0x0, d->base + INT_TC1_MASK); 204 writel_relaxed(0x0, d->base + INT_TC2_MASK); 205 writel_relaxed(0x0, d->base + INT_ERR1_MASK); 206 writel_relaxed(0x0, d->base + INT_ERR2_MASK); 207 } 208 } 209 210 static irqreturn_t k3_dma_int_handler(int irq, void *dev_id) 211 { 212 struct k3_dma_dev *d = (struct k3_dma_dev *)dev_id; 213 struct k3_dma_phy *p; 214 struct k3_dma_chan *c; 215 u32 stat = readl_relaxed(d->base + INT_STAT); 216 u32 tc1 = readl_relaxed(d->base + INT_TC1); 217 u32 tc2 = readl_relaxed(d->base + INT_TC2); 218 u32 err1 = readl_relaxed(d->base + INT_ERR1); 219 u32 err2 = readl_relaxed(d->base + INT_ERR2); 220 u32 i, irq_chan = 0; 221 222 while (stat) { 223 i = __ffs(stat); 224 stat &= ~BIT(i); 225 if (likely(tc1 & BIT(i)) || (tc2 & BIT(i))) { 226 unsigned long flags; 227 228 p = &d->phy[i]; 229 c = p->vchan; 230 if (c && (tc1 & BIT(i))) { 231 spin_lock_irqsave(&c->vc.lock, flags); 232 if (p->ds_run != NULL) { 233 vchan_cookie_complete(&p->ds_run->vd); 234 p->ds_done = p->ds_run; 235 p->ds_run = NULL; 236 } 237 spin_unlock_irqrestore(&c->vc.lock, flags); 238 } 239 if (c && (tc2 & BIT(i))) { 240 spin_lock_irqsave(&c->vc.lock, flags); 241 if (p->ds_run != NULL) 242 vchan_cyclic_callback(&p->ds_run->vd); 243 spin_unlock_irqrestore(&c->vc.lock, flags); 244 } 245 irq_chan |= BIT(i); 246 } 247 if (unlikely((err1 & BIT(i)) || (err2 & BIT(i)))) 248 dev_warn(d->slave.dev, "DMA ERR\n"); 249 } 250 251 writel_relaxed(irq_chan, d->base + INT_TC1_RAW); 252 writel_relaxed(irq_chan, d->base + INT_TC2_RAW); 253 writel_relaxed(err1, d->base + INT_ERR1_RAW); 254 writel_relaxed(err2, d->base + INT_ERR2_RAW); 255 256 if (irq_chan) 257 tasklet_schedule(&d->task); 258 259 if (irq_chan || err1 || err2) 260 return IRQ_HANDLED; 261 262 return IRQ_NONE; 263 } 264 265 static int k3_dma_start_txd(struct k3_dma_chan *c) 266 { 267 struct k3_dma_dev *d = to_k3_dma(c->vc.chan.device); 268 struct virt_dma_desc *vd = vchan_next_desc(&c->vc); 269 270 if (!c->phy) 271 return -EAGAIN; 272 273 if (BIT(c->phy->idx) & k3_dma_get_chan_stat(d)) 274 return -EAGAIN; 275 276 /* Avoid losing track of ds_run if a transaction is in flight */ 277 if (c->phy->ds_run) 278 return -EAGAIN; 279 280 if (vd) { 281 struct k3_dma_desc_sw *ds = 282 container_of(vd, struct k3_dma_desc_sw, vd); 283 /* 284 * fetch and remove request from vc->desc_issued 285 * so vc->desc_issued only contains desc pending 286 */ 287 list_del(&ds->vd.node); 288 289 c->phy->ds_run = ds; 290 c->phy->ds_done = NULL; 291 /* start dma */ 292 k3_dma_set_desc(c->phy, &ds->desc_hw[0]); 293 return 0; 294 } 295 c->phy->ds_run = NULL; 296 c->phy->ds_done = NULL; 297 return -EAGAIN; 298 } 299 300 static void k3_dma_tasklet(unsigned long arg) 301 { 302 struct k3_dma_dev *d = (struct k3_dma_dev *)arg; 303 struct k3_dma_phy *p; 304 struct k3_dma_chan *c, *cn; 305 unsigned pch, pch_alloc = 0; 306 307 /* check new dma request of running channel in vc->desc_issued */ 308 list_for_each_entry_safe(c, cn, &d->slave.channels, vc.chan.device_node) { 309 spin_lock_irq(&c->vc.lock); 310 p = c->phy; 311 if (p && p->ds_done) { 312 if (k3_dma_start_txd(c)) { 313 /* No current txd associated with this channel */ 314 dev_dbg(d->slave.dev, "pchan %u: free\n", p->idx); 315 /* Mark this channel free */ 316 c->phy = NULL; 317 p->vchan = NULL; 318 } 319 } 320 spin_unlock_irq(&c->vc.lock); 321 } 322 323 /* check new channel request in d->chan_pending */ 324 spin_lock_irq(&d->lock); 325 for (pch = 0; pch < d->dma_channels; pch++) { 326 if (!(d->dma_channel_mask & (1 << pch))) 327 continue; 328 329 p = &d->phy[pch]; 330 331 if (p->vchan == NULL && !list_empty(&d->chan_pending)) { 332 c = list_first_entry(&d->chan_pending, 333 struct k3_dma_chan, node); 334 /* remove from d->chan_pending */ 335 list_del_init(&c->node); 336 pch_alloc |= 1 << pch; 337 /* Mark this channel allocated */ 338 p->vchan = c; 339 c->phy = p; 340 dev_dbg(d->slave.dev, "pchan %u: alloc vchan %p\n", pch, &c->vc); 341 } 342 } 343 spin_unlock_irq(&d->lock); 344 345 for (pch = 0; pch < d->dma_channels; pch++) { 346 if (!(d->dma_channel_mask & (1 << pch))) 347 continue; 348 349 if (pch_alloc & (1 << pch)) { 350 p = &d->phy[pch]; 351 c = p->vchan; 352 if (c) { 353 spin_lock_irq(&c->vc.lock); 354 k3_dma_start_txd(c); 355 spin_unlock_irq(&c->vc.lock); 356 } 357 } 358 } 359 } 360 361 static void k3_dma_free_chan_resources(struct dma_chan *chan) 362 { 363 struct k3_dma_chan *c = to_k3_chan(chan); 364 struct k3_dma_dev *d = to_k3_dma(chan->device); 365 unsigned long flags; 366 367 spin_lock_irqsave(&d->lock, flags); 368 list_del_init(&c->node); 369 spin_unlock_irqrestore(&d->lock, flags); 370 371 vchan_free_chan_resources(&c->vc); 372 c->ccfg = 0; 373 } 374 375 static enum dma_status k3_dma_tx_status(struct dma_chan *chan, 376 dma_cookie_t cookie, struct dma_tx_state *state) 377 { 378 struct k3_dma_chan *c = to_k3_chan(chan); 379 struct k3_dma_dev *d = to_k3_dma(chan->device); 380 struct k3_dma_phy *p; 381 struct virt_dma_desc *vd; 382 unsigned long flags; 383 enum dma_status ret; 384 size_t bytes = 0; 385 386 ret = dma_cookie_status(&c->vc.chan, cookie, state); 387 if (ret == DMA_COMPLETE) 388 return ret; 389 390 spin_lock_irqsave(&c->vc.lock, flags); 391 p = c->phy; 392 ret = c->status; 393 394 /* 395 * If the cookie is on our issue queue, then the residue is 396 * its total size. 397 */ 398 vd = vchan_find_desc(&c->vc, cookie); 399 if (vd && !c->cyclic) { 400 bytes = container_of(vd, struct k3_dma_desc_sw, vd)->size; 401 } else if ((!p) || (!p->ds_run)) { 402 bytes = 0; 403 } else { 404 struct k3_dma_desc_sw *ds = p->ds_run; 405 u32 clli = 0, index = 0; 406 407 bytes = k3_dma_get_curr_cnt(d, p); 408 clli = k3_dma_get_curr_lli(p); 409 index = ((clli - ds->desc_hw_lli) / 410 sizeof(struct k3_desc_hw)) + 1; 411 for (; index < ds->desc_num; index++) { 412 bytes += ds->desc_hw[index].count; 413 /* end of lli */ 414 if (!ds->desc_hw[index].lli) 415 break; 416 } 417 } 418 spin_unlock_irqrestore(&c->vc.lock, flags); 419 dma_set_residue(state, bytes); 420 return ret; 421 } 422 423 static void k3_dma_issue_pending(struct dma_chan *chan) 424 { 425 struct k3_dma_chan *c = to_k3_chan(chan); 426 struct k3_dma_dev *d = to_k3_dma(chan->device); 427 unsigned long flags; 428 429 spin_lock_irqsave(&c->vc.lock, flags); 430 /* add request to vc->desc_issued */ 431 if (vchan_issue_pending(&c->vc)) { 432 spin_lock(&d->lock); 433 if (!c->phy) { 434 if (list_empty(&c->node)) { 435 /* if new channel, add chan_pending */ 436 list_add_tail(&c->node, &d->chan_pending); 437 /* check in tasklet */ 438 tasklet_schedule(&d->task); 439 dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc); 440 } 441 } 442 spin_unlock(&d->lock); 443 } else 444 dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc); 445 spin_unlock_irqrestore(&c->vc.lock, flags); 446 } 447 448 static void k3_dma_fill_desc(struct k3_dma_desc_sw *ds, dma_addr_t dst, 449 dma_addr_t src, size_t len, u32 num, u32 ccfg) 450 { 451 if (num != ds->desc_num - 1) 452 ds->desc_hw[num].lli = ds->desc_hw_lli + (num + 1) * 453 sizeof(struct k3_desc_hw); 454 455 ds->desc_hw[num].lli |= CX_LLI_CHAIN_EN; 456 ds->desc_hw[num].count = len; 457 ds->desc_hw[num].saddr = src; 458 ds->desc_hw[num].daddr = dst; 459 ds->desc_hw[num].config = ccfg; 460 } 461 462 static struct k3_dma_desc_sw *k3_dma_alloc_desc_resource(int num, 463 struct dma_chan *chan) 464 { 465 struct k3_dma_chan *c = to_k3_chan(chan); 466 struct k3_dma_desc_sw *ds; 467 struct k3_dma_dev *d = to_k3_dma(chan->device); 468 int lli_limit = LLI_BLOCK_SIZE / sizeof(struct k3_desc_hw); 469 470 if (num > lli_limit) { 471 dev_dbg(chan->device->dev, "vch %p: sg num %d exceed max %d\n", 472 &c->vc, num, lli_limit); 473 return NULL; 474 } 475 476 ds = kzalloc(sizeof(*ds), GFP_NOWAIT); 477 if (!ds) 478 return NULL; 479 480 ds->desc_hw = dma_pool_zalloc(d->pool, GFP_NOWAIT, &ds->desc_hw_lli); 481 if (!ds->desc_hw) { 482 dev_dbg(chan->device->dev, "vch %p: dma alloc fail\n", &c->vc); 483 kfree(ds); 484 return NULL; 485 } 486 ds->desc_num = num; 487 return ds; 488 } 489 490 static struct dma_async_tx_descriptor *k3_dma_prep_memcpy( 491 struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, 492 size_t len, unsigned long flags) 493 { 494 struct k3_dma_chan *c = to_k3_chan(chan); 495 struct k3_dma_desc_sw *ds; 496 size_t copy = 0; 497 int num = 0; 498 499 if (!len) 500 return NULL; 501 502 num = DIV_ROUND_UP(len, DMA_MAX_SIZE); 503 504 ds = k3_dma_alloc_desc_resource(num, chan); 505 if (!ds) 506 return NULL; 507 508 c->cyclic = 0; 509 ds->size = len; 510 num = 0; 511 512 if (!c->ccfg) { 513 /* default is memtomem, without calling device_config */ 514 c->ccfg = CX_CFG_SRCINCR | CX_CFG_DSTINCR | CX_CFG_EN; 515 c->ccfg |= (0xf << 20) | (0xf << 24); /* burst = 16 */ 516 c->ccfg |= (0x3 << 12) | (0x3 << 16); /* width = 64 bit */ 517 } 518 519 do { 520 copy = min_t(size_t, len, DMA_MAX_SIZE); 521 k3_dma_fill_desc(ds, dst, src, copy, num++, c->ccfg); 522 523 src += copy; 524 dst += copy; 525 len -= copy; 526 } while (len); 527 528 ds->desc_hw[num-1].lli = 0; /* end of link */ 529 return vchan_tx_prep(&c->vc, &ds->vd, flags); 530 } 531 532 static struct dma_async_tx_descriptor *k3_dma_prep_slave_sg( 533 struct dma_chan *chan, struct scatterlist *sgl, unsigned int sglen, 534 enum dma_transfer_direction dir, unsigned long flags, void *context) 535 { 536 struct k3_dma_chan *c = to_k3_chan(chan); 537 struct k3_dma_desc_sw *ds; 538 size_t len, avail, total = 0; 539 struct scatterlist *sg; 540 dma_addr_t addr, src = 0, dst = 0; 541 int num = sglen, i; 542 543 if (sgl == NULL) 544 return NULL; 545 546 c->cyclic = 0; 547 548 for_each_sg(sgl, sg, sglen, i) { 549 avail = sg_dma_len(sg); 550 if (avail > DMA_MAX_SIZE) 551 num += DIV_ROUND_UP(avail, DMA_MAX_SIZE) - 1; 552 } 553 554 ds = k3_dma_alloc_desc_resource(num, chan); 555 if (!ds) 556 return NULL; 557 num = 0; 558 k3_dma_config_write(chan, dir, &c->slave_config); 559 560 for_each_sg(sgl, sg, sglen, i) { 561 addr = sg_dma_address(sg); 562 avail = sg_dma_len(sg); 563 total += avail; 564 565 do { 566 len = min_t(size_t, avail, DMA_MAX_SIZE); 567 568 if (dir == DMA_MEM_TO_DEV) { 569 src = addr; 570 dst = c->dev_addr; 571 } else if (dir == DMA_DEV_TO_MEM) { 572 src = c->dev_addr; 573 dst = addr; 574 } 575 576 k3_dma_fill_desc(ds, dst, src, len, num++, c->ccfg); 577 578 addr += len; 579 avail -= len; 580 } while (avail); 581 } 582 583 ds->desc_hw[num-1].lli = 0; /* end of link */ 584 ds->size = total; 585 return vchan_tx_prep(&c->vc, &ds->vd, flags); 586 } 587 588 static struct dma_async_tx_descriptor * 589 k3_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, 590 size_t buf_len, size_t period_len, 591 enum dma_transfer_direction dir, 592 unsigned long flags) 593 { 594 struct k3_dma_chan *c = to_k3_chan(chan); 595 struct k3_dma_desc_sw *ds; 596 size_t len, avail, total = 0; 597 dma_addr_t addr, src = 0, dst = 0; 598 int num = 1, since = 0; 599 size_t modulo = DMA_CYCLIC_MAX_PERIOD; 600 u32 en_tc2 = 0; 601 602 dev_dbg(chan->device->dev, "%s: buf %pad, dst %pad, buf len %zu, period_len = %zu, dir %d\n", 603 __func__, &buf_addr, &to_k3_chan(chan)->dev_addr, 604 buf_len, period_len, (int)dir); 605 606 avail = buf_len; 607 if (avail > modulo) 608 num += DIV_ROUND_UP(avail, modulo) - 1; 609 610 ds = k3_dma_alloc_desc_resource(num, chan); 611 if (!ds) 612 return NULL; 613 614 c->cyclic = 1; 615 addr = buf_addr; 616 avail = buf_len; 617 total = avail; 618 num = 0; 619 k3_dma_config_write(chan, dir, &c->slave_config); 620 621 if (period_len < modulo) 622 modulo = period_len; 623 624 do { 625 len = min_t(size_t, avail, modulo); 626 627 if (dir == DMA_MEM_TO_DEV) { 628 src = addr; 629 dst = c->dev_addr; 630 } else if (dir == DMA_DEV_TO_MEM) { 631 src = c->dev_addr; 632 dst = addr; 633 } 634 since += len; 635 if (since >= period_len) { 636 /* descriptor asks for TC2 interrupt on completion */ 637 en_tc2 = CX_CFG_NODEIRQ; 638 since -= period_len; 639 } else 640 en_tc2 = 0; 641 642 k3_dma_fill_desc(ds, dst, src, len, num++, c->ccfg | en_tc2); 643 644 addr += len; 645 avail -= len; 646 } while (avail); 647 648 /* "Cyclic" == end of link points back to start of link */ 649 ds->desc_hw[num - 1].lli |= ds->desc_hw_lli; 650 651 ds->size = total; 652 653 return vchan_tx_prep(&c->vc, &ds->vd, flags); 654 } 655 656 static int k3_dma_config(struct dma_chan *chan, 657 struct dma_slave_config *cfg) 658 { 659 struct k3_dma_chan *c = to_k3_chan(chan); 660 661 memcpy(&c->slave_config, cfg, sizeof(*cfg)); 662 663 return 0; 664 } 665 666 static int k3_dma_config_write(struct dma_chan *chan, 667 enum dma_transfer_direction dir, 668 struct dma_slave_config *cfg) 669 { 670 struct k3_dma_chan *c = to_k3_chan(chan); 671 u32 maxburst = 0, val = 0; 672 enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED; 673 674 if (dir == DMA_DEV_TO_MEM) { 675 c->ccfg = CX_CFG_DSTINCR; 676 c->dev_addr = cfg->src_addr; 677 maxburst = cfg->src_maxburst; 678 width = cfg->src_addr_width; 679 } else if (dir == DMA_MEM_TO_DEV) { 680 c->ccfg = CX_CFG_SRCINCR; 681 c->dev_addr = cfg->dst_addr; 682 maxburst = cfg->dst_maxburst; 683 width = cfg->dst_addr_width; 684 } 685 switch (width) { 686 case DMA_SLAVE_BUSWIDTH_1_BYTE: 687 case DMA_SLAVE_BUSWIDTH_2_BYTES: 688 case DMA_SLAVE_BUSWIDTH_4_BYTES: 689 case DMA_SLAVE_BUSWIDTH_8_BYTES: 690 val = __ffs(width); 691 break; 692 default: 693 val = 3; 694 break; 695 } 696 c->ccfg |= (val << 12) | (val << 16); 697 698 if ((maxburst == 0) || (maxburst > 16)) 699 val = 15; 700 else 701 val = maxburst - 1; 702 c->ccfg |= (val << 20) | (val << 24); 703 c->ccfg |= CX_CFG_MEM2PER | CX_CFG_EN; 704 705 /* specific request line */ 706 c->ccfg |= c->vc.chan.chan_id << 4; 707 708 return 0; 709 } 710 711 static void k3_dma_free_desc(struct virt_dma_desc *vd) 712 { 713 struct k3_dma_desc_sw *ds = 714 container_of(vd, struct k3_dma_desc_sw, vd); 715 struct k3_dma_dev *d = to_k3_dma(vd->tx.chan->device); 716 717 dma_pool_free(d->pool, ds->desc_hw, ds->desc_hw_lli); 718 kfree(ds); 719 } 720 721 static int k3_dma_terminate_all(struct dma_chan *chan) 722 { 723 struct k3_dma_chan *c = to_k3_chan(chan); 724 struct k3_dma_dev *d = to_k3_dma(chan->device); 725 struct k3_dma_phy *p = c->phy; 726 unsigned long flags; 727 LIST_HEAD(head); 728 729 dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc); 730 731 /* Prevent this channel being scheduled */ 732 spin_lock(&d->lock); 733 list_del_init(&c->node); 734 spin_unlock(&d->lock); 735 736 /* Clear the tx descriptor lists */ 737 spin_lock_irqsave(&c->vc.lock, flags); 738 vchan_get_all_descriptors(&c->vc, &head); 739 if (p) { 740 /* vchan is assigned to a pchan - stop the channel */ 741 k3_dma_terminate_chan(p, d); 742 c->phy = NULL; 743 p->vchan = NULL; 744 if (p->ds_run) { 745 vchan_terminate_vdesc(&p->ds_run->vd); 746 p->ds_run = NULL; 747 } 748 p->ds_done = NULL; 749 } 750 spin_unlock_irqrestore(&c->vc.lock, flags); 751 vchan_dma_desc_free_list(&c->vc, &head); 752 753 return 0; 754 } 755 756 static void k3_dma_synchronize(struct dma_chan *chan) 757 { 758 struct k3_dma_chan *c = to_k3_chan(chan); 759 760 vchan_synchronize(&c->vc); 761 } 762 763 static int k3_dma_transfer_pause(struct dma_chan *chan) 764 { 765 struct k3_dma_chan *c = to_k3_chan(chan); 766 struct k3_dma_dev *d = to_k3_dma(chan->device); 767 struct k3_dma_phy *p = c->phy; 768 769 dev_dbg(d->slave.dev, "vchan %p: pause\n", &c->vc); 770 if (c->status == DMA_IN_PROGRESS) { 771 c->status = DMA_PAUSED; 772 if (p) { 773 k3_dma_pause_dma(p, false); 774 } else { 775 spin_lock(&d->lock); 776 list_del_init(&c->node); 777 spin_unlock(&d->lock); 778 } 779 } 780 781 return 0; 782 } 783 784 static int k3_dma_transfer_resume(struct dma_chan *chan) 785 { 786 struct k3_dma_chan *c = to_k3_chan(chan); 787 struct k3_dma_dev *d = to_k3_dma(chan->device); 788 struct k3_dma_phy *p = c->phy; 789 unsigned long flags; 790 791 dev_dbg(d->slave.dev, "vchan %p: resume\n", &c->vc); 792 spin_lock_irqsave(&c->vc.lock, flags); 793 if (c->status == DMA_PAUSED) { 794 c->status = DMA_IN_PROGRESS; 795 if (p) { 796 k3_dma_pause_dma(p, true); 797 } else if (!list_empty(&c->vc.desc_issued)) { 798 spin_lock(&d->lock); 799 list_add_tail(&c->node, &d->chan_pending); 800 spin_unlock(&d->lock); 801 } 802 } 803 spin_unlock_irqrestore(&c->vc.lock, flags); 804 805 return 0; 806 } 807 808 static const struct k3dma_soc_data k3_v1_dma_data = { 809 .flags = 0, 810 }; 811 812 static const struct k3dma_soc_data asp_v1_dma_data = { 813 .flags = K3_FLAG_NOCLK, 814 }; 815 816 static const struct of_device_id k3_pdma_dt_ids[] = { 817 { .compatible = "hisilicon,k3-dma-1.0", 818 .data = &k3_v1_dma_data 819 }, 820 { .compatible = "hisilicon,hisi-pcm-asp-dma-1.0", 821 .data = &asp_v1_dma_data 822 }, 823 {} 824 }; 825 MODULE_DEVICE_TABLE(of, k3_pdma_dt_ids); 826 827 static struct dma_chan *k3_of_dma_simple_xlate(struct of_phandle_args *dma_spec, 828 struct of_dma *ofdma) 829 { 830 struct k3_dma_dev *d = ofdma->of_dma_data; 831 unsigned int request = dma_spec->args[0]; 832 833 if (request >= d->dma_requests) 834 return NULL; 835 836 return dma_get_slave_channel(&(d->chans[request].vc.chan)); 837 } 838 839 static int k3_dma_probe(struct platform_device *op) 840 { 841 const struct k3dma_soc_data *soc_data; 842 struct k3_dma_dev *d; 843 const struct of_device_id *of_id; 844 int i, ret, irq = 0; 845 846 d = devm_kzalloc(&op->dev, sizeof(*d), GFP_KERNEL); 847 if (!d) 848 return -ENOMEM; 849 850 soc_data = device_get_match_data(&op->dev); 851 if (!soc_data) 852 return -EINVAL; 853 854 d->base = devm_platform_ioremap_resource(op, 0); 855 if (IS_ERR(d->base)) 856 return PTR_ERR(d->base); 857 858 of_id = of_match_device(k3_pdma_dt_ids, &op->dev); 859 if (of_id) { 860 of_property_read_u32((&op->dev)->of_node, 861 "dma-channels", &d->dma_channels); 862 of_property_read_u32((&op->dev)->of_node, 863 "dma-requests", &d->dma_requests); 864 ret = of_property_read_u32((&op->dev)->of_node, 865 "dma-channel-mask", &d->dma_channel_mask); 866 if (ret) { 867 dev_warn(&op->dev, 868 "dma-channel-mask doesn't exist, considering all as available.\n"); 869 d->dma_channel_mask = (u32)~0UL; 870 } 871 } 872 873 if (!(soc_data->flags & K3_FLAG_NOCLK)) { 874 d->clk = devm_clk_get(&op->dev, NULL); 875 if (IS_ERR(d->clk)) { 876 dev_err(&op->dev, "no dma clk\n"); 877 return PTR_ERR(d->clk); 878 } 879 } 880 881 irq = platform_get_irq(op, 0); 882 ret = devm_request_irq(&op->dev, irq, 883 k3_dma_int_handler, 0, DRIVER_NAME, d); 884 if (ret) 885 return ret; 886 887 d->irq = irq; 888 889 /* A DMA memory pool for LLIs, align on 32-byte boundary */ 890 d->pool = dmam_pool_create(DRIVER_NAME, &op->dev, 891 LLI_BLOCK_SIZE, 32, 0); 892 if (!d->pool) 893 return -ENOMEM; 894 895 /* init phy channel */ 896 d->phy = devm_kcalloc(&op->dev, 897 d->dma_channels, sizeof(struct k3_dma_phy), GFP_KERNEL); 898 if (d->phy == NULL) 899 return -ENOMEM; 900 901 for (i = 0; i < d->dma_channels; i++) { 902 struct k3_dma_phy *p; 903 904 if (!(d->dma_channel_mask & BIT(i))) 905 continue; 906 907 p = &d->phy[i]; 908 p->idx = i; 909 p->base = d->base + i * 0x40; 910 } 911 912 INIT_LIST_HEAD(&d->slave.channels); 913 dma_cap_set(DMA_SLAVE, d->slave.cap_mask); 914 dma_cap_set(DMA_MEMCPY, d->slave.cap_mask); 915 dma_cap_set(DMA_CYCLIC, d->slave.cap_mask); 916 d->slave.dev = &op->dev; 917 d->slave.device_free_chan_resources = k3_dma_free_chan_resources; 918 d->slave.device_tx_status = k3_dma_tx_status; 919 d->slave.device_prep_dma_memcpy = k3_dma_prep_memcpy; 920 d->slave.device_prep_slave_sg = k3_dma_prep_slave_sg; 921 d->slave.device_prep_dma_cyclic = k3_dma_prep_dma_cyclic; 922 d->slave.device_issue_pending = k3_dma_issue_pending; 923 d->slave.device_config = k3_dma_config; 924 d->slave.device_pause = k3_dma_transfer_pause; 925 d->slave.device_resume = k3_dma_transfer_resume; 926 d->slave.device_terminate_all = k3_dma_terminate_all; 927 d->slave.device_synchronize = k3_dma_synchronize; 928 d->slave.copy_align = DMAENGINE_ALIGN_8_BYTES; 929 930 /* init virtual channel */ 931 d->chans = devm_kcalloc(&op->dev, 932 d->dma_requests, sizeof(struct k3_dma_chan), GFP_KERNEL); 933 if (d->chans == NULL) 934 return -ENOMEM; 935 936 for (i = 0; i < d->dma_requests; i++) { 937 struct k3_dma_chan *c = &d->chans[i]; 938 939 c->status = DMA_IN_PROGRESS; 940 INIT_LIST_HEAD(&c->node); 941 c->vc.desc_free = k3_dma_free_desc; 942 vchan_init(&c->vc, &d->slave); 943 } 944 945 /* Enable clock before accessing registers */ 946 ret = clk_prepare_enable(d->clk); 947 if (ret < 0) { 948 dev_err(&op->dev, "clk_prepare_enable failed: %d\n", ret); 949 return ret; 950 } 951 952 k3_dma_enable_dma(d, true); 953 954 ret = dma_async_device_register(&d->slave); 955 if (ret) 956 goto dma_async_register_fail; 957 958 ret = of_dma_controller_register((&op->dev)->of_node, 959 k3_of_dma_simple_xlate, d); 960 if (ret) 961 goto of_dma_register_fail; 962 963 spin_lock_init(&d->lock); 964 INIT_LIST_HEAD(&d->chan_pending); 965 tasklet_init(&d->task, k3_dma_tasklet, (unsigned long)d); 966 platform_set_drvdata(op, d); 967 dev_info(&op->dev, "initialized\n"); 968 969 return 0; 970 971 of_dma_register_fail: 972 dma_async_device_unregister(&d->slave); 973 dma_async_register_fail: 974 clk_disable_unprepare(d->clk); 975 return ret; 976 } 977 978 static int k3_dma_remove(struct platform_device *op) 979 { 980 struct k3_dma_chan *c, *cn; 981 struct k3_dma_dev *d = platform_get_drvdata(op); 982 983 dma_async_device_unregister(&d->slave); 984 of_dma_controller_free((&op->dev)->of_node); 985 986 devm_free_irq(&op->dev, d->irq, d); 987 988 list_for_each_entry_safe(c, cn, &d->slave.channels, vc.chan.device_node) { 989 list_del(&c->vc.chan.device_node); 990 tasklet_kill(&c->vc.task); 991 } 992 tasklet_kill(&d->task); 993 clk_disable_unprepare(d->clk); 994 return 0; 995 } 996 997 #ifdef CONFIG_PM_SLEEP 998 static int k3_dma_suspend_dev(struct device *dev) 999 { 1000 struct k3_dma_dev *d = dev_get_drvdata(dev); 1001 u32 stat = 0; 1002 1003 stat = k3_dma_get_chan_stat(d); 1004 if (stat) { 1005 dev_warn(d->slave.dev, 1006 "chan %d is running fail to suspend\n", stat); 1007 return -1; 1008 } 1009 k3_dma_enable_dma(d, false); 1010 clk_disable_unprepare(d->clk); 1011 return 0; 1012 } 1013 1014 static int k3_dma_resume_dev(struct device *dev) 1015 { 1016 struct k3_dma_dev *d = dev_get_drvdata(dev); 1017 int ret = 0; 1018 1019 ret = clk_prepare_enable(d->clk); 1020 if (ret < 0) { 1021 dev_err(d->slave.dev, "clk_prepare_enable failed: %d\n", ret); 1022 return ret; 1023 } 1024 k3_dma_enable_dma(d, true); 1025 return 0; 1026 } 1027 #endif 1028 1029 static SIMPLE_DEV_PM_OPS(k3_dma_pmops, k3_dma_suspend_dev, k3_dma_resume_dev); 1030 1031 static struct platform_driver k3_pdma_driver = { 1032 .driver = { 1033 .name = DRIVER_NAME, 1034 .pm = &k3_dma_pmops, 1035 .of_match_table = k3_pdma_dt_ids, 1036 }, 1037 .probe = k3_dma_probe, 1038 .remove = k3_dma_remove, 1039 }; 1040 1041 module_platform_driver(k3_pdma_driver); 1042 1043 MODULE_DESCRIPTION("Hisilicon k3 DMA Driver"); 1044 MODULE_ALIAS("platform:k3dma"); 1045 MODULE_LICENSE("GPL v2"); 1046