1 /* 2 * IMG Multi-threaded DMA Controller (MDC) 3 * 4 * Copyright (C) 2009,2012,2013 Imagination Technologies Ltd. 5 * Copyright (C) 2014 Google, Inc. 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms and conditions of the GNU General Public License, 9 * version 2, as published by the Free Software Foundation. 10 */ 11 12 #include <linux/clk.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/dmaengine.h> 15 #include <linux/dmapool.h> 16 #include <linux/interrupt.h> 17 #include <linux/io.h> 18 #include <linux/irq.h> 19 #include <linux/kernel.h> 20 #include <linux/mfd/syscon.h> 21 #include <linux/module.h> 22 #include <linux/of.h> 23 #include <linux/of_device.h> 24 #include <linux/of_dma.h> 25 #include <linux/platform_device.h> 26 #include <linux/pm_runtime.h> 27 #include <linux/regmap.h> 28 #include <linux/slab.h> 29 #include <linux/spinlock.h> 30 31 #include "dmaengine.h" 32 #include "virt-dma.h" 33 34 #define MDC_MAX_DMA_CHANNELS 32 35 36 #define MDC_GENERAL_CONFIG 0x000 37 #define MDC_GENERAL_CONFIG_LIST_IEN BIT(31) 38 #define MDC_GENERAL_CONFIG_IEN BIT(29) 39 #define MDC_GENERAL_CONFIG_LEVEL_INT BIT(28) 40 #define MDC_GENERAL_CONFIG_INC_W BIT(12) 41 #define MDC_GENERAL_CONFIG_INC_R BIT(8) 42 #define MDC_GENERAL_CONFIG_PHYSICAL_W BIT(7) 43 #define MDC_GENERAL_CONFIG_WIDTH_W_SHIFT 4 44 #define MDC_GENERAL_CONFIG_WIDTH_W_MASK 0x7 45 #define MDC_GENERAL_CONFIG_PHYSICAL_R BIT(3) 46 #define MDC_GENERAL_CONFIG_WIDTH_R_SHIFT 0 47 #define MDC_GENERAL_CONFIG_WIDTH_R_MASK 0x7 48 49 #define MDC_READ_PORT_CONFIG 0x004 50 #define MDC_READ_PORT_CONFIG_STHREAD_SHIFT 28 51 #define MDC_READ_PORT_CONFIG_STHREAD_MASK 0xf 52 #define MDC_READ_PORT_CONFIG_RTHREAD_SHIFT 24 53 #define MDC_READ_PORT_CONFIG_RTHREAD_MASK 0xf 54 #define MDC_READ_PORT_CONFIG_WTHREAD_SHIFT 16 55 #define MDC_READ_PORT_CONFIG_WTHREAD_MASK 0xf 56 #define MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT 4 57 #define MDC_READ_PORT_CONFIG_BURST_SIZE_MASK 0xff 58 #define MDC_READ_PORT_CONFIG_DREQ_ENABLE BIT(1) 59 60 #define MDC_READ_ADDRESS 0x008 61 62 #define MDC_WRITE_ADDRESS 0x00c 63 64 #define MDC_TRANSFER_SIZE 0x010 65 #define MDC_TRANSFER_SIZE_MASK 0xffffff 66 67 #define MDC_LIST_NODE_ADDRESS 0x014 68 69 #define MDC_CMDS_PROCESSED 0x018 70 #define MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT 16 71 #define MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK 0x3f 72 #define MDC_CMDS_PROCESSED_INT_ACTIVE BIT(8) 73 #define MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT 0 74 #define MDC_CMDS_PROCESSED_CMDS_DONE_MASK 0x3f 75 76 #define MDC_CONTROL_AND_STATUS 0x01c 77 #define MDC_CONTROL_AND_STATUS_CANCEL BIT(20) 78 #define MDC_CONTROL_AND_STATUS_LIST_EN BIT(4) 79 #define MDC_CONTROL_AND_STATUS_EN BIT(0) 80 81 #define MDC_ACTIVE_TRANSFER_SIZE 0x030 82 83 #define MDC_GLOBAL_CONFIG_A 0x900 84 #define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT 16 85 #define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK 0xff 86 #define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT 8 87 #define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK 0xff 88 #define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT 0 89 #define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK 0xff 90 91 struct mdc_hw_list_desc { 92 u32 gen_conf; 93 u32 readport_conf; 94 u32 read_addr; 95 u32 write_addr; 96 u32 xfer_size; 97 u32 node_addr; 98 u32 cmds_done; 99 u32 ctrl_status; 100 /* 101 * Not part of the list descriptor, but instead used by the CPU to 102 * traverse the list. 103 */ 104 struct mdc_hw_list_desc *next_desc; 105 }; 106 107 struct mdc_tx_desc { 108 struct mdc_chan *chan; 109 struct virt_dma_desc vd; 110 dma_addr_t list_phys; 111 struct mdc_hw_list_desc *list; 112 bool cyclic; 113 bool cmd_loaded; 114 unsigned int list_len; 115 unsigned int list_period_len; 116 size_t list_xfer_size; 117 unsigned int list_cmds_done; 118 }; 119 120 struct mdc_chan { 121 struct mdc_dma *mdma; 122 struct virt_dma_chan vc; 123 struct dma_slave_config config; 124 struct mdc_tx_desc *desc; 125 int irq; 126 unsigned int periph; 127 unsigned int thread; 128 unsigned int chan_nr; 129 }; 130 131 struct mdc_dma_soc_data { 132 void (*enable_chan)(struct mdc_chan *mchan); 133 void (*disable_chan)(struct mdc_chan *mchan); 134 }; 135 136 struct mdc_dma { 137 struct dma_device dma_dev; 138 void __iomem *regs; 139 struct clk *clk; 140 struct dma_pool *desc_pool; 141 struct regmap *periph_regs; 142 spinlock_t lock; 143 unsigned int nr_threads; 144 unsigned int nr_channels; 145 unsigned int bus_width; 146 unsigned int max_burst_mult; 147 unsigned int max_xfer_size; 148 const struct mdc_dma_soc_data *soc; 149 struct mdc_chan channels[MDC_MAX_DMA_CHANNELS]; 150 }; 151 152 static inline u32 mdc_readl(struct mdc_dma *mdma, u32 reg) 153 { 154 return readl(mdma->regs + reg); 155 } 156 157 static inline void mdc_writel(struct mdc_dma *mdma, u32 val, u32 reg) 158 { 159 writel(val, mdma->regs + reg); 160 } 161 162 static inline u32 mdc_chan_readl(struct mdc_chan *mchan, u32 reg) 163 { 164 return mdc_readl(mchan->mdma, mchan->chan_nr * 0x040 + reg); 165 } 166 167 static inline void mdc_chan_writel(struct mdc_chan *mchan, u32 val, u32 reg) 168 { 169 mdc_writel(mchan->mdma, val, mchan->chan_nr * 0x040 + reg); 170 } 171 172 static inline struct mdc_chan *to_mdc_chan(struct dma_chan *c) 173 { 174 return container_of(to_virt_chan(c), struct mdc_chan, vc); 175 } 176 177 static inline struct mdc_tx_desc *to_mdc_desc(struct dma_async_tx_descriptor *t) 178 { 179 struct virt_dma_desc *vdesc = container_of(t, struct virt_dma_desc, tx); 180 181 return container_of(vdesc, struct mdc_tx_desc, vd); 182 } 183 184 static inline struct device *mdma2dev(struct mdc_dma *mdma) 185 { 186 return mdma->dma_dev.dev; 187 } 188 189 static inline unsigned int to_mdc_width(unsigned int bytes) 190 { 191 return ffs(bytes) - 1; 192 } 193 194 static inline void mdc_set_read_width(struct mdc_hw_list_desc *ldesc, 195 unsigned int bytes) 196 { 197 ldesc->gen_conf |= to_mdc_width(bytes) << 198 MDC_GENERAL_CONFIG_WIDTH_R_SHIFT; 199 } 200 201 static inline void mdc_set_write_width(struct mdc_hw_list_desc *ldesc, 202 unsigned int bytes) 203 { 204 ldesc->gen_conf |= to_mdc_width(bytes) << 205 MDC_GENERAL_CONFIG_WIDTH_W_SHIFT; 206 } 207 208 static void mdc_list_desc_config(struct mdc_chan *mchan, 209 struct mdc_hw_list_desc *ldesc, 210 enum dma_transfer_direction dir, 211 dma_addr_t src, dma_addr_t dst, size_t len) 212 { 213 struct mdc_dma *mdma = mchan->mdma; 214 unsigned int max_burst, burst_size; 215 216 ldesc->gen_conf = MDC_GENERAL_CONFIG_IEN | MDC_GENERAL_CONFIG_LIST_IEN | 217 MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W | 218 MDC_GENERAL_CONFIG_PHYSICAL_R; 219 ldesc->readport_conf = 220 (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) | 221 (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) | 222 (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT); 223 ldesc->read_addr = src; 224 ldesc->write_addr = dst; 225 ldesc->xfer_size = len - 1; 226 ldesc->node_addr = 0; 227 ldesc->cmds_done = 0; 228 ldesc->ctrl_status = MDC_CONTROL_AND_STATUS_LIST_EN | 229 MDC_CONTROL_AND_STATUS_EN; 230 ldesc->next_desc = NULL; 231 232 if (IS_ALIGNED(dst, mdma->bus_width) && 233 IS_ALIGNED(src, mdma->bus_width)) 234 max_burst = mdma->bus_width * mdma->max_burst_mult; 235 else 236 max_burst = mdma->bus_width * (mdma->max_burst_mult - 1); 237 238 if (dir == DMA_MEM_TO_DEV) { 239 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R; 240 ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE; 241 mdc_set_read_width(ldesc, mdma->bus_width); 242 mdc_set_write_width(ldesc, mchan->config.dst_addr_width); 243 burst_size = min(max_burst, mchan->config.dst_maxburst * 244 mchan->config.dst_addr_width); 245 } else if (dir == DMA_DEV_TO_MEM) { 246 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_W; 247 ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE; 248 mdc_set_read_width(ldesc, mchan->config.src_addr_width); 249 mdc_set_write_width(ldesc, mdma->bus_width); 250 burst_size = min(max_burst, mchan->config.src_maxburst * 251 mchan->config.src_addr_width); 252 } else { 253 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R | 254 MDC_GENERAL_CONFIG_INC_W; 255 mdc_set_read_width(ldesc, mdma->bus_width); 256 mdc_set_write_width(ldesc, mdma->bus_width); 257 burst_size = max_burst; 258 } 259 ldesc->readport_conf |= (burst_size - 1) << 260 MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT; 261 } 262 263 static void mdc_list_desc_free(struct mdc_tx_desc *mdesc) 264 { 265 struct mdc_dma *mdma = mdesc->chan->mdma; 266 struct mdc_hw_list_desc *curr, *next; 267 dma_addr_t curr_phys, next_phys; 268 269 curr = mdesc->list; 270 curr_phys = mdesc->list_phys; 271 while (curr) { 272 next = curr->next_desc; 273 next_phys = curr->node_addr; 274 dma_pool_free(mdma->desc_pool, curr, curr_phys); 275 curr = next; 276 curr_phys = next_phys; 277 } 278 } 279 280 static void mdc_desc_free(struct virt_dma_desc *vd) 281 { 282 struct mdc_tx_desc *mdesc = to_mdc_desc(&vd->tx); 283 284 mdc_list_desc_free(mdesc); 285 kfree(mdesc); 286 } 287 288 static struct dma_async_tx_descriptor *mdc_prep_dma_memcpy( 289 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len, 290 unsigned long flags) 291 { 292 struct mdc_chan *mchan = to_mdc_chan(chan); 293 struct mdc_dma *mdma = mchan->mdma; 294 struct mdc_tx_desc *mdesc; 295 struct mdc_hw_list_desc *curr, *prev = NULL; 296 dma_addr_t curr_phys; 297 298 if (!len) 299 return NULL; 300 301 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT); 302 if (!mdesc) 303 return NULL; 304 mdesc->chan = mchan; 305 mdesc->list_xfer_size = len; 306 307 while (len > 0) { 308 size_t xfer_size; 309 310 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, &curr_phys); 311 if (!curr) 312 goto free_desc; 313 314 if (prev) { 315 prev->node_addr = curr_phys; 316 prev->next_desc = curr; 317 } else { 318 mdesc->list_phys = curr_phys; 319 mdesc->list = curr; 320 } 321 322 xfer_size = min_t(size_t, mdma->max_xfer_size, len); 323 324 mdc_list_desc_config(mchan, curr, DMA_MEM_TO_MEM, src, dest, 325 xfer_size); 326 327 prev = curr; 328 329 mdesc->list_len++; 330 src += xfer_size; 331 dest += xfer_size; 332 len -= xfer_size; 333 } 334 335 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags); 336 337 free_desc: 338 mdc_desc_free(&mdesc->vd); 339 340 return NULL; 341 } 342 343 static int mdc_check_slave_width(struct mdc_chan *mchan, 344 enum dma_transfer_direction dir) 345 { 346 enum dma_slave_buswidth width; 347 348 if (dir == DMA_MEM_TO_DEV) 349 width = mchan->config.dst_addr_width; 350 else 351 width = mchan->config.src_addr_width; 352 353 switch (width) { 354 case DMA_SLAVE_BUSWIDTH_1_BYTE: 355 case DMA_SLAVE_BUSWIDTH_2_BYTES: 356 case DMA_SLAVE_BUSWIDTH_4_BYTES: 357 case DMA_SLAVE_BUSWIDTH_8_BYTES: 358 break; 359 default: 360 return -EINVAL; 361 } 362 363 if (width > mchan->mdma->bus_width) 364 return -EINVAL; 365 366 return 0; 367 } 368 369 static struct dma_async_tx_descriptor *mdc_prep_dma_cyclic( 370 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 371 size_t period_len, enum dma_transfer_direction dir, 372 unsigned long flags) 373 { 374 struct mdc_chan *mchan = to_mdc_chan(chan); 375 struct mdc_dma *mdma = mchan->mdma; 376 struct mdc_tx_desc *mdesc; 377 struct mdc_hw_list_desc *curr, *prev = NULL; 378 dma_addr_t curr_phys; 379 380 if (!buf_len && !period_len) 381 return NULL; 382 383 if (!is_slave_direction(dir)) 384 return NULL; 385 386 if (mdc_check_slave_width(mchan, dir) < 0) 387 return NULL; 388 389 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT); 390 if (!mdesc) 391 return NULL; 392 mdesc->chan = mchan; 393 mdesc->cyclic = true; 394 mdesc->list_xfer_size = buf_len; 395 mdesc->list_period_len = DIV_ROUND_UP(period_len, 396 mdma->max_xfer_size); 397 398 while (buf_len > 0) { 399 size_t remainder = min(period_len, buf_len); 400 401 while (remainder > 0) { 402 size_t xfer_size; 403 404 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, 405 &curr_phys); 406 if (!curr) 407 goto free_desc; 408 409 if (!prev) { 410 mdesc->list_phys = curr_phys; 411 mdesc->list = curr; 412 } else { 413 prev->node_addr = curr_phys; 414 prev->next_desc = curr; 415 } 416 417 xfer_size = min_t(size_t, mdma->max_xfer_size, 418 remainder); 419 420 if (dir == DMA_MEM_TO_DEV) { 421 mdc_list_desc_config(mchan, curr, dir, 422 buf_addr, 423 mchan->config.dst_addr, 424 xfer_size); 425 } else { 426 mdc_list_desc_config(mchan, curr, dir, 427 mchan->config.src_addr, 428 buf_addr, 429 xfer_size); 430 } 431 432 prev = curr; 433 434 mdesc->list_len++; 435 buf_addr += xfer_size; 436 buf_len -= xfer_size; 437 remainder -= xfer_size; 438 } 439 } 440 prev->node_addr = mdesc->list_phys; 441 442 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags); 443 444 free_desc: 445 mdc_desc_free(&mdesc->vd); 446 447 return NULL; 448 } 449 450 static struct dma_async_tx_descriptor *mdc_prep_slave_sg( 451 struct dma_chan *chan, struct scatterlist *sgl, 452 unsigned int sg_len, enum dma_transfer_direction dir, 453 unsigned long flags, void *context) 454 { 455 struct mdc_chan *mchan = to_mdc_chan(chan); 456 struct mdc_dma *mdma = mchan->mdma; 457 struct mdc_tx_desc *mdesc; 458 struct scatterlist *sg; 459 struct mdc_hw_list_desc *curr, *prev = NULL; 460 dma_addr_t curr_phys; 461 unsigned int i; 462 463 if (!sgl) 464 return NULL; 465 466 if (!is_slave_direction(dir)) 467 return NULL; 468 469 if (mdc_check_slave_width(mchan, dir) < 0) 470 return NULL; 471 472 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT); 473 if (!mdesc) 474 return NULL; 475 mdesc->chan = mchan; 476 477 for_each_sg(sgl, sg, sg_len, i) { 478 dma_addr_t buf = sg_dma_address(sg); 479 size_t buf_len = sg_dma_len(sg); 480 481 while (buf_len > 0) { 482 size_t xfer_size; 483 484 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, 485 &curr_phys); 486 if (!curr) 487 goto free_desc; 488 489 if (!prev) { 490 mdesc->list_phys = curr_phys; 491 mdesc->list = curr; 492 } else { 493 prev->node_addr = curr_phys; 494 prev->next_desc = curr; 495 } 496 497 xfer_size = min_t(size_t, mdma->max_xfer_size, 498 buf_len); 499 500 if (dir == DMA_MEM_TO_DEV) { 501 mdc_list_desc_config(mchan, curr, dir, buf, 502 mchan->config.dst_addr, 503 xfer_size); 504 } else { 505 mdc_list_desc_config(mchan, curr, dir, 506 mchan->config.src_addr, 507 buf, xfer_size); 508 } 509 510 prev = curr; 511 512 mdesc->list_len++; 513 mdesc->list_xfer_size += xfer_size; 514 buf += xfer_size; 515 buf_len -= xfer_size; 516 } 517 } 518 519 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags); 520 521 free_desc: 522 mdc_desc_free(&mdesc->vd); 523 524 return NULL; 525 } 526 527 static void mdc_issue_desc(struct mdc_chan *mchan) 528 { 529 struct mdc_dma *mdma = mchan->mdma; 530 struct virt_dma_desc *vd; 531 struct mdc_tx_desc *mdesc; 532 u32 val; 533 534 vd = vchan_next_desc(&mchan->vc); 535 if (!vd) 536 return; 537 538 list_del(&vd->node); 539 540 mdesc = to_mdc_desc(&vd->tx); 541 mchan->desc = mdesc; 542 543 dev_dbg(mdma2dev(mdma), "Issuing descriptor on channel %d\n", 544 mchan->chan_nr); 545 546 mdma->soc->enable_chan(mchan); 547 548 val = mdc_chan_readl(mchan, MDC_GENERAL_CONFIG); 549 val |= MDC_GENERAL_CONFIG_LIST_IEN | MDC_GENERAL_CONFIG_IEN | 550 MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W | 551 MDC_GENERAL_CONFIG_PHYSICAL_R; 552 mdc_chan_writel(mchan, val, MDC_GENERAL_CONFIG); 553 val = (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) | 554 (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) | 555 (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT); 556 mdc_chan_writel(mchan, val, MDC_READ_PORT_CONFIG); 557 mdc_chan_writel(mchan, mdesc->list_phys, MDC_LIST_NODE_ADDRESS); 558 val = mdc_chan_readl(mchan, MDC_CONTROL_AND_STATUS); 559 val |= MDC_CONTROL_AND_STATUS_LIST_EN; 560 mdc_chan_writel(mchan, val, MDC_CONTROL_AND_STATUS); 561 } 562 563 static void mdc_issue_pending(struct dma_chan *chan) 564 { 565 struct mdc_chan *mchan = to_mdc_chan(chan); 566 unsigned long flags; 567 568 spin_lock_irqsave(&mchan->vc.lock, flags); 569 if (vchan_issue_pending(&mchan->vc) && !mchan->desc) 570 mdc_issue_desc(mchan); 571 spin_unlock_irqrestore(&mchan->vc.lock, flags); 572 } 573 574 static enum dma_status mdc_tx_status(struct dma_chan *chan, 575 dma_cookie_t cookie, struct dma_tx_state *txstate) 576 { 577 struct mdc_chan *mchan = to_mdc_chan(chan); 578 struct mdc_tx_desc *mdesc; 579 struct virt_dma_desc *vd; 580 unsigned long flags; 581 size_t bytes = 0; 582 int ret; 583 584 ret = dma_cookie_status(chan, cookie, txstate); 585 if (ret == DMA_COMPLETE) 586 return ret; 587 588 if (!txstate) 589 return ret; 590 591 spin_lock_irqsave(&mchan->vc.lock, flags); 592 vd = vchan_find_desc(&mchan->vc, cookie); 593 if (vd) { 594 mdesc = to_mdc_desc(&vd->tx); 595 bytes = mdesc->list_xfer_size; 596 } else if (mchan->desc && mchan->desc->vd.tx.cookie == cookie) { 597 struct mdc_hw_list_desc *ldesc; 598 u32 val1, val2, done, processed, residue; 599 int i, cmds; 600 601 mdesc = mchan->desc; 602 603 /* 604 * Determine the number of commands that haven't been 605 * processed (handled by the IRQ handler) yet. 606 */ 607 do { 608 val1 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) & 609 ~MDC_CMDS_PROCESSED_INT_ACTIVE; 610 residue = mdc_chan_readl(mchan, 611 MDC_ACTIVE_TRANSFER_SIZE); 612 val2 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) & 613 ~MDC_CMDS_PROCESSED_INT_ACTIVE; 614 } while (val1 != val2); 615 616 done = (val1 >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) & 617 MDC_CMDS_PROCESSED_CMDS_DONE_MASK; 618 processed = (val1 >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) & 619 MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK; 620 cmds = (done - processed) % 621 (MDC_CMDS_PROCESSED_CMDS_DONE_MASK + 1); 622 623 /* 624 * If the command loaded event hasn't been processed yet, then 625 * the difference above includes an extra command. 626 */ 627 if (!mdesc->cmd_loaded) 628 cmds--; 629 else 630 cmds += mdesc->list_cmds_done; 631 632 bytes = mdesc->list_xfer_size; 633 ldesc = mdesc->list; 634 for (i = 0; i < cmds; i++) { 635 bytes -= ldesc->xfer_size + 1; 636 ldesc = ldesc->next_desc; 637 } 638 if (ldesc) { 639 if (residue != MDC_TRANSFER_SIZE_MASK) 640 bytes -= ldesc->xfer_size - residue; 641 else 642 bytes -= ldesc->xfer_size + 1; 643 } 644 } 645 spin_unlock_irqrestore(&mchan->vc.lock, flags); 646 647 dma_set_residue(txstate, bytes); 648 649 return ret; 650 } 651 652 static unsigned int mdc_get_new_events(struct mdc_chan *mchan) 653 { 654 u32 val, processed, done1, done2; 655 unsigned int ret; 656 657 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED); 658 processed = (val >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) & 659 MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK; 660 /* 661 * CMDS_DONE may have incremented between reading CMDS_PROCESSED 662 * and clearing INT_ACTIVE. Re-read CMDS_PROCESSED to ensure we 663 * didn't miss a command completion. 664 */ 665 do { 666 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED); 667 668 done1 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) & 669 MDC_CMDS_PROCESSED_CMDS_DONE_MASK; 670 671 val &= ~((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK << 672 MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) | 673 MDC_CMDS_PROCESSED_INT_ACTIVE); 674 675 val |= done1 << MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT; 676 677 mdc_chan_writel(mchan, val, MDC_CMDS_PROCESSED); 678 679 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED); 680 681 done2 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) & 682 MDC_CMDS_PROCESSED_CMDS_DONE_MASK; 683 } while (done1 != done2); 684 685 if (done1 >= processed) 686 ret = done1 - processed; 687 else 688 ret = ((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK + 1) - 689 processed) + done1; 690 691 return ret; 692 } 693 694 static int mdc_terminate_all(struct dma_chan *chan) 695 { 696 struct mdc_chan *mchan = to_mdc_chan(chan); 697 unsigned long flags; 698 LIST_HEAD(head); 699 700 spin_lock_irqsave(&mchan->vc.lock, flags); 701 702 mdc_chan_writel(mchan, MDC_CONTROL_AND_STATUS_CANCEL, 703 MDC_CONTROL_AND_STATUS); 704 705 if (mchan->desc) { 706 vchan_terminate_vdesc(&mchan->desc->vd); 707 mchan->desc = NULL; 708 } 709 vchan_get_all_descriptors(&mchan->vc, &head); 710 711 mdc_get_new_events(mchan); 712 713 spin_unlock_irqrestore(&mchan->vc.lock, flags); 714 715 vchan_dma_desc_free_list(&mchan->vc, &head); 716 717 return 0; 718 } 719 720 static void mdc_synchronize(struct dma_chan *chan) 721 { 722 struct mdc_chan *mchan = to_mdc_chan(chan); 723 724 vchan_synchronize(&mchan->vc); 725 } 726 727 static int mdc_slave_config(struct dma_chan *chan, 728 struct dma_slave_config *config) 729 { 730 struct mdc_chan *mchan = to_mdc_chan(chan); 731 unsigned long flags; 732 733 spin_lock_irqsave(&mchan->vc.lock, flags); 734 mchan->config = *config; 735 spin_unlock_irqrestore(&mchan->vc.lock, flags); 736 737 return 0; 738 } 739 740 static int mdc_alloc_chan_resources(struct dma_chan *chan) 741 { 742 struct mdc_chan *mchan = to_mdc_chan(chan); 743 struct device *dev = mdma2dev(mchan->mdma); 744 745 return pm_runtime_get_sync(dev); 746 } 747 748 static void mdc_free_chan_resources(struct dma_chan *chan) 749 { 750 struct mdc_chan *mchan = to_mdc_chan(chan); 751 struct mdc_dma *mdma = mchan->mdma; 752 struct device *dev = mdma2dev(mdma); 753 754 mdc_terminate_all(chan); 755 mdma->soc->disable_chan(mchan); 756 pm_runtime_put(dev); 757 } 758 759 static irqreturn_t mdc_chan_irq(int irq, void *dev_id) 760 { 761 struct mdc_chan *mchan = (struct mdc_chan *)dev_id; 762 struct mdc_tx_desc *mdesc; 763 unsigned int i, new_events; 764 765 spin_lock(&mchan->vc.lock); 766 767 dev_dbg(mdma2dev(mchan->mdma), "IRQ on channel %d\n", mchan->chan_nr); 768 769 new_events = mdc_get_new_events(mchan); 770 771 if (!new_events) 772 goto out; 773 774 mdesc = mchan->desc; 775 if (!mdesc) { 776 dev_warn(mdma2dev(mchan->mdma), 777 "IRQ with no active descriptor on channel %d\n", 778 mchan->chan_nr); 779 goto out; 780 } 781 782 for (i = 0; i < new_events; i++) { 783 /* 784 * The first interrupt in a transfer indicates that the 785 * command list has been loaded, not that a command has 786 * been completed. 787 */ 788 if (!mdesc->cmd_loaded) { 789 mdesc->cmd_loaded = true; 790 continue; 791 } 792 793 mdesc->list_cmds_done++; 794 if (mdesc->cyclic) { 795 mdesc->list_cmds_done %= mdesc->list_len; 796 if (mdesc->list_cmds_done % mdesc->list_period_len == 0) 797 vchan_cyclic_callback(&mdesc->vd); 798 } else if (mdesc->list_cmds_done == mdesc->list_len) { 799 mchan->desc = NULL; 800 vchan_cookie_complete(&mdesc->vd); 801 mdc_issue_desc(mchan); 802 break; 803 } 804 } 805 out: 806 spin_unlock(&mchan->vc.lock); 807 808 return IRQ_HANDLED; 809 } 810 811 static struct dma_chan *mdc_of_xlate(struct of_phandle_args *dma_spec, 812 struct of_dma *ofdma) 813 { 814 struct mdc_dma *mdma = ofdma->of_dma_data; 815 struct dma_chan *chan; 816 817 if (dma_spec->args_count != 3) 818 return NULL; 819 820 list_for_each_entry(chan, &mdma->dma_dev.channels, device_node) { 821 struct mdc_chan *mchan = to_mdc_chan(chan); 822 823 if (!(dma_spec->args[1] & BIT(mchan->chan_nr))) 824 continue; 825 if (dma_get_slave_channel(chan)) { 826 mchan->periph = dma_spec->args[0]; 827 mchan->thread = dma_spec->args[2]; 828 return chan; 829 } 830 } 831 832 return NULL; 833 } 834 835 #define PISTACHIO_CR_PERIPH_DMA_ROUTE(ch) (0x120 + 0x4 * ((ch) / 4)) 836 #define PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(ch) (8 * ((ch) % 4)) 837 #define PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK 0x3f 838 839 static void pistachio_mdc_enable_chan(struct mdc_chan *mchan) 840 { 841 struct mdc_dma *mdma = mchan->mdma; 842 843 regmap_update_bits(mdma->periph_regs, 844 PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr), 845 PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK << 846 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr), 847 mchan->periph << 848 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr)); 849 } 850 851 static void pistachio_mdc_disable_chan(struct mdc_chan *mchan) 852 { 853 struct mdc_dma *mdma = mchan->mdma; 854 855 regmap_update_bits(mdma->periph_regs, 856 PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr), 857 PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK << 858 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr), 859 0); 860 } 861 862 static const struct mdc_dma_soc_data pistachio_mdc_data = { 863 .enable_chan = pistachio_mdc_enable_chan, 864 .disable_chan = pistachio_mdc_disable_chan, 865 }; 866 867 static const struct of_device_id mdc_dma_of_match[] = { 868 { .compatible = "img,pistachio-mdc-dma", .data = &pistachio_mdc_data, }, 869 { }, 870 }; 871 MODULE_DEVICE_TABLE(of, mdc_dma_of_match); 872 873 static int img_mdc_runtime_suspend(struct device *dev) 874 { 875 struct mdc_dma *mdma = dev_get_drvdata(dev); 876 877 clk_disable_unprepare(mdma->clk); 878 879 return 0; 880 } 881 882 static int img_mdc_runtime_resume(struct device *dev) 883 { 884 struct mdc_dma *mdma = dev_get_drvdata(dev); 885 886 return clk_prepare_enable(mdma->clk); 887 } 888 889 static int mdc_dma_probe(struct platform_device *pdev) 890 { 891 struct mdc_dma *mdma; 892 struct resource *res; 893 unsigned int i; 894 u32 val; 895 int ret; 896 897 mdma = devm_kzalloc(&pdev->dev, sizeof(*mdma), GFP_KERNEL); 898 if (!mdma) 899 return -ENOMEM; 900 platform_set_drvdata(pdev, mdma); 901 902 mdma->soc = of_device_get_match_data(&pdev->dev); 903 904 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 905 mdma->regs = devm_ioremap_resource(&pdev->dev, res); 906 if (IS_ERR(mdma->regs)) 907 return PTR_ERR(mdma->regs); 908 909 mdma->periph_regs = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, 910 "img,cr-periph"); 911 if (IS_ERR(mdma->periph_regs)) 912 return PTR_ERR(mdma->periph_regs); 913 914 mdma->clk = devm_clk_get(&pdev->dev, "sys"); 915 if (IS_ERR(mdma->clk)) 916 return PTR_ERR(mdma->clk); 917 918 dma_cap_zero(mdma->dma_dev.cap_mask); 919 dma_cap_set(DMA_SLAVE, mdma->dma_dev.cap_mask); 920 dma_cap_set(DMA_PRIVATE, mdma->dma_dev.cap_mask); 921 dma_cap_set(DMA_CYCLIC, mdma->dma_dev.cap_mask); 922 dma_cap_set(DMA_MEMCPY, mdma->dma_dev.cap_mask); 923 924 val = mdc_readl(mdma, MDC_GLOBAL_CONFIG_A); 925 mdma->nr_channels = (val >> MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT) & 926 MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK; 927 mdma->nr_threads = 928 1 << ((val >> MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT) & 929 MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK); 930 mdma->bus_width = 931 (1 << ((val >> MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT) & 932 MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK)) / 8; 933 /* 934 * Although transfer sizes of up to MDC_TRANSFER_SIZE_MASK + 1 bytes 935 * are supported, this makes it possible for the value reported in 936 * MDC_ACTIVE_TRANSFER_SIZE to be ambiguous - an active transfer size 937 * of MDC_TRANSFER_SIZE_MASK may indicate either that 0 bytes or 938 * MDC_TRANSFER_SIZE_MASK + 1 bytes are remaining. To eliminate this 939 * ambiguity, restrict transfer sizes to one bus-width less than the 940 * actual maximum. 941 */ 942 mdma->max_xfer_size = MDC_TRANSFER_SIZE_MASK + 1 - mdma->bus_width; 943 944 of_property_read_u32(pdev->dev.of_node, "dma-channels", 945 &mdma->nr_channels); 946 ret = of_property_read_u32(pdev->dev.of_node, 947 "img,max-burst-multiplier", 948 &mdma->max_burst_mult); 949 if (ret) 950 return ret; 951 952 mdma->dma_dev.dev = &pdev->dev; 953 mdma->dma_dev.device_prep_slave_sg = mdc_prep_slave_sg; 954 mdma->dma_dev.device_prep_dma_cyclic = mdc_prep_dma_cyclic; 955 mdma->dma_dev.device_prep_dma_memcpy = mdc_prep_dma_memcpy; 956 mdma->dma_dev.device_alloc_chan_resources = mdc_alloc_chan_resources; 957 mdma->dma_dev.device_free_chan_resources = mdc_free_chan_resources; 958 mdma->dma_dev.device_tx_status = mdc_tx_status; 959 mdma->dma_dev.device_issue_pending = mdc_issue_pending; 960 mdma->dma_dev.device_terminate_all = mdc_terminate_all; 961 mdma->dma_dev.device_synchronize = mdc_synchronize; 962 mdma->dma_dev.device_config = mdc_slave_config; 963 964 mdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 965 mdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 966 for (i = 1; i <= mdma->bus_width; i <<= 1) { 967 mdma->dma_dev.src_addr_widths |= BIT(i); 968 mdma->dma_dev.dst_addr_widths |= BIT(i); 969 } 970 971 INIT_LIST_HEAD(&mdma->dma_dev.channels); 972 for (i = 0; i < mdma->nr_channels; i++) { 973 struct mdc_chan *mchan = &mdma->channels[i]; 974 975 mchan->mdma = mdma; 976 mchan->chan_nr = i; 977 mchan->irq = platform_get_irq(pdev, i); 978 if (mchan->irq < 0) 979 return mchan->irq; 980 981 ret = devm_request_irq(&pdev->dev, mchan->irq, mdc_chan_irq, 982 IRQ_TYPE_LEVEL_HIGH, 983 dev_name(&pdev->dev), mchan); 984 if (ret < 0) 985 return ret; 986 987 mchan->vc.desc_free = mdc_desc_free; 988 vchan_init(&mchan->vc, &mdma->dma_dev); 989 } 990 991 mdma->desc_pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev, 992 sizeof(struct mdc_hw_list_desc), 993 4, 0); 994 if (!mdma->desc_pool) 995 return -ENOMEM; 996 997 pm_runtime_enable(&pdev->dev); 998 if (!pm_runtime_enabled(&pdev->dev)) { 999 ret = img_mdc_runtime_resume(&pdev->dev); 1000 if (ret) 1001 return ret; 1002 } 1003 1004 ret = dma_async_device_register(&mdma->dma_dev); 1005 if (ret) 1006 goto suspend; 1007 1008 ret = of_dma_controller_register(pdev->dev.of_node, mdc_of_xlate, mdma); 1009 if (ret) 1010 goto unregister; 1011 1012 dev_info(&pdev->dev, "MDC with %u channels and %u threads\n", 1013 mdma->nr_channels, mdma->nr_threads); 1014 1015 return 0; 1016 1017 unregister: 1018 dma_async_device_unregister(&mdma->dma_dev); 1019 suspend: 1020 if (!pm_runtime_enabled(&pdev->dev)) 1021 img_mdc_runtime_suspend(&pdev->dev); 1022 pm_runtime_disable(&pdev->dev); 1023 return ret; 1024 } 1025 1026 static int mdc_dma_remove(struct platform_device *pdev) 1027 { 1028 struct mdc_dma *mdma = platform_get_drvdata(pdev); 1029 struct mdc_chan *mchan, *next; 1030 1031 of_dma_controller_free(pdev->dev.of_node); 1032 dma_async_device_unregister(&mdma->dma_dev); 1033 1034 list_for_each_entry_safe(mchan, next, &mdma->dma_dev.channels, 1035 vc.chan.device_node) { 1036 list_del(&mchan->vc.chan.device_node); 1037 1038 devm_free_irq(&pdev->dev, mchan->irq, mchan); 1039 1040 tasklet_kill(&mchan->vc.task); 1041 } 1042 1043 pm_runtime_disable(&pdev->dev); 1044 if (!pm_runtime_status_suspended(&pdev->dev)) 1045 img_mdc_runtime_suspend(&pdev->dev); 1046 1047 return 0; 1048 } 1049 1050 #ifdef CONFIG_PM_SLEEP 1051 static int img_mdc_suspend_late(struct device *dev) 1052 { 1053 struct mdc_dma *mdma = dev_get_drvdata(dev); 1054 int i; 1055 1056 /* Check that all channels are idle */ 1057 for (i = 0; i < mdma->nr_channels; i++) { 1058 struct mdc_chan *mchan = &mdma->channels[i]; 1059 1060 if (unlikely(mchan->desc)) 1061 return -EBUSY; 1062 } 1063 1064 return pm_runtime_force_suspend(dev); 1065 } 1066 1067 static int img_mdc_resume_early(struct device *dev) 1068 { 1069 return pm_runtime_force_resume(dev); 1070 } 1071 #endif /* CONFIG_PM_SLEEP */ 1072 1073 static const struct dev_pm_ops img_mdc_pm_ops = { 1074 SET_RUNTIME_PM_OPS(img_mdc_runtime_suspend, 1075 img_mdc_runtime_resume, NULL) 1076 SET_LATE_SYSTEM_SLEEP_PM_OPS(img_mdc_suspend_late, 1077 img_mdc_resume_early) 1078 }; 1079 1080 static struct platform_driver mdc_dma_driver = { 1081 .driver = { 1082 .name = "img-mdc-dma", 1083 .pm = &img_mdc_pm_ops, 1084 .of_match_table = of_match_ptr(mdc_dma_of_match), 1085 }, 1086 .probe = mdc_dma_probe, 1087 .remove = mdc_dma_remove, 1088 }; 1089 module_platform_driver(mdc_dma_driver); 1090 1091 MODULE_DESCRIPTION("IMG Multi-threaded DMA Controller (MDC) driver"); 1092 MODULE_AUTHOR("Andrew Bresticker <abrestic@chromium.org>"); 1093 MODULE_LICENSE("GPL v2"); 1094