1 /* 2 * IMG Multi-threaded DMA Controller (MDC) 3 * 4 * Copyright (C) 2009,2012,2013 Imagination Technologies Ltd. 5 * Copyright (C) 2014 Google, Inc. 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms and conditions of the GNU General Public License, 9 * version 2, as published by the Free Software Foundation. 10 */ 11 12 #include <linux/clk.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/dmaengine.h> 15 #include <linux/dmapool.h> 16 #include <linux/interrupt.h> 17 #include <linux/io.h> 18 #include <linux/irq.h> 19 #include <linux/kernel.h> 20 #include <linux/mfd/syscon.h> 21 #include <linux/module.h> 22 #include <linux/of.h> 23 #include <linux/of_device.h> 24 #include <linux/of_dma.h> 25 #include <linux/platform_device.h> 26 #include <linux/regmap.h> 27 #include <linux/slab.h> 28 #include <linux/spinlock.h> 29 30 #include "dmaengine.h" 31 #include "virt-dma.h" 32 33 #define MDC_MAX_DMA_CHANNELS 32 34 35 #define MDC_GENERAL_CONFIG 0x000 36 #define MDC_GENERAL_CONFIG_LIST_IEN BIT(31) 37 #define MDC_GENERAL_CONFIG_IEN BIT(29) 38 #define MDC_GENERAL_CONFIG_LEVEL_INT BIT(28) 39 #define MDC_GENERAL_CONFIG_INC_W BIT(12) 40 #define MDC_GENERAL_CONFIG_INC_R BIT(8) 41 #define MDC_GENERAL_CONFIG_PHYSICAL_W BIT(7) 42 #define MDC_GENERAL_CONFIG_WIDTH_W_SHIFT 4 43 #define MDC_GENERAL_CONFIG_WIDTH_W_MASK 0x7 44 #define MDC_GENERAL_CONFIG_PHYSICAL_R BIT(3) 45 #define MDC_GENERAL_CONFIG_WIDTH_R_SHIFT 0 46 #define MDC_GENERAL_CONFIG_WIDTH_R_MASK 0x7 47 48 #define MDC_READ_PORT_CONFIG 0x004 49 #define MDC_READ_PORT_CONFIG_STHREAD_SHIFT 28 50 #define MDC_READ_PORT_CONFIG_STHREAD_MASK 0xf 51 #define MDC_READ_PORT_CONFIG_RTHREAD_SHIFT 24 52 #define MDC_READ_PORT_CONFIG_RTHREAD_MASK 0xf 53 #define MDC_READ_PORT_CONFIG_WTHREAD_SHIFT 16 54 #define MDC_READ_PORT_CONFIG_WTHREAD_MASK 0xf 55 #define MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT 4 56 #define MDC_READ_PORT_CONFIG_BURST_SIZE_MASK 0xff 57 #define MDC_READ_PORT_CONFIG_DREQ_ENABLE BIT(1) 58 59 #define MDC_READ_ADDRESS 0x008 60 61 #define MDC_WRITE_ADDRESS 0x00c 62 63 #define MDC_TRANSFER_SIZE 0x010 64 #define MDC_TRANSFER_SIZE_MASK 0xffffff 65 66 #define MDC_LIST_NODE_ADDRESS 0x014 67 68 #define MDC_CMDS_PROCESSED 0x018 69 #define MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT 16 70 #define MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK 0x3f 71 #define MDC_CMDS_PROCESSED_INT_ACTIVE BIT(8) 72 #define MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT 0 73 #define MDC_CMDS_PROCESSED_CMDS_DONE_MASK 0x3f 74 75 #define MDC_CONTROL_AND_STATUS 0x01c 76 #define MDC_CONTROL_AND_STATUS_CANCEL BIT(20) 77 #define MDC_CONTROL_AND_STATUS_LIST_EN BIT(4) 78 #define MDC_CONTROL_AND_STATUS_EN BIT(0) 79 80 #define MDC_ACTIVE_TRANSFER_SIZE 0x030 81 82 #define MDC_GLOBAL_CONFIG_A 0x900 83 #define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT 16 84 #define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK 0xff 85 #define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT 8 86 #define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK 0xff 87 #define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT 0 88 #define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK 0xff 89 90 struct mdc_hw_list_desc { 91 u32 gen_conf; 92 u32 readport_conf; 93 u32 read_addr; 94 u32 write_addr; 95 u32 xfer_size; 96 u32 node_addr; 97 u32 cmds_done; 98 u32 ctrl_status; 99 /* 100 * Not part of the list descriptor, but instead used by the CPU to 101 * traverse the list. 102 */ 103 struct mdc_hw_list_desc *next_desc; 104 }; 105 106 struct mdc_tx_desc { 107 struct mdc_chan *chan; 108 struct virt_dma_desc vd; 109 dma_addr_t list_phys; 110 struct mdc_hw_list_desc *list; 111 bool cyclic; 112 bool cmd_loaded; 113 unsigned int list_len; 114 unsigned int list_period_len; 115 size_t list_xfer_size; 116 unsigned int list_cmds_done; 117 }; 118 119 struct mdc_chan { 120 struct mdc_dma *mdma; 121 struct virt_dma_chan vc; 122 struct dma_slave_config config; 123 struct mdc_tx_desc *desc; 124 int irq; 125 unsigned int periph; 126 unsigned int thread; 127 unsigned int chan_nr; 128 }; 129 130 struct mdc_dma_soc_data { 131 void (*enable_chan)(struct mdc_chan *mchan); 132 void (*disable_chan)(struct mdc_chan *mchan); 133 }; 134 135 struct mdc_dma { 136 struct dma_device dma_dev; 137 void __iomem *regs; 138 struct clk *clk; 139 struct dma_pool *desc_pool; 140 struct regmap *periph_regs; 141 spinlock_t lock; 142 unsigned int nr_threads; 143 unsigned int nr_channels; 144 unsigned int bus_width; 145 unsigned int max_burst_mult; 146 unsigned int max_xfer_size; 147 const struct mdc_dma_soc_data *soc; 148 struct mdc_chan channels[MDC_MAX_DMA_CHANNELS]; 149 }; 150 151 static inline u32 mdc_readl(struct mdc_dma *mdma, u32 reg) 152 { 153 return readl(mdma->regs + reg); 154 } 155 156 static inline void mdc_writel(struct mdc_dma *mdma, u32 val, u32 reg) 157 { 158 writel(val, mdma->regs + reg); 159 } 160 161 static inline u32 mdc_chan_readl(struct mdc_chan *mchan, u32 reg) 162 { 163 return mdc_readl(mchan->mdma, mchan->chan_nr * 0x040 + reg); 164 } 165 166 static inline void mdc_chan_writel(struct mdc_chan *mchan, u32 val, u32 reg) 167 { 168 mdc_writel(mchan->mdma, val, mchan->chan_nr * 0x040 + reg); 169 } 170 171 static inline struct mdc_chan *to_mdc_chan(struct dma_chan *c) 172 { 173 return container_of(to_virt_chan(c), struct mdc_chan, vc); 174 } 175 176 static inline struct mdc_tx_desc *to_mdc_desc(struct dma_async_tx_descriptor *t) 177 { 178 struct virt_dma_desc *vdesc = container_of(t, struct virt_dma_desc, tx); 179 180 return container_of(vdesc, struct mdc_tx_desc, vd); 181 } 182 183 static inline struct device *mdma2dev(struct mdc_dma *mdma) 184 { 185 return mdma->dma_dev.dev; 186 } 187 188 static inline unsigned int to_mdc_width(unsigned int bytes) 189 { 190 return ffs(bytes) - 1; 191 } 192 193 static inline void mdc_set_read_width(struct mdc_hw_list_desc *ldesc, 194 unsigned int bytes) 195 { 196 ldesc->gen_conf |= to_mdc_width(bytes) << 197 MDC_GENERAL_CONFIG_WIDTH_R_SHIFT; 198 } 199 200 static inline void mdc_set_write_width(struct mdc_hw_list_desc *ldesc, 201 unsigned int bytes) 202 { 203 ldesc->gen_conf |= to_mdc_width(bytes) << 204 MDC_GENERAL_CONFIG_WIDTH_W_SHIFT; 205 } 206 207 static void mdc_list_desc_config(struct mdc_chan *mchan, 208 struct mdc_hw_list_desc *ldesc, 209 enum dma_transfer_direction dir, 210 dma_addr_t src, dma_addr_t dst, size_t len) 211 { 212 struct mdc_dma *mdma = mchan->mdma; 213 unsigned int max_burst, burst_size; 214 215 ldesc->gen_conf = MDC_GENERAL_CONFIG_IEN | MDC_GENERAL_CONFIG_LIST_IEN | 216 MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W | 217 MDC_GENERAL_CONFIG_PHYSICAL_R; 218 ldesc->readport_conf = 219 (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) | 220 (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) | 221 (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT); 222 ldesc->read_addr = src; 223 ldesc->write_addr = dst; 224 ldesc->xfer_size = len - 1; 225 ldesc->node_addr = 0; 226 ldesc->cmds_done = 0; 227 ldesc->ctrl_status = MDC_CONTROL_AND_STATUS_LIST_EN | 228 MDC_CONTROL_AND_STATUS_EN; 229 ldesc->next_desc = NULL; 230 231 if (IS_ALIGNED(dst, mdma->bus_width) && 232 IS_ALIGNED(src, mdma->bus_width)) 233 max_burst = mdma->bus_width * mdma->max_burst_mult; 234 else 235 max_burst = mdma->bus_width * (mdma->max_burst_mult - 1); 236 237 if (dir == DMA_MEM_TO_DEV) { 238 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R; 239 ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE; 240 mdc_set_read_width(ldesc, mdma->bus_width); 241 mdc_set_write_width(ldesc, mchan->config.dst_addr_width); 242 burst_size = min(max_burst, mchan->config.dst_maxburst * 243 mchan->config.dst_addr_width); 244 } else if (dir == DMA_DEV_TO_MEM) { 245 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_W; 246 ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE; 247 mdc_set_read_width(ldesc, mchan->config.src_addr_width); 248 mdc_set_write_width(ldesc, mdma->bus_width); 249 burst_size = min(max_burst, mchan->config.src_maxburst * 250 mchan->config.src_addr_width); 251 } else { 252 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R | 253 MDC_GENERAL_CONFIG_INC_W; 254 mdc_set_read_width(ldesc, mdma->bus_width); 255 mdc_set_write_width(ldesc, mdma->bus_width); 256 burst_size = max_burst; 257 } 258 ldesc->readport_conf |= (burst_size - 1) << 259 MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT; 260 } 261 262 static void mdc_list_desc_free(struct mdc_tx_desc *mdesc) 263 { 264 struct mdc_dma *mdma = mdesc->chan->mdma; 265 struct mdc_hw_list_desc *curr, *next; 266 dma_addr_t curr_phys, next_phys; 267 268 curr = mdesc->list; 269 curr_phys = mdesc->list_phys; 270 while (curr) { 271 next = curr->next_desc; 272 next_phys = curr->node_addr; 273 dma_pool_free(mdma->desc_pool, curr, curr_phys); 274 curr = next; 275 curr_phys = next_phys; 276 } 277 } 278 279 static void mdc_desc_free(struct virt_dma_desc *vd) 280 { 281 struct mdc_tx_desc *mdesc = to_mdc_desc(&vd->tx); 282 283 mdc_list_desc_free(mdesc); 284 kfree(mdesc); 285 } 286 287 static struct dma_async_tx_descriptor *mdc_prep_dma_memcpy( 288 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len, 289 unsigned long flags) 290 { 291 struct mdc_chan *mchan = to_mdc_chan(chan); 292 struct mdc_dma *mdma = mchan->mdma; 293 struct mdc_tx_desc *mdesc; 294 struct mdc_hw_list_desc *curr, *prev = NULL; 295 dma_addr_t curr_phys; 296 297 if (!len) 298 return NULL; 299 300 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT); 301 if (!mdesc) 302 return NULL; 303 mdesc->chan = mchan; 304 mdesc->list_xfer_size = len; 305 306 while (len > 0) { 307 size_t xfer_size; 308 309 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, &curr_phys); 310 if (!curr) 311 goto free_desc; 312 313 if (prev) { 314 prev->node_addr = curr_phys; 315 prev->next_desc = curr; 316 } else { 317 mdesc->list_phys = curr_phys; 318 mdesc->list = curr; 319 } 320 321 xfer_size = min_t(size_t, mdma->max_xfer_size, len); 322 323 mdc_list_desc_config(mchan, curr, DMA_MEM_TO_MEM, src, dest, 324 xfer_size); 325 326 prev = curr; 327 328 mdesc->list_len++; 329 src += xfer_size; 330 dest += xfer_size; 331 len -= xfer_size; 332 } 333 334 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags); 335 336 free_desc: 337 mdc_desc_free(&mdesc->vd); 338 339 return NULL; 340 } 341 342 static int mdc_check_slave_width(struct mdc_chan *mchan, 343 enum dma_transfer_direction dir) 344 { 345 enum dma_slave_buswidth width; 346 347 if (dir == DMA_MEM_TO_DEV) 348 width = mchan->config.dst_addr_width; 349 else 350 width = mchan->config.src_addr_width; 351 352 switch (width) { 353 case DMA_SLAVE_BUSWIDTH_1_BYTE: 354 case DMA_SLAVE_BUSWIDTH_2_BYTES: 355 case DMA_SLAVE_BUSWIDTH_4_BYTES: 356 case DMA_SLAVE_BUSWIDTH_8_BYTES: 357 break; 358 default: 359 return -EINVAL; 360 } 361 362 if (width > mchan->mdma->bus_width) 363 return -EINVAL; 364 365 return 0; 366 } 367 368 static struct dma_async_tx_descriptor *mdc_prep_dma_cyclic( 369 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 370 size_t period_len, enum dma_transfer_direction dir, 371 unsigned long flags) 372 { 373 struct mdc_chan *mchan = to_mdc_chan(chan); 374 struct mdc_dma *mdma = mchan->mdma; 375 struct mdc_tx_desc *mdesc; 376 struct mdc_hw_list_desc *curr, *prev = NULL; 377 dma_addr_t curr_phys; 378 379 if (!buf_len && !period_len) 380 return NULL; 381 382 if (!is_slave_direction(dir)) 383 return NULL; 384 385 if (mdc_check_slave_width(mchan, dir) < 0) 386 return NULL; 387 388 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT); 389 if (!mdesc) 390 return NULL; 391 mdesc->chan = mchan; 392 mdesc->cyclic = true; 393 mdesc->list_xfer_size = buf_len; 394 mdesc->list_period_len = DIV_ROUND_UP(period_len, 395 mdma->max_xfer_size); 396 397 while (buf_len > 0) { 398 size_t remainder = min(period_len, buf_len); 399 400 while (remainder > 0) { 401 size_t xfer_size; 402 403 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, 404 &curr_phys); 405 if (!curr) 406 goto free_desc; 407 408 if (!prev) { 409 mdesc->list_phys = curr_phys; 410 mdesc->list = curr; 411 } else { 412 prev->node_addr = curr_phys; 413 prev->next_desc = curr; 414 } 415 416 xfer_size = min_t(size_t, mdma->max_xfer_size, 417 remainder); 418 419 if (dir == DMA_MEM_TO_DEV) { 420 mdc_list_desc_config(mchan, curr, dir, 421 buf_addr, 422 mchan->config.dst_addr, 423 xfer_size); 424 } else { 425 mdc_list_desc_config(mchan, curr, dir, 426 mchan->config.src_addr, 427 buf_addr, 428 xfer_size); 429 } 430 431 prev = curr; 432 433 mdesc->list_len++; 434 buf_addr += xfer_size; 435 buf_len -= xfer_size; 436 remainder -= xfer_size; 437 } 438 } 439 prev->node_addr = mdesc->list_phys; 440 441 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags); 442 443 free_desc: 444 mdc_desc_free(&mdesc->vd); 445 446 return NULL; 447 } 448 449 static struct dma_async_tx_descriptor *mdc_prep_slave_sg( 450 struct dma_chan *chan, struct scatterlist *sgl, 451 unsigned int sg_len, enum dma_transfer_direction dir, 452 unsigned long flags, void *context) 453 { 454 struct mdc_chan *mchan = to_mdc_chan(chan); 455 struct mdc_dma *mdma = mchan->mdma; 456 struct mdc_tx_desc *mdesc; 457 struct scatterlist *sg; 458 struct mdc_hw_list_desc *curr, *prev = NULL; 459 dma_addr_t curr_phys; 460 unsigned int i; 461 462 if (!sgl) 463 return NULL; 464 465 if (!is_slave_direction(dir)) 466 return NULL; 467 468 if (mdc_check_slave_width(mchan, dir) < 0) 469 return NULL; 470 471 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT); 472 if (!mdesc) 473 return NULL; 474 mdesc->chan = mchan; 475 476 for_each_sg(sgl, sg, sg_len, i) { 477 dma_addr_t buf = sg_dma_address(sg); 478 size_t buf_len = sg_dma_len(sg); 479 480 while (buf_len > 0) { 481 size_t xfer_size; 482 483 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, 484 &curr_phys); 485 if (!curr) 486 goto free_desc; 487 488 if (!prev) { 489 mdesc->list_phys = curr_phys; 490 mdesc->list = curr; 491 } else { 492 prev->node_addr = curr_phys; 493 prev->next_desc = curr; 494 } 495 496 xfer_size = min_t(size_t, mdma->max_xfer_size, 497 buf_len); 498 499 if (dir == DMA_MEM_TO_DEV) { 500 mdc_list_desc_config(mchan, curr, dir, buf, 501 mchan->config.dst_addr, 502 xfer_size); 503 } else { 504 mdc_list_desc_config(mchan, curr, dir, 505 mchan->config.src_addr, 506 buf, xfer_size); 507 } 508 509 prev = curr; 510 511 mdesc->list_len++; 512 mdesc->list_xfer_size += xfer_size; 513 buf += xfer_size; 514 buf_len -= xfer_size; 515 } 516 } 517 518 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags); 519 520 free_desc: 521 mdc_desc_free(&mdesc->vd); 522 523 return NULL; 524 } 525 526 static void mdc_issue_desc(struct mdc_chan *mchan) 527 { 528 struct mdc_dma *mdma = mchan->mdma; 529 struct virt_dma_desc *vd; 530 struct mdc_tx_desc *mdesc; 531 u32 val; 532 533 vd = vchan_next_desc(&mchan->vc); 534 if (!vd) 535 return; 536 537 list_del(&vd->node); 538 539 mdesc = to_mdc_desc(&vd->tx); 540 mchan->desc = mdesc; 541 542 dev_dbg(mdma2dev(mdma), "Issuing descriptor on channel %d\n", 543 mchan->chan_nr); 544 545 mdma->soc->enable_chan(mchan); 546 547 val = mdc_chan_readl(mchan, MDC_GENERAL_CONFIG); 548 val |= MDC_GENERAL_CONFIG_LIST_IEN | MDC_GENERAL_CONFIG_IEN | 549 MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W | 550 MDC_GENERAL_CONFIG_PHYSICAL_R; 551 mdc_chan_writel(mchan, val, MDC_GENERAL_CONFIG); 552 val = (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) | 553 (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) | 554 (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT); 555 mdc_chan_writel(mchan, val, MDC_READ_PORT_CONFIG); 556 mdc_chan_writel(mchan, mdesc->list_phys, MDC_LIST_NODE_ADDRESS); 557 val = mdc_chan_readl(mchan, MDC_CONTROL_AND_STATUS); 558 val |= MDC_CONTROL_AND_STATUS_LIST_EN; 559 mdc_chan_writel(mchan, val, MDC_CONTROL_AND_STATUS); 560 } 561 562 static void mdc_issue_pending(struct dma_chan *chan) 563 { 564 struct mdc_chan *mchan = to_mdc_chan(chan); 565 unsigned long flags; 566 567 spin_lock_irqsave(&mchan->vc.lock, flags); 568 if (vchan_issue_pending(&mchan->vc) && !mchan->desc) 569 mdc_issue_desc(mchan); 570 spin_unlock_irqrestore(&mchan->vc.lock, flags); 571 } 572 573 static enum dma_status mdc_tx_status(struct dma_chan *chan, 574 dma_cookie_t cookie, struct dma_tx_state *txstate) 575 { 576 struct mdc_chan *mchan = to_mdc_chan(chan); 577 struct mdc_tx_desc *mdesc; 578 struct virt_dma_desc *vd; 579 unsigned long flags; 580 size_t bytes = 0; 581 int ret; 582 583 ret = dma_cookie_status(chan, cookie, txstate); 584 if (ret == DMA_COMPLETE) 585 return ret; 586 587 if (!txstate) 588 return ret; 589 590 spin_lock_irqsave(&mchan->vc.lock, flags); 591 vd = vchan_find_desc(&mchan->vc, cookie); 592 if (vd) { 593 mdesc = to_mdc_desc(&vd->tx); 594 bytes = mdesc->list_xfer_size; 595 } else if (mchan->desc && mchan->desc->vd.tx.cookie == cookie) { 596 struct mdc_hw_list_desc *ldesc; 597 u32 val1, val2, done, processed, residue; 598 int i, cmds; 599 600 mdesc = mchan->desc; 601 602 /* 603 * Determine the number of commands that haven't been 604 * processed (handled by the IRQ handler) yet. 605 */ 606 do { 607 val1 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) & 608 ~MDC_CMDS_PROCESSED_INT_ACTIVE; 609 residue = mdc_chan_readl(mchan, 610 MDC_ACTIVE_TRANSFER_SIZE); 611 val2 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) & 612 ~MDC_CMDS_PROCESSED_INT_ACTIVE; 613 } while (val1 != val2); 614 615 done = (val1 >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) & 616 MDC_CMDS_PROCESSED_CMDS_DONE_MASK; 617 processed = (val1 >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) & 618 MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK; 619 cmds = (done - processed) % 620 (MDC_CMDS_PROCESSED_CMDS_DONE_MASK + 1); 621 622 /* 623 * If the command loaded event hasn't been processed yet, then 624 * the difference above includes an extra command. 625 */ 626 if (!mdesc->cmd_loaded) 627 cmds--; 628 else 629 cmds += mdesc->list_cmds_done; 630 631 bytes = mdesc->list_xfer_size; 632 ldesc = mdesc->list; 633 for (i = 0; i < cmds; i++) { 634 bytes -= ldesc->xfer_size + 1; 635 ldesc = ldesc->next_desc; 636 } 637 if (ldesc) { 638 if (residue != MDC_TRANSFER_SIZE_MASK) 639 bytes -= ldesc->xfer_size - residue; 640 else 641 bytes -= ldesc->xfer_size + 1; 642 } 643 } 644 spin_unlock_irqrestore(&mchan->vc.lock, flags); 645 646 dma_set_residue(txstate, bytes); 647 648 return ret; 649 } 650 651 static unsigned int mdc_get_new_events(struct mdc_chan *mchan) 652 { 653 u32 val, processed, done1, done2; 654 unsigned int ret; 655 656 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED); 657 processed = (val >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) & 658 MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK; 659 /* 660 * CMDS_DONE may have incremented between reading CMDS_PROCESSED 661 * and clearing INT_ACTIVE. Re-read CMDS_PROCESSED to ensure we 662 * didn't miss a command completion. 663 */ 664 do { 665 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED); 666 667 done1 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) & 668 MDC_CMDS_PROCESSED_CMDS_DONE_MASK; 669 670 val &= ~((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK << 671 MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) | 672 MDC_CMDS_PROCESSED_INT_ACTIVE); 673 674 val |= done1 << MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT; 675 676 mdc_chan_writel(mchan, val, MDC_CMDS_PROCESSED); 677 678 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED); 679 680 done2 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) & 681 MDC_CMDS_PROCESSED_CMDS_DONE_MASK; 682 } while (done1 != done2); 683 684 if (done1 >= processed) 685 ret = done1 - processed; 686 else 687 ret = ((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK + 1) - 688 processed) + done1; 689 690 return ret; 691 } 692 693 static int mdc_terminate_all(struct dma_chan *chan) 694 { 695 struct mdc_chan *mchan = to_mdc_chan(chan); 696 struct mdc_tx_desc *mdesc; 697 unsigned long flags; 698 LIST_HEAD(head); 699 700 spin_lock_irqsave(&mchan->vc.lock, flags); 701 702 mdc_chan_writel(mchan, MDC_CONTROL_AND_STATUS_CANCEL, 703 MDC_CONTROL_AND_STATUS); 704 705 mdesc = mchan->desc; 706 mchan->desc = NULL; 707 vchan_get_all_descriptors(&mchan->vc, &head); 708 709 mdc_get_new_events(mchan); 710 711 spin_unlock_irqrestore(&mchan->vc.lock, flags); 712 713 if (mdesc) 714 mdc_desc_free(&mdesc->vd); 715 vchan_dma_desc_free_list(&mchan->vc, &head); 716 717 return 0; 718 } 719 720 static int mdc_slave_config(struct dma_chan *chan, 721 struct dma_slave_config *config) 722 { 723 struct mdc_chan *mchan = to_mdc_chan(chan); 724 unsigned long flags; 725 726 spin_lock_irqsave(&mchan->vc.lock, flags); 727 mchan->config = *config; 728 spin_unlock_irqrestore(&mchan->vc.lock, flags); 729 730 return 0; 731 } 732 733 static void mdc_free_chan_resources(struct dma_chan *chan) 734 { 735 struct mdc_chan *mchan = to_mdc_chan(chan); 736 struct mdc_dma *mdma = mchan->mdma; 737 738 mdc_terminate_all(chan); 739 740 mdma->soc->disable_chan(mchan); 741 } 742 743 static irqreturn_t mdc_chan_irq(int irq, void *dev_id) 744 { 745 struct mdc_chan *mchan = (struct mdc_chan *)dev_id; 746 struct mdc_tx_desc *mdesc; 747 unsigned int i, new_events; 748 749 spin_lock(&mchan->vc.lock); 750 751 dev_dbg(mdma2dev(mchan->mdma), "IRQ on channel %d\n", mchan->chan_nr); 752 753 new_events = mdc_get_new_events(mchan); 754 755 if (!new_events) 756 goto out; 757 758 mdesc = mchan->desc; 759 if (!mdesc) { 760 dev_warn(mdma2dev(mchan->mdma), 761 "IRQ with no active descriptor on channel %d\n", 762 mchan->chan_nr); 763 goto out; 764 } 765 766 for (i = 0; i < new_events; i++) { 767 /* 768 * The first interrupt in a transfer indicates that the 769 * command list has been loaded, not that a command has 770 * been completed. 771 */ 772 if (!mdesc->cmd_loaded) { 773 mdesc->cmd_loaded = true; 774 continue; 775 } 776 777 mdesc->list_cmds_done++; 778 if (mdesc->cyclic) { 779 mdesc->list_cmds_done %= mdesc->list_len; 780 if (mdesc->list_cmds_done % mdesc->list_period_len == 0) 781 vchan_cyclic_callback(&mdesc->vd); 782 } else if (mdesc->list_cmds_done == mdesc->list_len) { 783 mchan->desc = NULL; 784 vchan_cookie_complete(&mdesc->vd); 785 mdc_issue_desc(mchan); 786 break; 787 } 788 } 789 out: 790 spin_unlock(&mchan->vc.lock); 791 792 return IRQ_HANDLED; 793 } 794 795 static struct dma_chan *mdc_of_xlate(struct of_phandle_args *dma_spec, 796 struct of_dma *ofdma) 797 { 798 struct mdc_dma *mdma = ofdma->of_dma_data; 799 struct dma_chan *chan; 800 801 if (dma_spec->args_count != 3) 802 return NULL; 803 804 list_for_each_entry(chan, &mdma->dma_dev.channels, device_node) { 805 struct mdc_chan *mchan = to_mdc_chan(chan); 806 807 if (!(dma_spec->args[1] & BIT(mchan->chan_nr))) 808 continue; 809 if (dma_get_slave_channel(chan)) { 810 mchan->periph = dma_spec->args[0]; 811 mchan->thread = dma_spec->args[2]; 812 return chan; 813 } 814 } 815 816 return NULL; 817 } 818 819 #define PISTACHIO_CR_PERIPH_DMA_ROUTE(ch) (0x120 + 0x4 * ((ch) / 4)) 820 #define PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(ch) (8 * ((ch) % 4)) 821 #define PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK 0x3f 822 823 static void pistachio_mdc_enable_chan(struct mdc_chan *mchan) 824 { 825 struct mdc_dma *mdma = mchan->mdma; 826 827 regmap_update_bits(mdma->periph_regs, 828 PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr), 829 PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK << 830 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr), 831 mchan->periph << 832 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr)); 833 } 834 835 static void pistachio_mdc_disable_chan(struct mdc_chan *mchan) 836 { 837 struct mdc_dma *mdma = mchan->mdma; 838 839 regmap_update_bits(mdma->periph_regs, 840 PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr), 841 PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK << 842 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr), 843 0); 844 } 845 846 static const struct mdc_dma_soc_data pistachio_mdc_data = { 847 .enable_chan = pistachio_mdc_enable_chan, 848 .disable_chan = pistachio_mdc_disable_chan, 849 }; 850 851 static const struct of_device_id mdc_dma_of_match[] = { 852 { .compatible = "img,pistachio-mdc-dma", .data = &pistachio_mdc_data, }, 853 { }, 854 }; 855 MODULE_DEVICE_TABLE(of, mdc_dma_of_match); 856 857 static int mdc_dma_probe(struct platform_device *pdev) 858 { 859 struct mdc_dma *mdma; 860 struct resource *res; 861 unsigned int i; 862 u32 val; 863 int ret; 864 865 mdma = devm_kzalloc(&pdev->dev, sizeof(*mdma), GFP_KERNEL); 866 if (!mdma) 867 return -ENOMEM; 868 platform_set_drvdata(pdev, mdma); 869 870 mdma->soc = of_device_get_match_data(&pdev->dev); 871 872 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 873 mdma->regs = devm_ioremap_resource(&pdev->dev, res); 874 if (IS_ERR(mdma->regs)) 875 return PTR_ERR(mdma->regs); 876 877 mdma->periph_regs = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, 878 "img,cr-periph"); 879 if (IS_ERR(mdma->periph_regs)) 880 return PTR_ERR(mdma->periph_regs); 881 882 mdma->clk = devm_clk_get(&pdev->dev, "sys"); 883 if (IS_ERR(mdma->clk)) 884 return PTR_ERR(mdma->clk); 885 886 ret = clk_prepare_enable(mdma->clk); 887 if (ret) 888 return ret; 889 890 dma_cap_zero(mdma->dma_dev.cap_mask); 891 dma_cap_set(DMA_SLAVE, mdma->dma_dev.cap_mask); 892 dma_cap_set(DMA_PRIVATE, mdma->dma_dev.cap_mask); 893 dma_cap_set(DMA_CYCLIC, mdma->dma_dev.cap_mask); 894 dma_cap_set(DMA_MEMCPY, mdma->dma_dev.cap_mask); 895 896 val = mdc_readl(mdma, MDC_GLOBAL_CONFIG_A); 897 mdma->nr_channels = (val >> MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT) & 898 MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK; 899 mdma->nr_threads = 900 1 << ((val >> MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT) & 901 MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK); 902 mdma->bus_width = 903 (1 << ((val >> MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT) & 904 MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK)) / 8; 905 /* 906 * Although transfer sizes of up to MDC_TRANSFER_SIZE_MASK + 1 bytes 907 * are supported, this makes it possible for the value reported in 908 * MDC_ACTIVE_TRANSFER_SIZE to be ambiguous - an active transfer size 909 * of MDC_TRANSFER_SIZE_MASK may indicate either that 0 bytes or 910 * MDC_TRANSFER_SIZE_MASK + 1 bytes are remaining. To eliminate this 911 * ambiguity, restrict transfer sizes to one bus-width less than the 912 * actual maximum. 913 */ 914 mdma->max_xfer_size = MDC_TRANSFER_SIZE_MASK + 1 - mdma->bus_width; 915 916 of_property_read_u32(pdev->dev.of_node, "dma-channels", 917 &mdma->nr_channels); 918 ret = of_property_read_u32(pdev->dev.of_node, 919 "img,max-burst-multiplier", 920 &mdma->max_burst_mult); 921 if (ret) 922 goto disable_clk; 923 924 mdma->dma_dev.dev = &pdev->dev; 925 mdma->dma_dev.device_prep_slave_sg = mdc_prep_slave_sg; 926 mdma->dma_dev.device_prep_dma_cyclic = mdc_prep_dma_cyclic; 927 mdma->dma_dev.device_prep_dma_memcpy = mdc_prep_dma_memcpy; 928 mdma->dma_dev.device_free_chan_resources = mdc_free_chan_resources; 929 mdma->dma_dev.device_tx_status = mdc_tx_status; 930 mdma->dma_dev.device_issue_pending = mdc_issue_pending; 931 mdma->dma_dev.device_terminate_all = mdc_terminate_all; 932 mdma->dma_dev.device_config = mdc_slave_config; 933 934 mdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 935 mdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 936 for (i = 1; i <= mdma->bus_width; i <<= 1) { 937 mdma->dma_dev.src_addr_widths |= BIT(i); 938 mdma->dma_dev.dst_addr_widths |= BIT(i); 939 } 940 941 INIT_LIST_HEAD(&mdma->dma_dev.channels); 942 for (i = 0; i < mdma->nr_channels; i++) { 943 struct mdc_chan *mchan = &mdma->channels[i]; 944 945 mchan->mdma = mdma; 946 mchan->chan_nr = i; 947 mchan->irq = platform_get_irq(pdev, i); 948 if (mchan->irq < 0) { 949 ret = mchan->irq; 950 goto disable_clk; 951 } 952 ret = devm_request_irq(&pdev->dev, mchan->irq, mdc_chan_irq, 953 IRQ_TYPE_LEVEL_HIGH, 954 dev_name(&pdev->dev), mchan); 955 if (ret < 0) 956 goto disable_clk; 957 958 mchan->vc.desc_free = mdc_desc_free; 959 vchan_init(&mchan->vc, &mdma->dma_dev); 960 } 961 962 mdma->desc_pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev, 963 sizeof(struct mdc_hw_list_desc), 964 4, 0); 965 if (!mdma->desc_pool) { 966 ret = -ENOMEM; 967 goto disable_clk; 968 } 969 970 ret = dma_async_device_register(&mdma->dma_dev); 971 if (ret) 972 goto disable_clk; 973 974 ret = of_dma_controller_register(pdev->dev.of_node, mdc_of_xlate, mdma); 975 if (ret) 976 goto unregister; 977 978 dev_info(&pdev->dev, "MDC with %u channels and %u threads\n", 979 mdma->nr_channels, mdma->nr_threads); 980 981 return 0; 982 983 unregister: 984 dma_async_device_unregister(&mdma->dma_dev); 985 disable_clk: 986 clk_disable_unprepare(mdma->clk); 987 return ret; 988 } 989 990 static int mdc_dma_remove(struct platform_device *pdev) 991 { 992 struct mdc_dma *mdma = platform_get_drvdata(pdev); 993 struct mdc_chan *mchan, *next; 994 995 of_dma_controller_free(pdev->dev.of_node); 996 dma_async_device_unregister(&mdma->dma_dev); 997 998 list_for_each_entry_safe(mchan, next, &mdma->dma_dev.channels, 999 vc.chan.device_node) { 1000 list_del(&mchan->vc.chan.device_node); 1001 1002 devm_free_irq(&pdev->dev, mchan->irq, mchan); 1003 1004 tasklet_kill(&mchan->vc.task); 1005 } 1006 1007 clk_disable_unprepare(mdma->clk); 1008 1009 return 0; 1010 } 1011 1012 static struct platform_driver mdc_dma_driver = { 1013 .driver = { 1014 .name = "img-mdc-dma", 1015 .of_match_table = of_match_ptr(mdc_dma_of_match), 1016 }, 1017 .probe = mdc_dma_probe, 1018 .remove = mdc_dma_remove, 1019 }; 1020 module_platform_driver(mdc_dma_driver); 1021 1022 MODULE_DESCRIPTION("IMG Multi-threaded DMA Controller (MDC) driver"); 1023 MODULE_AUTHOR("Andrew Bresticker <abrestic@chromium.org>"); 1024 MODULE_LICENSE("GPL v2"); 1025