1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * BCM2835 DMA engine support 4 * 5 * This driver only supports cyclic DMA transfers 6 * as needed for the I2S module. 7 * 8 * Author: Florian Meier <florian.meier@koalo.de> 9 * Copyright 2013 10 * 11 * Based on 12 * OMAP DMAengine support by Russell King 13 * 14 * BCM2708 DMA Driver 15 * Copyright (C) 2010 Broadcom 16 * 17 * Raspberry Pi PCM I2S ALSA Driver 18 * Copyright (c) by Phil Poole 2013 19 * 20 * MARVELL MMP Peripheral DMA Driver 21 * Copyright 2012 Marvell International Ltd. 22 */ 23 #include <linux/dmaengine.h> 24 #include <linux/dma-mapping.h> 25 #include <linux/dmapool.h> 26 #include <linux/err.h> 27 #include <linux/init.h> 28 #include <linux/interrupt.h> 29 #include <linux/list.h> 30 #include <linux/module.h> 31 #include <linux/platform_device.h> 32 #include <linux/slab.h> 33 #include <linux/io.h> 34 #include <linux/spinlock.h> 35 #include <linux/of.h> 36 #include <linux/of_dma.h> 37 38 #include "virt-dma.h" 39 40 #define BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED 14 41 #define BCM2835_DMA_CHAN_NAME_SIZE 8 42 43 struct bcm2835_dmadev { 44 struct dma_device ddev; 45 spinlock_t lock; 46 void __iomem *base; 47 struct device_dma_parameters dma_parms; 48 }; 49 50 struct bcm2835_dma_cb { 51 uint32_t info; 52 uint32_t src; 53 uint32_t dst; 54 uint32_t length; 55 uint32_t stride; 56 uint32_t next; 57 uint32_t pad[2]; 58 }; 59 60 struct bcm2835_cb_entry { 61 struct bcm2835_dma_cb *cb; 62 dma_addr_t paddr; 63 }; 64 65 struct bcm2835_chan { 66 struct virt_dma_chan vc; 67 struct list_head node; 68 69 struct dma_slave_config cfg; 70 unsigned int dreq; 71 72 int ch; 73 struct bcm2835_desc *desc; 74 struct dma_pool *cb_pool; 75 76 void __iomem *chan_base; 77 int irq_number; 78 unsigned int irq_flags; 79 80 bool is_lite_channel; 81 }; 82 83 struct bcm2835_desc { 84 struct bcm2835_chan *c; 85 struct virt_dma_desc vd; 86 enum dma_transfer_direction dir; 87 88 unsigned int frames; 89 size_t size; 90 91 bool cyclic; 92 93 struct bcm2835_cb_entry cb_list[]; 94 }; 95 96 #define BCM2835_DMA_CS 0x00 97 #define BCM2835_DMA_ADDR 0x04 98 #define BCM2835_DMA_TI 0x08 99 #define BCM2835_DMA_SOURCE_AD 0x0c 100 #define BCM2835_DMA_DEST_AD 0x10 101 #define BCM2835_DMA_LEN 0x14 102 #define BCM2835_DMA_STRIDE 0x18 103 #define BCM2835_DMA_NEXTCB 0x1c 104 #define BCM2835_DMA_DEBUG 0x20 105 106 /* DMA CS Control and Status bits */ 107 #define BCM2835_DMA_ACTIVE BIT(0) /* activate the DMA */ 108 #define BCM2835_DMA_END BIT(1) /* current CB has ended */ 109 #define BCM2835_DMA_INT BIT(2) /* interrupt status */ 110 #define BCM2835_DMA_DREQ BIT(3) /* DREQ state */ 111 #define BCM2835_DMA_ISPAUSED BIT(4) /* Pause requested or not active */ 112 #define BCM2835_DMA_ISHELD BIT(5) /* Is held by DREQ flow control */ 113 #define BCM2835_DMA_WAITING_FOR_WRITES BIT(6) /* waiting for last 114 * AXI-write to ack 115 */ 116 #define BCM2835_DMA_ERR BIT(8) 117 #define BCM2835_DMA_PRIORITY(x) ((x & 15) << 16) /* AXI priority */ 118 #define BCM2835_DMA_PANIC_PRIORITY(x) ((x & 15) << 20) /* panic priority */ 119 /* current value of TI.BCM2835_DMA_WAIT_RESP */ 120 #define BCM2835_DMA_WAIT_FOR_WRITES BIT(28) 121 #define BCM2835_DMA_DIS_DEBUG BIT(29) /* disable debug pause signal */ 122 #define BCM2835_DMA_ABORT BIT(30) /* Stop current CB, go to next, WO */ 123 #define BCM2835_DMA_RESET BIT(31) /* WO, self clearing */ 124 125 /* Transfer information bits - also bcm2835_cb.info field */ 126 #define BCM2835_DMA_INT_EN BIT(0) 127 #define BCM2835_DMA_TDMODE BIT(1) /* 2D-Mode */ 128 #define BCM2835_DMA_WAIT_RESP BIT(3) /* wait for AXI-write to be acked */ 129 #define BCM2835_DMA_D_INC BIT(4) 130 #define BCM2835_DMA_D_WIDTH BIT(5) /* 128bit writes if set */ 131 #define BCM2835_DMA_D_DREQ BIT(6) /* enable DREQ for destination */ 132 #define BCM2835_DMA_D_IGNORE BIT(7) /* ignore destination writes */ 133 #define BCM2835_DMA_S_INC BIT(8) 134 #define BCM2835_DMA_S_WIDTH BIT(9) /* 128bit writes if set */ 135 #define BCM2835_DMA_S_DREQ BIT(10) /* enable SREQ for source */ 136 #define BCM2835_DMA_S_IGNORE BIT(11) /* ignore source reads - read 0 */ 137 #define BCM2835_DMA_BURST_LENGTH(x) ((x & 15) << 12) 138 #define BCM2835_DMA_PER_MAP(x) ((x & 31) << 16) /* REQ source */ 139 #define BCM2835_DMA_WAIT(x) ((x & 31) << 21) /* add DMA-wait cycles */ 140 #define BCM2835_DMA_NO_WIDE_BURSTS BIT(26) /* no 2 beat write bursts */ 141 142 /* debug register bits */ 143 #define BCM2835_DMA_DEBUG_LAST_NOT_SET_ERR BIT(0) 144 #define BCM2835_DMA_DEBUG_FIFO_ERR BIT(1) 145 #define BCM2835_DMA_DEBUG_READ_ERR BIT(2) 146 #define BCM2835_DMA_DEBUG_OUTSTANDING_WRITES_SHIFT 4 147 #define BCM2835_DMA_DEBUG_OUTSTANDING_WRITES_BITS 4 148 #define BCM2835_DMA_DEBUG_ID_SHIFT 16 149 #define BCM2835_DMA_DEBUG_ID_BITS 9 150 #define BCM2835_DMA_DEBUG_STATE_SHIFT 16 151 #define BCM2835_DMA_DEBUG_STATE_BITS 9 152 #define BCM2835_DMA_DEBUG_VERSION_SHIFT 25 153 #define BCM2835_DMA_DEBUG_VERSION_BITS 3 154 #define BCM2835_DMA_DEBUG_LITE BIT(28) 155 156 /* shared registers for all dma channels */ 157 #define BCM2835_DMA_INT_STATUS 0xfe0 158 #define BCM2835_DMA_ENABLE 0xff0 159 160 #define BCM2835_DMA_DATA_TYPE_S8 1 161 #define BCM2835_DMA_DATA_TYPE_S16 2 162 #define BCM2835_DMA_DATA_TYPE_S32 4 163 #define BCM2835_DMA_DATA_TYPE_S128 16 164 165 /* Valid only for channels 0 - 14, 15 has its own base address */ 166 #define BCM2835_DMA_CHAN(n) ((n) << 8) /* Base address */ 167 #define BCM2835_DMA_CHANIO(base, n) ((base) + BCM2835_DMA_CHAN(n)) 168 169 /* the max dma length for different channels */ 170 #define MAX_DMA_LEN SZ_1G 171 #define MAX_LITE_DMA_LEN (SZ_64K - 4) 172 173 static inline size_t bcm2835_dma_max_frame_length(struct bcm2835_chan *c) 174 { 175 /* lite and normal channels have different max frame length */ 176 return c->is_lite_channel ? MAX_LITE_DMA_LEN : MAX_DMA_LEN; 177 } 178 179 /* how many frames of max_len size do we need to transfer len bytes */ 180 static inline size_t bcm2835_dma_frames_for_length(size_t len, 181 size_t max_len) 182 { 183 return DIV_ROUND_UP(len, max_len); 184 } 185 186 static inline struct bcm2835_dmadev *to_bcm2835_dma_dev(struct dma_device *d) 187 { 188 return container_of(d, struct bcm2835_dmadev, ddev); 189 } 190 191 static inline struct bcm2835_chan *to_bcm2835_dma_chan(struct dma_chan *c) 192 { 193 return container_of(c, struct bcm2835_chan, vc.chan); 194 } 195 196 static inline struct bcm2835_desc *to_bcm2835_dma_desc( 197 struct dma_async_tx_descriptor *t) 198 { 199 return container_of(t, struct bcm2835_desc, vd.tx); 200 } 201 202 static void bcm2835_dma_free_cb_chain(struct bcm2835_desc *desc) 203 { 204 size_t i; 205 206 for (i = 0; i < desc->frames; i++) 207 dma_pool_free(desc->c->cb_pool, desc->cb_list[i].cb, 208 desc->cb_list[i].paddr); 209 210 kfree(desc); 211 } 212 213 static void bcm2835_dma_desc_free(struct virt_dma_desc *vd) 214 { 215 bcm2835_dma_free_cb_chain( 216 container_of(vd, struct bcm2835_desc, vd)); 217 } 218 219 static void bcm2835_dma_create_cb_set_length( 220 struct bcm2835_chan *chan, 221 struct bcm2835_dma_cb *control_block, 222 size_t len, 223 size_t period_len, 224 size_t *total_len, 225 u32 finalextrainfo) 226 { 227 size_t max_len = bcm2835_dma_max_frame_length(chan); 228 229 /* set the length taking lite-channel limitations into account */ 230 control_block->length = min_t(u32, len, max_len); 231 232 /* finished if we have no period_length */ 233 if (!period_len) 234 return; 235 236 /* 237 * period_len means: that we need to generate 238 * transfers that are terminating at every 239 * multiple of period_len - this is typically 240 * used to set the interrupt flag in info 241 * which is required during cyclic transfers 242 */ 243 244 /* have we filled in period_length yet? */ 245 if (*total_len + control_block->length < period_len) { 246 /* update number of bytes in this period so far */ 247 *total_len += control_block->length; 248 return; 249 } 250 251 /* calculate the length that remains to reach period_length */ 252 control_block->length = period_len - *total_len; 253 254 /* reset total_length for next period */ 255 *total_len = 0; 256 257 /* add extrainfo bits in info */ 258 control_block->info |= finalextrainfo; 259 } 260 261 static inline size_t bcm2835_dma_count_frames_for_sg( 262 struct bcm2835_chan *c, 263 struct scatterlist *sgl, 264 unsigned int sg_len) 265 { 266 size_t frames = 0; 267 struct scatterlist *sgent; 268 unsigned int i; 269 size_t plength = bcm2835_dma_max_frame_length(c); 270 271 for_each_sg(sgl, sgent, sg_len, i) 272 frames += bcm2835_dma_frames_for_length( 273 sg_dma_len(sgent), plength); 274 275 return frames; 276 } 277 278 /** 279 * bcm2835_dma_create_cb_chain - create a control block and fills data in 280 * 281 * @chan: the @dma_chan for which we run this 282 * @direction: the direction in which we transfer 283 * @cyclic: it is a cyclic transfer 284 * @info: the default info bits to apply per controlblock 285 * @frames: number of controlblocks to allocate 286 * @src: the src address to assign (if the S_INC bit is set 287 * in @info, then it gets incremented) 288 * @dst: the dst address to assign (if the D_INC bit is set 289 * in @info, then it gets incremented) 290 * @buf_len: the full buffer length (may also be 0) 291 * @period_len: the period length when to apply @finalextrainfo 292 * in addition to the last transfer 293 * this will also break some control-blocks early 294 * @finalextrainfo: additional bits in last controlblock 295 * (or when period_len is reached in case of cyclic) 296 * @gfp: the GFP flag to use for allocation 297 */ 298 static struct bcm2835_desc *bcm2835_dma_create_cb_chain( 299 struct dma_chan *chan, enum dma_transfer_direction direction, 300 bool cyclic, u32 info, u32 finalextrainfo, size_t frames, 301 dma_addr_t src, dma_addr_t dst, size_t buf_len, 302 size_t period_len, gfp_t gfp) 303 { 304 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 305 size_t len = buf_len, total_len; 306 size_t frame; 307 struct bcm2835_desc *d; 308 struct bcm2835_cb_entry *cb_entry; 309 struct bcm2835_dma_cb *control_block; 310 311 if (!frames) 312 return NULL; 313 314 /* allocate and setup the descriptor. */ 315 d = kzalloc(sizeof(*d) + frames * sizeof(struct bcm2835_cb_entry), 316 gfp); 317 if (!d) 318 return NULL; 319 320 d->c = c; 321 d->dir = direction; 322 d->cyclic = cyclic; 323 324 /* 325 * Iterate over all frames, create a control block 326 * for each frame and link them together. 327 */ 328 for (frame = 0, total_len = 0; frame < frames; d->frames++, frame++) { 329 cb_entry = &d->cb_list[frame]; 330 cb_entry->cb = dma_pool_alloc(c->cb_pool, gfp, 331 &cb_entry->paddr); 332 if (!cb_entry->cb) 333 goto error_cb; 334 335 /* fill in the control block */ 336 control_block = cb_entry->cb; 337 control_block->info = info; 338 control_block->src = src; 339 control_block->dst = dst; 340 control_block->stride = 0; 341 control_block->next = 0; 342 /* set up length in control_block if requested */ 343 if (buf_len) { 344 /* calculate length honoring period_length */ 345 bcm2835_dma_create_cb_set_length( 346 c, control_block, 347 len, period_len, &total_len, 348 cyclic ? finalextrainfo : 0); 349 350 /* calculate new remaining length */ 351 len -= control_block->length; 352 } 353 354 /* link this the last controlblock */ 355 if (frame) 356 d->cb_list[frame - 1].cb->next = cb_entry->paddr; 357 358 /* update src and dst and length */ 359 if (src && (info & BCM2835_DMA_S_INC)) 360 src += control_block->length; 361 if (dst && (info & BCM2835_DMA_D_INC)) 362 dst += control_block->length; 363 364 /* Length of total transfer */ 365 d->size += control_block->length; 366 } 367 368 /* the last frame requires extra flags */ 369 d->cb_list[d->frames - 1].cb->info |= finalextrainfo; 370 371 /* detect a size missmatch */ 372 if (buf_len && (d->size != buf_len)) 373 goto error_cb; 374 375 return d; 376 error_cb: 377 bcm2835_dma_free_cb_chain(d); 378 379 return NULL; 380 } 381 382 static void bcm2835_dma_fill_cb_chain_with_sg( 383 struct dma_chan *chan, 384 enum dma_transfer_direction direction, 385 struct bcm2835_cb_entry *cb, 386 struct scatterlist *sgl, 387 unsigned int sg_len) 388 { 389 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 390 size_t len, max_len; 391 unsigned int i; 392 dma_addr_t addr; 393 struct scatterlist *sgent; 394 395 max_len = bcm2835_dma_max_frame_length(c); 396 for_each_sg(sgl, sgent, sg_len, i) { 397 for (addr = sg_dma_address(sgent), len = sg_dma_len(sgent); 398 len > 0; 399 addr += cb->cb->length, len -= cb->cb->length, cb++) { 400 if (direction == DMA_DEV_TO_MEM) 401 cb->cb->dst = addr; 402 else 403 cb->cb->src = addr; 404 cb->cb->length = min(len, max_len); 405 } 406 } 407 } 408 409 static int bcm2835_dma_abort(void __iomem *chan_base) 410 { 411 unsigned long cs; 412 long int timeout = 10000; 413 414 cs = readl(chan_base + BCM2835_DMA_CS); 415 if (!(cs & BCM2835_DMA_ACTIVE)) 416 return 0; 417 418 /* Write 0 to the active bit - Pause the DMA */ 419 writel(0, chan_base + BCM2835_DMA_CS); 420 421 /* Wait for any current AXI transfer to complete */ 422 while ((cs & BCM2835_DMA_ISPAUSED) && --timeout) { 423 cpu_relax(); 424 cs = readl(chan_base + BCM2835_DMA_CS); 425 } 426 427 /* We'll un-pause when we set of our next DMA */ 428 if (!timeout) 429 return -ETIMEDOUT; 430 431 if (!(cs & BCM2835_DMA_ACTIVE)) 432 return 0; 433 434 /* Terminate the control block chain */ 435 writel(0, chan_base + BCM2835_DMA_NEXTCB); 436 437 /* Abort the whole DMA */ 438 writel(BCM2835_DMA_ABORT | BCM2835_DMA_ACTIVE, 439 chan_base + BCM2835_DMA_CS); 440 441 return 0; 442 } 443 444 static void bcm2835_dma_start_desc(struct bcm2835_chan *c) 445 { 446 struct virt_dma_desc *vd = vchan_next_desc(&c->vc); 447 struct bcm2835_desc *d; 448 449 if (!vd) { 450 c->desc = NULL; 451 return; 452 } 453 454 list_del(&vd->node); 455 456 c->desc = d = to_bcm2835_dma_desc(&vd->tx); 457 458 writel(d->cb_list[0].paddr, c->chan_base + BCM2835_DMA_ADDR); 459 writel(BCM2835_DMA_ACTIVE, c->chan_base + BCM2835_DMA_CS); 460 } 461 462 static irqreturn_t bcm2835_dma_callback(int irq, void *data) 463 { 464 struct bcm2835_chan *c = data; 465 struct bcm2835_desc *d; 466 unsigned long flags; 467 468 /* check the shared interrupt */ 469 if (c->irq_flags & IRQF_SHARED) { 470 /* check if the interrupt is enabled */ 471 flags = readl(c->chan_base + BCM2835_DMA_CS); 472 /* if not set then we are not the reason for the irq */ 473 if (!(flags & BCM2835_DMA_INT)) 474 return IRQ_NONE; 475 } 476 477 spin_lock_irqsave(&c->vc.lock, flags); 478 479 /* Acknowledge interrupt */ 480 writel(BCM2835_DMA_INT, c->chan_base + BCM2835_DMA_CS); 481 482 d = c->desc; 483 484 if (d) { 485 if (d->cyclic) { 486 /* call the cyclic callback */ 487 vchan_cyclic_callback(&d->vd); 488 489 /* Keep the DMA engine running */ 490 writel(BCM2835_DMA_ACTIVE, 491 c->chan_base + BCM2835_DMA_CS); 492 } else { 493 vchan_cookie_complete(&c->desc->vd); 494 bcm2835_dma_start_desc(c); 495 } 496 } 497 498 spin_unlock_irqrestore(&c->vc.lock, flags); 499 500 return IRQ_HANDLED; 501 } 502 503 static int bcm2835_dma_alloc_chan_resources(struct dma_chan *chan) 504 { 505 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 506 struct device *dev = c->vc.chan.device->dev; 507 508 dev_dbg(dev, "Allocating DMA channel %d\n", c->ch); 509 510 c->cb_pool = dma_pool_create(dev_name(dev), dev, 511 sizeof(struct bcm2835_dma_cb), 0, 0); 512 if (!c->cb_pool) { 513 dev_err(dev, "unable to allocate descriptor pool\n"); 514 return -ENOMEM; 515 } 516 517 return request_irq(c->irq_number, bcm2835_dma_callback, 518 c->irq_flags, "DMA IRQ", c); 519 } 520 521 static void bcm2835_dma_free_chan_resources(struct dma_chan *chan) 522 { 523 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 524 525 vchan_free_chan_resources(&c->vc); 526 free_irq(c->irq_number, c); 527 dma_pool_destroy(c->cb_pool); 528 529 dev_dbg(c->vc.chan.device->dev, "Freeing DMA channel %u\n", c->ch); 530 } 531 532 static size_t bcm2835_dma_desc_size(struct bcm2835_desc *d) 533 { 534 return d->size; 535 } 536 537 static size_t bcm2835_dma_desc_size_pos(struct bcm2835_desc *d, dma_addr_t addr) 538 { 539 unsigned int i; 540 size_t size; 541 542 for (size = i = 0; i < d->frames; i++) { 543 struct bcm2835_dma_cb *control_block = d->cb_list[i].cb; 544 size_t this_size = control_block->length; 545 dma_addr_t dma; 546 547 if (d->dir == DMA_DEV_TO_MEM) 548 dma = control_block->dst; 549 else 550 dma = control_block->src; 551 552 if (size) 553 size += this_size; 554 else if (addr >= dma && addr < dma + this_size) 555 size += dma + this_size - addr; 556 } 557 558 return size; 559 } 560 561 static enum dma_status bcm2835_dma_tx_status(struct dma_chan *chan, 562 dma_cookie_t cookie, struct dma_tx_state *txstate) 563 { 564 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 565 struct virt_dma_desc *vd; 566 enum dma_status ret; 567 unsigned long flags; 568 569 ret = dma_cookie_status(chan, cookie, txstate); 570 if (ret == DMA_COMPLETE || !txstate) 571 return ret; 572 573 spin_lock_irqsave(&c->vc.lock, flags); 574 vd = vchan_find_desc(&c->vc, cookie); 575 if (vd) { 576 txstate->residue = 577 bcm2835_dma_desc_size(to_bcm2835_dma_desc(&vd->tx)); 578 } else if (c->desc && c->desc->vd.tx.cookie == cookie) { 579 struct bcm2835_desc *d = c->desc; 580 dma_addr_t pos; 581 582 if (d->dir == DMA_MEM_TO_DEV) 583 pos = readl(c->chan_base + BCM2835_DMA_SOURCE_AD); 584 else if (d->dir == DMA_DEV_TO_MEM) 585 pos = readl(c->chan_base + BCM2835_DMA_DEST_AD); 586 else 587 pos = 0; 588 589 txstate->residue = bcm2835_dma_desc_size_pos(d, pos); 590 } else { 591 txstate->residue = 0; 592 } 593 594 spin_unlock_irqrestore(&c->vc.lock, flags); 595 596 return ret; 597 } 598 599 static void bcm2835_dma_issue_pending(struct dma_chan *chan) 600 { 601 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 602 unsigned long flags; 603 604 spin_lock_irqsave(&c->vc.lock, flags); 605 if (vchan_issue_pending(&c->vc) && !c->desc) 606 bcm2835_dma_start_desc(c); 607 608 spin_unlock_irqrestore(&c->vc.lock, flags); 609 } 610 611 static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_memcpy( 612 struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, 613 size_t len, unsigned long flags) 614 { 615 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 616 struct bcm2835_desc *d; 617 u32 info = BCM2835_DMA_D_INC | BCM2835_DMA_S_INC; 618 u32 extra = BCM2835_DMA_INT_EN | BCM2835_DMA_WAIT_RESP; 619 size_t max_len = bcm2835_dma_max_frame_length(c); 620 size_t frames; 621 622 /* if src, dst or len is not given return with an error */ 623 if (!src || !dst || !len) 624 return NULL; 625 626 /* calculate number of frames */ 627 frames = bcm2835_dma_frames_for_length(len, max_len); 628 629 /* allocate the CB chain - this also fills in the pointers */ 630 d = bcm2835_dma_create_cb_chain(chan, DMA_MEM_TO_MEM, false, 631 info, extra, frames, 632 src, dst, len, 0, GFP_KERNEL); 633 if (!d) 634 return NULL; 635 636 return vchan_tx_prep(&c->vc, &d->vd, flags); 637 } 638 639 static struct dma_async_tx_descriptor *bcm2835_dma_prep_slave_sg( 640 struct dma_chan *chan, 641 struct scatterlist *sgl, unsigned int sg_len, 642 enum dma_transfer_direction direction, 643 unsigned long flags, void *context) 644 { 645 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 646 struct bcm2835_desc *d; 647 dma_addr_t src = 0, dst = 0; 648 u32 info = BCM2835_DMA_WAIT_RESP; 649 u32 extra = BCM2835_DMA_INT_EN; 650 size_t frames; 651 652 if (!is_slave_direction(direction)) { 653 dev_err(chan->device->dev, 654 "%s: bad direction?\n", __func__); 655 return NULL; 656 } 657 658 if (c->dreq != 0) 659 info |= BCM2835_DMA_PER_MAP(c->dreq); 660 661 if (direction == DMA_DEV_TO_MEM) { 662 if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) 663 return NULL; 664 src = c->cfg.src_addr; 665 info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC; 666 } else { 667 if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) 668 return NULL; 669 dst = c->cfg.dst_addr; 670 info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC; 671 } 672 673 /* count frames in sg list */ 674 frames = bcm2835_dma_count_frames_for_sg(c, sgl, sg_len); 675 676 /* allocate the CB chain */ 677 d = bcm2835_dma_create_cb_chain(chan, direction, false, 678 info, extra, 679 frames, src, dst, 0, 0, 680 GFP_KERNEL); 681 if (!d) 682 return NULL; 683 684 /* fill in frames with scatterlist pointers */ 685 bcm2835_dma_fill_cb_chain_with_sg(chan, direction, d->cb_list, 686 sgl, sg_len); 687 688 return vchan_tx_prep(&c->vc, &d->vd, flags); 689 } 690 691 static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_cyclic( 692 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 693 size_t period_len, enum dma_transfer_direction direction, 694 unsigned long flags) 695 { 696 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 697 struct bcm2835_desc *d; 698 dma_addr_t src, dst; 699 u32 info = BCM2835_DMA_WAIT_RESP; 700 u32 extra = BCM2835_DMA_INT_EN; 701 size_t max_len = bcm2835_dma_max_frame_length(c); 702 size_t frames; 703 704 /* Grab configuration */ 705 if (!is_slave_direction(direction)) { 706 dev_err(chan->device->dev, "%s: bad direction?\n", __func__); 707 return NULL; 708 } 709 710 if (!buf_len) { 711 dev_err(chan->device->dev, 712 "%s: bad buffer length (= 0)\n", __func__); 713 return NULL; 714 } 715 716 /* 717 * warn if buf_len is not a multiple of period_len - this may leed 718 * to unexpected latencies for interrupts and thus audiable clicks 719 */ 720 if (buf_len % period_len) 721 dev_warn_once(chan->device->dev, 722 "%s: buffer_length (%zd) is not a multiple of period_len (%zd)\n", 723 __func__, buf_len, period_len); 724 725 /* Setup DREQ channel */ 726 if (c->dreq != 0) 727 info |= BCM2835_DMA_PER_MAP(c->dreq); 728 729 if (direction == DMA_DEV_TO_MEM) { 730 if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) 731 return NULL; 732 src = c->cfg.src_addr; 733 dst = buf_addr; 734 info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC; 735 } else { 736 if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) 737 return NULL; 738 dst = c->cfg.dst_addr; 739 src = buf_addr; 740 info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC; 741 } 742 743 /* calculate number of frames */ 744 frames = /* number of periods */ 745 DIV_ROUND_UP(buf_len, period_len) * 746 /* number of frames per period */ 747 bcm2835_dma_frames_for_length(period_len, max_len); 748 749 /* 750 * allocate the CB chain 751 * note that we need to use GFP_NOWAIT, as the ALSA i2s dmaengine 752 * implementation calls prep_dma_cyclic with interrupts disabled. 753 */ 754 d = bcm2835_dma_create_cb_chain(chan, direction, true, 755 info, extra, 756 frames, src, dst, buf_len, 757 period_len, GFP_NOWAIT); 758 if (!d) 759 return NULL; 760 761 /* wrap around into a loop */ 762 d->cb_list[d->frames - 1].cb->next = d->cb_list[0].paddr; 763 764 return vchan_tx_prep(&c->vc, &d->vd, flags); 765 } 766 767 static int bcm2835_dma_slave_config(struct dma_chan *chan, 768 struct dma_slave_config *cfg) 769 { 770 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 771 772 c->cfg = *cfg; 773 774 return 0; 775 } 776 777 static int bcm2835_dma_terminate_all(struct dma_chan *chan) 778 { 779 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 780 struct bcm2835_dmadev *d = to_bcm2835_dma_dev(c->vc.chan.device); 781 unsigned long flags; 782 int timeout = 10000; 783 LIST_HEAD(head); 784 785 spin_lock_irqsave(&c->vc.lock, flags); 786 787 /* Prevent this channel being scheduled */ 788 spin_lock(&d->lock); 789 list_del_init(&c->node); 790 spin_unlock(&d->lock); 791 792 /* 793 * Stop DMA activity: we assume the callback will not be called 794 * after bcm_dma_abort() returns (even if it does, it will see 795 * c->desc is NULL and exit.) 796 */ 797 if (c->desc) { 798 vchan_terminate_vdesc(&c->desc->vd); 799 c->desc = NULL; 800 bcm2835_dma_abort(c->chan_base); 801 802 /* Wait for stopping */ 803 while (--timeout) { 804 if (!(readl(c->chan_base + BCM2835_DMA_CS) & 805 BCM2835_DMA_ACTIVE)) 806 break; 807 808 cpu_relax(); 809 } 810 811 if (!timeout) 812 dev_err(d->ddev.dev, "DMA transfer could not be terminated\n"); 813 } 814 815 vchan_get_all_descriptors(&c->vc, &head); 816 spin_unlock_irqrestore(&c->vc.lock, flags); 817 vchan_dma_desc_free_list(&c->vc, &head); 818 819 return 0; 820 } 821 822 static void bcm2835_dma_synchronize(struct dma_chan *chan) 823 { 824 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 825 826 vchan_synchronize(&c->vc); 827 } 828 829 static int bcm2835_dma_chan_init(struct bcm2835_dmadev *d, int chan_id, 830 int irq, unsigned int irq_flags) 831 { 832 struct bcm2835_chan *c; 833 834 c = devm_kzalloc(d->ddev.dev, sizeof(*c), GFP_KERNEL); 835 if (!c) 836 return -ENOMEM; 837 838 c->vc.desc_free = bcm2835_dma_desc_free; 839 vchan_init(&c->vc, &d->ddev); 840 INIT_LIST_HEAD(&c->node); 841 842 c->chan_base = BCM2835_DMA_CHANIO(d->base, chan_id); 843 c->ch = chan_id; 844 c->irq_number = irq; 845 c->irq_flags = irq_flags; 846 847 /* check in DEBUG register if this is a LITE channel */ 848 if (readl(c->chan_base + BCM2835_DMA_DEBUG) & 849 BCM2835_DMA_DEBUG_LITE) 850 c->is_lite_channel = true; 851 852 return 0; 853 } 854 855 static void bcm2835_dma_free(struct bcm2835_dmadev *od) 856 { 857 struct bcm2835_chan *c, *next; 858 859 list_for_each_entry_safe(c, next, &od->ddev.channels, 860 vc.chan.device_node) { 861 list_del(&c->vc.chan.device_node); 862 tasklet_kill(&c->vc.task); 863 } 864 } 865 866 static const struct of_device_id bcm2835_dma_of_match[] = { 867 { .compatible = "brcm,bcm2835-dma", }, 868 {}, 869 }; 870 MODULE_DEVICE_TABLE(of, bcm2835_dma_of_match); 871 872 static struct dma_chan *bcm2835_dma_xlate(struct of_phandle_args *spec, 873 struct of_dma *ofdma) 874 { 875 struct bcm2835_dmadev *d = ofdma->of_dma_data; 876 struct dma_chan *chan; 877 878 chan = dma_get_any_slave_channel(&d->ddev); 879 if (!chan) 880 return NULL; 881 882 /* Set DREQ from param */ 883 to_bcm2835_dma_chan(chan)->dreq = spec->args[0]; 884 885 return chan; 886 } 887 888 static int bcm2835_dma_probe(struct platform_device *pdev) 889 { 890 struct bcm2835_dmadev *od; 891 struct resource *res; 892 void __iomem *base; 893 int rc; 894 int i, j; 895 int irq[BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED + 1]; 896 int irq_flags; 897 uint32_t chans_available; 898 char chan_name[BCM2835_DMA_CHAN_NAME_SIZE]; 899 900 if (!pdev->dev.dma_mask) 901 pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask; 902 903 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 904 if (rc) 905 return rc; 906 907 od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL); 908 if (!od) 909 return -ENOMEM; 910 911 pdev->dev.dma_parms = &od->dma_parms; 912 dma_set_max_seg_size(&pdev->dev, 0x3FFFFFFF); 913 914 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 915 base = devm_ioremap_resource(&pdev->dev, res); 916 if (IS_ERR(base)) 917 return PTR_ERR(base); 918 919 od->base = base; 920 921 dma_cap_set(DMA_SLAVE, od->ddev.cap_mask); 922 dma_cap_set(DMA_PRIVATE, od->ddev.cap_mask); 923 dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask); 924 dma_cap_set(DMA_SLAVE, od->ddev.cap_mask); 925 dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask); 926 od->ddev.device_alloc_chan_resources = bcm2835_dma_alloc_chan_resources; 927 od->ddev.device_free_chan_resources = bcm2835_dma_free_chan_resources; 928 od->ddev.device_tx_status = bcm2835_dma_tx_status; 929 od->ddev.device_issue_pending = bcm2835_dma_issue_pending; 930 od->ddev.device_prep_dma_cyclic = bcm2835_dma_prep_dma_cyclic; 931 od->ddev.device_prep_slave_sg = bcm2835_dma_prep_slave_sg; 932 od->ddev.device_prep_dma_memcpy = bcm2835_dma_prep_dma_memcpy; 933 od->ddev.device_config = bcm2835_dma_slave_config; 934 od->ddev.device_terminate_all = bcm2835_dma_terminate_all; 935 od->ddev.device_synchronize = bcm2835_dma_synchronize; 936 od->ddev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 937 od->ddev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 938 od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) | 939 BIT(DMA_MEM_TO_MEM); 940 od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 941 od->ddev.dev = &pdev->dev; 942 INIT_LIST_HEAD(&od->ddev.channels); 943 spin_lock_init(&od->lock); 944 945 platform_set_drvdata(pdev, od); 946 947 /* Request DMA channel mask from device tree */ 948 if (of_property_read_u32(pdev->dev.of_node, 949 "brcm,dma-channel-mask", 950 &chans_available)) { 951 dev_err(&pdev->dev, "Failed to get channel mask\n"); 952 rc = -EINVAL; 953 goto err_no_dma; 954 } 955 956 /* get irqs for each channel that we support */ 957 for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) { 958 /* skip masked out channels */ 959 if (!(chans_available & (1 << i))) { 960 irq[i] = -1; 961 continue; 962 } 963 964 /* get the named irq */ 965 snprintf(chan_name, sizeof(chan_name), "dma%i", i); 966 irq[i] = platform_get_irq_byname(pdev, chan_name); 967 if (irq[i] >= 0) 968 continue; 969 970 /* legacy device tree case handling */ 971 dev_warn_once(&pdev->dev, 972 "missing interrupt-names property in device tree - legacy interpretation is used\n"); 973 /* 974 * in case of channel >= 11 975 * use the 11th interrupt and that is shared 976 */ 977 irq[i] = platform_get_irq(pdev, i < 11 ? i : 11); 978 } 979 980 /* get irqs for each channel */ 981 for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) { 982 /* skip channels without irq */ 983 if (irq[i] < 0) 984 continue; 985 986 /* check if there are other channels that also use this irq */ 987 irq_flags = 0; 988 for (j = 0; j <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; j++) 989 if ((i != j) && (irq[j] == irq[i])) { 990 irq_flags = IRQF_SHARED; 991 break; 992 } 993 994 /* initialize the channel */ 995 rc = bcm2835_dma_chan_init(od, i, irq[i], irq_flags); 996 if (rc) 997 goto err_no_dma; 998 } 999 1000 dev_dbg(&pdev->dev, "Initialized %i DMA channels\n", i); 1001 1002 /* Device-tree DMA controller registration */ 1003 rc = of_dma_controller_register(pdev->dev.of_node, 1004 bcm2835_dma_xlate, od); 1005 if (rc) { 1006 dev_err(&pdev->dev, "Failed to register DMA controller\n"); 1007 goto err_no_dma; 1008 } 1009 1010 rc = dma_async_device_register(&od->ddev); 1011 if (rc) { 1012 dev_err(&pdev->dev, 1013 "Failed to register slave DMA engine device: %d\n", rc); 1014 goto err_no_dma; 1015 } 1016 1017 dev_dbg(&pdev->dev, "Load BCM2835 DMA engine driver\n"); 1018 1019 return 0; 1020 1021 err_no_dma: 1022 bcm2835_dma_free(od); 1023 return rc; 1024 } 1025 1026 static int bcm2835_dma_remove(struct platform_device *pdev) 1027 { 1028 struct bcm2835_dmadev *od = platform_get_drvdata(pdev); 1029 1030 dma_async_device_unregister(&od->ddev); 1031 bcm2835_dma_free(od); 1032 1033 return 0; 1034 } 1035 1036 static struct platform_driver bcm2835_dma_driver = { 1037 .probe = bcm2835_dma_probe, 1038 .remove = bcm2835_dma_remove, 1039 .driver = { 1040 .name = "bcm2835-dma", 1041 .of_match_table = of_match_ptr(bcm2835_dma_of_match), 1042 }, 1043 }; 1044 1045 module_platform_driver(bcm2835_dma_driver); 1046 1047 MODULE_ALIAS("platform:bcm2835-dma"); 1048 MODULE_DESCRIPTION("BCM2835 DMA engine driver"); 1049 MODULE_AUTHOR("Florian Meier <florian.meier@koalo.de>"); 1050 MODULE_LICENSE("GPL"); 1051