1 /* 2 * Renesas R-Car Gen2 DMA Controller Driver 3 * 4 * Copyright (C) 2014 Renesas Electronics Inc. 5 * 6 * Author: Laurent Pinchart <laurent.pinchart@ideasonboard.com> 7 * 8 * This is free software; you can redistribute it and/or modify 9 * it under the terms of version 2 of the GNU General Public License as 10 * published by the Free Software Foundation. 11 */ 12 13 #include <linux/dma-mapping.h> 14 #include <linux/dmaengine.h> 15 #include <linux/interrupt.h> 16 #include <linux/list.h> 17 #include <linux/module.h> 18 #include <linux/mutex.h> 19 #include <linux/of.h> 20 #include <linux/of_dma.h> 21 #include <linux/of_platform.h> 22 #include <linux/platform_device.h> 23 #include <linux/pm_runtime.h> 24 #include <linux/slab.h> 25 #include <linux/spinlock.h> 26 27 #include "../dmaengine.h" 28 29 /* 30 * struct rcar_dmac_xfer_chunk - Descriptor for a hardware transfer 31 * @node: entry in the parent's chunks list 32 * @src_addr: device source address 33 * @dst_addr: device destination address 34 * @size: transfer size in bytes 35 */ 36 struct rcar_dmac_xfer_chunk { 37 struct list_head node; 38 39 dma_addr_t src_addr; 40 dma_addr_t dst_addr; 41 u32 size; 42 }; 43 44 /* 45 * struct rcar_dmac_hw_desc - Hardware descriptor for a transfer chunk 46 * @sar: value of the SAR register (source address) 47 * @dar: value of the DAR register (destination address) 48 * @tcr: value of the TCR register (transfer count) 49 */ 50 struct rcar_dmac_hw_desc { 51 u32 sar; 52 u32 dar; 53 u32 tcr; 54 u32 reserved; 55 } __attribute__((__packed__)); 56 57 /* 58 * struct rcar_dmac_desc - R-Car Gen2 DMA Transfer Descriptor 59 * @async_tx: base DMA asynchronous transaction descriptor 60 * @direction: direction of the DMA transfer 61 * @xfer_shift: log2 of the transfer size 62 * @chcr: value of the channel configuration register for this transfer 63 * @node: entry in the channel's descriptors lists 64 * @chunks: list of transfer chunks for this transfer 65 * @running: the transfer chunk being currently processed 66 * @nchunks: number of transfer chunks for this transfer 67 * @hwdescs.use: whether the transfer descriptor uses hardware descriptors 68 * @hwdescs.mem: hardware descriptors memory for the transfer 69 * @hwdescs.dma: device address of the hardware descriptors memory 70 * @hwdescs.size: size of the hardware descriptors in bytes 71 * @size: transfer size in bytes 72 * @cyclic: when set indicates that the DMA transfer is cyclic 73 */ 74 struct rcar_dmac_desc { 75 struct dma_async_tx_descriptor async_tx; 76 enum dma_transfer_direction direction; 77 unsigned int xfer_shift; 78 u32 chcr; 79 80 struct list_head node; 81 struct list_head chunks; 82 struct rcar_dmac_xfer_chunk *running; 83 unsigned int nchunks; 84 85 struct { 86 bool use; 87 struct rcar_dmac_hw_desc *mem; 88 dma_addr_t dma; 89 size_t size; 90 } hwdescs; 91 92 unsigned int size; 93 bool cyclic; 94 }; 95 96 #define to_rcar_dmac_desc(d) container_of(d, struct rcar_dmac_desc, async_tx) 97 98 /* 99 * struct rcar_dmac_desc_page - One page worth of descriptors 100 * @node: entry in the channel's pages list 101 * @descs: array of DMA descriptors 102 * @chunks: array of transfer chunk descriptors 103 */ 104 struct rcar_dmac_desc_page { 105 struct list_head node; 106 107 union { 108 struct rcar_dmac_desc descs[0]; 109 struct rcar_dmac_xfer_chunk chunks[0]; 110 }; 111 }; 112 113 #define RCAR_DMAC_DESCS_PER_PAGE \ 114 ((PAGE_SIZE - offsetof(struct rcar_dmac_desc_page, descs)) / \ 115 sizeof(struct rcar_dmac_desc)) 116 #define RCAR_DMAC_XFER_CHUNKS_PER_PAGE \ 117 ((PAGE_SIZE - offsetof(struct rcar_dmac_desc_page, chunks)) / \ 118 sizeof(struct rcar_dmac_xfer_chunk)) 119 120 /* 121 * struct rcar_dmac_chan - R-Car Gen2 DMA Controller Channel 122 * @chan: base DMA channel object 123 * @iomem: channel I/O memory base 124 * @index: index of this channel in the controller 125 * @src_xfer_size: size (in bytes) of hardware transfers on the source side 126 * @dst_xfer_size: size (in bytes) of hardware transfers on the destination side 127 * @src_slave_addr: slave source memory address 128 * @dst_slave_addr: slave destination memory address 129 * @mid_rid: hardware MID/RID for the DMA client using this channel 130 * @lock: protects the channel CHCR register and the desc members 131 * @desc.free: list of free descriptors 132 * @desc.pending: list of pending descriptors (submitted with tx_submit) 133 * @desc.active: list of active descriptors (activated with issue_pending) 134 * @desc.done: list of completed descriptors 135 * @desc.wait: list of descriptors waiting for an ack 136 * @desc.running: the descriptor being processed (a member of the active list) 137 * @desc.chunks_free: list of free transfer chunk descriptors 138 * @desc.pages: list of pages used by allocated descriptors 139 */ 140 struct rcar_dmac_chan { 141 struct dma_chan chan; 142 void __iomem *iomem; 143 unsigned int index; 144 145 unsigned int src_xfer_size; 146 unsigned int dst_xfer_size; 147 dma_addr_t src_slave_addr; 148 dma_addr_t dst_slave_addr; 149 int mid_rid; 150 151 spinlock_t lock; 152 153 struct { 154 struct list_head free; 155 struct list_head pending; 156 struct list_head active; 157 struct list_head done; 158 struct list_head wait; 159 struct rcar_dmac_desc *running; 160 161 struct list_head chunks_free; 162 163 struct list_head pages; 164 } desc; 165 }; 166 167 #define to_rcar_dmac_chan(c) container_of(c, struct rcar_dmac_chan, chan) 168 169 /* 170 * struct rcar_dmac - R-Car Gen2 DMA Controller 171 * @engine: base DMA engine object 172 * @dev: the hardware device 173 * @iomem: remapped I/O memory base 174 * @n_channels: number of available channels 175 * @channels: array of DMAC channels 176 * @modules: bitmask of client modules in use 177 */ 178 struct rcar_dmac { 179 struct dma_device engine; 180 struct device *dev; 181 void __iomem *iomem; 182 183 unsigned int n_channels; 184 struct rcar_dmac_chan *channels; 185 186 DECLARE_BITMAP(modules, 256); 187 }; 188 189 #define to_rcar_dmac(d) container_of(d, struct rcar_dmac, engine) 190 191 /* ----------------------------------------------------------------------------- 192 * Registers 193 */ 194 195 #define RCAR_DMAC_CHAN_OFFSET(i) (0x8000 + 0x80 * (i)) 196 197 #define RCAR_DMAISTA 0x0020 198 #define RCAR_DMASEC 0x0030 199 #define RCAR_DMAOR 0x0060 200 #define RCAR_DMAOR_PRI_FIXED (0 << 8) 201 #define RCAR_DMAOR_PRI_ROUND_ROBIN (3 << 8) 202 #define RCAR_DMAOR_AE (1 << 2) 203 #define RCAR_DMAOR_DME (1 << 0) 204 #define RCAR_DMACHCLR 0x0080 205 #define RCAR_DMADPSEC 0x00a0 206 207 #define RCAR_DMASAR 0x0000 208 #define RCAR_DMADAR 0x0004 209 #define RCAR_DMATCR 0x0008 210 #define RCAR_DMATCR_MASK 0x00ffffff 211 #define RCAR_DMATSR 0x0028 212 #define RCAR_DMACHCR 0x000c 213 #define RCAR_DMACHCR_CAE (1 << 31) 214 #define RCAR_DMACHCR_CAIE (1 << 30) 215 #define RCAR_DMACHCR_DPM_DISABLED (0 << 28) 216 #define RCAR_DMACHCR_DPM_ENABLED (1 << 28) 217 #define RCAR_DMACHCR_DPM_REPEAT (2 << 28) 218 #define RCAR_DMACHCR_DPM_INFINITE (3 << 28) 219 #define RCAR_DMACHCR_RPT_SAR (1 << 27) 220 #define RCAR_DMACHCR_RPT_DAR (1 << 26) 221 #define RCAR_DMACHCR_RPT_TCR (1 << 25) 222 #define RCAR_DMACHCR_DPB (1 << 22) 223 #define RCAR_DMACHCR_DSE (1 << 19) 224 #define RCAR_DMACHCR_DSIE (1 << 18) 225 #define RCAR_DMACHCR_TS_1B ((0 << 20) | (0 << 3)) 226 #define RCAR_DMACHCR_TS_2B ((0 << 20) | (1 << 3)) 227 #define RCAR_DMACHCR_TS_4B ((0 << 20) | (2 << 3)) 228 #define RCAR_DMACHCR_TS_16B ((0 << 20) | (3 << 3)) 229 #define RCAR_DMACHCR_TS_32B ((1 << 20) | (0 << 3)) 230 #define RCAR_DMACHCR_TS_64B ((1 << 20) | (1 << 3)) 231 #define RCAR_DMACHCR_TS_8B ((1 << 20) | (3 << 3)) 232 #define RCAR_DMACHCR_DM_FIXED (0 << 14) 233 #define RCAR_DMACHCR_DM_INC (1 << 14) 234 #define RCAR_DMACHCR_DM_DEC (2 << 14) 235 #define RCAR_DMACHCR_SM_FIXED (0 << 12) 236 #define RCAR_DMACHCR_SM_INC (1 << 12) 237 #define RCAR_DMACHCR_SM_DEC (2 << 12) 238 #define RCAR_DMACHCR_RS_AUTO (4 << 8) 239 #define RCAR_DMACHCR_RS_DMARS (8 << 8) 240 #define RCAR_DMACHCR_IE (1 << 2) 241 #define RCAR_DMACHCR_TE (1 << 1) 242 #define RCAR_DMACHCR_DE (1 << 0) 243 #define RCAR_DMATCRB 0x0018 244 #define RCAR_DMATSRB 0x0038 245 #define RCAR_DMACHCRB 0x001c 246 #define RCAR_DMACHCRB_DCNT(n) ((n) << 24) 247 #define RCAR_DMACHCRB_DPTR_MASK (0xff << 16) 248 #define RCAR_DMACHCRB_DPTR_SHIFT 16 249 #define RCAR_DMACHCRB_DRST (1 << 15) 250 #define RCAR_DMACHCRB_DTS (1 << 8) 251 #define RCAR_DMACHCRB_SLM_NORMAL (0 << 4) 252 #define RCAR_DMACHCRB_SLM_CLK(n) ((8 | (n)) << 4) 253 #define RCAR_DMACHCRB_PRI(n) ((n) << 0) 254 #define RCAR_DMARS 0x0040 255 #define RCAR_DMABUFCR 0x0048 256 #define RCAR_DMABUFCR_MBU(n) ((n) << 16) 257 #define RCAR_DMABUFCR_ULB(n) ((n) << 0) 258 #define RCAR_DMADPBASE 0x0050 259 #define RCAR_DMADPBASE_MASK 0xfffffff0 260 #define RCAR_DMADPBASE_SEL (1 << 0) 261 #define RCAR_DMADPCR 0x0054 262 #define RCAR_DMADPCR_DIPT(n) ((n) << 24) 263 #define RCAR_DMAFIXSAR 0x0010 264 #define RCAR_DMAFIXDAR 0x0014 265 #define RCAR_DMAFIXDPBASE 0x0060 266 267 /* Hardcode the MEMCPY transfer size to 4 bytes. */ 268 #define RCAR_DMAC_MEMCPY_XFER_SIZE 4 269 270 /* ----------------------------------------------------------------------------- 271 * Device access 272 */ 273 274 static void rcar_dmac_write(struct rcar_dmac *dmac, u32 reg, u32 data) 275 { 276 if (reg == RCAR_DMAOR) 277 writew(data, dmac->iomem + reg); 278 else 279 writel(data, dmac->iomem + reg); 280 } 281 282 static u32 rcar_dmac_read(struct rcar_dmac *dmac, u32 reg) 283 { 284 if (reg == RCAR_DMAOR) 285 return readw(dmac->iomem + reg); 286 else 287 return readl(dmac->iomem + reg); 288 } 289 290 static u32 rcar_dmac_chan_read(struct rcar_dmac_chan *chan, u32 reg) 291 { 292 if (reg == RCAR_DMARS) 293 return readw(chan->iomem + reg); 294 else 295 return readl(chan->iomem + reg); 296 } 297 298 static void rcar_dmac_chan_write(struct rcar_dmac_chan *chan, u32 reg, u32 data) 299 { 300 if (reg == RCAR_DMARS) 301 writew(data, chan->iomem + reg); 302 else 303 writel(data, chan->iomem + reg); 304 } 305 306 /* ----------------------------------------------------------------------------- 307 * Initialization and configuration 308 */ 309 310 static bool rcar_dmac_chan_is_busy(struct rcar_dmac_chan *chan) 311 { 312 u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 313 314 return !!(chcr & (RCAR_DMACHCR_DE | RCAR_DMACHCR_TE)); 315 } 316 317 static void rcar_dmac_chan_start_xfer(struct rcar_dmac_chan *chan) 318 { 319 struct rcar_dmac_desc *desc = chan->desc.running; 320 u32 chcr = desc->chcr; 321 322 WARN_ON_ONCE(rcar_dmac_chan_is_busy(chan)); 323 324 if (chan->mid_rid >= 0) 325 rcar_dmac_chan_write(chan, RCAR_DMARS, chan->mid_rid); 326 327 if (desc->hwdescs.use) { 328 struct rcar_dmac_xfer_chunk *chunk; 329 330 dev_dbg(chan->chan.device->dev, 331 "chan%u: queue desc %p: %u@%pad\n", 332 chan->index, desc, desc->nchunks, &desc->hwdescs.dma); 333 334 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 335 rcar_dmac_chan_write(chan, RCAR_DMAFIXDPBASE, 336 desc->hwdescs.dma >> 32); 337 #endif 338 rcar_dmac_chan_write(chan, RCAR_DMADPBASE, 339 (desc->hwdescs.dma & 0xfffffff0) | 340 RCAR_DMADPBASE_SEL); 341 rcar_dmac_chan_write(chan, RCAR_DMACHCRB, 342 RCAR_DMACHCRB_DCNT(desc->nchunks - 1) | 343 RCAR_DMACHCRB_DRST); 344 345 /* 346 * Errata: When descriptor memory is accessed through an IOMMU 347 * the DMADAR register isn't initialized automatically from the 348 * first descriptor at beginning of transfer by the DMAC like it 349 * should. Initialize it manually with the destination address 350 * of the first chunk. 351 */ 352 chunk = list_first_entry(&desc->chunks, 353 struct rcar_dmac_xfer_chunk, node); 354 rcar_dmac_chan_write(chan, RCAR_DMADAR, 355 chunk->dst_addr & 0xffffffff); 356 357 /* 358 * Program the descriptor stage interrupt to occur after the end 359 * of the first stage. 360 */ 361 rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(1)); 362 363 chcr |= RCAR_DMACHCR_RPT_SAR | RCAR_DMACHCR_RPT_DAR 364 | RCAR_DMACHCR_RPT_TCR | RCAR_DMACHCR_DPB; 365 366 /* 367 * If the descriptor isn't cyclic enable normal descriptor mode 368 * and the transfer completion interrupt. 369 */ 370 if (!desc->cyclic) 371 chcr |= RCAR_DMACHCR_DPM_ENABLED | RCAR_DMACHCR_IE; 372 /* 373 * If the descriptor is cyclic and has a callback enable the 374 * descriptor stage interrupt in infinite repeat mode. 375 */ 376 else if (desc->async_tx.callback) 377 chcr |= RCAR_DMACHCR_DPM_INFINITE | RCAR_DMACHCR_DSIE; 378 /* 379 * Otherwise just select infinite repeat mode without any 380 * interrupt. 381 */ 382 else 383 chcr |= RCAR_DMACHCR_DPM_INFINITE; 384 } else { 385 struct rcar_dmac_xfer_chunk *chunk = desc->running; 386 387 dev_dbg(chan->chan.device->dev, 388 "chan%u: queue chunk %p: %u@%pad -> %pad\n", 389 chan->index, chunk, chunk->size, &chunk->src_addr, 390 &chunk->dst_addr); 391 392 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 393 rcar_dmac_chan_write(chan, RCAR_DMAFIXSAR, 394 chunk->src_addr >> 32); 395 rcar_dmac_chan_write(chan, RCAR_DMAFIXDAR, 396 chunk->dst_addr >> 32); 397 #endif 398 rcar_dmac_chan_write(chan, RCAR_DMASAR, 399 chunk->src_addr & 0xffffffff); 400 rcar_dmac_chan_write(chan, RCAR_DMADAR, 401 chunk->dst_addr & 0xffffffff); 402 rcar_dmac_chan_write(chan, RCAR_DMATCR, 403 chunk->size >> desc->xfer_shift); 404 405 chcr |= RCAR_DMACHCR_DPM_DISABLED | RCAR_DMACHCR_IE; 406 } 407 408 rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr | RCAR_DMACHCR_DE); 409 } 410 411 static int rcar_dmac_init(struct rcar_dmac *dmac) 412 { 413 u16 dmaor; 414 415 /* Clear all channels and enable the DMAC globally. */ 416 rcar_dmac_write(dmac, RCAR_DMACHCLR, GENMASK(dmac->n_channels - 1, 0)); 417 rcar_dmac_write(dmac, RCAR_DMAOR, 418 RCAR_DMAOR_PRI_FIXED | RCAR_DMAOR_DME); 419 420 dmaor = rcar_dmac_read(dmac, RCAR_DMAOR); 421 if ((dmaor & (RCAR_DMAOR_AE | RCAR_DMAOR_DME)) != RCAR_DMAOR_DME) { 422 dev_warn(dmac->dev, "DMAOR initialization failed.\n"); 423 return -EIO; 424 } 425 426 return 0; 427 } 428 429 /* ----------------------------------------------------------------------------- 430 * Descriptors submission 431 */ 432 433 static dma_cookie_t rcar_dmac_tx_submit(struct dma_async_tx_descriptor *tx) 434 { 435 struct rcar_dmac_chan *chan = to_rcar_dmac_chan(tx->chan); 436 struct rcar_dmac_desc *desc = to_rcar_dmac_desc(tx); 437 unsigned long flags; 438 dma_cookie_t cookie; 439 440 spin_lock_irqsave(&chan->lock, flags); 441 442 cookie = dma_cookie_assign(tx); 443 444 dev_dbg(chan->chan.device->dev, "chan%u: submit #%d@%p\n", 445 chan->index, tx->cookie, desc); 446 447 list_add_tail(&desc->node, &chan->desc.pending); 448 desc->running = list_first_entry(&desc->chunks, 449 struct rcar_dmac_xfer_chunk, node); 450 451 spin_unlock_irqrestore(&chan->lock, flags); 452 453 return cookie; 454 } 455 456 /* ----------------------------------------------------------------------------- 457 * Descriptors allocation and free 458 */ 459 460 /* 461 * rcar_dmac_desc_alloc - Allocate a page worth of DMA descriptors 462 * @chan: the DMA channel 463 * @gfp: allocation flags 464 */ 465 static int rcar_dmac_desc_alloc(struct rcar_dmac_chan *chan, gfp_t gfp) 466 { 467 struct rcar_dmac_desc_page *page; 468 unsigned long flags; 469 LIST_HEAD(list); 470 unsigned int i; 471 472 page = (void *)get_zeroed_page(gfp); 473 if (!page) 474 return -ENOMEM; 475 476 for (i = 0; i < RCAR_DMAC_DESCS_PER_PAGE; ++i) { 477 struct rcar_dmac_desc *desc = &page->descs[i]; 478 479 dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan); 480 desc->async_tx.tx_submit = rcar_dmac_tx_submit; 481 INIT_LIST_HEAD(&desc->chunks); 482 483 list_add_tail(&desc->node, &list); 484 } 485 486 spin_lock_irqsave(&chan->lock, flags); 487 list_splice_tail(&list, &chan->desc.free); 488 list_add_tail(&page->node, &chan->desc.pages); 489 spin_unlock_irqrestore(&chan->lock, flags); 490 491 return 0; 492 } 493 494 /* 495 * rcar_dmac_desc_put - Release a DMA transfer descriptor 496 * @chan: the DMA channel 497 * @desc: the descriptor 498 * 499 * Put the descriptor and its transfer chunk descriptors back in the channel's 500 * free descriptors lists. The descriptor's chunks list will be reinitialized to 501 * an empty list as a result. 502 * 503 * The descriptor must have been removed from the channel's lists before calling 504 * this function. 505 */ 506 static void rcar_dmac_desc_put(struct rcar_dmac_chan *chan, 507 struct rcar_dmac_desc *desc) 508 { 509 unsigned long flags; 510 511 spin_lock_irqsave(&chan->lock, flags); 512 list_splice_tail_init(&desc->chunks, &chan->desc.chunks_free); 513 list_add(&desc->node, &chan->desc.free); 514 spin_unlock_irqrestore(&chan->lock, flags); 515 } 516 517 static void rcar_dmac_desc_recycle_acked(struct rcar_dmac_chan *chan) 518 { 519 struct rcar_dmac_desc *desc, *_desc; 520 unsigned long flags; 521 LIST_HEAD(list); 522 523 /* 524 * We have to temporarily move all descriptors from the wait list to a 525 * local list as iterating over the wait list, even with 526 * list_for_each_entry_safe, isn't safe if we release the channel lock 527 * around the rcar_dmac_desc_put() call. 528 */ 529 spin_lock_irqsave(&chan->lock, flags); 530 list_splice_init(&chan->desc.wait, &list); 531 spin_unlock_irqrestore(&chan->lock, flags); 532 533 list_for_each_entry_safe(desc, _desc, &list, node) { 534 if (async_tx_test_ack(&desc->async_tx)) { 535 list_del(&desc->node); 536 rcar_dmac_desc_put(chan, desc); 537 } 538 } 539 540 if (list_empty(&list)) 541 return; 542 543 /* Put the remaining descriptors back in the wait list. */ 544 spin_lock_irqsave(&chan->lock, flags); 545 list_splice(&list, &chan->desc.wait); 546 spin_unlock_irqrestore(&chan->lock, flags); 547 } 548 549 /* 550 * rcar_dmac_desc_get - Allocate a descriptor for a DMA transfer 551 * @chan: the DMA channel 552 * 553 * Locking: This function must be called in a non-atomic context. 554 * 555 * Return: A pointer to the allocated descriptor or NULL if no descriptor can 556 * be allocated. 557 */ 558 static struct rcar_dmac_desc *rcar_dmac_desc_get(struct rcar_dmac_chan *chan) 559 { 560 struct rcar_dmac_desc *desc; 561 unsigned long flags; 562 int ret; 563 564 /* Recycle acked descriptors before attempting allocation. */ 565 rcar_dmac_desc_recycle_acked(chan); 566 567 spin_lock_irqsave(&chan->lock, flags); 568 569 while (list_empty(&chan->desc.free)) { 570 /* 571 * No free descriptors, allocate a page worth of them and try 572 * again, as someone else could race us to get the newly 573 * allocated descriptors. If the allocation fails return an 574 * error. 575 */ 576 spin_unlock_irqrestore(&chan->lock, flags); 577 ret = rcar_dmac_desc_alloc(chan, GFP_NOWAIT); 578 if (ret < 0) 579 return NULL; 580 spin_lock_irqsave(&chan->lock, flags); 581 } 582 583 desc = list_first_entry(&chan->desc.free, struct rcar_dmac_desc, node); 584 list_del(&desc->node); 585 586 spin_unlock_irqrestore(&chan->lock, flags); 587 588 return desc; 589 } 590 591 /* 592 * rcar_dmac_xfer_chunk_alloc - Allocate a page worth of transfer chunks 593 * @chan: the DMA channel 594 * @gfp: allocation flags 595 */ 596 static int rcar_dmac_xfer_chunk_alloc(struct rcar_dmac_chan *chan, gfp_t gfp) 597 { 598 struct rcar_dmac_desc_page *page; 599 unsigned long flags; 600 LIST_HEAD(list); 601 unsigned int i; 602 603 page = (void *)get_zeroed_page(gfp); 604 if (!page) 605 return -ENOMEM; 606 607 for (i = 0; i < RCAR_DMAC_XFER_CHUNKS_PER_PAGE; ++i) { 608 struct rcar_dmac_xfer_chunk *chunk = &page->chunks[i]; 609 610 list_add_tail(&chunk->node, &list); 611 } 612 613 spin_lock_irqsave(&chan->lock, flags); 614 list_splice_tail(&list, &chan->desc.chunks_free); 615 list_add_tail(&page->node, &chan->desc.pages); 616 spin_unlock_irqrestore(&chan->lock, flags); 617 618 return 0; 619 } 620 621 /* 622 * rcar_dmac_xfer_chunk_get - Allocate a transfer chunk for a DMA transfer 623 * @chan: the DMA channel 624 * 625 * Locking: This function must be called in a non-atomic context. 626 * 627 * Return: A pointer to the allocated transfer chunk descriptor or NULL if no 628 * descriptor can be allocated. 629 */ 630 static struct rcar_dmac_xfer_chunk * 631 rcar_dmac_xfer_chunk_get(struct rcar_dmac_chan *chan) 632 { 633 struct rcar_dmac_xfer_chunk *chunk; 634 unsigned long flags; 635 int ret; 636 637 spin_lock_irqsave(&chan->lock, flags); 638 639 while (list_empty(&chan->desc.chunks_free)) { 640 /* 641 * No free descriptors, allocate a page worth of them and try 642 * again, as someone else could race us to get the newly 643 * allocated descriptors. If the allocation fails return an 644 * error. 645 */ 646 spin_unlock_irqrestore(&chan->lock, flags); 647 ret = rcar_dmac_xfer_chunk_alloc(chan, GFP_NOWAIT); 648 if (ret < 0) 649 return NULL; 650 spin_lock_irqsave(&chan->lock, flags); 651 } 652 653 chunk = list_first_entry(&chan->desc.chunks_free, 654 struct rcar_dmac_xfer_chunk, node); 655 list_del(&chunk->node); 656 657 spin_unlock_irqrestore(&chan->lock, flags); 658 659 return chunk; 660 } 661 662 static void rcar_dmac_realloc_hwdesc(struct rcar_dmac_chan *chan, 663 struct rcar_dmac_desc *desc, size_t size) 664 { 665 /* 666 * dma_alloc_coherent() allocates memory in page size increments. To 667 * avoid reallocating the hardware descriptors when the allocated size 668 * wouldn't change align the requested size to a multiple of the page 669 * size. 670 */ 671 size = PAGE_ALIGN(size); 672 673 if (desc->hwdescs.size == size) 674 return; 675 676 if (desc->hwdescs.mem) { 677 dma_free_coherent(chan->chan.device->dev, desc->hwdescs.size, 678 desc->hwdescs.mem, desc->hwdescs.dma); 679 desc->hwdescs.mem = NULL; 680 desc->hwdescs.size = 0; 681 } 682 683 if (!size) 684 return; 685 686 desc->hwdescs.mem = dma_alloc_coherent(chan->chan.device->dev, size, 687 &desc->hwdescs.dma, GFP_NOWAIT); 688 if (!desc->hwdescs.mem) 689 return; 690 691 desc->hwdescs.size = size; 692 } 693 694 static int rcar_dmac_fill_hwdesc(struct rcar_dmac_chan *chan, 695 struct rcar_dmac_desc *desc) 696 { 697 struct rcar_dmac_xfer_chunk *chunk; 698 struct rcar_dmac_hw_desc *hwdesc; 699 700 rcar_dmac_realloc_hwdesc(chan, desc, desc->nchunks * sizeof(*hwdesc)); 701 702 hwdesc = desc->hwdescs.mem; 703 if (!hwdesc) 704 return -ENOMEM; 705 706 list_for_each_entry(chunk, &desc->chunks, node) { 707 hwdesc->sar = chunk->src_addr; 708 hwdesc->dar = chunk->dst_addr; 709 hwdesc->tcr = chunk->size >> desc->xfer_shift; 710 hwdesc++; 711 } 712 713 return 0; 714 } 715 716 /* ----------------------------------------------------------------------------- 717 * Stop and reset 718 */ 719 720 static void rcar_dmac_chan_halt(struct rcar_dmac_chan *chan) 721 { 722 u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 723 724 chcr &= ~(RCAR_DMACHCR_DSE | RCAR_DMACHCR_DSIE | RCAR_DMACHCR_IE | 725 RCAR_DMACHCR_TE | RCAR_DMACHCR_DE); 726 rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr); 727 } 728 729 static void rcar_dmac_chan_reinit(struct rcar_dmac_chan *chan) 730 { 731 struct rcar_dmac_desc *desc, *_desc; 732 unsigned long flags; 733 LIST_HEAD(descs); 734 735 spin_lock_irqsave(&chan->lock, flags); 736 737 /* Move all non-free descriptors to the local lists. */ 738 list_splice_init(&chan->desc.pending, &descs); 739 list_splice_init(&chan->desc.active, &descs); 740 list_splice_init(&chan->desc.done, &descs); 741 list_splice_init(&chan->desc.wait, &descs); 742 743 chan->desc.running = NULL; 744 745 spin_unlock_irqrestore(&chan->lock, flags); 746 747 list_for_each_entry_safe(desc, _desc, &descs, node) { 748 list_del(&desc->node); 749 rcar_dmac_desc_put(chan, desc); 750 } 751 } 752 753 static void rcar_dmac_stop(struct rcar_dmac *dmac) 754 { 755 rcar_dmac_write(dmac, RCAR_DMAOR, 0); 756 } 757 758 static void rcar_dmac_abort(struct rcar_dmac *dmac) 759 { 760 unsigned int i; 761 762 /* Stop all channels. */ 763 for (i = 0; i < dmac->n_channels; ++i) { 764 struct rcar_dmac_chan *chan = &dmac->channels[i]; 765 766 /* Stop and reinitialize the channel. */ 767 spin_lock(&chan->lock); 768 rcar_dmac_chan_halt(chan); 769 spin_unlock(&chan->lock); 770 771 rcar_dmac_chan_reinit(chan); 772 } 773 } 774 775 /* ----------------------------------------------------------------------------- 776 * Descriptors preparation 777 */ 778 779 static void rcar_dmac_chan_configure_desc(struct rcar_dmac_chan *chan, 780 struct rcar_dmac_desc *desc) 781 { 782 static const u32 chcr_ts[] = { 783 RCAR_DMACHCR_TS_1B, RCAR_DMACHCR_TS_2B, 784 RCAR_DMACHCR_TS_4B, RCAR_DMACHCR_TS_8B, 785 RCAR_DMACHCR_TS_16B, RCAR_DMACHCR_TS_32B, 786 RCAR_DMACHCR_TS_64B, 787 }; 788 789 unsigned int xfer_size; 790 u32 chcr; 791 792 switch (desc->direction) { 793 case DMA_DEV_TO_MEM: 794 chcr = RCAR_DMACHCR_DM_INC | RCAR_DMACHCR_SM_FIXED 795 | RCAR_DMACHCR_RS_DMARS; 796 xfer_size = chan->src_xfer_size; 797 break; 798 799 case DMA_MEM_TO_DEV: 800 chcr = RCAR_DMACHCR_DM_FIXED | RCAR_DMACHCR_SM_INC 801 | RCAR_DMACHCR_RS_DMARS; 802 xfer_size = chan->dst_xfer_size; 803 break; 804 805 case DMA_MEM_TO_MEM: 806 default: 807 chcr = RCAR_DMACHCR_DM_INC | RCAR_DMACHCR_SM_INC 808 | RCAR_DMACHCR_RS_AUTO; 809 xfer_size = RCAR_DMAC_MEMCPY_XFER_SIZE; 810 break; 811 } 812 813 desc->xfer_shift = ilog2(xfer_size); 814 desc->chcr = chcr | chcr_ts[desc->xfer_shift]; 815 } 816 817 /* 818 * rcar_dmac_chan_prep_sg - prepare transfer descriptors from an SG list 819 * 820 * Common routine for public (MEMCPY) and slave DMA. The MEMCPY case is also 821 * converted to scatter-gather to guarantee consistent locking and a correct 822 * list manipulation. For slave DMA direction carries the usual meaning, and, 823 * logically, the SG list is RAM and the addr variable contains slave address, 824 * e.g., the FIFO I/O register. For MEMCPY direction equals DMA_MEM_TO_MEM 825 * and the SG list contains only one element and points at the source buffer. 826 */ 827 static struct dma_async_tx_descriptor * 828 rcar_dmac_chan_prep_sg(struct rcar_dmac_chan *chan, struct scatterlist *sgl, 829 unsigned int sg_len, dma_addr_t dev_addr, 830 enum dma_transfer_direction dir, unsigned long dma_flags, 831 bool cyclic) 832 { 833 struct rcar_dmac_xfer_chunk *chunk; 834 struct rcar_dmac_desc *desc; 835 struct scatterlist *sg; 836 unsigned int nchunks = 0; 837 unsigned int max_chunk_size; 838 unsigned int full_size = 0; 839 bool highmem = false; 840 unsigned int i; 841 842 desc = rcar_dmac_desc_get(chan); 843 if (!desc) 844 return NULL; 845 846 desc->async_tx.flags = dma_flags; 847 desc->async_tx.cookie = -EBUSY; 848 849 desc->cyclic = cyclic; 850 desc->direction = dir; 851 852 rcar_dmac_chan_configure_desc(chan, desc); 853 854 max_chunk_size = (RCAR_DMATCR_MASK + 1) << desc->xfer_shift; 855 856 /* 857 * Allocate and fill the transfer chunk descriptors. We own the only 858 * reference to the DMA descriptor, there's no need for locking. 859 */ 860 for_each_sg(sgl, sg, sg_len, i) { 861 dma_addr_t mem_addr = sg_dma_address(sg); 862 unsigned int len = sg_dma_len(sg); 863 864 full_size += len; 865 866 while (len) { 867 unsigned int size = min(len, max_chunk_size); 868 869 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 870 /* 871 * Prevent individual transfers from crossing 4GB 872 * boundaries. 873 */ 874 if (dev_addr >> 32 != (dev_addr + size - 1) >> 32) 875 size = ALIGN(dev_addr, 1ULL << 32) - dev_addr; 876 if (mem_addr >> 32 != (mem_addr + size - 1) >> 32) 877 size = ALIGN(mem_addr, 1ULL << 32) - mem_addr; 878 879 /* 880 * Check if either of the source or destination address 881 * can't be expressed in 32 bits. If so we can't use 882 * hardware descriptor lists. 883 */ 884 if (dev_addr >> 32 || mem_addr >> 32) 885 highmem = true; 886 #endif 887 888 chunk = rcar_dmac_xfer_chunk_get(chan); 889 if (!chunk) { 890 rcar_dmac_desc_put(chan, desc); 891 return NULL; 892 } 893 894 if (dir == DMA_DEV_TO_MEM) { 895 chunk->src_addr = dev_addr; 896 chunk->dst_addr = mem_addr; 897 } else { 898 chunk->src_addr = mem_addr; 899 chunk->dst_addr = dev_addr; 900 } 901 902 chunk->size = size; 903 904 dev_dbg(chan->chan.device->dev, 905 "chan%u: chunk %p/%p sgl %u@%p, %u/%u %pad -> %pad\n", 906 chan->index, chunk, desc, i, sg, size, len, 907 &chunk->src_addr, &chunk->dst_addr); 908 909 mem_addr += size; 910 if (dir == DMA_MEM_TO_MEM) 911 dev_addr += size; 912 913 len -= size; 914 915 list_add_tail(&chunk->node, &desc->chunks); 916 nchunks++; 917 } 918 } 919 920 desc->nchunks = nchunks; 921 desc->size = full_size; 922 923 /* 924 * Use hardware descriptor lists if possible when more than one chunk 925 * needs to be transferred (otherwise they don't make much sense). 926 * 927 * The highmem check currently covers the whole transfer. As an 928 * optimization we could use descriptor lists for consecutive lowmem 929 * chunks and direct manual mode for highmem chunks. Whether the 930 * performance improvement would be significant enough compared to the 931 * additional complexity remains to be investigated. 932 */ 933 desc->hwdescs.use = !highmem && nchunks > 1; 934 if (desc->hwdescs.use) { 935 if (rcar_dmac_fill_hwdesc(chan, desc) < 0) 936 desc->hwdescs.use = false; 937 } 938 939 return &desc->async_tx; 940 } 941 942 /* ----------------------------------------------------------------------------- 943 * DMA engine operations 944 */ 945 946 static int rcar_dmac_alloc_chan_resources(struct dma_chan *chan) 947 { 948 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 949 int ret; 950 951 INIT_LIST_HEAD(&rchan->desc.chunks_free); 952 INIT_LIST_HEAD(&rchan->desc.pages); 953 954 /* Preallocate descriptors. */ 955 ret = rcar_dmac_xfer_chunk_alloc(rchan, GFP_KERNEL); 956 if (ret < 0) 957 return -ENOMEM; 958 959 ret = rcar_dmac_desc_alloc(rchan, GFP_KERNEL); 960 if (ret < 0) 961 return -ENOMEM; 962 963 return pm_runtime_get_sync(chan->device->dev); 964 } 965 966 static void rcar_dmac_free_chan_resources(struct dma_chan *chan) 967 { 968 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 969 struct rcar_dmac *dmac = to_rcar_dmac(chan->device); 970 struct rcar_dmac_desc_page *page, *_page; 971 struct rcar_dmac_desc *desc; 972 LIST_HEAD(list); 973 974 /* Protect against ISR */ 975 spin_lock_irq(&rchan->lock); 976 rcar_dmac_chan_halt(rchan); 977 spin_unlock_irq(&rchan->lock); 978 979 /* Now no new interrupts will occur */ 980 981 if (rchan->mid_rid >= 0) { 982 /* The caller is holding dma_list_mutex */ 983 clear_bit(rchan->mid_rid, dmac->modules); 984 rchan->mid_rid = -EINVAL; 985 } 986 987 list_splice_init(&rchan->desc.free, &list); 988 list_splice_init(&rchan->desc.pending, &list); 989 list_splice_init(&rchan->desc.active, &list); 990 list_splice_init(&rchan->desc.done, &list); 991 list_splice_init(&rchan->desc.wait, &list); 992 993 rchan->desc.running = NULL; 994 995 list_for_each_entry(desc, &list, node) 996 rcar_dmac_realloc_hwdesc(rchan, desc, 0); 997 998 list_for_each_entry_safe(page, _page, &rchan->desc.pages, node) { 999 list_del(&page->node); 1000 free_page((unsigned long)page); 1001 } 1002 1003 pm_runtime_put(chan->device->dev); 1004 } 1005 1006 static struct dma_async_tx_descriptor * 1007 rcar_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest, 1008 dma_addr_t dma_src, size_t len, unsigned long flags) 1009 { 1010 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1011 struct scatterlist sgl; 1012 1013 if (!len) 1014 return NULL; 1015 1016 sg_init_table(&sgl, 1); 1017 sg_set_page(&sgl, pfn_to_page(PFN_DOWN(dma_src)), len, 1018 offset_in_page(dma_src)); 1019 sg_dma_address(&sgl) = dma_src; 1020 sg_dma_len(&sgl) = len; 1021 1022 return rcar_dmac_chan_prep_sg(rchan, &sgl, 1, dma_dest, 1023 DMA_MEM_TO_MEM, flags, false); 1024 } 1025 1026 static struct dma_async_tx_descriptor * 1027 rcar_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 1028 unsigned int sg_len, enum dma_transfer_direction dir, 1029 unsigned long flags, void *context) 1030 { 1031 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1032 dma_addr_t dev_addr; 1033 1034 /* Someone calling slave DMA on a generic channel? */ 1035 if (rchan->mid_rid < 0 || !sg_len) { 1036 dev_warn(chan->device->dev, 1037 "%s: bad parameter: len=%d, id=%d\n", 1038 __func__, sg_len, rchan->mid_rid); 1039 return NULL; 1040 } 1041 1042 dev_addr = dir == DMA_DEV_TO_MEM 1043 ? rchan->src_slave_addr : rchan->dst_slave_addr; 1044 return rcar_dmac_chan_prep_sg(rchan, sgl, sg_len, dev_addr, 1045 dir, flags, false); 1046 } 1047 1048 #define RCAR_DMAC_MAX_SG_LEN 32 1049 1050 static struct dma_async_tx_descriptor * 1051 rcar_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, 1052 size_t buf_len, size_t period_len, 1053 enum dma_transfer_direction dir, unsigned long flags) 1054 { 1055 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1056 struct dma_async_tx_descriptor *desc; 1057 struct scatterlist *sgl; 1058 dma_addr_t dev_addr; 1059 unsigned int sg_len; 1060 unsigned int i; 1061 1062 /* Someone calling slave DMA on a generic channel? */ 1063 if (rchan->mid_rid < 0 || buf_len < period_len) { 1064 dev_warn(chan->device->dev, 1065 "%s: bad parameter: buf_len=%zu, period_len=%zu, id=%d\n", 1066 __func__, buf_len, period_len, rchan->mid_rid); 1067 return NULL; 1068 } 1069 1070 sg_len = buf_len / period_len; 1071 if (sg_len > RCAR_DMAC_MAX_SG_LEN) { 1072 dev_err(chan->device->dev, 1073 "chan%u: sg length %d exceds limit %d", 1074 rchan->index, sg_len, RCAR_DMAC_MAX_SG_LEN); 1075 return NULL; 1076 } 1077 1078 /* 1079 * Allocate the sg list dynamically as it would consume too much stack 1080 * space. 1081 */ 1082 sgl = kcalloc(sg_len, sizeof(*sgl), GFP_NOWAIT); 1083 if (!sgl) 1084 return NULL; 1085 1086 sg_init_table(sgl, sg_len); 1087 1088 for (i = 0; i < sg_len; ++i) { 1089 dma_addr_t src = buf_addr + (period_len * i); 1090 1091 sg_set_page(&sgl[i], pfn_to_page(PFN_DOWN(src)), period_len, 1092 offset_in_page(src)); 1093 sg_dma_address(&sgl[i]) = src; 1094 sg_dma_len(&sgl[i]) = period_len; 1095 } 1096 1097 dev_addr = dir == DMA_DEV_TO_MEM 1098 ? rchan->src_slave_addr : rchan->dst_slave_addr; 1099 desc = rcar_dmac_chan_prep_sg(rchan, sgl, sg_len, dev_addr, 1100 dir, flags, true); 1101 1102 kfree(sgl); 1103 return desc; 1104 } 1105 1106 static int rcar_dmac_device_config(struct dma_chan *chan, 1107 struct dma_slave_config *cfg) 1108 { 1109 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1110 1111 /* 1112 * We could lock this, but you shouldn't be configuring the 1113 * channel, while using it... 1114 */ 1115 rchan->src_slave_addr = cfg->src_addr; 1116 rchan->dst_slave_addr = cfg->dst_addr; 1117 rchan->src_xfer_size = cfg->src_addr_width; 1118 rchan->dst_xfer_size = cfg->dst_addr_width; 1119 1120 return 0; 1121 } 1122 1123 static int rcar_dmac_chan_terminate_all(struct dma_chan *chan) 1124 { 1125 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1126 unsigned long flags; 1127 1128 spin_lock_irqsave(&rchan->lock, flags); 1129 rcar_dmac_chan_halt(rchan); 1130 spin_unlock_irqrestore(&rchan->lock, flags); 1131 1132 /* 1133 * FIXME: No new interrupt can occur now, but the IRQ thread might still 1134 * be running. 1135 */ 1136 1137 rcar_dmac_chan_reinit(rchan); 1138 1139 return 0; 1140 } 1141 1142 static unsigned int rcar_dmac_chan_get_residue(struct rcar_dmac_chan *chan, 1143 dma_cookie_t cookie) 1144 { 1145 struct rcar_dmac_desc *desc = chan->desc.running; 1146 struct rcar_dmac_xfer_chunk *running = NULL; 1147 struct rcar_dmac_xfer_chunk *chunk; 1148 enum dma_status status; 1149 unsigned int residue = 0; 1150 unsigned int dptr = 0; 1151 1152 if (!desc) 1153 return 0; 1154 1155 /* 1156 * If the cookie corresponds to a descriptor that has been completed 1157 * there is no residue. The same check has already been performed by the 1158 * caller but without holding the channel lock, so the descriptor could 1159 * now be complete. 1160 */ 1161 status = dma_cookie_status(&chan->chan, cookie, NULL); 1162 if (status == DMA_COMPLETE) 1163 return 0; 1164 1165 /* 1166 * If the cookie doesn't correspond to the currently running transfer 1167 * then the descriptor hasn't been processed yet, and the residue is 1168 * equal to the full descriptor size. 1169 */ 1170 if (cookie != desc->async_tx.cookie) { 1171 list_for_each_entry(desc, &chan->desc.pending, node) { 1172 if (cookie == desc->async_tx.cookie) 1173 return desc->size; 1174 } 1175 list_for_each_entry(desc, &chan->desc.active, node) { 1176 if (cookie == desc->async_tx.cookie) 1177 return desc->size; 1178 } 1179 1180 /* 1181 * No descriptor found for the cookie, there's thus no residue. 1182 * This shouldn't happen if the calling driver passes a correct 1183 * cookie value. 1184 */ 1185 WARN(1, "No descriptor for cookie!"); 1186 return 0; 1187 } 1188 1189 /* 1190 * In descriptor mode the descriptor running pointer is not maintained 1191 * by the interrupt handler, find the running descriptor from the 1192 * descriptor pointer field in the CHCRB register. In non-descriptor 1193 * mode just use the running descriptor pointer. 1194 */ 1195 if (desc->hwdescs.use) { 1196 dptr = (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) & 1197 RCAR_DMACHCRB_DPTR_MASK) >> RCAR_DMACHCRB_DPTR_SHIFT; 1198 WARN_ON(dptr >= desc->nchunks); 1199 } else { 1200 running = desc->running; 1201 } 1202 1203 /* Compute the size of all chunks still to be transferred. */ 1204 list_for_each_entry_reverse(chunk, &desc->chunks, node) { 1205 if (chunk == running || ++dptr == desc->nchunks) 1206 break; 1207 1208 residue += chunk->size; 1209 } 1210 1211 /* Add the residue for the current chunk. */ 1212 residue += rcar_dmac_chan_read(chan, RCAR_DMATCR) << desc->xfer_shift; 1213 1214 return residue; 1215 } 1216 1217 static enum dma_status rcar_dmac_tx_status(struct dma_chan *chan, 1218 dma_cookie_t cookie, 1219 struct dma_tx_state *txstate) 1220 { 1221 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1222 enum dma_status status; 1223 unsigned long flags; 1224 unsigned int residue; 1225 1226 status = dma_cookie_status(chan, cookie, txstate); 1227 if (status == DMA_COMPLETE || !txstate) 1228 return status; 1229 1230 spin_lock_irqsave(&rchan->lock, flags); 1231 residue = rcar_dmac_chan_get_residue(rchan, cookie); 1232 spin_unlock_irqrestore(&rchan->lock, flags); 1233 1234 /* if there's no residue, the cookie is complete */ 1235 if (!residue) 1236 return DMA_COMPLETE; 1237 1238 dma_set_residue(txstate, residue); 1239 1240 return status; 1241 } 1242 1243 static void rcar_dmac_issue_pending(struct dma_chan *chan) 1244 { 1245 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1246 unsigned long flags; 1247 1248 spin_lock_irqsave(&rchan->lock, flags); 1249 1250 if (list_empty(&rchan->desc.pending)) 1251 goto done; 1252 1253 /* Append the pending list to the active list. */ 1254 list_splice_tail_init(&rchan->desc.pending, &rchan->desc.active); 1255 1256 /* 1257 * If no transfer is running pick the first descriptor from the active 1258 * list and start the transfer. 1259 */ 1260 if (!rchan->desc.running) { 1261 struct rcar_dmac_desc *desc; 1262 1263 desc = list_first_entry(&rchan->desc.active, 1264 struct rcar_dmac_desc, node); 1265 rchan->desc.running = desc; 1266 1267 rcar_dmac_chan_start_xfer(rchan); 1268 } 1269 1270 done: 1271 spin_unlock_irqrestore(&rchan->lock, flags); 1272 } 1273 1274 /* ----------------------------------------------------------------------------- 1275 * IRQ handling 1276 */ 1277 1278 static irqreturn_t rcar_dmac_isr_desc_stage_end(struct rcar_dmac_chan *chan) 1279 { 1280 struct rcar_dmac_desc *desc = chan->desc.running; 1281 unsigned int stage; 1282 1283 if (WARN_ON(!desc || !desc->cyclic)) { 1284 /* 1285 * This should never happen, there should always be a running 1286 * cyclic descriptor when a descriptor stage end interrupt is 1287 * triggered. Warn and return. 1288 */ 1289 return IRQ_NONE; 1290 } 1291 1292 /* Program the interrupt pointer to the next stage. */ 1293 stage = (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) & 1294 RCAR_DMACHCRB_DPTR_MASK) >> RCAR_DMACHCRB_DPTR_SHIFT; 1295 rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(stage)); 1296 1297 return IRQ_WAKE_THREAD; 1298 } 1299 1300 static irqreturn_t rcar_dmac_isr_transfer_end(struct rcar_dmac_chan *chan) 1301 { 1302 struct rcar_dmac_desc *desc = chan->desc.running; 1303 irqreturn_t ret = IRQ_WAKE_THREAD; 1304 1305 if (WARN_ON_ONCE(!desc)) { 1306 /* 1307 * This should never happen, there should always be a running 1308 * descriptor when a transfer end interrupt is triggered. Warn 1309 * and return. 1310 */ 1311 return IRQ_NONE; 1312 } 1313 1314 /* 1315 * The transfer end interrupt isn't generated for each chunk when using 1316 * descriptor mode. Only update the running chunk pointer in 1317 * non-descriptor mode. 1318 */ 1319 if (!desc->hwdescs.use) { 1320 /* 1321 * If we haven't completed the last transfer chunk simply move 1322 * to the next one. Only wake the IRQ thread if the transfer is 1323 * cyclic. 1324 */ 1325 if (!list_is_last(&desc->running->node, &desc->chunks)) { 1326 desc->running = list_next_entry(desc->running, node); 1327 if (!desc->cyclic) 1328 ret = IRQ_HANDLED; 1329 goto done; 1330 } 1331 1332 /* 1333 * We've completed the last transfer chunk. If the transfer is 1334 * cyclic, move back to the first one. 1335 */ 1336 if (desc->cyclic) { 1337 desc->running = 1338 list_first_entry(&desc->chunks, 1339 struct rcar_dmac_xfer_chunk, 1340 node); 1341 goto done; 1342 } 1343 } 1344 1345 /* The descriptor is complete, move it to the done list. */ 1346 list_move_tail(&desc->node, &chan->desc.done); 1347 1348 /* Queue the next descriptor, if any. */ 1349 if (!list_empty(&chan->desc.active)) 1350 chan->desc.running = list_first_entry(&chan->desc.active, 1351 struct rcar_dmac_desc, 1352 node); 1353 else 1354 chan->desc.running = NULL; 1355 1356 done: 1357 if (chan->desc.running) 1358 rcar_dmac_chan_start_xfer(chan); 1359 1360 return ret; 1361 } 1362 1363 static irqreturn_t rcar_dmac_isr_channel(int irq, void *dev) 1364 { 1365 u32 mask = RCAR_DMACHCR_DSE | RCAR_DMACHCR_TE; 1366 struct rcar_dmac_chan *chan = dev; 1367 irqreturn_t ret = IRQ_NONE; 1368 u32 chcr; 1369 1370 spin_lock(&chan->lock); 1371 1372 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 1373 if (chcr & RCAR_DMACHCR_TE) 1374 mask |= RCAR_DMACHCR_DE; 1375 rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr & ~mask); 1376 1377 if (chcr & RCAR_DMACHCR_DSE) 1378 ret |= rcar_dmac_isr_desc_stage_end(chan); 1379 1380 if (chcr & RCAR_DMACHCR_TE) 1381 ret |= rcar_dmac_isr_transfer_end(chan); 1382 1383 spin_unlock(&chan->lock); 1384 1385 return ret; 1386 } 1387 1388 static irqreturn_t rcar_dmac_isr_channel_thread(int irq, void *dev) 1389 { 1390 struct rcar_dmac_chan *chan = dev; 1391 struct rcar_dmac_desc *desc; 1392 1393 spin_lock_irq(&chan->lock); 1394 1395 /* For cyclic transfers notify the user after every chunk. */ 1396 if (chan->desc.running && chan->desc.running->cyclic) { 1397 dma_async_tx_callback callback; 1398 void *callback_param; 1399 1400 desc = chan->desc.running; 1401 callback = desc->async_tx.callback; 1402 callback_param = desc->async_tx.callback_param; 1403 1404 if (callback) { 1405 spin_unlock_irq(&chan->lock); 1406 callback(callback_param); 1407 spin_lock_irq(&chan->lock); 1408 } 1409 } 1410 1411 /* 1412 * Call the callback function for all descriptors on the done list and 1413 * move them to the ack wait list. 1414 */ 1415 while (!list_empty(&chan->desc.done)) { 1416 desc = list_first_entry(&chan->desc.done, struct rcar_dmac_desc, 1417 node); 1418 dma_cookie_complete(&desc->async_tx); 1419 list_del(&desc->node); 1420 1421 if (desc->async_tx.callback) { 1422 spin_unlock_irq(&chan->lock); 1423 /* 1424 * We own the only reference to this descriptor, we can 1425 * safely dereference it without holding the channel 1426 * lock. 1427 */ 1428 desc->async_tx.callback(desc->async_tx.callback_param); 1429 spin_lock_irq(&chan->lock); 1430 } 1431 1432 list_add_tail(&desc->node, &chan->desc.wait); 1433 } 1434 1435 spin_unlock_irq(&chan->lock); 1436 1437 /* Recycle all acked descriptors. */ 1438 rcar_dmac_desc_recycle_acked(chan); 1439 1440 return IRQ_HANDLED; 1441 } 1442 1443 static irqreturn_t rcar_dmac_isr_error(int irq, void *data) 1444 { 1445 struct rcar_dmac *dmac = data; 1446 1447 if (!(rcar_dmac_read(dmac, RCAR_DMAOR) & RCAR_DMAOR_AE)) 1448 return IRQ_NONE; 1449 1450 /* 1451 * An unrecoverable error occurred on an unknown channel. Halt the DMAC, 1452 * abort transfers on all channels, and reinitialize the DMAC. 1453 */ 1454 rcar_dmac_stop(dmac); 1455 rcar_dmac_abort(dmac); 1456 rcar_dmac_init(dmac); 1457 1458 return IRQ_HANDLED; 1459 } 1460 1461 /* ----------------------------------------------------------------------------- 1462 * OF xlate and channel filter 1463 */ 1464 1465 static bool rcar_dmac_chan_filter(struct dma_chan *chan, void *arg) 1466 { 1467 struct rcar_dmac *dmac = to_rcar_dmac(chan->device); 1468 struct of_phandle_args *dma_spec = arg; 1469 1470 /* 1471 * FIXME: Using a filter on OF platforms is a nonsense. The OF xlate 1472 * function knows from which device it wants to allocate a channel from, 1473 * and would be perfectly capable of selecting the channel it wants. 1474 * Forcing it to call dma_request_channel() and iterate through all 1475 * channels from all controllers is just pointless. 1476 */ 1477 if (chan->device->device_config != rcar_dmac_device_config || 1478 dma_spec->np != chan->device->dev->of_node) 1479 return false; 1480 1481 return !test_and_set_bit(dma_spec->args[0], dmac->modules); 1482 } 1483 1484 static struct dma_chan *rcar_dmac_of_xlate(struct of_phandle_args *dma_spec, 1485 struct of_dma *ofdma) 1486 { 1487 struct rcar_dmac_chan *rchan; 1488 struct dma_chan *chan; 1489 dma_cap_mask_t mask; 1490 1491 if (dma_spec->args_count != 1) 1492 return NULL; 1493 1494 /* Only slave DMA channels can be allocated via DT */ 1495 dma_cap_zero(mask); 1496 dma_cap_set(DMA_SLAVE, mask); 1497 1498 chan = dma_request_channel(mask, rcar_dmac_chan_filter, dma_spec); 1499 if (!chan) 1500 return NULL; 1501 1502 rchan = to_rcar_dmac_chan(chan); 1503 rchan->mid_rid = dma_spec->args[0]; 1504 1505 return chan; 1506 } 1507 1508 /* ----------------------------------------------------------------------------- 1509 * Power management 1510 */ 1511 1512 #ifdef CONFIG_PM_SLEEP 1513 static int rcar_dmac_sleep_suspend(struct device *dev) 1514 { 1515 /* 1516 * TODO: Wait for the current transfer to complete and stop the device. 1517 */ 1518 return 0; 1519 } 1520 1521 static int rcar_dmac_sleep_resume(struct device *dev) 1522 { 1523 /* TODO: Resume transfers, if any. */ 1524 return 0; 1525 } 1526 #endif 1527 1528 #ifdef CONFIG_PM 1529 static int rcar_dmac_runtime_suspend(struct device *dev) 1530 { 1531 return 0; 1532 } 1533 1534 static int rcar_dmac_runtime_resume(struct device *dev) 1535 { 1536 struct rcar_dmac *dmac = dev_get_drvdata(dev); 1537 1538 return rcar_dmac_init(dmac); 1539 } 1540 #endif 1541 1542 static const struct dev_pm_ops rcar_dmac_pm = { 1543 SET_SYSTEM_SLEEP_PM_OPS(rcar_dmac_sleep_suspend, rcar_dmac_sleep_resume) 1544 SET_RUNTIME_PM_OPS(rcar_dmac_runtime_suspend, rcar_dmac_runtime_resume, 1545 NULL) 1546 }; 1547 1548 /* ----------------------------------------------------------------------------- 1549 * Probe and remove 1550 */ 1551 1552 static int rcar_dmac_chan_probe(struct rcar_dmac *dmac, 1553 struct rcar_dmac_chan *rchan, 1554 unsigned int index) 1555 { 1556 struct platform_device *pdev = to_platform_device(dmac->dev); 1557 struct dma_chan *chan = &rchan->chan; 1558 char pdev_irqname[5]; 1559 char *irqname; 1560 int irq; 1561 int ret; 1562 1563 rchan->index = index; 1564 rchan->iomem = dmac->iomem + RCAR_DMAC_CHAN_OFFSET(index); 1565 rchan->mid_rid = -EINVAL; 1566 1567 spin_lock_init(&rchan->lock); 1568 1569 INIT_LIST_HEAD(&rchan->desc.free); 1570 INIT_LIST_HEAD(&rchan->desc.pending); 1571 INIT_LIST_HEAD(&rchan->desc.active); 1572 INIT_LIST_HEAD(&rchan->desc.done); 1573 INIT_LIST_HEAD(&rchan->desc.wait); 1574 1575 /* Request the channel interrupt. */ 1576 sprintf(pdev_irqname, "ch%u", index); 1577 irq = platform_get_irq_byname(pdev, pdev_irqname); 1578 if (irq < 0) { 1579 dev_err(dmac->dev, "no IRQ specified for channel %u\n", index); 1580 return -ENODEV; 1581 } 1582 1583 irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", 1584 dev_name(dmac->dev), index); 1585 if (!irqname) 1586 return -ENOMEM; 1587 1588 ret = devm_request_threaded_irq(dmac->dev, irq, rcar_dmac_isr_channel, 1589 rcar_dmac_isr_channel_thread, 0, 1590 irqname, rchan); 1591 if (ret) { 1592 dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", irq, ret); 1593 return ret; 1594 } 1595 1596 /* 1597 * Initialize the DMA engine channel and add it to the DMA engine 1598 * channels list. 1599 */ 1600 chan->device = &dmac->engine; 1601 dma_cookie_init(chan); 1602 1603 list_add_tail(&chan->device_node, &dmac->engine.channels); 1604 1605 return 0; 1606 } 1607 1608 static int rcar_dmac_parse_of(struct device *dev, struct rcar_dmac *dmac) 1609 { 1610 struct device_node *np = dev->of_node; 1611 int ret; 1612 1613 ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels); 1614 if (ret < 0) { 1615 dev_err(dev, "unable to read dma-channels property\n"); 1616 return ret; 1617 } 1618 1619 if (dmac->n_channels <= 0 || dmac->n_channels >= 100) { 1620 dev_err(dev, "invalid number of channels %u\n", 1621 dmac->n_channels); 1622 return -EINVAL; 1623 } 1624 1625 return 0; 1626 } 1627 1628 static int rcar_dmac_probe(struct platform_device *pdev) 1629 { 1630 const enum dma_slave_buswidth widths = DMA_SLAVE_BUSWIDTH_1_BYTE | 1631 DMA_SLAVE_BUSWIDTH_2_BYTES | DMA_SLAVE_BUSWIDTH_4_BYTES | 1632 DMA_SLAVE_BUSWIDTH_8_BYTES | DMA_SLAVE_BUSWIDTH_16_BYTES | 1633 DMA_SLAVE_BUSWIDTH_32_BYTES | DMA_SLAVE_BUSWIDTH_64_BYTES; 1634 unsigned int channels_offset = 0; 1635 struct dma_device *engine; 1636 struct rcar_dmac *dmac; 1637 struct resource *mem; 1638 unsigned int i; 1639 char *irqname; 1640 int irq; 1641 int ret; 1642 1643 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); 1644 if (!dmac) 1645 return -ENOMEM; 1646 1647 dmac->dev = &pdev->dev; 1648 platform_set_drvdata(pdev, dmac); 1649 1650 ret = rcar_dmac_parse_of(&pdev->dev, dmac); 1651 if (ret < 0) 1652 return ret; 1653 1654 /* 1655 * A still unconfirmed hardware bug prevents the IPMMU microTLB 0 to be 1656 * flushed correctly, resulting in memory corruption. DMAC 0 channel 0 1657 * is connected to microTLB 0 on currently supported platforms, so we 1658 * can't use it with the IPMMU. As the IOMMU API operates at the device 1659 * level we can't disable it selectively, so ignore channel 0 for now if 1660 * the device is part of an IOMMU group. 1661 */ 1662 if (pdev->dev.iommu_group) { 1663 dmac->n_channels--; 1664 channels_offset = 1; 1665 } 1666 1667 dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels, 1668 sizeof(*dmac->channels), GFP_KERNEL); 1669 if (!dmac->channels) 1670 return -ENOMEM; 1671 1672 /* Request resources. */ 1673 mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1674 dmac->iomem = devm_ioremap_resource(&pdev->dev, mem); 1675 if (IS_ERR(dmac->iomem)) 1676 return PTR_ERR(dmac->iomem); 1677 1678 irq = platform_get_irq_byname(pdev, "error"); 1679 if (irq < 0) { 1680 dev_err(&pdev->dev, "no error IRQ specified\n"); 1681 return -ENODEV; 1682 } 1683 1684 irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:error", 1685 dev_name(dmac->dev)); 1686 if (!irqname) 1687 return -ENOMEM; 1688 1689 ret = devm_request_irq(&pdev->dev, irq, rcar_dmac_isr_error, 0, 1690 irqname, dmac); 1691 if (ret) { 1692 dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n", 1693 irq, ret); 1694 return ret; 1695 } 1696 1697 /* Enable runtime PM and initialize the device. */ 1698 pm_runtime_enable(&pdev->dev); 1699 ret = pm_runtime_get_sync(&pdev->dev); 1700 if (ret < 0) { 1701 dev_err(&pdev->dev, "runtime PM get sync failed (%d)\n", ret); 1702 return ret; 1703 } 1704 1705 ret = rcar_dmac_init(dmac); 1706 pm_runtime_put(&pdev->dev); 1707 1708 if (ret) { 1709 dev_err(&pdev->dev, "failed to reset device\n"); 1710 goto error; 1711 } 1712 1713 /* Initialize the channels. */ 1714 INIT_LIST_HEAD(&dmac->engine.channels); 1715 1716 for (i = 0; i < dmac->n_channels; ++i) { 1717 ret = rcar_dmac_chan_probe(dmac, &dmac->channels[i], 1718 i + channels_offset); 1719 if (ret < 0) 1720 goto error; 1721 } 1722 1723 /* Register the DMAC as a DMA provider for DT. */ 1724 ret = of_dma_controller_register(pdev->dev.of_node, rcar_dmac_of_xlate, 1725 NULL); 1726 if (ret < 0) 1727 goto error; 1728 1729 /* 1730 * Register the DMA engine device. 1731 * 1732 * Default transfer size of 32 bytes requires 32-byte alignment. 1733 */ 1734 engine = &dmac->engine; 1735 dma_cap_set(DMA_MEMCPY, engine->cap_mask); 1736 dma_cap_set(DMA_SLAVE, engine->cap_mask); 1737 1738 engine->dev = &pdev->dev; 1739 engine->copy_align = ilog2(RCAR_DMAC_MEMCPY_XFER_SIZE); 1740 1741 engine->src_addr_widths = widths; 1742 engine->dst_addr_widths = widths; 1743 engine->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); 1744 engine->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1745 1746 engine->device_alloc_chan_resources = rcar_dmac_alloc_chan_resources; 1747 engine->device_free_chan_resources = rcar_dmac_free_chan_resources; 1748 engine->device_prep_dma_memcpy = rcar_dmac_prep_dma_memcpy; 1749 engine->device_prep_slave_sg = rcar_dmac_prep_slave_sg; 1750 engine->device_prep_dma_cyclic = rcar_dmac_prep_dma_cyclic; 1751 engine->device_config = rcar_dmac_device_config; 1752 engine->device_terminate_all = rcar_dmac_chan_terminate_all; 1753 engine->device_tx_status = rcar_dmac_tx_status; 1754 engine->device_issue_pending = rcar_dmac_issue_pending; 1755 1756 ret = dma_async_device_register(engine); 1757 if (ret < 0) 1758 goto error; 1759 1760 return 0; 1761 1762 error: 1763 of_dma_controller_free(pdev->dev.of_node); 1764 pm_runtime_disable(&pdev->dev); 1765 return ret; 1766 } 1767 1768 static int rcar_dmac_remove(struct platform_device *pdev) 1769 { 1770 struct rcar_dmac *dmac = platform_get_drvdata(pdev); 1771 1772 of_dma_controller_free(pdev->dev.of_node); 1773 dma_async_device_unregister(&dmac->engine); 1774 1775 pm_runtime_disable(&pdev->dev); 1776 1777 return 0; 1778 } 1779 1780 static void rcar_dmac_shutdown(struct platform_device *pdev) 1781 { 1782 struct rcar_dmac *dmac = platform_get_drvdata(pdev); 1783 1784 rcar_dmac_stop(dmac); 1785 } 1786 1787 static const struct of_device_id rcar_dmac_of_ids[] = { 1788 { .compatible = "renesas,rcar-dmac", }, 1789 { /* Sentinel */ } 1790 }; 1791 MODULE_DEVICE_TABLE(of, rcar_dmac_of_ids); 1792 1793 static struct platform_driver rcar_dmac_driver = { 1794 .driver = { 1795 .pm = &rcar_dmac_pm, 1796 .name = "rcar-dmac", 1797 .of_match_table = rcar_dmac_of_ids, 1798 }, 1799 .probe = rcar_dmac_probe, 1800 .remove = rcar_dmac_remove, 1801 .shutdown = rcar_dmac_shutdown, 1802 }; 1803 1804 module_platform_driver(rcar_dmac_driver); 1805 1806 MODULE_DESCRIPTION("R-Car Gen2 DMA Controller Driver"); 1807 MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>"); 1808 MODULE_LICENSE("GPL v2"); 1809