1 /* 2 * Renesas R-Car Gen2 DMA Controller Driver 3 * 4 * Copyright (C) 2014 Renesas Electronics Inc. 5 * 6 * Author: Laurent Pinchart <laurent.pinchart@ideasonboard.com> 7 * 8 * This is free software; you can redistribute it and/or modify 9 * it under the terms of version 2 of the GNU General Public License as 10 * published by the Free Software Foundation. 11 */ 12 13 #include <linux/delay.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/dmaengine.h> 16 #include <linux/interrupt.h> 17 #include <linux/list.h> 18 #include <linux/module.h> 19 #include <linux/mutex.h> 20 #include <linux/of.h> 21 #include <linux/of_dma.h> 22 #include <linux/of_platform.h> 23 #include <linux/platform_device.h> 24 #include <linux/pm_runtime.h> 25 #include <linux/slab.h> 26 #include <linux/spinlock.h> 27 28 #include "../dmaengine.h" 29 30 /* 31 * struct rcar_dmac_xfer_chunk - Descriptor for a hardware transfer 32 * @node: entry in the parent's chunks list 33 * @src_addr: device source address 34 * @dst_addr: device destination address 35 * @size: transfer size in bytes 36 */ 37 struct rcar_dmac_xfer_chunk { 38 struct list_head node; 39 40 dma_addr_t src_addr; 41 dma_addr_t dst_addr; 42 u32 size; 43 }; 44 45 /* 46 * struct rcar_dmac_hw_desc - Hardware descriptor for a transfer chunk 47 * @sar: value of the SAR register (source address) 48 * @dar: value of the DAR register (destination address) 49 * @tcr: value of the TCR register (transfer count) 50 */ 51 struct rcar_dmac_hw_desc { 52 u32 sar; 53 u32 dar; 54 u32 tcr; 55 u32 reserved; 56 } __attribute__((__packed__)); 57 58 /* 59 * struct rcar_dmac_desc - R-Car Gen2 DMA Transfer Descriptor 60 * @async_tx: base DMA asynchronous transaction descriptor 61 * @direction: direction of the DMA transfer 62 * @xfer_shift: log2 of the transfer size 63 * @chcr: value of the channel configuration register for this transfer 64 * @node: entry in the channel's descriptors lists 65 * @chunks: list of transfer chunks for this transfer 66 * @running: the transfer chunk being currently processed 67 * @nchunks: number of transfer chunks for this transfer 68 * @hwdescs.use: whether the transfer descriptor uses hardware descriptors 69 * @hwdescs.mem: hardware descriptors memory for the transfer 70 * @hwdescs.dma: device address of the hardware descriptors memory 71 * @hwdescs.size: size of the hardware descriptors in bytes 72 * @size: transfer size in bytes 73 * @cyclic: when set indicates that the DMA transfer is cyclic 74 */ 75 struct rcar_dmac_desc { 76 struct dma_async_tx_descriptor async_tx; 77 enum dma_transfer_direction direction; 78 unsigned int xfer_shift; 79 u32 chcr; 80 81 struct list_head node; 82 struct list_head chunks; 83 struct rcar_dmac_xfer_chunk *running; 84 unsigned int nchunks; 85 86 struct { 87 bool use; 88 struct rcar_dmac_hw_desc *mem; 89 dma_addr_t dma; 90 size_t size; 91 } hwdescs; 92 93 unsigned int size; 94 bool cyclic; 95 }; 96 97 #define to_rcar_dmac_desc(d) container_of(d, struct rcar_dmac_desc, async_tx) 98 99 /* 100 * struct rcar_dmac_desc_page - One page worth of descriptors 101 * @node: entry in the channel's pages list 102 * @descs: array of DMA descriptors 103 * @chunks: array of transfer chunk descriptors 104 */ 105 struct rcar_dmac_desc_page { 106 struct list_head node; 107 108 union { 109 struct rcar_dmac_desc descs[0]; 110 struct rcar_dmac_xfer_chunk chunks[0]; 111 }; 112 }; 113 114 #define RCAR_DMAC_DESCS_PER_PAGE \ 115 ((PAGE_SIZE - offsetof(struct rcar_dmac_desc_page, descs)) / \ 116 sizeof(struct rcar_dmac_desc)) 117 #define RCAR_DMAC_XFER_CHUNKS_PER_PAGE \ 118 ((PAGE_SIZE - offsetof(struct rcar_dmac_desc_page, chunks)) / \ 119 sizeof(struct rcar_dmac_xfer_chunk)) 120 121 /* 122 * struct rcar_dmac_chan_slave - Slave configuration 123 * @slave_addr: slave memory address 124 * @xfer_size: size (in bytes) of hardware transfers 125 */ 126 struct rcar_dmac_chan_slave { 127 phys_addr_t slave_addr; 128 unsigned int xfer_size; 129 }; 130 131 /* 132 * struct rcar_dmac_chan_map - Map of slave device phys to dma address 133 * @addr: slave dma address 134 * @dir: direction of mapping 135 * @slave: slave configuration that is mapped 136 */ 137 struct rcar_dmac_chan_map { 138 dma_addr_t addr; 139 enum dma_data_direction dir; 140 struct rcar_dmac_chan_slave slave; 141 }; 142 143 /* 144 * struct rcar_dmac_chan - R-Car Gen2 DMA Controller Channel 145 * @chan: base DMA channel object 146 * @iomem: channel I/O memory base 147 * @index: index of this channel in the controller 148 * @irq: channel IRQ 149 * @src: slave memory address and size on the source side 150 * @dst: slave memory address and size on the destination side 151 * @mid_rid: hardware MID/RID for the DMA client using this channel 152 * @lock: protects the channel CHCR register and the desc members 153 * @desc.free: list of free descriptors 154 * @desc.pending: list of pending descriptors (submitted with tx_submit) 155 * @desc.active: list of active descriptors (activated with issue_pending) 156 * @desc.done: list of completed descriptors 157 * @desc.wait: list of descriptors waiting for an ack 158 * @desc.running: the descriptor being processed (a member of the active list) 159 * @desc.chunks_free: list of free transfer chunk descriptors 160 * @desc.pages: list of pages used by allocated descriptors 161 */ 162 struct rcar_dmac_chan { 163 struct dma_chan chan; 164 void __iomem *iomem; 165 unsigned int index; 166 int irq; 167 168 struct rcar_dmac_chan_slave src; 169 struct rcar_dmac_chan_slave dst; 170 struct rcar_dmac_chan_map map; 171 int mid_rid; 172 173 spinlock_t lock; 174 175 struct { 176 struct list_head free; 177 struct list_head pending; 178 struct list_head active; 179 struct list_head done; 180 struct list_head wait; 181 struct rcar_dmac_desc *running; 182 183 struct list_head chunks_free; 184 185 struct list_head pages; 186 } desc; 187 }; 188 189 #define to_rcar_dmac_chan(c) container_of(c, struct rcar_dmac_chan, chan) 190 191 /* 192 * struct rcar_dmac - R-Car Gen2 DMA Controller 193 * @engine: base DMA engine object 194 * @dev: the hardware device 195 * @iomem: remapped I/O memory base 196 * @n_channels: number of available channels 197 * @channels: array of DMAC channels 198 * @modules: bitmask of client modules in use 199 */ 200 struct rcar_dmac { 201 struct dma_device engine; 202 struct device *dev; 203 void __iomem *iomem; 204 205 unsigned int n_channels; 206 struct rcar_dmac_chan *channels; 207 208 DECLARE_BITMAP(modules, 256); 209 }; 210 211 #define to_rcar_dmac(d) container_of(d, struct rcar_dmac, engine) 212 213 /* ----------------------------------------------------------------------------- 214 * Registers 215 */ 216 217 #define RCAR_DMAC_CHAN_OFFSET(i) (0x8000 + 0x80 * (i)) 218 219 #define RCAR_DMAISTA 0x0020 220 #define RCAR_DMASEC 0x0030 221 #define RCAR_DMAOR 0x0060 222 #define RCAR_DMAOR_PRI_FIXED (0 << 8) 223 #define RCAR_DMAOR_PRI_ROUND_ROBIN (3 << 8) 224 #define RCAR_DMAOR_AE (1 << 2) 225 #define RCAR_DMAOR_DME (1 << 0) 226 #define RCAR_DMACHCLR 0x0080 227 #define RCAR_DMADPSEC 0x00a0 228 229 #define RCAR_DMASAR 0x0000 230 #define RCAR_DMADAR 0x0004 231 #define RCAR_DMATCR 0x0008 232 #define RCAR_DMATCR_MASK 0x00ffffff 233 #define RCAR_DMATSR 0x0028 234 #define RCAR_DMACHCR 0x000c 235 #define RCAR_DMACHCR_CAE (1 << 31) 236 #define RCAR_DMACHCR_CAIE (1 << 30) 237 #define RCAR_DMACHCR_DPM_DISABLED (0 << 28) 238 #define RCAR_DMACHCR_DPM_ENABLED (1 << 28) 239 #define RCAR_DMACHCR_DPM_REPEAT (2 << 28) 240 #define RCAR_DMACHCR_DPM_INFINITE (3 << 28) 241 #define RCAR_DMACHCR_RPT_SAR (1 << 27) 242 #define RCAR_DMACHCR_RPT_DAR (1 << 26) 243 #define RCAR_DMACHCR_RPT_TCR (1 << 25) 244 #define RCAR_DMACHCR_DPB (1 << 22) 245 #define RCAR_DMACHCR_DSE (1 << 19) 246 #define RCAR_DMACHCR_DSIE (1 << 18) 247 #define RCAR_DMACHCR_TS_1B ((0 << 20) | (0 << 3)) 248 #define RCAR_DMACHCR_TS_2B ((0 << 20) | (1 << 3)) 249 #define RCAR_DMACHCR_TS_4B ((0 << 20) | (2 << 3)) 250 #define RCAR_DMACHCR_TS_16B ((0 << 20) | (3 << 3)) 251 #define RCAR_DMACHCR_TS_32B ((1 << 20) | (0 << 3)) 252 #define RCAR_DMACHCR_TS_64B ((1 << 20) | (1 << 3)) 253 #define RCAR_DMACHCR_TS_8B ((1 << 20) | (3 << 3)) 254 #define RCAR_DMACHCR_DM_FIXED (0 << 14) 255 #define RCAR_DMACHCR_DM_INC (1 << 14) 256 #define RCAR_DMACHCR_DM_DEC (2 << 14) 257 #define RCAR_DMACHCR_SM_FIXED (0 << 12) 258 #define RCAR_DMACHCR_SM_INC (1 << 12) 259 #define RCAR_DMACHCR_SM_DEC (2 << 12) 260 #define RCAR_DMACHCR_RS_AUTO (4 << 8) 261 #define RCAR_DMACHCR_RS_DMARS (8 << 8) 262 #define RCAR_DMACHCR_IE (1 << 2) 263 #define RCAR_DMACHCR_TE (1 << 1) 264 #define RCAR_DMACHCR_DE (1 << 0) 265 #define RCAR_DMATCRB 0x0018 266 #define RCAR_DMATSRB 0x0038 267 #define RCAR_DMACHCRB 0x001c 268 #define RCAR_DMACHCRB_DCNT(n) ((n) << 24) 269 #define RCAR_DMACHCRB_DPTR_MASK (0xff << 16) 270 #define RCAR_DMACHCRB_DPTR_SHIFT 16 271 #define RCAR_DMACHCRB_DRST (1 << 15) 272 #define RCAR_DMACHCRB_DTS (1 << 8) 273 #define RCAR_DMACHCRB_SLM_NORMAL (0 << 4) 274 #define RCAR_DMACHCRB_SLM_CLK(n) ((8 | (n)) << 4) 275 #define RCAR_DMACHCRB_PRI(n) ((n) << 0) 276 #define RCAR_DMARS 0x0040 277 #define RCAR_DMABUFCR 0x0048 278 #define RCAR_DMABUFCR_MBU(n) ((n) << 16) 279 #define RCAR_DMABUFCR_ULB(n) ((n) << 0) 280 #define RCAR_DMADPBASE 0x0050 281 #define RCAR_DMADPBASE_MASK 0xfffffff0 282 #define RCAR_DMADPBASE_SEL (1 << 0) 283 #define RCAR_DMADPCR 0x0054 284 #define RCAR_DMADPCR_DIPT(n) ((n) << 24) 285 #define RCAR_DMAFIXSAR 0x0010 286 #define RCAR_DMAFIXDAR 0x0014 287 #define RCAR_DMAFIXDPBASE 0x0060 288 289 /* Hardcode the MEMCPY transfer size to 4 bytes. */ 290 #define RCAR_DMAC_MEMCPY_XFER_SIZE 4 291 292 /* ----------------------------------------------------------------------------- 293 * Device access 294 */ 295 296 static void rcar_dmac_write(struct rcar_dmac *dmac, u32 reg, u32 data) 297 { 298 if (reg == RCAR_DMAOR) 299 writew(data, dmac->iomem + reg); 300 else 301 writel(data, dmac->iomem + reg); 302 } 303 304 static u32 rcar_dmac_read(struct rcar_dmac *dmac, u32 reg) 305 { 306 if (reg == RCAR_DMAOR) 307 return readw(dmac->iomem + reg); 308 else 309 return readl(dmac->iomem + reg); 310 } 311 312 static u32 rcar_dmac_chan_read(struct rcar_dmac_chan *chan, u32 reg) 313 { 314 if (reg == RCAR_DMARS) 315 return readw(chan->iomem + reg); 316 else 317 return readl(chan->iomem + reg); 318 } 319 320 static void rcar_dmac_chan_write(struct rcar_dmac_chan *chan, u32 reg, u32 data) 321 { 322 if (reg == RCAR_DMARS) 323 writew(data, chan->iomem + reg); 324 else 325 writel(data, chan->iomem + reg); 326 } 327 328 /* ----------------------------------------------------------------------------- 329 * Initialization and configuration 330 */ 331 332 static bool rcar_dmac_chan_is_busy(struct rcar_dmac_chan *chan) 333 { 334 u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 335 336 return !!(chcr & (RCAR_DMACHCR_DE | RCAR_DMACHCR_TE)); 337 } 338 339 static void rcar_dmac_chan_start_xfer(struct rcar_dmac_chan *chan) 340 { 341 struct rcar_dmac_desc *desc = chan->desc.running; 342 u32 chcr = desc->chcr; 343 344 WARN_ON_ONCE(rcar_dmac_chan_is_busy(chan)); 345 346 if (chan->mid_rid >= 0) 347 rcar_dmac_chan_write(chan, RCAR_DMARS, chan->mid_rid); 348 349 if (desc->hwdescs.use) { 350 struct rcar_dmac_xfer_chunk *chunk = 351 list_first_entry(&desc->chunks, 352 struct rcar_dmac_xfer_chunk, node); 353 354 dev_dbg(chan->chan.device->dev, 355 "chan%u: queue desc %p: %u@%pad\n", 356 chan->index, desc, desc->nchunks, &desc->hwdescs.dma); 357 358 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 359 rcar_dmac_chan_write(chan, RCAR_DMAFIXSAR, 360 chunk->src_addr >> 32); 361 rcar_dmac_chan_write(chan, RCAR_DMAFIXDAR, 362 chunk->dst_addr >> 32); 363 rcar_dmac_chan_write(chan, RCAR_DMAFIXDPBASE, 364 desc->hwdescs.dma >> 32); 365 #endif 366 rcar_dmac_chan_write(chan, RCAR_DMADPBASE, 367 (desc->hwdescs.dma & 0xfffffff0) | 368 RCAR_DMADPBASE_SEL); 369 rcar_dmac_chan_write(chan, RCAR_DMACHCRB, 370 RCAR_DMACHCRB_DCNT(desc->nchunks - 1) | 371 RCAR_DMACHCRB_DRST); 372 373 /* 374 * Errata: When descriptor memory is accessed through an IOMMU 375 * the DMADAR register isn't initialized automatically from the 376 * first descriptor at beginning of transfer by the DMAC like it 377 * should. Initialize it manually with the destination address 378 * of the first chunk. 379 */ 380 rcar_dmac_chan_write(chan, RCAR_DMADAR, 381 chunk->dst_addr & 0xffffffff); 382 383 /* 384 * Program the descriptor stage interrupt to occur after the end 385 * of the first stage. 386 */ 387 rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(1)); 388 389 chcr |= RCAR_DMACHCR_RPT_SAR | RCAR_DMACHCR_RPT_DAR 390 | RCAR_DMACHCR_RPT_TCR | RCAR_DMACHCR_DPB; 391 392 /* 393 * If the descriptor isn't cyclic enable normal descriptor mode 394 * and the transfer completion interrupt. 395 */ 396 if (!desc->cyclic) 397 chcr |= RCAR_DMACHCR_DPM_ENABLED | RCAR_DMACHCR_IE; 398 /* 399 * If the descriptor is cyclic and has a callback enable the 400 * descriptor stage interrupt in infinite repeat mode. 401 */ 402 else if (desc->async_tx.callback) 403 chcr |= RCAR_DMACHCR_DPM_INFINITE | RCAR_DMACHCR_DSIE; 404 /* 405 * Otherwise just select infinite repeat mode without any 406 * interrupt. 407 */ 408 else 409 chcr |= RCAR_DMACHCR_DPM_INFINITE; 410 } else { 411 struct rcar_dmac_xfer_chunk *chunk = desc->running; 412 413 dev_dbg(chan->chan.device->dev, 414 "chan%u: queue chunk %p: %u@%pad -> %pad\n", 415 chan->index, chunk, chunk->size, &chunk->src_addr, 416 &chunk->dst_addr); 417 418 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 419 rcar_dmac_chan_write(chan, RCAR_DMAFIXSAR, 420 chunk->src_addr >> 32); 421 rcar_dmac_chan_write(chan, RCAR_DMAFIXDAR, 422 chunk->dst_addr >> 32); 423 #endif 424 rcar_dmac_chan_write(chan, RCAR_DMASAR, 425 chunk->src_addr & 0xffffffff); 426 rcar_dmac_chan_write(chan, RCAR_DMADAR, 427 chunk->dst_addr & 0xffffffff); 428 rcar_dmac_chan_write(chan, RCAR_DMATCR, 429 chunk->size >> desc->xfer_shift); 430 431 chcr |= RCAR_DMACHCR_DPM_DISABLED | RCAR_DMACHCR_IE; 432 } 433 434 rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr | RCAR_DMACHCR_DE); 435 } 436 437 static int rcar_dmac_init(struct rcar_dmac *dmac) 438 { 439 u16 dmaor; 440 441 /* Clear all channels and enable the DMAC globally. */ 442 rcar_dmac_write(dmac, RCAR_DMACHCLR, GENMASK(dmac->n_channels - 1, 0)); 443 rcar_dmac_write(dmac, RCAR_DMAOR, 444 RCAR_DMAOR_PRI_FIXED | RCAR_DMAOR_DME); 445 446 dmaor = rcar_dmac_read(dmac, RCAR_DMAOR); 447 if ((dmaor & (RCAR_DMAOR_AE | RCAR_DMAOR_DME)) != RCAR_DMAOR_DME) { 448 dev_warn(dmac->dev, "DMAOR initialization failed.\n"); 449 return -EIO; 450 } 451 452 return 0; 453 } 454 455 /* ----------------------------------------------------------------------------- 456 * Descriptors submission 457 */ 458 459 static dma_cookie_t rcar_dmac_tx_submit(struct dma_async_tx_descriptor *tx) 460 { 461 struct rcar_dmac_chan *chan = to_rcar_dmac_chan(tx->chan); 462 struct rcar_dmac_desc *desc = to_rcar_dmac_desc(tx); 463 unsigned long flags; 464 dma_cookie_t cookie; 465 466 spin_lock_irqsave(&chan->lock, flags); 467 468 cookie = dma_cookie_assign(tx); 469 470 dev_dbg(chan->chan.device->dev, "chan%u: submit #%d@%p\n", 471 chan->index, tx->cookie, desc); 472 473 list_add_tail(&desc->node, &chan->desc.pending); 474 desc->running = list_first_entry(&desc->chunks, 475 struct rcar_dmac_xfer_chunk, node); 476 477 spin_unlock_irqrestore(&chan->lock, flags); 478 479 return cookie; 480 } 481 482 /* ----------------------------------------------------------------------------- 483 * Descriptors allocation and free 484 */ 485 486 /* 487 * rcar_dmac_desc_alloc - Allocate a page worth of DMA descriptors 488 * @chan: the DMA channel 489 * @gfp: allocation flags 490 */ 491 static int rcar_dmac_desc_alloc(struct rcar_dmac_chan *chan, gfp_t gfp) 492 { 493 struct rcar_dmac_desc_page *page; 494 unsigned long flags; 495 LIST_HEAD(list); 496 unsigned int i; 497 498 page = (void *)get_zeroed_page(gfp); 499 if (!page) 500 return -ENOMEM; 501 502 for (i = 0; i < RCAR_DMAC_DESCS_PER_PAGE; ++i) { 503 struct rcar_dmac_desc *desc = &page->descs[i]; 504 505 dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan); 506 desc->async_tx.tx_submit = rcar_dmac_tx_submit; 507 INIT_LIST_HEAD(&desc->chunks); 508 509 list_add_tail(&desc->node, &list); 510 } 511 512 spin_lock_irqsave(&chan->lock, flags); 513 list_splice_tail(&list, &chan->desc.free); 514 list_add_tail(&page->node, &chan->desc.pages); 515 spin_unlock_irqrestore(&chan->lock, flags); 516 517 return 0; 518 } 519 520 /* 521 * rcar_dmac_desc_put - Release a DMA transfer descriptor 522 * @chan: the DMA channel 523 * @desc: the descriptor 524 * 525 * Put the descriptor and its transfer chunk descriptors back in the channel's 526 * free descriptors lists. The descriptor's chunks list will be reinitialized to 527 * an empty list as a result. 528 * 529 * The descriptor must have been removed from the channel's lists before calling 530 * this function. 531 */ 532 static void rcar_dmac_desc_put(struct rcar_dmac_chan *chan, 533 struct rcar_dmac_desc *desc) 534 { 535 unsigned long flags; 536 537 spin_lock_irqsave(&chan->lock, flags); 538 list_splice_tail_init(&desc->chunks, &chan->desc.chunks_free); 539 list_add(&desc->node, &chan->desc.free); 540 spin_unlock_irqrestore(&chan->lock, flags); 541 } 542 543 static void rcar_dmac_desc_recycle_acked(struct rcar_dmac_chan *chan) 544 { 545 struct rcar_dmac_desc *desc, *_desc; 546 unsigned long flags; 547 LIST_HEAD(list); 548 549 /* 550 * We have to temporarily move all descriptors from the wait list to a 551 * local list as iterating over the wait list, even with 552 * list_for_each_entry_safe, isn't safe if we release the channel lock 553 * around the rcar_dmac_desc_put() call. 554 */ 555 spin_lock_irqsave(&chan->lock, flags); 556 list_splice_init(&chan->desc.wait, &list); 557 spin_unlock_irqrestore(&chan->lock, flags); 558 559 list_for_each_entry_safe(desc, _desc, &list, node) { 560 if (async_tx_test_ack(&desc->async_tx)) { 561 list_del(&desc->node); 562 rcar_dmac_desc_put(chan, desc); 563 } 564 } 565 566 if (list_empty(&list)) 567 return; 568 569 /* Put the remaining descriptors back in the wait list. */ 570 spin_lock_irqsave(&chan->lock, flags); 571 list_splice(&list, &chan->desc.wait); 572 spin_unlock_irqrestore(&chan->lock, flags); 573 } 574 575 /* 576 * rcar_dmac_desc_get - Allocate a descriptor for a DMA transfer 577 * @chan: the DMA channel 578 * 579 * Locking: This function must be called in a non-atomic context. 580 * 581 * Return: A pointer to the allocated descriptor or NULL if no descriptor can 582 * be allocated. 583 */ 584 static struct rcar_dmac_desc *rcar_dmac_desc_get(struct rcar_dmac_chan *chan) 585 { 586 struct rcar_dmac_desc *desc; 587 unsigned long flags; 588 int ret; 589 590 /* Recycle acked descriptors before attempting allocation. */ 591 rcar_dmac_desc_recycle_acked(chan); 592 593 spin_lock_irqsave(&chan->lock, flags); 594 595 while (list_empty(&chan->desc.free)) { 596 /* 597 * No free descriptors, allocate a page worth of them and try 598 * again, as someone else could race us to get the newly 599 * allocated descriptors. If the allocation fails return an 600 * error. 601 */ 602 spin_unlock_irqrestore(&chan->lock, flags); 603 ret = rcar_dmac_desc_alloc(chan, GFP_NOWAIT); 604 if (ret < 0) 605 return NULL; 606 spin_lock_irqsave(&chan->lock, flags); 607 } 608 609 desc = list_first_entry(&chan->desc.free, struct rcar_dmac_desc, node); 610 list_del(&desc->node); 611 612 spin_unlock_irqrestore(&chan->lock, flags); 613 614 return desc; 615 } 616 617 /* 618 * rcar_dmac_xfer_chunk_alloc - Allocate a page worth of transfer chunks 619 * @chan: the DMA channel 620 * @gfp: allocation flags 621 */ 622 static int rcar_dmac_xfer_chunk_alloc(struct rcar_dmac_chan *chan, gfp_t gfp) 623 { 624 struct rcar_dmac_desc_page *page; 625 unsigned long flags; 626 LIST_HEAD(list); 627 unsigned int i; 628 629 page = (void *)get_zeroed_page(gfp); 630 if (!page) 631 return -ENOMEM; 632 633 for (i = 0; i < RCAR_DMAC_XFER_CHUNKS_PER_PAGE; ++i) { 634 struct rcar_dmac_xfer_chunk *chunk = &page->chunks[i]; 635 636 list_add_tail(&chunk->node, &list); 637 } 638 639 spin_lock_irqsave(&chan->lock, flags); 640 list_splice_tail(&list, &chan->desc.chunks_free); 641 list_add_tail(&page->node, &chan->desc.pages); 642 spin_unlock_irqrestore(&chan->lock, flags); 643 644 return 0; 645 } 646 647 /* 648 * rcar_dmac_xfer_chunk_get - Allocate a transfer chunk for a DMA transfer 649 * @chan: the DMA channel 650 * 651 * Locking: This function must be called in a non-atomic context. 652 * 653 * Return: A pointer to the allocated transfer chunk descriptor or NULL if no 654 * descriptor can be allocated. 655 */ 656 static struct rcar_dmac_xfer_chunk * 657 rcar_dmac_xfer_chunk_get(struct rcar_dmac_chan *chan) 658 { 659 struct rcar_dmac_xfer_chunk *chunk; 660 unsigned long flags; 661 int ret; 662 663 spin_lock_irqsave(&chan->lock, flags); 664 665 while (list_empty(&chan->desc.chunks_free)) { 666 /* 667 * No free descriptors, allocate a page worth of them and try 668 * again, as someone else could race us to get the newly 669 * allocated descriptors. If the allocation fails return an 670 * error. 671 */ 672 spin_unlock_irqrestore(&chan->lock, flags); 673 ret = rcar_dmac_xfer_chunk_alloc(chan, GFP_NOWAIT); 674 if (ret < 0) 675 return NULL; 676 spin_lock_irqsave(&chan->lock, flags); 677 } 678 679 chunk = list_first_entry(&chan->desc.chunks_free, 680 struct rcar_dmac_xfer_chunk, node); 681 list_del(&chunk->node); 682 683 spin_unlock_irqrestore(&chan->lock, flags); 684 685 return chunk; 686 } 687 688 static void rcar_dmac_realloc_hwdesc(struct rcar_dmac_chan *chan, 689 struct rcar_dmac_desc *desc, size_t size) 690 { 691 /* 692 * dma_alloc_coherent() allocates memory in page size increments. To 693 * avoid reallocating the hardware descriptors when the allocated size 694 * wouldn't change align the requested size to a multiple of the page 695 * size. 696 */ 697 size = PAGE_ALIGN(size); 698 699 if (desc->hwdescs.size == size) 700 return; 701 702 if (desc->hwdescs.mem) { 703 dma_free_coherent(chan->chan.device->dev, desc->hwdescs.size, 704 desc->hwdescs.mem, desc->hwdescs.dma); 705 desc->hwdescs.mem = NULL; 706 desc->hwdescs.size = 0; 707 } 708 709 if (!size) 710 return; 711 712 desc->hwdescs.mem = dma_alloc_coherent(chan->chan.device->dev, size, 713 &desc->hwdescs.dma, GFP_NOWAIT); 714 if (!desc->hwdescs.mem) 715 return; 716 717 desc->hwdescs.size = size; 718 } 719 720 static int rcar_dmac_fill_hwdesc(struct rcar_dmac_chan *chan, 721 struct rcar_dmac_desc *desc) 722 { 723 struct rcar_dmac_xfer_chunk *chunk; 724 struct rcar_dmac_hw_desc *hwdesc; 725 726 rcar_dmac_realloc_hwdesc(chan, desc, desc->nchunks * sizeof(*hwdesc)); 727 728 hwdesc = desc->hwdescs.mem; 729 if (!hwdesc) 730 return -ENOMEM; 731 732 list_for_each_entry(chunk, &desc->chunks, node) { 733 hwdesc->sar = chunk->src_addr; 734 hwdesc->dar = chunk->dst_addr; 735 hwdesc->tcr = chunk->size >> desc->xfer_shift; 736 hwdesc++; 737 } 738 739 return 0; 740 } 741 742 /* ----------------------------------------------------------------------------- 743 * Stop and reset 744 */ 745 static void rcar_dmac_chcr_de_barrier(struct rcar_dmac_chan *chan) 746 { 747 u32 chcr; 748 unsigned int i; 749 750 /* 751 * Ensure that the setting of the DE bit is actually 0 after 752 * clearing it. 753 */ 754 for (i = 0; i < 1024; i++) { 755 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 756 if (!(chcr & RCAR_DMACHCR_DE)) 757 return; 758 udelay(1); 759 } 760 761 dev_err(chan->chan.device->dev, "CHCR DE check error\n"); 762 } 763 764 static void rcar_dmac_sync_tcr(struct rcar_dmac_chan *chan) 765 { 766 u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 767 768 if (!(chcr & RCAR_DMACHCR_DE)) 769 return; 770 771 /* set DE=0 and flush remaining data */ 772 rcar_dmac_chan_write(chan, RCAR_DMACHCR, (chcr & ~RCAR_DMACHCR_DE)); 773 774 /* make sure all remaining data was flushed */ 775 rcar_dmac_chcr_de_barrier(chan); 776 777 /* back DE */ 778 rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr); 779 } 780 781 static void rcar_dmac_chan_halt(struct rcar_dmac_chan *chan) 782 { 783 u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 784 785 chcr &= ~(RCAR_DMACHCR_DSE | RCAR_DMACHCR_DSIE | RCAR_DMACHCR_IE | 786 RCAR_DMACHCR_TE | RCAR_DMACHCR_DE); 787 rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr); 788 rcar_dmac_chcr_de_barrier(chan); 789 } 790 791 static void rcar_dmac_chan_reinit(struct rcar_dmac_chan *chan) 792 { 793 struct rcar_dmac_desc *desc, *_desc; 794 unsigned long flags; 795 LIST_HEAD(descs); 796 797 spin_lock_irqsave(&chan->lock, flags); 798 799 /* Move all non-free descriptors to the local lists. */ 800 list_splice_init(&chan->desc.pending, &descs); 801 list_splice_init(&chan->desc.active, &descs); 802 list_splice_init(&chan->desc.done, &descs); 803 list_splice_init(&chan->desc.wait, &descs); 804 805 chan->desc.running = NULL; 806 807 spin_unlock_irqrestore(&chan->lock, flags); 808 809 list_for_each_entry_safe(desc, _desc, &descs, node) { 810 list_del(&desc->node); 811 rcar_dmac_desc_put(chan, desc); 812 } 813 } 814 815 static void rcar_dmac_stop(struct rcar_dmac *dmac) 816 { 817 rcar_dmac_write(dmac, RCAR_DMAOR, 0); 818 } 819 820 static void rcar_dmac_abort(struct rcar_dmac *dmac) 821 { 822 unsigned int i; 823 824 /* Stop all channels. */ 825 for (i = 0; i < dmac->n_channels; ++i) { 826 struct rcar_dmac_chan *chan = &dmac->channels[i]; 827 828 /* Stop and reinitialize the channel. */ 829 spin_lock(&chan->lock); 830 rcar_dmac_chan_halt(chan); 831 spin_unlock(&chan->lock); 832 833 rcar_dmac_chan_reinit(chan); 834 } 835 } 836 837 /* ----------------------------------------------------------------------------- 838 * Descriptors preparation 839 */ 840 841 static void rcar_dmac_chan_configure_desc(struct rcar_dmac_chan *chan, 842 struct rcar_dmac_desc *desc) 843 { 844 static const u32 chcr_ts[] = { 845 RCAR_DMACHCR_TS_1B, RCAR_DMACHCR_TS_2B, 846 RCAR_DMACHCR_TS_4B, RCAR_DMACHCR_TS_8B, 847 RCAR_DMACHCR_TS_16B, RCAR_DMACHCR_TS_32B, 848 RCAR_DMACHCR_TS_64B, 849 }; 850 851 unsigned int xfer_size; 852 u32 chcr; 853 854 switch (desc->direction) { 855 case DMA_DEV_TO_MEM: 856 chcr = RCAR_DMACHCR_DM_INC | RCAR_DMACHCR_SM_FIXED 857 | RCAR_DMACHCR_RS_DMARS; 858 xfer_size = chan->src.xfer_size; 859 break; 860 861 case DMA_MEM_TO_DEV: 862 chcr = RCAR_DMACHCR_DM_FIXED | RCAR_DMACHCR_SM_INC 863 | RCAR_DMACHCR_RS_DMARS; 864 xfer_size = chan->dst.xfer_size; 865 break; 866 867 case DMA_MEM_TO_MEM: 868 default: 869 chcr = RCAR_DMACHCR_DM_INC | RCAR_DMACHCR_SM_INC 870 | RCAR_DMACHCR_RS_AUTO; 871 xfer_size = RCAR_DMAC_MEMCPY_XFER_SIZE; 872 break; 873 } 874 875 desc->xfer_shift = ilog2(xfer_size); 876 desc->chcr = chcr | chcr_ts[desc->xfer_shift]; 877 } 878 879 /* 880 * rcar_dmac_chan_prep_sg - prepare transfer descriptors from an SG list 881 * 882 * Common routine for public (MEMCPY) and slave DMA. The MEMCPY case is also 883 * converted to scatter-gather to guarantee consistent locking and a correct 884 * list manipulation. For slave DMA direction carries the usual meaning, and, 885 * logically, the SG list is RAM and the addr variable contains slave address, 886 * e.g., the FIFO I/O register. For MEMCPY direction equals DMA_MEM_TO_MEM 887 * and the SG list contains only one element and points at the source buffer. 888 */ 889 static struct dma_async_tx_descriptor * 890 rcar_dmac_chan_prep_sg(struct rcar_dmac_chan *chan, struct scatterlist *sgl, 891 unsigned int sg_len, dma_addr_t dev_addr, 892 enum dma_transfer_direction dir, unsigned long dma_flags, 893 bool cyclic) 894 { 895 struct rcar_dmac_xfer_chunk *chunk; 896 struct rcar_dmac_desc *desc; 897 struct scatterlist *sg; 898 unsigned int nchunks = 0; 899 unsigned int max_chunk_size; 900 unsigned int full_size = 0; 901 bool cross_boundary = false; 902 unsigned int i; 903 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 904 u32 high_dev_addr; 905 u32 high_mem_addr; 906 #endif 907 908 desc = rcar_dmac_desc_get(chan); 909 if (!desc) 910 return NULL; 911 912 desc->async_tx.flags = dma_flags; 913 desc->async_tx.cookie = -EBUSY; 914 915 desc->cyclic = cyclic; 916 desc->direction = dir; 917 918 rcar_dmac_chan_configure_desc(chan, desc); 919 920 max_chunk_size = RCAR_DMATCR_MASK << desc->xfer_shift; 921 922 /* 923 * Allocate and fill the transfer chunk descriptors. We own the only 924 * reference to the DMA descriptor, there's no need for locking. 925 */ 926 for_each_sg(sgl, sg, sg_len, i) { 927 dma_addr_t mem_addr = sg_dma_address(sg); 928 unsigned int len = sg_dma_len(sg); 929 930 full_size += len; 931 932 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 933 if (i == 0) { 934 high_dev_addr = dev_addr >> 32; 935 high_mem_addr = mem_addr >> 32; 936 } 937 938 if ((dev_addr >> 32 != high_dev_addr) || 939 (mem_addr >> 32 != high_mem_addr)) 940 cross_boundary = true; 941 #endif 942 while (len) { 943 unsigned int size = min(len, max_chunk_size); 944 945 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 946 /* 947 * Prevent individual transfers from crossing 4GB 948 * boundaries. 949 */ 950 if (dev_addr >> 32 != (dev_addr + size - 1) >> 32) { 951 size = ALIGN(dev_addr, 1ULL << 32) - dev_addr; 952 cross_boundary = true; 953 } 954 if (mem_addr >> 32 != (mem_addr + size - 1) >> 32) { 955 size = ALIGN(mem_addr, 1ULL << 32) - mem_addr; 956 cross_boundary = true; 957 } 958 #endif 959 960 chunk = rcar_dmac_xfer_chunk_get(chan); 961 if (!chunk) { 962 rcar_dmac_desc_put(chan, desc); 963 return NULL; 964 } 965 966 if (dir == DMA_DEV_TO_MEM) { 967 chunk->src_addr = dev_addr; 968 chunk->dst_addr = mem_addr; 969 } else { 970 chunk->src_addr = mem_addr; 971 chunk->dst_addr = dev_addr; 972 } 973 974 chunk->size = size; 975 976 dev_dbg(chan->chan.device->dev, 977 "chan%u: chunk %p/%p sgl %u@%p, %u/%u %pad -> %pad\n", 978 chan->index, chunk, desc, i, sg, size, len, 979 &chunk->src_addr, &chunk->dst_addr); 980 981 mem_addr += size; 982 if (dir == DMA_MEM_TO_MEM) 983 dev_addr += size; 984 985 len -= size; 986 987 list_add_tail(&chunk->node, &desc->chunks); 988 nchunks++; 989 } 990 } 991 992 desc->nchunks = nchunks; 993 desc->size = full_size; 994 995 /* 996 * Use hardware descriptor lists if possible when more than one chunk 997 * needs to be transferred (otherwise they don't make much sense). 998 * 999 * Source/Destination address should be located in same 4GiB region 1000 * in the 40bit address space when it uses Hardware descriptor, 1001 * and cross_boundary is checking it. 1002 */ 1003 desc->hwdescs.use = !cross_boundary && nchunks > 1; 1004 if (desc->hwdescs.use) { 1005 if (rcar_dmac_fill_hwdesc(chan, desc) < 0) 1006 desc->hwdescs.use = false; 1007 } 1008 1009 return &desc->async_tx; 1010 } 1011 1012 /* ----------------------------------------------------------------------------- 1013 * DMA engine operations 1014 */ 1015 1016 static int rcar_dmac_alloc_chan_resources(struct dma_chan *chan) 1017 { 1018 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1019 int ret; 1020 1021 INIT_LIST_HEAD(&rchan->desc.chunks_free); 1022 INIT_LIST_HEAD(&rchan->desc.pages); 1023 1024 /* Preallocate descriptors. */ 1025 ret = rcar_dmac_xfer_chunk_alloc(rchan, GFP_KERNEL); 1026 if (ret < 0) 1027 return -ENOMEM; 1028 1029 ret = rcar_dmac_desc_alloc(rchan, GFP_KERNEL); 1030 if (ret < 0) 1031 return -ENOMEM; 1032 1033 return pm_runtime_get_sync(chan->device->dev); 1034 } 1035 1036 static void rcar_dmac_free_chan_resources(struct dma_chan *chan) 1037 { 1038 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1039 struct rcar_dmac *dmac = to_rcar_dmac(chan->device); 1040 struct rcar_dmac_chan_map *map = &rchan->map; 1041 struct rcar_dmac_desc_page *page, *_page; 1042 struct rcar_dmac_desc *desc; 1043 LIST_HEAD(list); 1044 1045 /* Protect against ISR */ 1046 spin_lock_irq(&rchan->lock); 1047 rcar_dmac_chan_halt(rchan); 1048 spin_unlock_irq(&rchan->lock); 1049 1050 /* 1051 * Now no new interrupts will occur, but one might already be 1052 * running. Wait for it to finish before freeing resources. 1053 */ 1054 synchronize_irq(rchan->irq); 1055 1056 if (rchan->mid_rid >= 0) { 1057 /* The caller is holding dma_list_mutex */ 1058 clear_bit(rchan->mid_rid, dmac->modules); 1059 rchan->mid_rid = -EINVAL; 1060 } 1061 1062 list_splice_init(&rchan->desc.free, &list); 1063 list_splice_init(&rchan->desc.pending, &list); 1064 list_splice_init(&rchan->desc.active, &list); 1065 list_splice_init(&rchan->desc.done, &list); 1066 list_splice_init(&rchan->desc.wait, &list); 1067 1068 rchan->desc.running = NULL; 1069 1070 list_for_each_entry(desc, &list, node) 1071 rcar_dmac_realloc_hwdesc(rchan, desc, 0); 1072 1073 list_for_each_entry_safe(page, _page, &rchan->desc.pages, node) { 1074 list_del(&page->node); 1075 free_page((unsigned long)page); 1076 } 1077 1078 /* Remove slave mapping if present. */ 1079 if (map->slave.xfer_size) { 1080 dma_unmap_resource(chan->device->dev, map->addr, 1081 map->slave.xfer_size, map->dir, 0); 1082 map->slave.xfer_size = 0; 1083 } 1084 1085 pm_runtime_put(chan->device->dev); 1086 } 1087 1088 static struct dma_async_tx_descriptor * 1089 rcar_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest, 1090 dma_addr_t dma_src, size_t len, unsigned long flags) 1091 { 1092 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1093 struct scatterlist sgl; 1094 1095 if (!len) 1096 return NULL; 1097 1098 sg_init_table(&sgl, 1); 1099 sg_set_page(&sgl, pfn_to_page(PFN_DOWN(dma_src)), len, 1100 offset_in_page(dma_src)); 1101 sg_dma_address(&sgl) = dma_src; 1102 sg_dma_len(&sgl) = len; 1103 1104 return rcar_dmac_chan_prep_sg(rchan, &sgl, 1, dma_dest, 1105 DMA_MEM_TO_MEM, flags, false); 1106 } 1107 1108 static int rcar_dmac_map_slave_addr(struct dma_chan *chan, 1109 enum dma_transfer_direction dir) 1110 { 1111 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1112 struct rcar_dmac_chan_map *map = &rchan->map; 1113 phys_addr_t dev_addr; 1114 size_t dev_size; 1115 enum dma_data_direction dev_dir; 1116 1117 if (dir == DMA_DEV_TO_MEM) { 1118 dev_addr = rchan->src.slave_addr; 1119 dev_size = rchan->src.xfer_size; 1120 dev_dir = DMA_TO_DEVICE; 1121 } else { 1122 dev_addr = rchan->dst.slave_addr; 1123 dev_size = rchan->dst.xfer_size; 1124 dev_dir = DMA_FROM_DEVICE; 1125 } 1126 1127 /* Reuse current map if possible. */ 1128 if (dev_addr == map->slave.slave_addr && 1129 dev_size == map->slave.xfer_size && 1130 dev_dir == map->dir) 1131 return 0; 1132 1133 /* Remove old mapping if present. */ 1134 if (map->slave.xfer_size) 1135 dma_unmap_resource(chan->device->dev, map->addr, 1136 map->slave.xfer_size, map->dir, 0); 1137 map->slave.xfer_size = 0; 1138 1139 /* Create new slave address map. */ 1140 map->addr = dma_map_resource(chan->device->dev, dev_addr, dev_size, 1141 dev_dir, 0); 1142 1143 if (dma_mapping_error(chan->device->dev, map->addr)) { 1144 dev_err(chan->device->dev, 1145 "chan%u: failed to map %zx@%pap", rchan->index, 1146 dev_size, &dev_addr); 1147 return -EIO; 1148 } 1149 1150 dev_dbg(chan->device->dev, "chan%u: map %zx@%pap to %pad dir: %s\n", 1151 rchan->index, dev_size, &dev_addr, &map->addr, 1152 dev_dir == DMA_TO_DEVICE ? "DMA_TO_DEVICE" : "DMA_FROM_DEVICE"); 1153 1154 map->slave.slave_addr = dev_addr; 1155 map->slave.xfer_size = dev_size; 1156 map->dir = dev_dir; 1157 1158 return 0; 1159 } 1160 1161 static struct dma_async_tx_descriptor * 1162 rcar_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 1163 unsigned int sg_len, enum dma_transfer_direction dir, 1164 unsigned long flags, void *context) 1165 { 1166 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1167 1168 /* Someone calling slave DMA on a generic channel? */ 1169 if (rchan->mid_rid < 0 || !sg_len) { 1170 dev_warn(chan->device->dev, 1171 "%s: bad parameter: len=%d, id=%d\n", 1172 __func__, sg_len, rchan->mid_rid); 1173 return NULL; 1174 } 1175 1176 if (rcar_dmac_map_slave_addr(chan, dir)) 1177 return NULL; 1178 1179 return rcar_dmac_chan_prep_sg(rchan, sgl, sg_len, rchan->map.addr, 1180 dir, flags, false); 1181 } 1182 1183 #define RCAR_DMAC_MAX_SG_LEN 32 1184 1185 static struct dma_async_tx_descriptor * 1186 rcar_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, 1187 size_t buf_len, size_t period_len, 1188 enum dma_transfer_direction dir, unsigned long flags) 1189 { 1190 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1191 struct dma_async_tx_descriptor *desc; 1192 struct scatterlist *sgl; 1193 unsigned int sg_len; 1194 unsigned int i; 1195 1196 /* Someone calling slave DMA on a generic channel? */ 1197 if (rchan->mid_rid < 0 || buf_len < period_len) { 1198 dev_warn(chan->device->dev, 1199 "%s: bad parameter: buf_len=%zu, period_len=%zu, id=%d\n", 1200 __func__, buf_len, period_len, rchan->mid_rid); 1201 return NULL; 1202 } 1203 1204 if (rcar_dmac_map_slave_addr(chan, dir)) 1205 return NULL; 1206 1207 sg_len = buf_len / period_len; 1208 if (sg_len > RCAR_DMAC_MAX_SG_LEN) { 1209 dev_err(chan->device->dev, 1210 "chan%u: sg length %d exceds limit %d", 1211 rchan->index, sg_len, RCAR_DMAC_MAX_SG_LEN); 1212 return NULL; 1213 } 1214 1215 /* 1216 * Allocate the sg list dynamically as it would consume too much stack 1217 * space. 1218 */ 1219 sgl = kcalloc(sg_len, sizeof(*sgl), GFP_NOWAIT); 1220 if (!sgl) 1221 return NULL; 1222 1223 sg_init_table(sgl, sg_len); 1224 1225 for (i = 0; i < sg_len; ++i) { 1226 dma_addr_t src = buf_addr + (period_len * i); 1227 1228 sg_set_page(&sgl[i], pfn_to_page(PFN_DOWN(src)), period_len, 1229 offset_in_page(src)); 1230 sg_dma_address(&sgl[i]) = src; 1231 sg_dma_len(&sgl[i]) = period_len; 1232 } 1233 1234 desc = rcar_dmac_chan_prep_sg(rchan, sgl, sg_len, rchan->map.addr, 1235 dir, flags, true); 1236 1237 kfree(sgl); 1238 return desc; 1239 } 1240 1241 static int rcar_dmac_device_config(struct dma_chan *chan, 1242 struct dma_slave_config *cfg) 1243 { 1244 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1245 1246 /* 1247 * We could lock this, but you shouldn't be configuring the 1248 * channel, while using it... 1249 */ 1250 rchan->src.slave_addr = cfg->src_addr; 1251 rchan->dst.slave_addr = cfg->dst_addr; 1252 rchan->src.xfer_size = cfg->src_addr_width; 1253 rchan->dst.xfer_size = cfg->dst_addr_width; 1254 1255 return 0; 1256 } 1257 1258 static int rcar_dmac_chan_terminate_all(struct dma_chan *chan) 1259 { 1260 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1261 unsigned long flags; 1262 1263 spin_lock_irqsave(&rchan->lock, flags); 1264 rcar_dmac_chan_halt(rchan); 1265 spin_unlock_irqrestore(&rchan->lock, flags); 1266 1267 /* 1268 * FIXME: No new interrupt can occur now, but the IRQ thread might still 1269 * be running. 1270 */ 1271 1272 rcar_dmac_chan_reinit(rchan); 1273 1274 return 0; 1275 } 1276 1277 static unsigned int rcar_dmac_chan_get_residue(struct rcar_dmac_chan *chan, 1278 dma_cookie_t cookie) 1279 { 1280 struct rcar_dmac_desc *desc = chan->desc.running; 1281 struct rcar_dmac_xfer_chunk *running = NULL; 1282 struct rcar_dmac_xfer_chunk *chunk; 1283 enum dma_status status; 1284 unsigned int residue = 0; 1285 unsigned int dptr = 0; 1286 1287 if (!desc) 1288 return 0; 1289 1290 /* 1291 * If the cookie corresponds to a descriptor that has been completed 1292 * there is no residue. The same check has already been performed by the 1293 * caller but without holding the channel lock, so the descriptor could 1294 * now be complete. 1295 */ 1296 status = dma_cookie_status(&chan->chan, cookie, NULL); 1297 if (status == DMA_COMPLETE) 1298 return 0; 1299 1300 /* 1301 * If the cookie doesn't correspond to the currently running transfer 1302 * then the descriptor hasn't been processed yet, and the residue is 1303 * equal to the full descriptor size. 1304 * Also, a client driver is possible to call this function before 1305 * rcar_dmac_isr_channel_thread() runs. In this case, the "desc.running" 1306 * will be the next descriptor, and the done list will appear. So, if 1307 * the argument cookie matches the done list's cookie, we can assume 1308 * the residue is zero. 1309 */ 1310 if (cookie != desc->async_tx.cookie) { 1311 list_for_each_entry(desc, &chan->desc.done, node) { 1312 if (cookie == desc->async_tx.cookie) 1313 return 0; 1314 } 1315 list_for_each_entry(desc, &chan->desc.pending, node) { 1316 if (cookie == desc->async_tx.cookie) 1317 return desc->size; 1318 } 1319 list_for_each_entry(desc, &chan->desc.active, node) { 1320 if (cookie == desc->async_tx.cookie) 1321 return desc->size; 1322 } 1323 1324 /* 1325 * No descriptor found for the cookie, there's thus no residue. 1326 * This shouldn't happen if the calling driver passes a correct 1327 * cookie value. 1328 */ 1329 WARN(1, "No descriptor for cookie!"); 1330 return 0; 1331 } 1332 1333 /* 1334 * In descriptor mode the descriptor running pointer is not maintained 1335 * by the interrupt handler, find the running descriptor from the 1336 * descriptor pointer field in the CHCRB register. In non-descriptor 1337 * mode just use the running descriptor pointer. 1338 */ 1339 if (desc->hwdescs.use) { 1340 dptr = (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) & 1341 RCAR_DMACHCRB_DPTR_MASK) >> RCAR_DMACHCRB_DPTR_SHIFT; 1342 if (dptr == 0) 1343 dptr = desc->nchunks; 1344 dptr--; 1345 WARN_ON(dptr >= desc->nchunks); 1346 } else { 1347 running = desc->running; 1348 } 1349 1350 /* Compute the size of all chunks still to be transferred. */ 1351 list_for_each_entry_reverse(chunk, &desc->chunks, node) { 1352 if (chunk == running || ++dptr == desc->nchunks) 1353 break; 1354 1355 residue += chunk->size; 1356 } 1357 1358 if (desc->direction == DMA_DEV_TO_MEM) 1359 rcar_dmac_sync_tcr(chan); 1360 1361 /* Add the residue for the current chunk. */ 1362 residue += rcar_dmac_chan_read(chan, RCAR_DMATCRB) << desc->xfer_shift; 1363 1364 return residue; 1365 } 1366 1367 static enum dma_status rcar_dmac_tx_status(struct dma_chan *chan, 1368 dma_cookie_t cookie, 1369 struct dma_tx_state *txstate) 1370 { 1371 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1372 enum dma_status status; 1373 unsigned long flags; 1374 unsigned int residue; 1375 1376 status = dma_cookie_status(chan, cookie, txstate); 1377 if (status == DMA_COMPLETE || !txstate) 1378 return status; 1379 1380 spin_lock_irqsave(&rchan->lock, flags); 1381 residue = rcar_dmac_chan_get_residue(rchan, cookie); 1382 spin_unlock_irqrestore(&rchan->lock, flags); 1383 1384 /* if there's no residue, the cookie is complete */ 1385 if (!residue) 1386 return DMA_COMPLETE; 1387 1388 dma_set_residue(txstate, residue); 1389 1390 return status; 1391 } 1392 1393 static void rcar_dmac_issue_pending(struct dma_chan *chan) 1394 { 1395 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1396 unsigned long flags; 1397 1398 spin_lock_irqsave(&rchan->lock, flags); 1399 1400 if (list_empty(&rchan->desc.pending)) 1401 goto done; 1402 1403 /* Append the pending list to the active list. */ 1404 list_splice_tail_init(&rchan->desc.pending, &rchan->desc.active); 1405 1406 /* 1407 * If no transfer is running pick the first descriptor from the active 1408 * list and start the transfer. 1409 */ 1410 if (!rchan->desc.running) { 1411 struct rcar_dmac_desc *desc; 1412 1413 desc = list_first_entry(&rchan->desc.active, 1414 struct rcar_dmac_desc, node); 1415 rchan->desc.running = desc; 1416 1417 rcar_dmac_chan_start_xfer(rchan); 1418 } 1419 1420 done: 1421 spin_unlock_irqrestore(&rchan->lock, flags); 1422 } 1423 1424 static void rcar_dmac_device_synchronize(struct dma_chan *chan) 1425 { 1426 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1427 1428 synchronize_irq(rchan->irq); 1429 } 1430 1431 /* ----------------------------------------------------------------------------- 1432 * IRQ handling 1433 */ 1434 1435 static irqreturn_t rcar_dmac_isr_desc_stage_end(struct rcar_dmac_chan *chan) 1436 { 1437 struct rcar_dmac_desc *desc = chan->desc.running; 1438 unsigned int stage; 1439 1440 if (WARN_ON(!desc || !desc->cyclic)) { 1441 /* 1442 * This should never happen, there should always be a running 1443 * cyclic descriptor when a descriptor stage end interrupt is 1444 * triggered. Warn and return. 1445 */ 1446 return IRQ_NONE; 1447 } 1448 1449 /* Program the interrupt pointer to the next stage. */ 1450 stage = (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) & 1451 RCAR_DMACHCRB_DPTR_MASK) >> RCAR_DMACHCRB_DPTR_SHIFT; 1452 rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(stage)); 1453 1454 return IRQ_WAKE_THREAD; 1455 } 1456 1457 static irqreturn_t rcar_dmac_isr_transfer_end(struct rcar_dmac_chan *chan) 1458 { 1459 struct rcar_dmac_desc *desc = chan->desc.running; 1460 irqreturn_t ret = IRQ_WAKE_THREAD; 1461 1462 if (WARN_ON_ONCE(!desc)) { 1463 /* 1464 * This should never happen, there should always be a running 1465 * descriptor when a transfer end interrupt is triggered. Warn 1466 * and return. 1467 */ 1468 return IRQ_NONE; 1469 } 1470 1471 /* 1472 * The transfer end interrupt isn't generated for each chunk when using 1473 * descriptor mode. Only update the running chunk pointer in 1474 * non-descriptor mode. 1475 */ 1476 if (!desc->hwdescs.use) { 1477 /* 1478 * If we haven't completed the last transfer chunk simply move 1479 * to the next one. Only wake the IRQ thread if the transfer is 1480 * cyclic. 1481 */ 1482 if (!list_is_last(&desc->running->node, &desc->chunks)) { 1483 desc->running = list_next_entry(desc->running, node); 1484 if (!desc->cyclic) 1485 ret = IRQ_HANDLED; 1486 goto done; 1487 } 1488 1489 /* 1490 * We've completed the last transfer chunk. If the transfer is 1491 * cyclic, move back to the first one. 1492 */ 1493 if (desc->cyclic) { 1494 desc->running = 1495 list_first_entry(&desc->chunks, 1496 struct rcar_dmac_xfer_chunk, 1497 node); 1498 goto done; 1499 } 1500 } 1501 1502 /* The descriptor is complete, move it to the done list. */ 1503 list_move_tail(&desc->node, &chan->desc.done); 1504 1505 /* Queue the next descriptor, if any. */ 1506 if (!list_empty(&chan->desc.active)) 1507 chan->desc.running = list_first_entry(&chan->desc.active, 1508 struct rcar_dmac_desc, 1509 node); 1510 else 1511 chan->desc.running = NULL; 1512 1513 done: 1514 if (chan->desc.running) 1515 rcar_dmac_chan_start_xfer(chan); 1516 1517 return ret; 1518 } 1519 1520 static irqreturn_t rcar_dmac_isr_channel(int irq, void *dev) 1521 { 1522 u32 mask = RCAR_DMACHCR_DSE | RCAR_DMACHCR_TE; 1523 struct rcar_dmac_chan *chan = dev; 1524 irqreturn_t ret = IRQ_NONE; 1525 u32 chcr; 1526 1527 spin_lock(&chan->lock); 1528 1529 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 1530 if (chcr & RCAR_DMACHCR_TE) 1531 mask |= RCAR_DMACHCR_DE; 1532 rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr & ~mask); 1533 if (mask & RCAR_DMACHCR_DE) 1534 rcar_dmac_chcr_de_barrier(chan); 1535 1536 if (chcr & RCAR_DMACHCR_DSE) 1537 ret |= rcar_dmac_isr_desc_stage_end(chan); 1538 1539 if (chcr & RCAR_DMACHCR_TE) 1540 ret |= rcar_dmac_isr_transfer_end(chan); 1541 1542 spin_unlock(&chan->lock); 1543 1544 return ret; 1545 } 1546 1547 static irqreturn_t rcar_dmac_isr_channel_thread(int irq, void *dev) 1548 { 1549 struct rcar_dmac_chan *chan = dev; 1550 struct rcar_dmac_desc *desc; 1551 struct dmaengine_desc_callback cb; 1552 1553 spin_lock_irq(&chan->lock); 1554 1555 /* For cyclic transfers notify the user after every chunk. */ 1556 if (chan->desc.running && chan->desc.running->cyclic) { 1557 desc = chan->desc.running; 1558 dmaengine_desc_get_callback(&desc->async_tx, &cb); 1559 1560 if (dmaengine_desc_callback_valid(&cb)) { 1561 spin_unlock_irq(&chan->lock); 1562 dmaengine_desc_callback_invoke(&cb, NULL); 1563 spin_lock_irq(&chan->lock); 1564 } 1565 } 1566 1567 /* 1568 * Call the callback function for all descriptors on the done list and 1569 * move them to the ack wait list. 1570 */ 1571 while (!list_empty(&chan->desc.done)) { 1572 desc = list_first_entry(&chan->desc.done, struct rcar_dmac_desc, 1573 node); 1574 dma_cookie_complete(&desc->async_tx); 1575 list_del(&desc->node); 1576 1577 dmaengine_desc_get_callback(&desc->async_tx, &cb); 1578 if (dmaengine_desc_callback_valid(&cb)) { 1579 spin_unlock_irq(&chan->lock); 1580 /* 1581 * We own the only reference to this descriptor, we can 1582 * safely dereference it without holding the channel 1583 * lock. 1584 */ 1585 dmaengine_desc_callback_invoke(&cb, NULL); 1586 spin_lock_irq(&chan->lock); 1587 } 1588 1589 list_add_tail(&desc->node, &chan->desc.wait); 1590 } 1591 1592 spin_unlock_irq(&chan->lock); 1593 1594 /* Recycle all acked descriptors. */ 1595 rcar_dmac_desc_recycle_acked(chan); 1596 1597 return IRQ_HANDLED; 1598 } 1599 1600 static irqreturn_t rcar_dmac_isr_error(int irq, void *data) 1601 { 1602 struct rcar_dmac *dmac = data; 1603 1604 if (!(rcar_dmac_read(dmac, RCAR_DMAOR) & RCAR_DMAOR_AE)) 1605 return IRQ_NONE; 1606 1607 /* 1608 * An unrecoverable error occurred on an unknown channel. Halt the DMAC, 1609 * abort transfers on all channels, and reinitialize the DMAC. 1610 */ 1611 rcar_dmac_stop(dmac); 1612 rcar_dmac_abort(dmac); 1613 rcar_dmac_init(dmac); 1614 1615 return IRQ_HANDLED; 1616 } 1617 1618 /* ----------------------------------------------------------------------------- 1619 * OF xlate and channel filter 1620 */ 1621 1622 static bool rcar_dmac_chan_filter(struct dma_chan *chan, void *arg) 1623 { 1624 struct rcar_dmac *dmac = to_rcar_dmac(chan->device); 1625 struct of_phandle_args *dma_spec = arg; 1626 1627 /* 1628 * FIXME: Using a filter on OF platforms is a nonsense. The OF xlate 1629 * function knows from which device it wants to allocate a channel from, 1630 * and would be perfectly capable of selecting the channel it wants. 1631 * Forcing it to call dma_request_channel() and iterate through all 1632 * channels from all controllers is just pointless. 1633 */ 1634 if (chan->device->device_config != rcar_dmac_device_config || 1635 dma_spec->np != chan->device->dev->of_node) 1636 return false; 1637 1638 return !test_and_set_bit(dma_spec->args[0], dmac->modules); 1639 } 1640 1641 static struct dma_chan *rcar_dmac_of_xlate(struct of_phandle_args *dma_spec, 1642 struct of_dma *ofdma) 1643 { 1644 struct rcar_dmac_chan *rchan; 1645 struct dma_chan *chan; 1646 dma_cap_mask_t mask; 1647 1648 if (dma_spec->args_count != 1) 1649 return NULL; 1650 1651 /* Only slave DMA channels can be allocated via DT */ 1652 dma_cap_zero(mask); 1653 dma_cap_set(DMA_SLAVE, mask); 1654 1655 chan = dma_request_channel(mask, rcar_dmac_chan_filter, dma_spec); 1656 if (!chan) 1657 return NULL; 1658 1659 rchan = to_rcar_dmac_chan(chan); 1660 rchan->mid_rid = dma_spec->args[0]; 1661 1662 return chan; 1663 } 1664 1665 /* ----------------------------------------------------------------------------- 1666 * Power management 1667 */ 1668 1669 #ifdef CONFIG_PM 1670 static int rcar_dmac_runtime_suspend(struct device *dev) 1671 { 1672 return 0; 1673 } 1674 1675 static int rcar_dmac_runtime_resume(struct device *dev) 1676 { 1677 struct rcar_dmac *dmac = dev_get_drvdata(dev); 1678 1679 return rcar_dmac_init(dmac); 1680 } 1681 #endif 1682 1683 static const struct dev_pm_ops rcar_dmac_pm = { 1684 /* 1685 * TODO for system sleep/resume: 1686 * - Wait for the current transfer to complete and stop the device, 1687 * - Resume transfers, if any. 1688 */ 1689 SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, 1690 pm_runtime_force_resume) 1691 SET_RUNTIME_PM_OPS(rcar_dmac_runtime_suspend, rcar_dmac_runtime_resume, 1692 NULL) 1693 }; 1694 1695 /* ----------------------------------------------------------------------------- 1696 * Probe and remove 1697 */ 1698 1699 static int rcar_dmac_chan_probe(struct rcar_dmac *dmac, 1700 struct rcar_dmac_chan *rchan, 1701 unsigned int index) 1702 { 1703 struct platform_device *pdev = to_platform_device(dmac->dev); 1704 struct dma_chan *chan = &rchan->chan; 1705 char pdev_irqname[5]; 1706 char *irqname; 1707 int ret; 1708 1709 rchan->index = index; 1710 rchan->iomem = dmac->iomem + RCAR_DMAC_CHAN_OFFSET(index); 1711 rchan->mid_rid = -EINVAL; 1712 1713 spin_lock_init(&rchan->lock); 1714 1715 INIT_LIST_HEAD(&rchan->desc.free); 1716 INIT_LIST_HEAD(&rchan->desc.pending); 1717 INIT_LIST_HEAD(&rchan->desc.active); 1718 INIT_LIST_HEAD(&rchan->desc.done); 1719 INIT_LIST_HEAD(&rchan->desc.wait); 1720 1721 /* Request the channel interrupt. */ 1722 sprintf(pdev_irqname, "ch%u", index); 1723 rchan->irq = platform_get_irq_byname(pdev, pdev_irqname); 1724 if (rchan->irq < 0) { 1725 dev_err(dmac->dev, "no IRQ specified for channel %u\n", index); 1726 return -ENODEV; 1727 } 1728 1729 irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", 1730 dev_name(dmac->dev), index); 1731 if (!irqname) 1732 return -ENOMEM; 1733 1734 /* 1735 * Initialize the DMA engine channel and add it to the DMA engine 1736 * channels list. 1737 */ 1738 chan->device = &dmac->engine; 1739 dma_cookie_init(chan); 1740 1741 list_add_tail(&chan->device_node, &dmac->engine.channels); 1742 1743 ret = devm_request_threaded_irq(dmac->dev, rchan->irq, 1744 rcar_dmac_isr_channel, 1745 rcar_dmac_isr_channel_thread, 0, 1746 irqname, rchan); 1747 if (ret) { 1748 dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", 1749 rchan->irq, ret); 1750 return ret; 1751 } 1752 1753 return 0; 1754 } 1755 1756 static int rcar_dmac_parse_of(struct device *dev, struct rcar_dmac *dmac) 1757 { 1758 struct device_node *np = dev->of_node; 1759 int ret; 1760 1761 ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels); 1762 if (ret < 0) { 1763 dev_err(dev, "unable to read dma-channels property\n"); 1764 return ret; 1765 } 1766 1767 if (dmac->n_channels <= 0 || dmac->n_channels >= 100) { 1768 dev_err(dev, "invalid number of channels %u\n", 1769 dmac->n_channels); 1770 return -EINVAL; 1771 } 1772 1773 return 0; 1774 } 1775 1776 static int rcar_dmac_probe(struct platform_device *pdev) 1777 { 1778 const enum dma_slave_buswidth widths = DMA_SLAVE_BUSWIDTH_1_BYTE | 1779 DMA_SLAVE_BUSWIDTH_2_BYTES | DMA_SLAVE_BUSWIDTH_4_BYTES | 1780 DMA_SLAVE_BUSWIDTH_8_BYTES | DMA_SLAVE_BUSWIDTH_16_BYTES | 1781 DMA_SLAVE_BUSWIDTH_32_BYTES | DMA_SLAVE_BUSWIDTH_64_BYTES; 1782 unsigned int channels_offset = 0; 1783 struct dma_device *engine; 1784 struct rcar_dmac *dmac; 1785 struct resource *mem; 1786 unsigned int i; 1787 char *irqname; 1788 int irq; 1789 int ret; 1790 1791 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); 1792 if (!dmac) 1793 return -ENOMEM; 1794 1795 dmac->dev = &pdev->dev; 1796 platform_set_drvdata(pdev, dmac); 1797 dma_set_mask_and_coherent(dmac->dev, DMA_BIT_MASK(40)); 1798 1799 ret = rcar_dmac_parse_of(&pdev->dev, dmac); 1800 if (ret < 0) 1801 return ret; 1802 1803 /* 1804 * A still unconfirmed hardware bug prevents the IPMMU microTLB 0 to be 1805 * flushed correctly, resulting in memory corruption. DMAC 0 channel 0 1806 * is connected to microTLB 0 on currently supported platforms, so we 1807 * can't use it with the IPMMU. As the IOMMU API operates at the device 1808 * level we can't disable it selectively, so ignore channel 0 for now if 1809 * the device is part of an IOMMU group. 1810 */ 1811 if (pdev->dev.iommu_group) { 1812 dmac->n_channels--; 1813 channels_offset = 1; 1814 } 1815 1816 dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels, 1817 sizeof(*dmac->channels), GFP_KERNEL); 1818 if (!dmac->channels) 1819 return -ENOMEM; 1820 1821 /* Request resources. */ 1822 mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1823 dmac->iomem = devm_ioremap_resource(&pdev->dev, mem); 1824 if (IS_ERR(dmac->iomem)) 1825 return PTR_ERR(dmac->iomem); 1826 1827 irq = platform_get_irq_byname(pdev, "error"); 1828 if (irq < 0) { 1829 dev_err(&pdev->dev, "no error IRQ specified\n"); 1830 return -ENODEV; 1831 } 1832 1833 irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:error", 1834 dev_name(dmac->dev)); 1835 if (!irqname) 1836 return -ENOMEM; 1837 1838 /* Enable runtime PM and initialize the device. */ 1839 pm_runtime_enable(&pdev->dev); 1840 ret = pm_runtime_get_sync(&pdev->dev); 1841 if (ret < 0) { 1842 dev_err(&pdev->dev, "runtime PM get sync failed (%d)\n", ret); 1843 return ret; 1844 } 1845 1846 ret = rcar_dmac_init(dmac); 1847 pm_runtime_put(&pdev->dev); 1848 1849 if (ret) { 1850 dev_err(&pdev->dev, "failed to reset device\n"); 1851 goto error; 1852 } 1853 1854 /* Initialize engine */ 1855 engine = &dmac->engine; 1856 1857 dma_cap_set(DMA_MEMCPY, engine->cap_mask); 1858 dma_cap_set(DMA_SLAVE, engine->cap_mask); 1859 1860 engine->dev = &pdev->dev; 1861 engine->copy_align = ilog2(RCAR_DMAC_MEMCPY_XFER_SIZE); 1862 1863 engine->src_addr_widths = widths; 1864 engine->dst_addr_widths = widths; 1865 engine->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); 1866 engine->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1867 1868 engine->device_alloc_chan_resources = rcar_dmac_alloc_chan_resources; 1869 engine->device_free_chan_resources = rcar_dmac_free_chan_resources; 1870 engine->device_prep_dma_memcpy = rcar_dmac_prep_dma_memcpy; 1871 engine->device_prep_slave_sg = rcar_dmac_prep_slave_sg; 1872 engine->device_prep_dma_cyclic = rcar_dmac_prep_dma_cyclic; 1873 engine->device_config = rcar_dmac_device_config; 1874 engine->device_terminate_all = rcar_dmac_chan_terminate_all; 1875 engine->device_tx_status = rcar_dmac_tx_status; 1876 engine->device_issue_pending = rcar_dmac_issue_pending; 1877 engine->device_synchronize = rcar_dmac_device_synchronize; 1878 1879 INIT_LIST_HEAD(&engine->channels); 1880 1881 for (i = 0; i < dmac->n_channels; ++i) { 1882 ret = rcar_dmac_chan_probe(dmac, &dmac->channels[i], 1883 i + channels_offset); 1884 if (ret < 0) 1885 goto error; 1886 } 1887 1888 ret = devm_request_irq(&pdev->dev, irq, rcar_dmac_isr_error, 0, 1889 irqname, dmac); 1890 if (ret) { 1891 dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n", 1892 irq, ret); 1893 return ret; 1894 } 1895 1896 /* Register the DMAC as a DMA provider for DT. */ 1897 ret = of_dma_controller_register(pdev->dev.of_node, rcar_dmac_of_xlate, 1898 NULL); 1899 if (ret < 0) 1900 goto error; 1901 1902 /* 1903 * Register the DMA engine device. 1904 * 1905 * Default transfer size of 32 bytes requires 32-byte alignment. 1906 */ 1907 ret = dma_async_device_register(engine); 1908 if (ret < 0) 1909 goto error; 1910 1911 return 0; 1912 1913 error: 1914 of_dma_controller_free(pdev->dev.of_node); 1915 pm_runtime_disable(&pdev->dev); 1916 return ret; 1917 } 1918 1919 static int rcar_dmac_remove(struct platform_device *pdev) 1920 { 1921 struct rcar_dmac *dmac = platform_get_drvdata(pdev); 1922 1923 of_dma_controller_free(pdev->dev.of_node); 1924 dma_async_device_unregister(&dmac->engine); 1925 1926 pm_runtime_disable(&pdev->dev); 1927 1928 return 0; 1929 } 1930 1931 static void rcar_dmac_shutdown(struct platform_device *pdev) 1932 { 1933 struct rcar_dmac *dmac = platform_get_drvdata(pdev); 1934 1935 rcar_dmac_stop(dmac); 1936 } 1937 1938 static const struct of_device_id rcar_dmac_of_ids[] = { 1939 { .compatible = "renesas,rcar-dmac", }, 1940 { /* Sentinel */ } 1941 }; 1942 MODULE_DEVICE_TABLE(of, rcar_dmac_of_ids); 1943 1944 static struct platform_driver rcar_dmac_driver = { 1945 .driver = { 1946 .pm = &rcar_dmac_pm, 1947 .name = "rcar-dmac", 1948 .of_match_table = rcar_dmac_of_ids, 1949 }, 1950 .probe = rcar_dmac_probe, 1951 .remove = rcar_dmac_remove, 1952 .shutdown = rcar_dmac_shutdown, 1953 }; 1954 1955 module_platform_driver(rcar_dmac_driver); 1956 1957 MODULE_DESCRIPTION("R-Car Gen2 DMA Controller Driver"); 1958 MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>"); 1959 MODULE_LICENSE("GPL v2"); 1960