1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Renesas R-Car Gen2/Gen3 DMA Controller Driver 4 * 5 * Copyright (C) 2014-2019 Renesas Electronics Inc. 6 * 7 * Author: Laurent Pinchart <laurent.pinchart@ideasonboard.com> 8 */ 9 10 #include <linux/delay.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/dmaengine.h> 13 #include <linux/interrupt.h> 14 #include <linux/list.h> 15 #include <linux/module.h> 16 #include <linux/mutex.h> 17 #include <linux/of.h> 18 #include <linux/of_dma.h> 19 #include <linux/of_platform.h> 20 #include <linux/platform_device.h> 21 #include <linux/pm_runtime.h> 22 #include <linux/slab.h> 23 #include <linux/spinlock.h> 24 25 #include "../dmaengine.h" 26 27 /* 28 * struct rcar_dmac_xfer_chunk - Descriptor for a hardware transfer 29 * @node: entry in the parent's chunks list 30 * @src_addr: device source address 31 * @dst_addr: device destination address 32 * @size: transfer size in bytes 33 */ 34 struct rcar_dmac_xfer_chunk { 35 struct list_head node; 36 37 dma_addr_t src_addr; 38 dma_addr_t dst_addr; 39 u32 size; 40 }; 41 42 /* 43 * struct rcar_dmac_hw_desc - Hardware descriptor for a transfer chunk 44 * @sar: value of the SAR register (source address) 45 * @dar: value of the DAR register (destination address) 46 * @tcr: value of the TCR register (transfer count) 47 */ 48 struct rcar_dmac_hw_desc { 49 u32 sar; 50 u32 dar; 51 u32 tcr; 52 u32 reserved; 53 } __attribute__((__packed__)); 54 55 /* 56 * struct rcar_dmac_desc - R-Car Gen2 DMA Transfer Descriptor 57 * @async_tx: base DMA asynchronous transaction descriptor 58 * @direction: direction of the DMA transfer 59 * @xfer_shift: log2 of the transfer size 60 * @chcr: value of the channel configuration register for this transfer 61 * @node: entry in the channel's descriptors lists 62 * @chunks: list of transfer chunks for this transfer 63 * @running: the transfer chunk being currently processed 64 * @nchunks: number of transfer chunks for this transfer 65 * @hwdescs.use: whether the transfer descriptor uses hardware descriptors 66 * @hwdescs.mem: hardware descriptors memory for the transfer 67 * @hwdescs.dma: device address of the hardware descriptors memory 68 * @hwdescs.size: size of the hardware descriptors in bytes 69 * @size: transfer size in bytes 70 * @cyclic: when set indicates that the DMA transfer is cyclic 71 */ 72 struct rcar_dmac_desc { 73 struct dma_async_tx_descriptor async_tx; 74 enum dma_transfer_direction direction; 75 unsigned int xfer_shift; 76 u32 chcr; 77 78 struct list_head node; 79 struct list_head chunks; 80 struct rcar_dmac_xfer_chunk *running; 81 unsigned int nchunks; 82 83 struct { 84 bool use; 85 struct rcar_dmac_hw_desc *mem; 86 dma_addr_t dma; 87 size_t size; 88 } hwdescs; 89 90 unsigned int size; 91 bool cyclic; 92 }; 93 94 #define to_rcar_dmac_desc(d) container_of(d, struct rcar_dmac_desc, async_tx) 95 96 /* 97 * struct rcar_dmac_desc_page - One page worth of descriptors 98 * @node: entry in the channel's pages list 99 * @descs: array of DMA descriptors 100 * @chunks: array of transfer chunk descriptors 101 */ 102 struct rcar_dmac_desc_page { 103 struct list_head node; 104 105 union { 106 struct rcar_dmac_desc descs[0]; 107 struct rcar_dmac_xfer_chunk chunks[0]; 108 }; 109 }; 110 111 #define RCAR_DMAC_DESCS_PER_PAGE \ 112 ((PAGE_SIZE - offsetof(struct rcar_dmac_desc_page, descs)) / \ 113 sizeof(struct rcar_dmac_desc)) 114 #define RCAR_DMAC_XFER_CHUNKS_PER_PAGE \ 115 ((PAGE_SIZE - offsetof(struct rcar_dmac_desc_page, chunks)) / \ 116 sizeof(struct rcar_dmac_xfer_chunk)) 117 118 /* 119 * struct rcar_dmac_chan_slave - Slave configuration 120 * @slave_addr: slave memory address 121 * @xfer_size: size (in bytes) of hardware transfers 122 */ 123 struct rcar_dmac_chan_slave { 124 phys_addr_t slave_addr; 125 unsigned int xfer_size; 126 }; 127 128 /* 129 * struct rcar_dmac_chan_map - Map of slave device phys to dma address 130 * @addr: slave dma address 131 * @dir: direction of mapping 132 * @slave: slave configuration that is mapped 133 */ 134 struct rcar_dmac_chan_map { 135 dma_addr_t addr; 136 enum dma_data_direction dir; 137 struct rcar_dmac_chan_slave slave; 138 }; 139 140 /* 141 * struct rcar_dmac_chan - R-Car Gen2 DMA Controller Channel 142 * @chan: base DMA channel object 143 * @iomem: channel I/O memory base 144 * @index: index of this channel in the controller 145 * @irq: channel IRQ 146 * @src: slave memory address and size on the source side 147 * @dst: slave memory address and size on the destination side 148 * @mid_rid: hardware MID/RID for the DMA client using this channel 149 * @lock: protects the channel CHCR register and the desc members 150 * @desc.free: list of free descriptors 151 * @desc.pending: list of pending descriptors (submitted with tx_submit) 152 * @desc.active: list of active descriptors (activated with issue_pending) 153 * @desc.done: list of completed descriptors 154 * @desc.wait: list of descriptors waiting for an ack 155 * @desc.running: the descriptor being processed (a member of the active list) 156 * @desc.chunks_free: list of free transfer chunk descriptors 157 * @desc.pages: list of pages used by allocated descriptors 158 */ 159 struct rcar_dmac_chan { 160 struct dma_chan chan; 161 void __iomem *iomem; 162 unsigned int index; 163 int irq; 164 165 struct rcar_dmac_chan_slave src; 166 struct rcar_dmac_chan_slave dst; 167 struct rcar_dmac_chan_map map; 168 int mid_rid; 169 170 spinlock_t lock; 171 172 struct { 173 struct list_head free; 174 struct list_head pending; 175 struct list_head active; 176 struct list_head done; 177 struct list_head wait; 178 struct rcar_dmac_desc *running; 179 180 struct list_head chunks_free; 181 182 struct list_head pages; 183 } desc; 184 }; 185 186 #define to_rcar_dmac_chan(c) container_of(c, struct rcar_dmac_chan, chan) 187 188 /* 189 * struct rcar_dmac - R-Car Gen2 DMA Controller 190 * @engine: base DMA engine object 191 * @dev: the hardware device 192 * @iomem: remapped I/O memory base 193 * @n_channels: number of available channels 194 * @channels: array of DMAC channels 195 * @modules: bitmask of client modules in use 196 */ 197 struct rcar_dmac { 198 struct dma_device engine; 199 struct device *dev; 200 void __iomem *iomem; 201 struct device_dma_parameters parms; 202 203 unsigned int n_channels; 204 struct rcar_dmac_chan *channels; 205 206 DECLARE_BITMAP(modules, 256); 207 }; 208 209 #define to_rcar_dmac(d) container_of(d, struct rcar_dmac, engine) 210 211 /* ----------------------------------------------------------------------------- 212 * Registers 213 */ 214 215 #define RCAR_DMAC_CHAN_OFFSET(i) (0x8000 + 0x80 * (i)) 216 217 #define RCAR_DMAISTA 0x0020 218 #define RCAR_DMASEC 0x0030 219 #define RCAR_DMAOR 0x0060 220 #define RCAR_DMAOR_PRI_FIXED (0 << 8) 221 #define RCAR_DMAOR_PRI_ROUND_ROBIN (3 << 8) 222 #define RCAR_DMAOR_AE (1 << 2) 223 #define RCAR_DMAOR_DME (1 << 0) 224 #define RCAR_DMACHCLR 0x0080 225 #define RCAR_DMADPSEC 0x00a0 226 227 #define RCAR_DMASAR 0x0000 228 #define RCAR_DMADAR 0x0004 229 #define RCAR_DMATCR 0x0008 230 #define RCAR_DMATCR_MASK 0x00ffffff 231 #define RCAR_DMATSR 0x0028 232 #define RCAR_DMACHCR 0x000c 233 #define RCAR_DMACHCR_CAE (1 << 31) 234 #define RCAR_DMACHCR_CAIE (1 << 30) 235 #define RCAR_DMACHCR_DPM_DISABLED (0 << 28) 236 #define RCAR_DMACHCR_DPM_ENABLED (1 << 28) 237 #define RCAR_DMACHCR_DPM_REPEAT (2 << 28) 238 #define RCAR_DMACHCR_DPM_INFINITE (3 << 28) 239 #define RCAR_DMACHCR_RPT_SAR (1 << 27) 240 #define RCAR_DMACHCR_RPT_DAR (1 << 26) 241 #define RCAR_DMACHCR_RPT_TCR (1 << 25) 242 #define RCAR_DMACHCR_DPB (1 << 22) 243 #define RCAR_DMACHCR_DSE (1 << 19) 244 #define RCAR_DMACHCR_DSIE (1 << 18) 245 #define RCAR_DMACHCR_TS_1B ((0 << 20) | (0 << 3)) 246 #define RCAR_DMACHCR_TS_2B ((0 << 20) | (1 << 3)) 247 #define RCAR_DMACHCR_TS_4B ((0 << 20) | (2 << 3)) 248 #define RCAR_DMACHCR_TS_16B ((0 << 20) | (3 << 3)) 249 #define RCAR_DMACHCR_TS_32B ((1 << 20) | (0 << 3)) 250 #define RCAR_DMACHCR_TS_64B ((1 << 20) | (1 << 3)) 251 #define RCAR_DMACHCR_TS_8B ((1 << 20) | (3 << 3)) 252 #define RCAR_DMACHCR_DM_FIXED (0 << 14) 253 #define RCAR_DMACHCR_DM_INC (1 << 14) 254 #define RCAR_DMACHCR_DM_DEC (2 << 14) 255 #define RCAR_DMACHCR_SM_FIXED (0 << 12) 256 #define RCAR_DMACHCR_SM_INC (1 << 12) 257 #define RCAR_DMACHCR_SM_DEC (2 << 12) 258 #define RCAR_DMACHCR_RS_AUTO (4 << 8) 259 #define RCAR_DMACHCR_RS_DMARS (8 << 8) 260 #define RCAR_DMACHCR_IE (1 << 2) 261 #define RCAR_DMACHCR_TE (1 << 1) 262 #define RCAR_DMACHCR_DE (1 << 0) 263 #define RCAR_DMATCRB 0x0018 264 #define RCAR_DMATSRB 0x0038 265 #define RCAR_DMACHCRB 0x001c 266 #define RCAR_DMACHCRB_DCNT(n) ((n) << 24) 267 #define RCAR_DMACHCRB_DPTR_MASK (0xff << 16) 268 #define RCAR_DMACHCRB_DPTR_SHIFT 16 269 #define RCAR_DMACHCRB_DRST (1 << 15) 270 #define RCAR_DMACHCRB_DTS (1 << 8) 271 #define RCAR_DMACHCRB_SLM_NORMAL (0 << 4) 272 #define RCAR_DMACHCRB_SLM_CLK(n) ((8 | (n)) << 4) 273 #define RCAR_DMACHCRB_PRI(n) ((n) << 0) 274 #define RCAR_DMARS 0x0040 275 #define RCAR_DMABUFCR 0x0048 276 #define RCAR_DMABUFCR_MBU(n) ((n) << 16) 277 #define RCAR_DMABUFCR_ULB(n) ((n) << 0) 278 #define RCAR_DMADPBASE 0x0050 279 #define RCAR_DMADPBASE_MASK 0xfffffff0 280 #define RCAR_DMADPBASE_SEL (1 << 0) 281 #define RCAR_DMADPCR 0x0054 282 #define RCAR_DMADPCR_DIPT(n) ((n) << 24) 283 #define RCAR_DMAFIXSAR 0x0010 284 #define RCAR_DMAFIXDAR 0x0014 285 #define RCAR_DMAFIXDPBASE 0x0060 286 287 /* Hardcode the MEMCPY transfer size to 4 bytes. */ 288 #define RCAR_DMAC_MEMCPY_XFER_SIZE 4 289 290 /* ----------------------------------------------------------------------------- 291 * Device access 292 */ 293 294 static void rcar_dmac_write(struct rcar_dmac *dmac, u32 reg, u32 data) 295 { 296 if (reg == RCAR_DMAOR) 297 writew(data, dmac->iomem + reg); 298 else 299 writel(data, dmac->iomem + reg); 300 } 301 302 static u32 rcar_dmac_read(struct rcar_dmac *dmac, u32 reg) 303 { 304 if (reg == RCAR_DMAOR) 305 return readw(dmac->iomem + reg); 306 else 307 return readl(dmac->iomem + reg); 308 } 309 310 static u32 rcar_dmac_chan_read(struct rcar_dmac_chan *chan, u32 reg) 311 { 312 if (reg == RCAR_DMARS) 313 return readw(chan->iomem + reg); 314 else 315 return readl(chan->iomem + reg); 316 } 317 318 static void rcar_dmac_chan_write(struct rcar_dmac_chan *chan, u32 reg, u32 data) 319 { 320 if (reg == RCAR_DMARS) 321 writew(data, chan->iomem + reg); 322 else 323 writel(data, chan->iomem + reg); 324 } 325 326 /* ----------------------------------------------------------------------------- 327 * Initialization and configuration 328 */ 329 330 static bool rcar_dmac_chan_is_busy(struct rcar_dmac_chan *chan) 331 { 332 u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 333 334 return !!(chcr & (RCAR_DMACHCR_DE | RCAR_DMACHCR_TE)); 335 } 336 337 static void rcar_dmac_chan_start_xfer(struct rcar_dmac_chan *chan) 338 { 339 struct rcar_dmac_desc *desc = chan->desc.running; 340 u32 chcr = desc->chcr; 341 342 WARN_ON_ONCE(rcar_dmac_chan_is_busy(chan)); 343 344 if (chan->mid_rid >= 0) 345 rcar_dmac_chan_write(chan, RCAR_DMARS, chan->mid_rid); 346 347 if (desc->hwdescs.use) { 348 struct rcar_dmac_xfer_chunk *chunk = 349 list_first_entry(&desc->chunks, 350 struct rcar_dmac_xfer_chunk, node); 351 352 dev_dbg(chan->chan.device->dev, 353 "chan%u: queue desc %p: %u@%pad\n", 354 chan->index, desc, desc->nchunks, &desc->hwdescs.dma); 355 356 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 357 rcar_dmac_chan_write(chan, RCAR_DMAFIXSAR, 358 chunk->src_addr >> 32); 359 rcar_dmac_chan_write(chan, RCAR_DMAFIXDAR, 360 chunk->dst_addr >> 32); 361 rcar_dmac_chan_write(chan, RCAR_DMAFIXDPBASE, 362 desc->hwdescs.dma >> 32); 363 #endif 364 rcar_dmac_chan_write(chan, RCAR_DMADPBASE, 365 (desc->hwdescs.dma & 0xfffffff0) | 366 RCAR_DMADPBASE_SEL); 367 rcar_dmac_chan_write(chan, RCAR_DMACHCRB, 368 RCAR_DMACHCRB_DCNT(desc->nchunks - 1) | 369 RCAR_DMACHCRB_DRST); 370 371 /* 372 * Errata: When descriptor memory is accessed through an IOMMU 373 * the DMADAR register isn't initialized automatically from the 374 * first descriptor at beginning of transfer by the DMAC like it 375 * should. Initialize it manually with the destination address 376 * of the first chunk. 377 */ 378 rcar_dmac_chan_write(chan, RCAR_DMADAR, 379 chunk->dst_addr & 0xffffffff); 380 381 /* 382 * Program the descriptor stage interrupt to occur after the end 383 * of the first stage. 384 */ 385 rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(1)); 386 387 chcr |= RCAR_DMACHCR_RPT_SAR | RCAR_DMACHCR_RPT_DAR 388 | RCAR_DMACHCR_RPT_TCR | RCAR_DMACHCR_DPB; 389 390 /* 391 * If the descriptor isn't cyclic enable normal descriptor mode 392 * and the transfer completion interrupt. 393 */ 394 if (!desc->cyclic) 395 chcr |= RCAR_DMACHCR_DPM_ENABLED | RCAR_DMACHCR_IE; 396 /* 397 * If the descriptor is cyclic and has a callback enable the 398 * descriptor stage interrupt in infinite repeat mode. 399 */ 400 else if (desc->async_tx.callback) 401 chcr |= RCAR_DMACHCR_DPM_INFINITE | RCAR_DMACHCR_DSIE; 402 /* 403 * Otherwise just select infinite repeat mode without any 404 * interrupt. 405 */ 406 else 407 chcr |= RCAR_DMACHCR_DPM_INFINITE; 408 } else { 409 struct rcar_dmac_xfer_chunk *chunk = desc->running; 410 411 dev_dbg(chan->chan.device->dev, 412 "chan%u: queue chunk %p: %u@%pad -> %pad\n", 413 chan->index, chunk, chunk->size, &chunk->src_addr, 414 &chunk->dst_addr); 415 416 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 417 rcar_dmac_chan_write(chan, RCAR_DMAFIXSAR, 418 chunk->src_addr >> 32); 419 rcar_dmac_chan_write(chan, RCAR_DMAFIXDAR, 420 chunk->dst_addr >> 32); 421 #endif 422 rcar_dmac_chan_write(chan, RCAR_DMASAR, 423 chunk->src_addr & 0xffffffff); 424 rcar_dmac_chan_write(chan, RCAR_DMADAR, 425 chunk->dst_addr & 0xffffffff); 426 rcar_dmac_chan_write(chan, RCAR_DMATCR, 427 chunk->size >> desc->xfer_shift); 428 429 chcr |= RCAR_DMACHCR_DPM_DISABLED | RCAR_DMACHCR_IE; 430 } 431 432 rcar_dmac_chan_write(chan, RCAR_DMACHCR, 433 chcr | RCAR_DMACHCR_DE | RCAR_DMACHCR_CAIE); 434 } 435 436 static int rcar_dmac_init(struct rcar_dmac *dmac) 437 { 438 u16 dmaor; 439 440 /* Clear all channels and enable the DMAC globally. */ 441 rcar_dmac_write(dmac, RCAR_DMACHCLR, GENMASK(dmac->n_channels - 1, 0)); 442 rcar_dmac_write(dmac, RCAR_DMAOR, 443 RCAR_DMAOR_PRI_FIXED | RCAR_DMAOR_DME); 444 445 dmaor = rcar_dmac_read(dmac, RCAR_DMAOR); 446 if ((dmaor & (RCAR_DMAOR_AE | RCAR_DMAOR_DME)) != RCAR_DMAOR_DME) { 447 dev_warn(dmac->dev, "DMAOR initialization failed.\n"); 448 return -EIO; 449 } 450 451 return 0; 452 } 453 454 /* ----------------------------------------------------------------------------- 455 * Descriptors submission 456 */ 457 458 static dma_cookie_t rcar_dmac_tx_submit(struct dma_async_tx_descriptor *tx) 459 { 460 struct rcar_dmac_chan *chan = to_rcar_dmac_chan(tx->chan); 461 struct rcar_dmac_desc *desc = to_rcar_dmac_desc(tx); 462 unsigned long flags; 463 dma_cookie_t cookie; 464 465 spin_lock_irqsave(&chan->lock, flags); 466 467 cookie = dma_cookie_assign(tx); 468 469 dev_dbg(chan->chan.device->dev, "chan%u: submit #%d@%p\n", 470 chan->index, tx->cookie, desc); 471 472 list_add_tail(&desc->node, &chan->desc.pending); 473 desc->running = list_first_entry(&desc->chunks, 474 struct rcar_dmac_xfer_chunk, node); 475 476 spin_unlock_irqrestore(&chan->lock, flags); 477 478 return cookie; 479 } 480 481 /* ----------------------------------------------------------------------------- 482 * Descriptors allocation and free 483 */ 484 485 /* 486 * rcar_dmac_desc_alloc - Allocate a page worth of DMA descriptors 487 * @chan: the DMA channel 488 * @gfp: allocation flags 489 */ 490 static int rcar_dmac_desc_alloc(struct rcar_dmac_chan *chan, gfp_t gfp) 491 { 492 struct rcar_dmac_desc_page *page; 493 unsigned long flags; 494 LIST_HEAD(list); 495 unsigned int i; 496 497 page = (void *)get_zeroed_page(gfp); 498 if (!page) 499 return -ENOMEM; 500 501 for (i = 0; i < RCAR_DMAC_DESCS_PER_PAGE; ++i) { 502 struct rcar_dmac_desc *desc = &page->descs[i]; 503 504 dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan); 505 desc->async_tx.tx_submit = rcar_dmac_tx_submit; 506 INIT_LIST_HEAD(&desc->chunks); 507 508 list_add_tail(&desc->node, &list); 509 } 510 511 spin_lock_irqsave(&chan->lock, flags); 512 list_splice_tail(&list, &chan->desc.free); 513 list_add_tail(&page->node, &chan->desc.pages); 514 spin_unlock_irqrestore(&chan->lock, flags); 515 516 return 0; 517 } 518 519 /* 520 * rcar_dmac_desc_put - Release a DMA transfer descriptor 521 * @chan: the DMA channel 522 * @desc: the descriptor 523 * 524 * Put the descriptor and its transfer chunk descriptors back in the channel's 525 * free descriptors lists. The descriptor's chunks list will be reinitialized to 526 * an empty list as a result. 527 * 528 * The descriptor must have been removed from the channel's lists before calling 529 * this function. 530 */ 531 static void rcar_dmac_desc_put(struct rcar_dmac_chan *chan, 532 struct rcar_dmac_desc *desc) 533 { 534 unsigned long flags; 535 536 spin_lock_irqsave(&chan->lock, flags); 537 list_splice_tail_init(&desc->chunks, &chan->desc.chunks_free); 538 list_add(&desc->node, &chan->desc.free); 539 spin_unlock_irqrestore(&chan->lock, flags); 540 } 541 542 static void rcar_dmac_desc_recycle_acked(struct rcar_dmac_chan *chan) 543 { 544 struct rcar_dmac_desc *desc, *_desc; 545 unsigned long flags; 546 LIST_HEAD(list); 547 548 /* 549 * We have to temporarily move all descriptors from the wait list to a 550 * local list as iterating over the wait list, even with 551 * list_for_each_entry_safe, isn't safe if we release the channel lock 552 * around the rcar_dmac_desc_put() call. 553 */ 554 spin_lock_irqsave(&chan->lock, flags); 555 list_splice_init(&chan->desc.wait, &list); 556 spin_unlock_irqrestore(&chan->lock, flags); 557 558 list_for_each_entry_safe(desc, _desc, &list, node) { 559 if (async_tx_test_ack(&desc->async_tx)) { 560 list_del(&desc->node); 561 rcar_dmac_desc_put(chan, desc); 562 } 563 } 564 565 if (list_empty(&list)) 566 return; 567 568 /* Put the remaining descriptors back in the wait list. */ 569 spin_lock_irqsave(&chan->lock, flags); 570 list_splice(&list, &chan->desc.wait); 571 spin_unlock_irqrestore(&chan->lock, flags); 572 } 573 574 /* 575 * rcar_dmac_desc_get - Allocate a descriptor for a DMA transfer 576 * @chan: the DMA channel 577 * 578 * Locking: This function must be called in a non-atomic context. 579 * 580 * Return: A pointer to the allocated descriptor or NULL if no descriptor can 581 * be allocated. 582 */ 583 static struct rcar_dmac_desc *rcar_dmac_desc_get(struct rcar_dmac_chan *chan) 584 { 585 struct rcar_dmac_desc *desc; 586 unsigned long flags; 587 int ret; 588 589 /* Recycle acked descriptors before attempting allocation. */ 590 rcar_dmac_desc_recycle_acked(chan); 591 592 spin_lock_irqsave(&chan->lock, flags); 593 594 while (list_empty(&chan->desc.free)) { 595 /* 596 * No free descriptors, allocate a page worth of them and try 597 * again, as someone else could race us to get the newly 598 * allocated descriptors. If the allocation fails return an 599 * error. 600 */ 601 spin_unlock_irqrestore(&chan->lock, flags); 602 ret = rcar_dmac_desc_alloc(chan, GFP_NOWAIT); 603 if (ret < 0) 604 return NULL; 605 spin_lock_irqsave(&chan->lock, flags); 606 } 607 608 desc = list_first_entry(&chan->desc.free, struct rcar_dmac_desc, node); 609 list_del(&desc->node); 610 611 spin_unlock_irqrestore(&chan->lock, flags); 612 613 return desc; 614 } 615 616 /* 617 * rcar_dmac_xfer_chunk_alloc - Allocate a page worth of transfer chunks 618 * @chan: the DMA channel 619 * @gfp: allocation flags 620 */ 621 static int rcar_dmac_xfer_chunk_alloc(struct rcar_dmac_chan *chan, gfp_t gfp) 622 { 623 struct rcar_dmac_desc_page *page; 624 unsigned long flags; 625 LIST_HEAD(list); 626 unsigned int i; 627 628 page = (void *)get_zeroed_page(gfp); 629 if (!page) 630 return -ENOMEM; 631 632 for (i = 0; i < RCAR_DMAC_XFER_CHUNKS_PER_PAGE; ++i) { 633 struct rcar_dmac_xfer_chunk *chunk = &page->chunks[i]; 634 635 list_add_tail(&chunk->node, &list); 636 } 637 638 spin_lock_irqsave(&chan->lock, flags); 639 list_splice_tail(&list, &chan->desc.chunks_free); 640 list_add_tail(&page->node, &chan->desc.pages); 641 spin_unlock_irqrestore(&chan->lock, flags); 642 643 return 0; 644 } 645 646 /* 647 * rcar_dmac_xfer_chunk_get - Allocate a transfer chunk for a DMA transfer 648 * @chan: the DMA channel 649 * 650 * Locking: This function must be called in a non-atomic context. 651 * 652 * Return: A pointer to the allocated transfer chunk descriptor or NULL if no 653 * descriptor can be allocated. 654 */ 655 static struct rcar_dmac_xfer_chunk * 656 rcar_dmac_xfer_chunk_get(struct rcar_dmac_chan *chan) 657 { 658 struct rcar_dmac_xfer_chunk *chunk; 659 unsigned long flags; 660 int ret; 661 662 spin_lock_irqsave(&chan->lock, flags); 663 664 while (list_empty(&chan->desc.chunks_free)) { 665 /* 666 * No free descriptors, allocate a page worth of them and try 667 * again, as someone else could race us to get the newly 668 * allocated descriptors. If the allocation fails return an 669 * error. 670 */ 671 spin_unlock_irqrestore(&chan->lock, flags); 672 ret = rcar_dmac_xfer_chunk_alloc(chan, GFP_NOWAIT); 673 if (ret < 0) 674 return NULL; 675 spin_lock_irqsave(&chan->lock, flags); 676 } 677 678 chunk = list_first_entry(&chan->desc.chunks_free, 679 struct rcar_dmac_xfer_chunk, node); 680 list_del(&chunk->node); 681 682 spin_unlock_irqrestore(&chan->lock, flags); 683 684 return chunk; 685 } 686 687 static void rcar_dmac_realloc_hwdesc(struct rcar_dmac_chan *chan, 688 struct rcar_dmac_desc *desc, size_t size) 689 { 690 /* 691 * dma_alloc_coherent() allocates memory in page size increments. To 692 * avoid reallocating the hardware descriptors when the allocated size 693 * wouldn't change align the requested size to a multiple of the page 694 * size. 695 */ 696 size = PAGE_ALIGN(size); 697 698 if (desc->hwdescs.size == size) 699 return; 700 701 if (desc->hwdescs.mem) { 702 dma_free_coherent(chan->chan.device->dev, desc->hwdescs.size, 703 desc->hwdescs.mem, desc->hwdescs.dma); 704 desc->hwdescs.mem = NULL; 705 desc->hwdescs.size = 0; 706 } 707 708 if (!size) 709 return; 710 711 desc->hwdescs.mem = dma_alloc_coherent(chan->chan.device->dev, size, 712 &desc->hwdescs.dma, GFP_NOWAIT); 713 if (!desc->hwdescs.mem) 714 return; 715 716 desc->hwdescs.size = size; 717 } 718 719 static int rcar_dmac_fill_hwdesc(struct rcar_dmac_chan *chan, 720 struct rcar_dmac_desc *desc) 721 { 722 struct rcar_dmac_xfer_chunk *chunk; 723 struct rcar_dmac_hw_desc *hwdesc; 724 725 rcar_dmac_realloc_hwdesc(chan, desc, desc->nchunks * sizeof(*hwdesc)); 726 727 hwdesc = desc->hwdescs.mem; 728 if (!hwdesc) 729 return -ENOMEM; 730 731 list_for_each_entry(chunk, &desc->chunks, node) { 732 hwdesc->sar = chunk->src_addr; 733 hwdesc->dar = chunk->dst_addr; 734 hwdesc->tcr = chunk->size >> desc->xfer_shift; 735 hwdesc++; 736 } 737 738 return 0; 739 } 740 741 /* ----------------------------------------------------------------------------- 742 * Stop and reset 743 */ 744 static void rcar_dmac_chcr_de_barrier(struct rcar_dmac_chan *chan) 745 { 746 u32 chcr; 747 unsigned int i; 748 749 /* 750 * Ensure that the setting of the DE bit is actually 0 after 751 * clearing it. 752 */ 753 for (i = 0; i < 1024; i++) { 754 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 755 if (!(chcr & RCAR_DMACHCR_DE)) 756 return; 757 udelay(1); 758 } 759 760 dev_err(chan->chan.device->dev, "CHCR DE check error\n"); 761 } 762 763 static void rcar_dmac_clear_chcr_de(struct rcar_dmac_chan *chan) 764 { 765 u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 766 767 /* set DE=0 and flush remaining data */ 768 rcar_dmac_chan_write(chan, RCAR_DMACHCR, (chcr & ~RCAR_DMACHCR_DE)); 769 770 /* make sure all remaining data was flushed */ 771 rcar_dmac_chcr_de_barrier(chan); 772 } 773 774 static void rcar_dmac_chan_halt(struct rcar_dmac_chan *chan) 775 { 776 u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 777 778 chcr &= ~(RCAR_DMACHCR_DSE | RCAR_DMACHCR_DSIE | RCAR_DMACHCR_IE | 779 RCAR_DMACHCR_TE | RCAR_DMACHCR_DE | 780 RCAR_DMACHCR_CAE | RCAR_DMACHCR_CAIE); 781 rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr); 782 rcar_dmac_chcr_de_barrier(chan); 783 } 784 785 static void rcar_dmac_chan_reinit(struct rcar_dmac_chan *chan) 786 { 787 struct rcar_dmac_desc *desc, *_desc; 788 unsigned long flags; 789 LIST_HEAD(descs); 790 791 spin_lock_irqsave(&chan->lock, flags); 792 793 /* Move all non-free descriptors to the local lists. */ 794 list_splice_init(&chan->desc.pending, &descs); 795 list_splice_init(&chan->desc.active, &descs); 796 list_splice_init(&chan->desc.done, &descs); 797 list_splice_init(&chan->desc.wait, &descs); 798 799 chan->desc.running = NULL; 800 801 spin_unlock_irqrestore(&chan->lock, flags); 802 803 list_for_each_entry_safe(desc, _desc, &descs, node) { 804 list_del(&desc->node); 805 rcar_dmac_desc_put(chan, desc); 806 } 807 } 808 809 static void rcar_dmac_stop_all_chan(struct rcar_dmac *dmac) 810 { 811 unsigned int i; 812 813 /* Stop all channels. */ 814 for (i = 0; i < dmac->n_channels; ++i) { 815 struct rcar_dmac_chan *chan = &dmac->channels[i]; 816 817 /* Stop and reinitialize the channel. */ 818 spin_lock_irq(&chan->lock); 819 rcar_dmac_chan_halt(chan); 820 spin_unlock_irq(&chan->lock); 821 } 822 } 823 824 static int rcar_dmac_chan_pause(struct dma_chan *chan) 825 { 826 unsigned long flags; 827 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 828 829 spin_lock_irqsave(&rchan->lock, flags); 830 rcar_dmac_clear_chcr_de(rchan); 831 spin_unlock_irqrestore(&rchan->lock, flags); 832 833 return 0; 834 } 835 836 /* ----------------------------------------------------------------------------- 837 * Descriptors preparation 838 */ 839 840 static void rcar_dmac_chan_configure_desc(struct rcar_dmac_chan *chan, 841 struct rcar_dmac_desc *desc) 842 { 843 static const u32 chcr_ts[] = { 844 RCAR_DMACHCR_TS_1B, RCAR_DMACHCR_TS_2B, 845 RCAR_DMACHCR_TS_4B, RCAR_DMACHCR_TS_8B, 846 RCAR_DMACHCR_TS_16B, RCAR_DMACHCR_TS_32B, 847 RCAR_DMACHCR_TS_64B, 848 }; 849 850 unsigned int xfer_size; 851 u32 chcr; 852 853 switch (desc->direction) { 854 case DMA_DEV_TO_MEM: 855 chcr = RCAR_DMACHCR_DM_INC | RCAR_DMACHCR_SM_FIXED 856 | RCAR_DMACHCR_RS_DMARS; 857 xfer_size = chan->src.xfer_size; 858 break; 859 860 case DMA_MEM_TO_DEV: 861 chcr = RCAR_DMACHCR_DM_FIXED | RCAR_DMACHCR_SM_INC 862 | RCAR_DMACHCR_RS_DMARS; 863 xfer_size = chan->dst.xfer_size; 864 break; 865 866 case DMA_MEM_TO_MEM: 867 default: 868 chcr = RCAR_DMACHCR_DM_INC | RCAR_DMACHCR_SM_INC 869 | RCAR_DMACHCR_RS_AUTO; 870 xfer_size = RCAR_DMAC_MEMCPY_XFER_SIZE; 871 break; 872 } 873 874 desc->xfer_shift = ilog2(xfer_size); 875 desc->chcr = chcr | chcr_ts[desc->xfer_shift]; 876 } 877 878 /* 879 * rcar_dmac_chan_prep_sg - prepare transfer descriptors from an SG list 880 * 881 * Common routine for public (MEMCPY) and slave DMA. The MEMCPY case is also 882 * converted to scatter-gather to guarantee consistent locking and a correct 883 * list manipulation. For slave DMA direction carries the usual meaning, and, 884 * logically, the SG list is RAM and the addr variable contains slave address, 885 * e.g., the FIFO I/O register. For MEMCPY direction equals DMA_MEM_TO_MEM 886 * and the SG list contains only one element and points at the source buffer. 887 */ 888 static struct dma_async_tx_descriptor * 889 rcar_dmac_chan_prep_sg(struct rcar_dmac_chan *chan, struct scatterlist *sgl, 890 unsigned int sg_len, dma_addr_t dev_addr, 891 enum dma_transfer_direction dir, unsigned long dma_flags, 892 bool cyclic) 893 { 894 struct rcar_dmac_xfer_chunk *chunk; 895 struct rcar_dmac_desc *desc; 896 struct scatterlist *sg; 897 unsigned int nchunks = 0; 898 unsigned int max_chunk_size; 899 unsigned int full_size = 0; 900 bool cross_boundary = false; 901 unsigned int i; 902 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 903 u32 high_dev_addr; 904 u32 high_mem_addr; 905 #endif 906 907 desc = rcar_dmac_desc_get(chan); 908 if (!desc) 909 return NULL; 910 911 desc->async_tx.flags = dma_flags; 912 desc->async_tx.cookie = -EBUSY; 913 914 desc->cyclic = cyclic; 915 desc->direction = dir; 916 917 rcar_dmac_chan_configure_desc(chan, desc); 918 919 max_chunk_size = RCAR_DMATCR_MASK << desc->xfer_shift; 920 921 /* 922 * Allocate and fill the transfer chunk descriptors. We own the only 923 * reference to the DMA descriptor, there's no need for locking. 924 */ 925 for_each_sg(sgl, sg, sg_len, i) { 926 dma_addr_t mem_addr = sg_dma_address(sg); 927 unsigned int len = sg_dma_len(sg); 928 929 full_size += len; 930 931 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 932 if (i == 0) { 933 high_dev_addr = dev_addr >> 32; 934 high_mem_addr = mem_addr >> 32; 935 } 936 937 if ((dev_addr >> 32 != high_dev_addr) || 938 (mem_addr >> 32 != high_mem_addr)) 939 cross_boundary = true; 940 #endif 941 while (len) { 942 unsigned int size = min(len, max_chunk_size); 943 944 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 945 /* 946 * Prevent individual transfers from crossing 4GB 947 * boundaries. 948 */ 949 if (dev_addr >> 32 != (dev_addr + size - 1) >> 32) { 950 size = ALIGN(dev_addr, 1ULL << 32) - dev_addr; 951 cross_boundary = true; 952 } 953 if (mem_addr >> 32 != (mem_addr + size - 1) >> 32) { 954 size = ALIGN(mem_addr, 1ULL << 32) - mem_addr; 955 cross_boundary = true; 956 } 957 #endif 958 959 chunk = rcar_dmac_xfer_chunk_get(chan); 960 if (!chunk) { 961 rcar_dmac_desc_put(chan, desc); 962 return NULL; 963 } 964 965 if (dir == DMA_DEV_TO_MEM) { 966 chunk->src_addr = dev_addr; 967 chunk->dst_addr = mem_addr; 968 } else { 969 chunk->src_addr = mem_addr; 970 chunk->dst_addr = dev_addr; 971 } 972 973 chunk->size = size; 974 975 dev_dbg(chan->chan.device->dev, 976 "chan%u: chunk %p/%p sgl %u@%p, %u/%u %pad -> %pad\n", 977 chan->index, chunk, desc, i, sg, size, len, 978 &chunk->src_addr, &chunk->dst_addr); 979 980 mem_addr += size; 981 if (dir == DMA_MEM_TO_MEM) 982 dev_addr += size; 983 984 len -= size; 985 986 list_add_tail(&chunk->node, &desc->chunks); 987 nchunks++; 988 } 989 } 990 991 desc->nchunks = nchunks; 992 desc->size = full_size; 993 994 /* 995 * Use hardware descriptor lists if possible when more than one chunk 996 * needs to be transferred (otherwise they don't make much sense). 997 * 998 * Source/Destination address should be located in same 4GiB region 999 * in the 40bit address space when it uses Hardware descriptor, 1000 * and cross_boundary is checking it. 1001 */ 1002 desc->hwdescs.use = !cross_boundary && nchunks > 1; 1003 if (desc->hwdescs.use) { 1004 if (rcar_dmac_fill_hwdesc(chan, desc) < 0) 1005 desc->hwdescs.use = false; 1006 } 1007 1008 return &desc->async_tx; 1009 } 1010 1011 /* ----------------------------------------------------------------------------- 1012 * DMA engine operations 1013 */ 1014 1015 static int rcar_dmac_alloc_chan_resources(struct dma_chan *chan) 1016 { 1017 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1018 int ret; 1019 1020 INIT_LIST_HEAD(&rchan->desc.chunks_free); 1021 INIT_LIST_HEAD(&rchan->desc.pages); 1022 1023 /* Preallocate descriptors. */ 1024 ret = rcar_dmac_xfer_chunk_alloc(rchan, GFP_KERNEL); 1025 if (ret < 0) 1026 return -ENOMEM; 1027 1028 ret = rcar_dmac_desc_alloc(rchan, GFP_KERNEL); 1029 if (ret < 0) 1030 return -ENOMEM; 1031 1032 return pm_runtime_get_sync(chan->device->dev); 1033 } 1034 1035 static void rcar_dmac_free_chan_resources(struct dma_chan *chan) 1036 { 1037 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1038 struct rcar_dmac *dmac = to_rcar_dmac(chan->device); 1039 struct rcar_dmac_chan_map *map = &rchan->map; 1040 struct rcar_dmac_desc_page *page, *_page; 1041 struct rcar_dmac_desc *desc; 1042 LIST_HEAD(list); 1043 1044 /* Protect against ISR */ 1045 spin_lock_irq(&rchan->lock); 1046 rcar_dmac_chan_halt(rchan); 1047 spin_unlock_irq(&rchan->lock); 1048 1049 /* 1050 * Now no new interrupts will occur, but one might already be 1051 * running. Wait for it to finish before freeing resources. 1052 */ 1053 synchronize_irq(rchan->irq); 1054 1055 if (rchan->mid_rid >= 0) { 1056 /* The caller is holding dma_list_mutex */ 1057 clear_bit(rchan->mid_rid, dmac->modules); 1058 rchan->mid_rid = -EINVAL; 1059 } 1060 1061 list_splice_init(&rchan->desc.free, &list); 1062 list_splice_init(&rchan->desc.pending, &list); 1063 list_splice_init(&rchan->desc.active, &list); 1064 list_splice_init(&rchan->desc.done, &list); 1065 list_splice_init(&rchan->desc.wait, &list); 1066 1067 rchan->desc.running = NULL; 1068 1069 list_for_each_entry(desc, &list, node) 1070 rcar_dmac_realloc_hwdesc(rchan, desc, 0); 1071 1072 list_for_each_entry_safe(page, _page, &rchan->desc.pages, node) { 1073 list_del(&page->node); 1074 free_page((unsigned long)page); 1075 } 1076 1077 /* Remove slave mapping if present. */ 1078 if (map->slave.xfer_size) { 1079 dma_unmap_resource(chan->device->dev, map->addr, 1080 map->slave.xfer_size, map->dir, 0); 1081 map->slave.xfer_size = 0; 1082 } 1083 1084 pm_runtime_put(chan->device->dev); 1085 } 1086 1087 static struct dma_async_tx_descriptor * 1088 rcar_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest, 1089 dma_addr_t dma_src, size_t len, unsigned long flags) 1090 { 1091 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1092 struct scatterlist sgl; 1093 1094 if (!len) 1095 return NULL; 1096 1097 sg_init_table(&sgl, 1); 1098 sg_set_page(&sgl, pfn_to_page(PFN_DOWN(dma_src)), len, 1099 offset_in_page(dma_src)); 1100 sg_dma_address(&sgl) = dma_src; 1101 sg_dma_len(&sgl) = len; 1102 1103 return rcar_dmac_chan_prep_sg(rchan, &sgl, 1, dma_dest, 1104 DMA_MEM_TO_MEM, flags, false); 1105 } 1106 1107 static int rcar_dmac_map_slave_addr(struct dma_chan *chan, 1108 enum dma_transfer_direction dir) 1109 { 1110 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1111 struct rcar_dmac_chan_map *map = &rchan->map; 1112 phys_addr_t dev_addr; 1113 size_t dev_size; 1114 enum dma_data_direction dev_dir; 1115 1116 if (dir == DMA_DEV_TO_MEM) { 1117 dev_addr = rchan->src.slave_addr; 1118 dev_size = rchan->src.xfer_size; 1119 dev_dir = DMA_TO_DEVICE; 1120 } else { 1121 dev_addr = rchan->dst.slave_addr; 1122 dev_size = rchan->dst.xfer_size; 1123 dev_dir = DMA_FROM_DEVICE; 1124 } 1125 1126 /* Reuse current map if possible. */ 1127 if (dev_addr == map->slave.slave_addr && 1128 dev_size == map->slave.xfer_size && 1129 dev_dir == map->dir) 1130 return 0; 1131 1132 /* Remove old mapping if present. */ 1133 if (map->slave.xfer_size) 1134 dma_unmap_resource(chan->device->dev, map->addr, 1135 map->slave.xfer_size, map->dir, 0); 1136 map->slave.xfer_size = 0; 1137 1138 /* Create new slave address map. */ 1139 map->addr = dma_map_resource(chan->device->dev, dev_addr, dev_size, 1140 dev_dir, 0); 1141 1142 if (dma_mapping_error(chan->device->dev, map->addr)) { 1143 dev_err(chan->device->dev, 1144 "chan%u: failed to map %zx@%pap", rchan->index, 1145 dev_size, &dev_addr); 1146 return -EIO; 1147 } 1148 1149 dev_dbg(chan->device->dev, "chan%u: map %zx@%pap to %pad dir: %s\n", 1150 rchan->index, dev_size, &dev_addr, &map->addr, 1151 dev_dir == DMA_TO_DEVICE ? "DMA_TO_DEVICE" : "DMA_FROM_DEVICE"); 1152 1153 map->slave.slave_addr = dev_addr; 1154 map->slave.xfer_size = dev_size; 1155 map->dir = dev_dir; 1156 1157 return 0; 1158 } 1159 1160 static struct dma_async_tx_descriptor * 1161 rcar_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 1162 unsigned int sg_len, enum dma_transfer_direction dir, 1163 unsigned long flags, void *context) 1164 { 1165 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1166 1167 /* Someone calling slave DMA on a generic channel? */ 1168 if (rchan->mid_rid < 0 || !sg_len) { 1169 dev_warn(chan->device->dev, 1170 "%s: bad parameter: len=%d, id=%d\n", 1171 __func__, sg_len, rchan->mid_rid); 1172 return NULL; 1173 } 1174 1175 if (rcar_dmac_map_slave_addr(chan, dir)) 1176 return NULL; 1177 1178 return rcar_dmac_chan_prep_sg(rchan, sgl, sg_len, rchan->map.addr, 1179 dir, flags, false); 1180 } 1181 1182 #define RCAR_DMAC_MAX_SG_LEN 32 1183 1184 static struct dma_async_tx_descriptor * 1185 rcar_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, 1186 size_t buf_len, size_t period_len, 1187 enum dma_transfer_direction dir, unsigned long flags) 1188 { 1189 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1190 struct dma_async_tx_descriptor *desc; 1191 struct scatterlist *sgl; 1192 unsigned int sg_len; 1193 unsigned int i; 1194 1195 /* Someone calling slave DMA on a generic channel? */ 1196 if (rchan->mid_rid < 0 || buf_len < period_len) { 1197 dev_warn(chan->device->dev, 1198 "%s: bad parameter: buf_len=%zu, period_len=%zu, id=%d\n", 1199 __func__, buf_len, period_len, rchan->mid_rid); 1200 return NULL; 1201 } 1202 1203 if (rcar_dmac_map_slave_addr(chan, dir)) 1204 return NULL; 1205 1206 sg_len = buf_len / period_len; 1207 if (sg_len > RCAR_DMAC_MAX_SG_LEN) { 1208 dev_err(chan->device->dev, 1209 "chan%u: sg length %d exceds limit %d", 1210 rchan->index, sg_len, RCAR_DMAC_MAX_SG_LEN); 1211 return NULL; 1212 } 1213 1214 /* 1215 * Allocate the sg list dynamically as it would consume too much stack 1216 * space. 1217 */ 1218 sgl = kcalloc(sg_len, sizeof(*sgl), GFP_NOWAIT); 1219 if (!sgl) 1220 return NULL; 1221 1222 sg_init_table(sgl, sg_len); 1223 1224 for (i = 0; i < sg_len; ++i) { 1225 dma_addr_t src = buf_addr + (period_len * i); 1226 1227 sg_set_page(&sgl[i], pfn_to_page(PFN_DOWN(src)), period_len, 1228 offset_in_page(src)); 1229 sg_dma_address(&sgl[i]) = src; 1230 sg_dma_len(&sgl[i]) = period_len; 1231 } 1232 1233 desc = rcar_dmac_chan_prep_sg(rchan, sgl, sg_len, rchan->map.addr, 1234 dir, flags, true); 1235 1236 kfree(sgl); 1237 return desc; 1238 } 1239 1240 static int rcar_dmac_device_config(struct dma_chan *chan, 1241 struct dma_slave_config *cfg) 1242 { 1243 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1244 1245 /* 1246 * We could lock this, but you shouldn't be configuring the 1247 * channel, while using it... 1248 */ 1249 rchan->src.slave_addr = cfg->src_addr; 1250 rchan->dst.slave_addr = cfg->dst_addr; 1251 rchan->src.xfer_size = cfg->src_addr_width; 1252 rchan->dst.xfer_size = cfg->dst_addr_width; 1253 1254 return 0; 1255 } 1256 1257 static int rcar_dmac_chan_terminate_all(struct dma_chan *chan) 1258 { 1259 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1260 unsigned long flags; 1261 1262 spin_lock_irqsave(&rchan->lock, flags); 1263 rcar_dmac_chan_halt(rchan); 1264 spin_unlock_irqrestore(&rchan->lock, flags); 1265 1266 /* 1267 * FIXME: No new interrupt can occur now, but the IRQ thread might still 1268 * be running. 1269 */ 1270 1271 rcar_dmac_chan_reinit(rchan); 1272 1273 return 0; 1274 } 1275 1276 static unsigned int rcar_dmac_chan_get_residue(struct rcar_dmac_chan *chan, 1277 dma_cookie_t cookie) 1278 { 1279 struct rcar_dmac_desc *desc = chan->desc.running; 1280 struct rcar_dmac_xfer_chunk *running = NULL; 1281 struct rcar_dmac_xfer_chunk *chunk; 1282 enum dma_status status; 1283 unsigned int residue = 0; 1284 unsigned int dptr = 0; 1285 unsigned int chcrb; 1286 unsigned int tcrb; 1287 unsigned int i; 1288 1289 if (!desc) 1290 return 0; 1291 1292 /* 1293 * If the cookie corresponds to a descriptor that has been completed 1294 * there is no residue. The same check has already been performed by the 1295 * caller but without holding the channel lock, so the descriptor could 1296 * now be complete. 1297 */ 1298 status = dma_cookie_status(&chan->chan, cookie, NULL); 1299 if (status == DMA_COMPLETE) 1300 return 0; 1301 1302 /* 1303 * If the cookie doesn't correspond to the currently running transfer 1304 * then the descriptor hasn't been processed yet, and the residue is 1305 * equal to the full descriptor size. 1306 * Also, a client driver is possible to call this function before 1307 * rcar_dmac_isr_channel_thread() runs. In this case, the "desc.running" 1308 * will be the next descriptor, and the done list will appear. So, if 1309 * the argument cookie matches the done list's cookie, we can assume 1310 * the residue is zero. 1311 */ 1312 if (cookie != desc->async_tx.cookie) { 1313 list_for_each_entry(desc, &chan->desc.done, node) { 1314 if (cookie == desc->async_tx.cookie) 1315 return 0; 1316 } 1317 list_for_each_entry(desc, &chan->desc.pending, node) { 1318 if (cookie == desc->async_tx.cookie) 1319 return desc->size; 1320 } 1321 list_for_each_entry(desc, &chan->desc.active, node) { 1322 if (cookie == desc->async_tx.cookie) 1323 return desc->size; 1324 } 1325 1326 /* 1327 * No descriptor found for the cookie, there's thus no residue. 1328 * This shouldn't happen if the calling driver passes a correct 1329 * cookie value. 1330 */ 1331 WARN(1, "No descriptor for cookie!"); 1332 return 0; 1333 } 1334 1335 /* 1336 * We need to read two registers. 1337 * Make sure the control register does not skip to next chunk 1338 * while reading the counter. 1339 * Trying it 3 times should be enough: Initial read, retry, retry 1340 * for the paranoid. 1341 */ 1342 for (i = 0; i < 3; i++) { 1343 chcrb = rcar_dmac_chan_read(chan, RCAR_DMACHCRB) & 1344 RCAR_DMACHCRB_DPTR_MASK; 1345 tcrb = rcar_dmac_chan_read(chan, RCAR_DMATCRB); 1346 /* Still the same? */ 1347 if (chcrb == (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) & 1348 RCAR_DMACHCRB_DPTR_MASK)) 1349 break; 1350 } 1351 WARN_ONCE(i >= 3, "residue might be not continuous!"); 1352 1353 /* 1354 * In descriptor mode the descriptor running pointer is not maintained 1355 * by the interrupt handler, find the running descriptor from the 1356 * descriptor pointer field in the CHCRB register. In non-descriptor 1357 * mode just use the running descriptor pointer. 1358 */ 1359 if (desc->hwdescs.use) { 1360 dptr = chcrb >> RCAR_DMACHCRB_DPTR_SHIFT; 1361 if (dptr == 0) 1362 dptr = desc->nchunks; 1363 dptr--; 1364 WARN_ON(dptr >= desc->nchunks); 1365 } else { 1366 running = desc->running; 1367 } 1368 1369 /* Compute the size of all chunks still to be transferred. */ 1370 list_for_each_entry_reverse(chunk, &desc->chunks, node) { 1371 if (chunk == running || ++dptr == desc->nchunks) 1372 break; 1373 1374 residue += chunk->size; 1375 } 1376 1377 /* Add the residue for the current chunk. */ 1378 residue += tcrb << desc->xfer_shift; 1379 1380 return residue; 1381 } 1382 1383 static enum dma_status rcar_dmac_tx_status(struct dma_chan *chan, 1384 dma_cookie_t cookie, 1385 struct dma_tx_state *txstate) 1386 { 1387 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1388 enum dma_status status; 1389 unsigned long flags; 1390 unsigned int residue; 1391 bool cyclic; 1392 1393 status = dma_cookie_status(chan, cookie, txstate); 1394 if (status == DMA_COMPLETE || !txstate) 1395 return status; 1396 1397 spin_lock_irqsave(&rchan->lock, flags); 1398 residue = rcar_dmac_chan_get_residue(rchan, cookie); 1399 cyclic = rchan->desc.running ? rchan->desc.running->cyclic : false; 1400 spin_unlock_irqrestore(&rchan->lock, flags); 1401 1402 /* if there's no residue, the cookie is complete */ 1403 if (!residue && !cyclic) 1404 return DMA_COMPLETE; 1405 1406 dma_set_residue(txstate, residue); 1407 1408 return status; 1409 } 1410 1411 static void rcar_dmac_issue_pending(struct dma_chan *chan) 1412 { 1413 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1414 unsigned long flags; 1415 1416 spin_lock_irqsave(&rchan->lock, flags); 1417 1418 if (list_empty(&rchan->desc.pending)) 1419 goto done; 1420 1421 /* Append the pending list to the active list. */ 1422 list_splice_tail_init(&rchan->desc.pending, &rchan->desc.active); 1423 1424 /* 1425 * If no transfer is running pick the first descriptor from the active 1426 * list and start the transfer. 1427 */ 1428 if (!rchan->desc.running) { 1429 struct rcar_dmac_desc *desc; 1430 1431 desc = list_first_entry(&rchan->desc.active, 1432 struct rcar_dmac_desc, node); 1433 rchan->desc.running = desc; 1434 1435 rcar_dmac_chan_start_xfer(rchan); 1436 } 1437 1438 done: 1439 spin_unlock_irqrestore(&rchan->lock, flags); 1440 } 1441 1442 static void rcar_dmac_device_synchronize(struct dma_chan *chan) 1443 { 1444 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1445 1446 synchronize_irq(rchan->irq); 1447 } 1448 1449 /* ----------------------------------------------------------------------------- 1450 * IRQ handling 1451 */ 1452 1453 static irqreturn_t rcar_dmac_isr_desc_stage_end(struct rcar_dmac_chan *chan) 1454 { 1455 struct rcar_dmac_desc *desc = chan->desc.running; 1456 unsigned int stage; 1457 1458 if (WARN_ON(!desc || !desc->cyclic)) { 1459 /* 1460 * This should never happen, there should always be a running 1461 * cyclic descriptor when a descriptor stage end interrupt is 1462 * triggered. Warn and return. 1463 */ 1464 return IRQ_NONE; 1465 } 1466 1467 /* Program the interrupt pointer to the next stage. */ 1468 stage = (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) & 1469 RCAR_DMACHCRB_DPTR_MASK) >> RCAR_DMACHCRB_DPTR_SHIFT; 1470 rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(stage)); 1471 1472 return IRQ_WAKE_THREAD; 1473 } 1474 1475 static irqreturn_t rcar_dmac_isr_transfer_end(struct rcar_dmac_chan *chan) 1476 { 1477 struct rcar_dmac_desc *desc = chan->desc.running; 1478 irqreturn_t ret = IRQ_WAKE_THREAD; 1479 1480 if (WARN_ON_ONCE(!desc)) { 1481 /* 1482 * This should never happen, there should always be a running 1483 * descriptor when a transfer end interrupt is triggered. Warn 1484 * and return. 1485 */ 1486 return IRQ_NONE; 1487 } 1488 1489 /* 1490 * The transfer end interrupt isn't generated for each chunk when using 1491 * descriptor mode. Only update the running chunk pointer in 1492 * non-descriptor mode. 1493 */ 1494 if (!desc->hwdescs.use) { 1495 /* 1496 * If we haven't completed the last transfer chunk simply move 1497 * to the next one. Only wake the IRQ thread if the transfer is 1498 * cyclic. 1499 */ 1500 if (!list_is_last(&desc->running->node, &desc->chunks)) { 1501 desc->running = list_next_entry(desc->running, node); 1502 if (!desc->cyclic) 1503 ret = IRQ_HANDLED; 1504 goto done; 1505 } 1506 1507 /* 1508 * We've completed the last transfer chunk. If the transfer is 1509 * cyclic, move back to the first one. 1510 */ 1511 if (desc->cyclic) { 1512 desc->running = 1513 list_first_entry(&desc->chunks, 1514 struct rcar_dmac_xfer_chunk, 1515 node); 1516 goto done; 1517 } 1518 } 1519 1520 /* The descriptor is complete, move it to the done list. */ 1521 list_move_tail(&desc->node, &chan->desc.done); 1522 1523 /* Queue the next descriptor, if any. */ 1524 if (!list_empty(&chan->desc.active)) 1525 chan->desc.running = list_first_entry(&chan->desc.active, 1526 struct rcar_dmac_desc, 1527 node); 1528 else 1529 chan->desc.running = NULL; 1530 1531 done: 1532 if (chan->desc.running) 1533 rcar_dmac_chan_start_xfer(chan); 1534 1535 return ret; 1536 } 1537 1538 static irqreturn_t rcar_dmac_isr_channel(int irq, void *dev) 1539 { 1540 u32 mask = RCAR_DMACHCR_DSE | RCAR_DMACHCR_TE; 1541 struct rcar_dmac_chan *chan = dev; 1542 irqreturn_t ret = IRQ_NONE; 1543 bool reinit = false; 1544 u32 chcr; 1545 1546 spin_lock(&chan->lock); 1547 1548 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 1549 if (chcr & RCAR_DMACHCR_CAE) { 1550 struct rcar_dmac *dmac = to_rcar_dmac(chan->chan.device); 1551 1552 /* 1553 * We don't need to call rcar_dmac_chan_halt() 1554 * because channel is already stopped in error case. 1555 * We need to clear register and check DE bit as recovery. 1556 */ 1557 rcar_dmac_write(dmac, RCAR_DMACHCLR, 1 << chan->index); 1558 rcar_dmac_chcr_de_barrier(chan); 1559 reinit = true; 1560 goto spin_lock_end; 1561 } 1562 1563 if (chcr & RCAR_DMACHCR_TE) 1564 mask |= RCAR_DMACHCR_DE; 1565 rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr & ~mask); 1566 if (mask & RCAR_DMACHCR_DE) 1567 rcar_dmac_chcr_de_barrier(chan); 1568 1569 if (chcr & RCAR_DMACHCR_DSE) 1570 ret |= rcar_dmac_isr_desc_stage_end(chan); 1571 1572 if (chcr & RCAR_DMACHCR_TE) 1573 ret |= rcar_dmac_isr_transfer_end(chan); 1574 1575 spin_lock_end: 1576 spin_unlock(&chan->lock); 1577 1578 if (reinit) { 1579 dev_err(chan->chan.device->dev, "Channel Address Error\n"); 1580 1581 rcar_dmac_chan_reinit(chan); 1582 ret = IRQ_HANDLED; 1583 } 1584 1585 return ret; 1586 } 1587 1588 static irqreturn_t rcar_dmac_isr_channel_thread(int irq, void *dev) 1589 { 1590 struct rcar_dmac_chan *chan = dev; 1591 struct rcar_dmac_desc *desc; 1592 struct dmaengine_desc_callback cb; 1593 1594 spin_lock_irq(&chan->lock); 1595 1596 /* For cyclic transfers notify the user after every chunk. */ 1597 if (chan->desc.running && chan->desc.running->cyclic) { 1598 desc = chan->desc.running; 1599 dmaengine_desc_get_callback(&desc->async_tx, &cb); 1600 1601 if (dmaengine_desc_callback_valid(&cb)) { 1602 spin_unlock_irq(&chan->lock); 1603 dmaengine_desc_callback_invoke(&cb, NULL); 1604 spin_lock_irq(&chan->lock); 1605 } 1606 } 1607 1608 /* 1609 * Call the callback function for all descriptors on the done list and 1610 * move them to the ack wait list. 1611 */ 1612 while (!list_empty(&chan->desc.done)) { 1613 desc = list_first_entry(&chan->desc.done, struct rcar_dmac_desc, 1614 node); 1615 dma_cookie_complete(&desc->async_tx); 1616 list_del(&desc->node); 1617 1618 dmaengine_desc_get_callback(&desc->async_tx, &cb); 1619 if (dmaengine_desc_callback_valid(&cb)) { 1620 spin_unlock_irq(&chan->lock); 1621 /* 1622 * We own the only reference to this descriptor, we can 1623 * safely dereference it without holding the channel 1624 * lock. 1625 */ 1626 dmaengine_desc_callback_invoke(&cb, NULL); 1627 spin_lock_irq(&chan->lock); 1628 } 1629 1630 list_add_tail(&desc->node, &chan->desc.wait); 1631 } 1632 1633 spin_unlock_irq(&chan->lock); 1634 1635 /* Recycle all acked descriptors. */ 1636 rcar_dmac_desc_recycle_acked(chan); 1637 1638 return IRQ_HANDLED; 1639 } 1640 1641 /* ----------------------------------------------------------------------------- 1642 * OF xlate and channel filter 1643 */ 1644 1645 static bool rcar_dmac_chan_filter(struct dma_chan *chan, void *arg) 1646 { 1647 struct rcar_dmac *dmac = to_rcar_dmac(chan->device); 1648 struct of_phandle_args *dma_spec = arg; 1649 1650 /* 1651 * FIXME: Using a filter on OF platforms is a nonsense. The OF xlate 1652 * function knows from which device it wants to allocate a channel from, 1653 * and would be perfectly capable of selecting the channel it wants. 1654 * Forcing it to call dma_request_channel() and iterate through all 1655 * channels from all controllers is just pointless. 1656 */ 1657 if (chan->device->device_config != rcar_dmac_device_config || 1658 dma_spec->np != chan->device->dev->of_node) 1659 return false; 1660 1661 return !test_and_set_bit(dma_spec->args[0], dmac->modules); 1662 } 1663 1664 static struct dma_chan *rcar_dmac_of_xlate(struct of_phandle_args *dma_spec, 1665 struct of_dma *ofdma) 1666 { 1667 struct rcar_dmac_chan *rchan; 1668 struct dma_chan *chan; 1669 dma_cap_mask_t mask; 1670 1671 if (dma_spec->args_count != 1) 1672 return NULL; 1673 1674 /* Only slave DMA channels can be allocated via DT */ 1675 dma_cap_zero(mask); 1676 dma_cap_set(DMA_SLAVE, mask); 1677 1678 chan = dma_request_channel(mask, rcar_dmac_chan_filter, dma_spec); 1679 if (!chan) 1680 return NULL; 1681 1682 rchan = to_rcar_dmac_chan(chan); 1683 rchan->mid_rid = dma_spec->args[0]; 1684 1685 return chan; 1686 } 1687 1688 /* ----------------------------------------------------------------------------- 1689 * Power management 1690 */ 1691 1692 #ifdef CONFIG_PM 1693 static int rcar_dmac_runtime_suspend(struct device *dev) 1694 { 1695 return 0; 1696 } 1697 1698 static int rcar_dmac_runtime_resume(struct device *dev) 1699 { 1700 struct rcar_dmac *dmac = dev_get_drvdata(dev); 1701 1702 return rcar_dmac_init(dmac); 1703 } 1704 #endif 1705 1706 static const struct dev_pm_ops rcar_dmac_pm = { 1707 /* 1708 * TODO for system sleep/resume: 1709 * - Wait for the current transfer to complete and stop the device, 1710 * - Resume transfers, if any. 1711 */ 1712 SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, 1713 pm_runtime_force_resume) 1714 SET_RUNTIME_PM_OPS(rcar_dmac_runtime_suspend, rcar_dmac_runtime_resume, 1715 NULL) 1716 }; 1717 1718 /* ----------------------------------------------------------------------------- 1719 * Probe and remove 1720 */ 1721 1722 static int rcar_dmac_chan_probe(struct rcar_dmac *dmac, 1723 struct rcar_dmac_chan *rchan, 1724 unsigned int index) 1725 { 1726 struct platform_device *pdev = to_platform_device(dmac->dev); 1727 struct dma_chan *chan = &rchan->chan; 1728 char pdev_irqname[5]; 1729 char *irqname; 1730 int ret; 1731 1732 rchan->index = index; 1733 rchan->iomem = dmac->iomem + RCAR_DMAC_CHAN_OFFSET(index); 1734 rchan->mid_rid = -EINVAL; 1735 1736 spin_lock_init(&rchan->lock); 1737 1738 INIT_LIST_HEAD(&rchan->desc.free); 1739 INIT_LIST_HEAD(&rchan->desc.pending); 1740 INIT_LIST_HEAD(&rchan->desc.active); 1741 INIT_LIST_HEAD(&rchan->desc.done); 1742 INIT_LIST_HEAD(&rchan->desc.wait); 1743 1744 /* Request the channel interrupt. */ 1745 sprintf(pdev_irqname, "ch%u", index); 1746 rchan->irq = platform_get_irq_byname(pdev, pdev_irqname); 1747 if (rchan->irq < 0) { 1748 dev_err(dmac->dev, "no IRQ specified for channel %u\n", index); 1749 return -ENODEV; 1750 } 1751 1752 irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", 1753 dev_name(dmac->dev), index); 1754 if (!irqname) 1755 return -ENOMEM; 1756 1757 /* 1758 * Initialize the DMA engine channel and add it to the DMA engine 1759 * channels list. 1760 */ 1761 chan->device = &dmac->engine; 1762 dma_cookie_init(chan); 1763 1764 list_add_tail(&chan->device_node, &dmac->engine.channels); 1765 1766 ret = devm_request_threaded_irq(dmac->dev, rchan->irq, 1767 rcar_dmac_isr_channel, 1768 rcar_dmac_isr_channel_thread, 0, 1769 irqname, rchan); 1770 if (ret) { 1771 dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", 1772 rchan->irq, ret); 1773 return ret; 1774 } 1775 1776 return 0; 1777 } 1778 1779 static int rcar_dmac_parse_of(struct device *dev, struct rcar_dmac *dmac) 1780 { 1781 struct device_node *np = dev->of_node; 1782 int ret; 1783 1784 ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels); 1785 if (ret < 0) { 1786 dev_err(dev, "unable to read dma-channels property\n"); 1787 return ret; 1788 } 1789 1790 if (dmac->n_channels <= 0 || dmac->n_channels >= 100) { 1791 dev_err(dev, "invalid number of channels %u\n", 1792 dmac->n_channels); 1793 return -EINVAL; 1794 } 1795 1796 return 0; 1797 } 1798 1799 static int rcar_dmac_probe(struct platform_device *pdev) 1800 { 1801 const enum dma_slave_buswidth widths = DMA_SLAVE_BUSWIDTH_1_BYTE | 1802 DMA_SLAVE_BUSWIDTH_2_BYTES | DMA_SLAVE_BUSWIDTH_4_BYTES | 1803 DMA_SLAVE_BUSWIDTH_8_BYTES | DMA_SLAVE_BUSWIDTH_16_BYTES | 1804 DMA_SLAVE_BUSWIDTH_32_BYTES | DMA_SLAVE_BUSWIDTH_64_BYTES; 1805 unsigned int channels_offset = 0; 1806 struct dma_device *engine; 1807 struct rcar_dmac *dmac; 1808 struct resource *mem; 1809 unsigned int i; 1810 int ret; 1811 1812 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); 1813 if (!dmac) 1814 return -ENOMEM; 1815 1816 dmac->dev = &pdev->dev; 1817 platform_set_drvdata(pdev, dmac); 1818 dmac->dev->dma_parms = &dmac->parms; 1819 dma_set_max_seg_size(dmac->dev, RCAR_DMATCR_MASK); 1820 dma_set_mask_and_coherent(dmac->dev, DMA_BIT_MASK(40)); 1821 1822 ret = rcar_dmac_parse_of(&pdev->dev, dmac); 1823 if (ret < 0) 1824 return ret; 1825 1826 /* 1827 * A still unconfirmed hardware bug prevents the IPMMU microTLB 0 to be 1828 * flushed correctly, resulting in memory corruption. DMAC 0 channel 0 1829 * is connected to microTLB 0 on currently supported platforms, so we 1830 * can't use it with the IPMMU. As the IOMMU API operates at the device 1831 * level we can't disable it selectively, so ignore channel 0 for now if 1832 * the device is part of an IOMMU group. 1833 */ 1834 if (device_iommu_mapped(&pdev->dev)) { 1835 dmac->n_channels--; 1836 channels_offset = 1; 1837 } 1838 1839 dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels, 1840 sizeof(*dmac->channels), GFP_KERNEL); 1841 if (!dmac->channels) 1842 return -ENOMEM; 1843 1844 /* Request resources. */ 1845 mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1846 dmac->iomem = devm_ioremap_resource(&pdev->dev, mem); 1847 if (IS_ERR(dmac->iomem)) 1848 return PTR_ERR(dmac->iomem); 1849 1850 /* Enable runtime PM and initialize the device. */ 1851 pm_runtime_enable(&pdev->dev); 1852 ret = pm_runtime_get_sync(&pdev->dev); 1853 if (ret < 0) { 1854 dev_err(&pdev->dev, "runtime PM get sync failed (%d)\n", ret); 1855 return ret; 1856 } 1857 1858 ret = rcar_dmac_init(dmac); 1859 pm_runtime_put(&pdev->dev); 1860 1861 if (ret) { 1862 dev_err(&pdev->dev, "failed to reset device\n"); 1863 goto error; 1864 } 1865 1866 /* Initialize engine */ 1867 engine = &dmac->engine; 1868 1869 dma_cap_set(DMA_MEMCPY, engine->cap_mask); 1870 dma_cap_set(DMA_SLAVE, engine->cap_mask); 1871 1872 engine->dev = &pdev->dev; 1873 engine->copy_align = ilog2(RCAR_DMAC_MEMCPY_XFER_SIZE); 1874 1875 engine->src_addr_widths = widths; 1876 engine->dst_addr_widths = widths; 1877 engine->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); 1878 engine->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1879 1880 engine->device_alloc_chan_resources = rcar_dmac_alloc_chan_resources; 1881 engine->device_free_chan_resources = rcar_dmac_free_chan_resources; 1882 engine->device_prep_dma_memcpy = rcar_dmac_prep_dma_memcpy; 1883 engine->device_prep_slave_sg = rcar_dmac_prep_slave_sg; 1884 engine->device_prep_dma_cyclic = rcar_dmac_prep_dma_cyclic; 1885 engine->device_config = rcar_dmac_device_config; 1886 engine->device_pause = rcar_dmac_chan_pause; 1887 engine->device_terminate_all = rcar_dmac_chan_terminate_all; 1888 engine->device_tx_status = rcar_dmac_tx_status; 1889 engine->device_issue_pending = rcar_dmac_issue_pending; 1890 engine->device_synchronize = rcar_dmac_device_synchronize; 1891 1892 INIT_LIST_HEAD(&engine->channels); 1893 1894 for (i = 0; i < dmac->n_channels; ++i) { 1895 ret = rcar_dmac_chan_probe(dmac, &dmac->channels[i], 1896 i + channels_offset); 1897 if (ret < 0) 1898 goto error; 1899 } 1900 1901 /* Register the DMAC as a DMA provider for DT. */ 1902 ret = of_dma_controller_register(pdev->dev.of_node, rcar_dmac_of_xlate, 1903 NULL); 1904 if (ret < 0) 1905 goto error; 1906 1907 /* 1908 * Register the DMA engine device. 1909 * 1910 * Default transfer size of 32 bytes requires 32-byte alignment. 1911 */ 1912 ret = dma_async_device_register(engine); 1913 if (ret < 0) 1914 goto error; 1915 1916 return 0; 1917 1918 error: 1919 of_dma_controller_free(pdev->dev.of_node); 1920 pm_runtime_disable(&pdev->dev); 1921 return ret; 1922 } 1923 1924 static int rcar_dmac_remove(struct platform_device *pdev) 1925 { 1926 struct rcar_dmac *dmac = platform_get_drvdata(pdev); 1927 1928 of_dma_controller_free(pdev->dev.of_node); 1929 dma_async_device_unregister(&dmac->engine); 1930 1931 pm_runtime_disable(&pdev->dev); 1932 1933 return 0; 1934 } 1935 1936 static void rcar_dmac_shutdown(struct platform_device *pdev) 1937 { 1938 struct rcar_dmac *dmac = platform_get_drvdata(pdev); 1939 1940 rcar_dmac_stop_all_chan(dmac); 1941 } 1942 1943 static const struct of_device_id rcar_dmac_of_ids[] = { 1944 { .compatible = "renesas,rcar-dmac", }, 1945 { /* Sentinel */ } 1946 }; 1947 MODULE_DEVICE_TABLE(of, rcar_dmac_of_ids); 1948 1949 static struct platform_driver rcar_dmac_driver = { 1950 .driver = { 1951 .pm = &rcar_dmac_pm, 1952 .name = "rcar-dmac", 1953 .of_match_table = rcar_dmac_of_ids, 1954 }, 1955 .probe = rcar_dmac_probe, 1956 .remove = rcar_dmac_remove, 1957 .shutdown = rcar_dmac_shutdown, 1958 }; 1959 1960 module_platform_driver(rcar_dmac_driver); 1961 1962 MODULE_DESCRIPTION("R-Car Gen2 DMA Controller Driver"); 1963 MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>"); 1964 MODULE_LICENSE("GPL v2"); 1965