1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Renesas R-Car Gen2/Gen3 DMA Controller Driver 4 * 5 * Copyright (C) 2014-2019 Renesas Electronics Inc. 6 * 7 * Author: Laurent Pinchart <laurent.pinchart@ideasonboard.com> 8 */ 9 10 #include <linux/delay.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/dmaengine.h> 13 #include <linux/interrupt.h> 14 #include <linux/list.h> 15 #include <linux/module.h> 16 #include <linux/mutex.h> 17 #include <linux/of.h> 18 #include <linux/of_dma.h> 19 #include <linux/of_platform.h> 20 #include <linux/platform_device.h> 21 #include <linux/pm_runtime.h> 22 #include <linux/slab.h> 23 #include <linux/spinlock.h> 24 25 #include "../dmaengine.h" 26 27 /* 28 * struct rcar_dmac_xfer_chunk - Descriptor for a hardware transfer 29 * @node: entry in the parent's chunks list 30 * @src_addr: device source address 31 * @dst_addr: device destination address 32 * @size: transfer size in bytes 33 */ 34 struct rcar_dmac_xfer_chunk { 35 struct list_head node; 36 37 dma_addr_t src_addr; 38 dma_addr_t dst_addr; 39 u32 size; 40 }; 41 42 /* 43 * struct rcar_dmac_hw_desc - Hardware descriptor for a transfer chunk 44 * @sar: value of the SAR register (source address) 45 * @dar: value of the DAR register (destination address) 46 * @tcr: value of the TCR register (transfer count) 47 */ 48 struct rcar_dmac_hw_desc { 49 u32 sar; 50 u32 dar; 51 u32 tcr; 52 u32 reserved; 53 } __attribute__((__packed__)); 54 55 /* 56 * struct rcar_dmac_desc - R-Car Gen2 DMA Transfer Descriptor 57 * @async_tx: base DMA asynchronous transaction descriptor 58 * @direction: direction of the DMA transfer 59 * @xfer_shift: log2 of the transfer size 60 * @chcr: value of the channel configuration register for this transfer 61 * @node: entry in the channel's descriptors lists 62 * @chunks: list of transfer chunks for this transfer 63 * @running: the transfer chunk being currently processed 64 * @nchunks: number of transfer chunks for this transfer 65 * @hwdescs.use: whether the transfer descriptor uses hardware descriptors 66 * @hwdescs.mem: hardware descriptors memory for the transfer 67 * @hwdescs.dma: device address of the hardware descriptors memory 68 * @hwdescs.size: size of the hardware descriptors in bytes 69 * @size: transfer size in bytes 70 * @cyclic: when set indicates that the DMA transfer is cyclic 71 */ 72 struct rcar_dmac_desc { 73 struct dma_async_tx_descriptor async_tx; 74 enum dma_transfer_direction direction; 75 unsigned int xfer_shift; 76 u32 chcr; 77 78 struct list_head node; 79 struct list_head chunks; 80 struct rcar_dmac_xfer_chunk *running; 81 unsigned int nchunks; 82 83 struct { 84 bool use; 85 struct rcar_dmac_hw_desc *mem; 86 dma_addr_t dma; 87 size_t size; 88 } hwdescs; 89 90 unsigned int size; 91 bool cyclic; 92 }; 93 94 #define to_rcar_dmac_desc(d) container_of(d, struct rcar_dmac_desc, async_tx) 95 96 /* 97 * struct rcar_dmac_desc_page - One page worth of descriptors 98 * @node: entry in the channel's pages list 99 * @descs: array of DMA descriptors 100 * @chunks: array of transfer chunk descriptors 101 */ 102 struct rcar_dmac_desc_page { 103 struct list_head node; 104 105 union { 106 struct rcar_dmac_desc descs[0]; 107 struct rcar_dmac_xfer_chunk chunks[0]; 108 }; 109 }; 110 111 #define RCAR_DMAC_DESCS_PER_PAGE \ 112 ((PAGE_SIZE - offsetof(struct rcar_dmac_desc_page, descs)) / \ 113 sizeof(struct rcar_dmac_desc)) 114 #define RCAR_DMAC_XFER_CHUNKS_PER_PAGE \ 115 ((PAGE_SIZE - offsetof(struct rcar_dmac_desc_page, chunks)) / \ 116 sizeof(struct rcar_dmac_xfer_chunk)) 117 118 /* 119 * struct rcar_dmac_chan_slave - Slave configuration 120 * @slave_addr: slave memory address 121 * @xfer_size: size (in bytes) of hardware transfers 122 */ 123 struct rcar_dmac_chan_slave { 124 phys_addr_t slave_addr; 125 unsigned int xfer_size; 126 }; 127 128 /* 129 * struct rcar_dmac_chan_map - Map of slave device phys to dma address 130 * @addr: slave dma address 131 * @dir: direction of mapping 132 * @slave: slave configuration that is mapped 133 */ 134 struct rcar_dmac_chan_map { 135 dma_addr_t addr; 136 enum dma_data_direction dir; 137 struct rcar_dmac_chan_slave slave; 138 }; 139 140 /* 141 * struct rcar_dmac_chan - R-Car Gen2 DMA Controller Channel 142 * @chan: base DMA channel object 143 * @iomem: channel I/O memory base 144 * @index: index of this channel in the controller 145 * @irq: channel IRQ 146 * @src: slave memory address and size on the source side 147 * @dst: slave memory address and size on the destination side 148 * @mid_rid: hardware MID/RID for the DMA client using this channel 149 * @lock: protects the channel CHCR register and the desc members 150 * @desc.free: list of free descriptors 151 * @desc.pending: list of pending descriptors (submitted with tx_submit) 152 * @desc.active: list of active descriptors (activated with issue_pending) 153 * @desc.done: list of completed descriptors 154 * @desc.wait: list of descriptors waiting for an ack 155 * @desc.running: the descriptor being processed (a member of the active list) 156 * @desc.chunks_free: list of free transfer chunk descriptors 157 * @desc.pages: list of pages used by allocated descriptors 158 */ 159 struct rcar_dmac_chan { 160 struct dma_chan chan; 161 void __iomem *iomem; 162 unsigned int index; 163 int irq; 164 165 struct rcar_dmac_chan_slave src; 166 struct rcar_dmac_chan_slave dst; 167 struct rcar_dmac_chan_map map; 168 int mid_rid; 169 170 spinlock_t lock; 171 172 struct { 173 struct list_head free; 174 struct list_head pending; 175 struct list_head active; 176 struct list_head done; 177 struct list_head wait; 178 struct rcar_dmac_desc *running; 179 180 struct list_head chunks_free; 181 182 struct list_head pages; 183 } desc; 184 }; 185 186 #define to_rcar_dmac_chan(c) container_of(c, struct rcar_dmac_chan, chan) 187 188 /* 189 * struct rcar_dmac - R-Car Gen2 DMA Controller 190 * @engine: base DMA engine object 191 * @dev: the hardware device 192 * @iomem: remapped I/O memory base 193 * @n_channels: number of available channels 194 * @channels: array of DMAC channels 195 * @channels_mask: bitfield of which DMA channels are managed by this driver 196 * @modules: bitmask of client modules in use 197 */ 198 struct rcar_dmac { 199 struct dma_device engine; 200 struct device *dev; 201 void __iomem *iomem; 202 203 unsigned int n_channels; 204 struct rcar_dmac_chan *channels; 205 u32 channels_mask; 206 207 DECLARE_BITMAP(modules, 256); 208 }; 209 210 #define to_rcar_dmac(d) container_of(d, struct rcar_dmac, engine) 211 212 /* 213 * struct rcar_dmac_of_data - This driver's OF data 214 * @chan_offset_base: DMAC channels base offset 215 * @chan_offset_stride: DMAC channels offset stride 216 */ 217 struct rcar_dmac_of_data { 218 u32 chan_offset_base; 219 u32 chan_offset_stride; 220 }; 221 222 /* ----------------------------------------------------------------------------- 223 * Registers 224 */ 225 226 #define RCAR_DMAISTA 0x0020 227 #define RCAR_DMASEC 0x0030 228 #define RCAR_DMAOR 0x0060 229 #define RCAR_DMAOR_PRI_FIXED (0 << 8) 230 #define RCAR_DMAOR_PRI_ROUND_ROBIN (3 << 8) 231 #define RCAR_DMAOR_AE (1 << 2) 232 #define RCAR_DMAOR_DME (1 << 0) 233 #define RCAR_DMACHCLR 0x0080 234 #define RCAR_DMADPSEC 0x00a0 235 236 #define RCAR_DMASAR 0x0000 237 #define RCAR_DMADAR 0x0004 238 #define RCAR_DMATCR 0x0008 239 #define RCAR_DMATCR_MASK 0x00ffffff 240 #define RCAR_DMATSR 0x0028 241 #define RCAR_DMACHCR 0x000c 242 #define RCAR_DMACHCR_CAE (1 << 31) 243 #define RCAR_DMACHCR_CAIE (1 << 30) 244 #define RCAR_DMACHCR_DPM_DISABLED (0 << 28) 245 #define RCAR_DMACHCR_DPM_ENABLED (1 << 28) 246 #define RCAR_DMACHCR_DPM_REPEAT (2 << 28) 247 #define RCAR_DMACHCR_DPM_INFINITE (3 << 28) 248 #define RCAR_DMACHCR_RPT_SAR (1 << 27) 249 #define RCAR_DMACHCR_RPT_DAR (1 << 26) 250 #define RCAR_DMACHCR_RPT_TCR (1 << 25) 251 #define RCAR_DMACHCR_DPB (1 << 22) 252 #define RCAR_DMACHCR_DSE (1 << 19) 253 #define RCAR_DMACHCR_DSIE (1 << 18) 254 #define RCAR_DMACHCR_TS_1B ((0 << 20) | (0 << 3)) 255 #define RCAR_DMACHCR_TS_2B ((0 << 20) | (1 << 3)) 256 #define RCAR_DMACHCR_TS_4B ((0 << 20) | (2 << 3)) 257 #define RCAR_DMACHCR_TS_16B ((0 << 20) | (3 << 3)) 258 #define RCAR_DMACHCR_TS_32B ((1 << 20) | (0 << 3)) 259 #define RCAR_DMACHCR_TS_64B ((1 << 20) | (1 << 3)) 260 #define RCAR_DMACHCR_TS_8B ((1 << 20) | (3 << 3)) 261 #define RCAR_DMACHCR_DM_FIXED (0 << 14) 262 #define RCAR_DMACHCR_DM_INC (1 << 14) 263 #define RCAR_DMACHCR_DM_DEC (2 << 14) 264 #define RCAR_DMACHCR_SM_FIXED (0 << 12) 265 #define RCAR_DMACHCR_SM_INC (1 << 12) 266 #define RCAR_DMACHCR_SM_DEC (2 << 12) 267 #define RCAR_DMACHCR_RS_AUTO (4 << 8) 268 #define RCAR_DMACHCR_RS_DMARS (8 << 8) 269 #define RCAR_DMACHCR_IE (1 << 2) 270 #define RCAR_DMACHCR_TE (1 << 1) 271 #define RCAR_DMACHCR_DE (1 << 0) 272 #define RCAR_DMATCRB 0x0018 273 #define RCAR_DMATSRB 0x0038 274 #define RCAR_DMACHCRB 0x001c 275 #define RCAR_DMACHCRB_DCNT(n) ((n) << 24) 276 #define RCAR_DMACHCRB_DPTR_MASK (0xff << 16) 277 #define RCAR_DMACHCRB_DPTR_SHIFT 16 278 #define RCAR_DMACHCRB_DRST (1 << 15) 279 #define RCAR_DMACHCRB_DTS (1 << 8) 280 #define RCAR_DMACHCRB_SLM_NORMAL (0 << 4) 281 #define RCAR_DMACHCRB_SLM_CLK(n) ((8 | (n)) << 4) 282 #define RCAR_DMACHCRB_PRI(n) ((n) << 0) 283 #define RCAR_DMARS 0x0040 284 #define RCAR_DMABUFCR 0x0048 285 #define RCAR_DMABUFCR_MBU(n) ((n) << 16) 286 #define RCAR_DMABUFCR_ULB(n) ((n) << 0) 287 #define RCAR_DMADPBASE 0x0050 288 #define RCAR_DMADPBASE_MASK 0xfffffff0 289 #define RCAR_DMADPBASE_SEL (1 << 0) 290 #define RCAR_DMADPCR 0x0054 291 #define RCAR_DMADPCR_DIPT(n) ((n) << 24) 292 #define RCAR_DMAFIXSAR 0x0010 293 #define RCAR_DMAFIXDAR 0x0014 294 #define RCAR_DMAFIXDPBASE 0x0060 295 296 /* Hardcode the MEMCPY transfer size to 4 bytes. */ 297 #define RCAR_DMAC_MEMCPY_XFER_SIZE 4 298 299 /* ----------------------------------------------------------------------------- 300 * Device access 301 */ 302 303 static void rcar_dmac_write(struct rcar_dmac *dmac, u32 reg, u32 data) 304 { 305 if (reg == RCAR_DMAOR) 306 writew(data, dmac->iomem + reg); 307 else 308 writel(data, dmac->iomem + reg); 309 } 310 311 static u32 rcar_dmac_read(struct rcar_dmac *dmac, u32 reg) 312 { 313 if (reg == RCAR_DMAOR) 314 return readw(dmac->iomem + reg); 315 else 316 return readl(dmac->iomem + reg); 317 } 318 319 static u32 rcar_dmac_chan_read(struct rcar_dmac_chan *chan, u32 reg) 320 { 321 if (reg == RCAR_DMARS) 322 return readw(chan->iomem + reg); 323 else 324 return readl(chan->iomem + reg); 325 } 326 327 static void rcar_dmac_chan_write(struct rcar_dmac_chan *chan, u32 reg, u32 data) 328 { 329 if (reg == RCAR_DMARS) 330 writew(data, chan->iomem + reg); 331 else 332 writel(data, chan->iomem + reg); 333 } 334 335 /* ----------------------------------------------------------------------------- 336 * Initialization and configuration 337 */ 338 339 static bool rcar_dmac_chan_is_busy(struct rcar_dmac_chan *chan) 340 { 341 u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 342 343 return !!(chcr & (RCAR_DMACHCR_DE | RCAR_DMACHCR_TE)); 344 } 345 346 static void rcar_dmac_chan_start_xfer(struct rcar_dmac_chan *chan) 347 { 348 struct rcar_dmac_desc *desc = chan->desc.running; 349 u32 chcr = desc->chcr; 350 351 WARN_ON_ONCE(rcar_dmac_chan_is_busy(chan)); 352 353 if (chan->mid_rid >= 0) 354 rcar_dmac_chan_write(chan, RCAR_DMARS, chan->mid_rid); 355 356 if (desc->hwdescs.use) { 357 struct rcar_dmac_xfer_chunk *chunk = 358 list_first_entry(&desc->chunks, 359 struct rcar_dmac_xfer_chunk, node); 360 361 dev_dbg(chan->chan.device->dev, 362 "chan%u: queue desc %p: %u@%pad\n", 363 chan->index, desc, desc->nchunks, &desc->hwdescs.dma); 364 365 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 366 rcar_dmac_chan_write(chan, RCAR_DMAFIXSAR, 367 chunk->src_addr >> 32); 368 rcar_dmac_chan_write(chan, RCAR_DMAFIXDAR, 369 chunk->dst_addr >> 32); 370 rcar_dmac_chan_write(chan, RCAR_DMAFIXDPBASE, 371 desc->hwdescs.dma >> 32); 372 #endif 373 rcar_dmac_chan_write(chan, RCAR_DMADPBASE, 374 (desc->hwdescs.dma & 0xfffffff0) | 375 RCAR_DMADPBASE_SEL); 376 rcar_dmac_chan_write(chan, RCAR_DMACHCRB, 377 RCAR_DMACHCRB_DCNT(desc->nchunks - 1) | 378 RCAR_DMACHCRB_DRST); 379 380 /* 381 * Errata: When descriptor memory is accessed through an IOMMU 382 * the DMADAR register isn't initialized automatically from the 383 * first descriptor at beginning of transfer by the DMAC like it 384 * should. Initialize it manually with the destination address 385 * of the first chunk. 386 */ 387 rcar_dmac_chan_write(chan, RCAR_DMADAR, 388 chunk->dst_addr & 0xffffffff); 389 390 /* 391 * Program the descriptor stage interrupt to occur after the end 392 * of the first stage. 393 */ 394 rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(1)); 395 396 chcr |= RCAR_DMACHCR_RPT_SAR | RCAR_DMACHCR_RPT_DAR 397 | RCAR_DMACHCR_RPT_TCR | RCAR_DMACHCR_DPB; 398 399 /* 400 * If the descriptor isn't cyclic enable normal descriptor mode 401 * and the transfer completion interrupt. 402 */ 403 if (!desc->cyclic) 404 chcr |= RCAR_DMACHCR_DPM_ENABLED | RCAR_DMACHCR_IE; 405 /* 406 * If the descriptor is cyclic and has a callback enable the 407 * descriptor stage interrupt in infinite repeat mode. 408 */ 409 else if (desc->async_tx.callback) 410 chcr |= RCAR_DMACHCR_DPM_INFINITE | RCAR_DMACHCR_DSIE; 411 /* 412 * Otherwise just select infinite repeat mode without any 413 * interrupt. 414 */ 415 else 416 chcr |= RCAR_DMACHCR_DPM_INFINITE; 417 } else { 418 struct rcar_dmac_xfer_chunk *chunk = desc->running; 419 420 dev_dbg(chan->chan.device->dev, 421 "chan%u: queue chunk %p: %u@%pad -> %pad\n", 422 chan->index, chunk, chunk->size, &chunk->src_addr, 423 &chunk->dst_addr); 424 425 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 426 rcar_dmac_chan_write(chan, RCAR_DMAFIXSAR, 427 chunk->src_addr >> 32); 428 rcar_dmac_chan_write(chan, RCAR_DMAFIXDAR, 429 chunk->dst_addr >> 32); 430 #endif 431 rcar_dmac_chan_write(chan, RCAR_DMASAR, 432 chunk->src_addr & 0xffffffff); 433 rcar_dmac_chan_write(chan, RCAR_DMADAR, 434 chunk->dst_addr & 0xffffffff); 435 rcar_dmac_chan_write(chan, RCAR_DMATCR, 436 chunk->size >> desc->xfer_shift); 437 438 chcr |= RCAR_DMACHCR_DPM_DISABLED | RCAR_DMACHCR_IE; 439 } 440 441 rcar_dmac_chan_write(chan, RCAR_DMACHCR, 442 chcr | RCAR_DMACHCR_DE | RCAR_DMACHCR_CAIE); 443 } 444 445 static int rcar_dmac_init(struct rcar_dmac *dmac) 446 { 447 u16 dmaor; 448 449 /* Clear all channels and enable the DMAC globally. */ 450 rcar_dmac_write(dmac, RCAR_DMACHCLR, dmac->channels_mask); 451 rcar_dmac_write(dmac, RCAR_DMAOR, 452 RCAR_DMAOR_PRI_FIXED | RCAR_DMAOR_DME); 453 454 dmaor = rcar_dmac_read(dmac, RCAR_DMAOR); 455 if ((dmaor & (RCAR_DMAOR_AE | RCAR_DMAOR_DME)) != RCAR_DMAOR_DME) { 456 dev_warn(dmac->dev, "DMAOR initialization failed.\n"); 457 return -EIO; 458 } 459 460 return 0; 461 } 462 463 /* ----------------------------------------------------------------------------- 464 * Descriptors submission 465 */ 466 467 static dma_cookie_t rcar_dmac_tx_submit(struct dma_async_tx_descriptor *tx) 468 { 469 struct rcar_dmac_chan *chan = to_rcar_dmac_chan(tx->chan); 470 struct rcar_dmac_desc *desc = to_rcar_dmac_desc(tx); 471 unsigned long flags; 472 dma_cookie_t cookie; 473 474 spin_lock_irqsave(&chan->lock, flags); 475 476 cookie = dma_cookie_assign(tx); 477 478 dev_dbg(chan->chan.device->dev, "chan%u: submit #%d@%p\n", 479 chan->index, tx->cookie, desc); 480 481 list_add_tail(&desc->node, &chan->desc.pending); 482 desc->running = list_first_entry(&desc->chunks, 483 struct rcar_dmac_xfer_chunk, node); 484 485 spin_unlock_irqrestore(&chan->lock, flags); 486 487 return cookie; 488 } 489 490 /* ----------------------------------------------------------------------------- 491 * Descriptors allocation and free 492 */ 493 494 /* 495 * rcar_dmac_desc_alloc - Allocate a page worth of DMA descriptors 496 * @chan: the DMA channel 497 * @gfp: allocation flags 498 */ 499 static int rcar_dmac_desc_alloc(struct rcar_dmac_chan *chan, gfp_t gfp) 500 { 501 struct rcar_dmac_desc_page *page; 502 unsigned long flags; 503 LIST_HEAD(list); 504 unsigned int i; 505 506 page = (void *)get_zeroed_page(gfp); 507 if (!page) 508 return -ENOMEM; 509 510 for (i = 0; i < RCAR_DMAC_DESCS_PER_PAGE; ++i) { 511 struct rcar_dmac_desc *desc = &page->descs[i]; 512 513 dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan); 514 desc->async_tx.tx_submit = rcar_dmac_tx_submit; 515 INIT_LIST_HEAD(&desc->chunks); 516 517 list_add_tail(&desc->node, &list); 518 } 519 520 spin_lock_irqsave(&chan->lock, flags); 521 list_splice_tail(&list, &chan->desc.free); 522 list_add_tail(&page->node, &chan->desc.pages); 523 spin_unlock_irqrestore(&chan->lock, flags); 524 525 return 0; 526 } 527 528 /* 529 * rcar_dmac_desc_put - Release a DMA transfer descriptor 530 * @chan: the DMA channel 531 * @desc: the descriptor 532 * 533 * Put the descriptor and its transfer chunk descriptors back in the channel's 534 * free descriptors lists. The descriptor's chunks list will be reinitialized to 535 * an empty list as a result. 536 * 537 * The descriptor must have been removed from the channel's lists before calling 538 * this function. 539 */ 540 static void rcar_dmac_desc_put(struct rcar_dmac_chan *chan, 541 struct rcar_dmac_desc *desc) 542 { 543 unsigned long flags; 544 545 spin_lock_irqsave(&chan->lock, flags); 546 list_splice_tail_init(&desc->chunks, &chan->desc.chunks_free); 547 list_add(&desc->node, &chan->desc.free); 548 spin_unlock_irqrestore(&chan->lock, flags); 549 } 550 551 static void rcar_dmac_desc_recycle_acked(struct rcar_dmac_chan *chan) 552 { 553 struct rcar_dmac_desc *desc, *_desc; 554 unsigned long flags; 555 LIST_HEAD(list); 556 557 /* 558 * We have to temporarily move all descriptors from the wait list to a 559 * local list as iterating over the wait list, even with 560 * list_for_each_entry_safe, isn't safe if we release the channel lock 561 * around the rcar_dmac_desc_put() call. 562 */ 563 spin_lock_irqsave(&chan->lock, flags); 564 list_splice_init(&chan->desc.wait, &list); 565 spin_unlock_irqrestore(&chan->lock, flags); 566 567 list_for_each_entry_safe(desc, _desc, &list, node) { 568 if (async_tx_test_ack(&desc->async_tx)) { 569 list_del(&desc->node); 570 rcar_dmac_desc_put(chan, desc); 571 } 572 } 573 574 if (list_empty(&list)) 575 return; 576 577 /* Put the remaining descriptors back in the wait list. */ 578 spin_lock_irqsave(&chan->lock, flags); 579 list_splice(&list, &chan->desc.wait); 580 spin_unlock_irqrestore(&chan->lock, flags); 581 } 582 583 /* 584 * rcar_dmac_desc_get - Allocate a descriptor for a DMA transfer 585 * @chan: the DMA channel 586 * 587 * Locking: This function must be called in a non-atomic context. 588 * 589 * Return: A pointer to the allocated descriptor or NULL if no descriptor can 590 * be allocated. 591 */ 592 static struct rcar_dmac_desc *rcar_dmac_desc_get(struct rcar_dmac_chan *chan) 593 { 594 struct rcar_dmac_desc *desc; 595 unsigned long flags; 596 int ret; 597 598 /* Recycle acked descriptors before attempting allocation. */ 599 rcar_dmac_desc_recycle_acked(chan); 600 601 spin_lock_irqsave(&chan->lock, flags); 602 603 while (list_empty(&chan->desc.free)) { 604 /* 605 * No free descriptors, allocate a page worth of them and try 606 * again, as someone else could race us to get the newly 607 * allocated descriptors. If the allocation fails return an 608 * error. 609 */ 610 spin_unlock_irqrestore(&chan->lock, flags); 611 ret = rcar_dmac_desc_alloc(chan, GFP_NOWAIT); 612 if (ret < 0) 613 return NULL; 614 spin_lock_irqsave(&chan->lock, flags); 615 } 616 617 desc = list_first_entry(&chan->desc.free, struct rcar_dmac_desc, node); 618 list_del(&desc->node); 619 620 spin_unlock_irqrestore(&chan->lock, flags); 621 622 return desc; 623 } 624 625 /* 626 * rcar_dmac_xfer_chunk_alloc - Allocate a page worth of transfer chunks 627 * @chan: the DMA channel 628 * @gfp: allocation flags 629 */ 630 static int rcar_dmac_xfer_chunk_alloc(struct rcar_dmac_chan *chan, gfp_t gfp) 631 { 632 struct rcar_dmac_desc_page *page; 633 unsigned long flags; 634 LIST_HEAD(list); 635 unsigned int i; 636 637 page = (void *)get_zeroed_page(gfp); 638 if (!page) 639 return -ENOMEM; 640 641 for (i = 0; i < RCAR_DMAC_XFER_CHUNKS_PER_PAGE; ++i) { 642 struct rcar_dmac_xfer_chunk *chunk = &page->chunks[i]; 643 644 list_add_tail(&chunk->node, &list); 645 } 646 647 spin_lock_irqsave(&chan->lock, flags); 648 list_splice_tail(&list, &chan->desc.chunks_free); 649 list_add_tail(&page->node, &chan->desc.pages); 650 spin_unlock_irqrestore(&chan->lock, flags); 651 652 return 0; 653 } 654 655 /* 656 * rcar_dmac_xfer_chunk_get - Allocate a transfer chunk for a DMA transfer 657 * @chan: the DMA channel 658 * 659 * Locking: This function must be called in a non-atomic context. 660 * 661 * Return: A pointer to the allocated transfer chunk descriptor or NULL if no 662 * descriptor can be allocated. 663 */ 664 static struct rcar_dmac_xfer_chunk * 665 rcar_dmac_xfer_chunk_get(struct rcar_dmac_chan *chan) 666 { 667 struct rcar_dmac_xfer_chunk *chunk; 668 unsigned long flags; 669 int ret; 670 671 spin_lock_irqsave(&chan->lock, flags); 672 673 while (list_empty(&chan->desc.chunks_free)) { 674 /* 675 * No free descriptors, allocate a page worth of them and try 676 * again, as someone else could race us to get the newly 677 * allocated descriptors. If the allocation fails return an 678 * error. 679 */ 680 spin_unlock_irqrestore(&chan->lock, flags); 681 ret = rcar_dmac_xfer_chunk_alloc(chan, GFP_NOWAIT); 682 if (ret < 0) 683 return NULL; 684 spin_lock_irqsave(&chan->lock, flags); 685 } 686 687 chunk = list_first_entry(&chan->desc.chunks_free, 688 struct rcar_dmac_xfer_chunk, node); 689 list_del(&chunk->node); 690 691 spin_unlock_irqrestore(&chan->lock, flags); 692 693 return chunk; 694 } 695 696 static void rcar_dmac_realloc_hwdesc(struct rcar_dmac_chan *chan, 697 struct rcar_dmac_desc *desc, size_t size) 698 { 699 /* 700 * dma_alloc_coherent() allocates memory in page size increments. To 701 * avoid reallocating the hardware descriptors when the allocated size 702 * wouldn't change align the requested size to a multiple of the page 703 * size. 704 */ 705 size = PAGE_ALIGN(size); 706 707 if (desc->hwdescs.size == size) 708 return; 709 710 if (desc->hwdescs.mem) { 711 dma_free_coherent(chan->chan.device->dev, desc->hwdescs.size, 712 desc->hwdescs.mem, desc->hwdescs.dma); 713 desc->hwdescs.mem = NULL; 714 desc->hwdescs.size = 0; 715 } 716 717 if (!size) 718 return; 719 720 desc->hwdescs.mem = dma_alloc_coherent(chan->chan.device->dev, size, 721 &desc->hwdescs.dma, GFP_NOWAIT); 722 if (!desc->hwdescs.mem) 723 return; 724 725 desc->hwdescs.size = size; 726 } 727 728 static int rcar_dmac_fill_hwdesc(struct rcar_dmac_chan *chan, 729 struct rcar_dmac_desc *desc) 730 { 731 struct rcar_dmac_xfer_chunk *chunk; 732 struct rcar_dmac_hw_desc *hwdesc; 733 734 rcar_dmac_realloc_hwdesc(chan, desc, desc->nchunks * sizeof(*hwdesc)); 735 736 hwdesc = desc->hwdescs.mem; 737 if (!hwdesc) 738 return -ENOMEM; 739 740 list_for_each_entry(chunk, &desc->chunks, node) { 741 hwdesc->sar = chunk->src_addr; 742 hwdesc->dar = chunk->dst_addr; 743 hwdesc->tcr = chunk->size >> desc->xfer_shift; 744 hwdesc++; 745 } 746 747 return 0; 748 } 749 750 /* ----------------------------------------------------------------------------- 751 * Stop and reset 752 */ 753 static void rcar_dmac_chcr_de_barrier(struct rcar_dmac_chan *chan) 754 { 755 u32 chcr; 756 unsigned int i; 757 758 /* 759 * Ensure that the setting of the DE bit is actually 0 after 760 * clearing it. 761 */ 762 for (i = 0; i < 1024; i++) { 763 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 764 if (!(chcr & RCAR_DMACHCR_DE)) 765 return; 766 udelay(1); 767 } 768 769 dev_err(chan->chan.device->dev, "CHCR DE check error\n"); 770 } 771 772 static void rcar_dmac_clear_chcr_de(struct rcar_dmac_chan *chan) 773 { 774 u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 775 776 /* set DE=0 and flush remaining data */ 777 rcar_dmac_chan_write(chan, RCAR_DMACHCR, (chcr & ~RCAR_DMACHCR_DE)); 778 779 /* make sure all remaining data was flushed */ 780 rcar_dmac_chcr_de_barrier(chan); 781 } 782 783 static void rcar_dmac_chan_halt(struct rcar_dmac_chan *chan) 784 { 785 u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 786 787 chcr &= ~(RCAR_DMACHCR_DSE | RCAR_DMACHCR_DSIE | RCAR_DMACHCR_IE | 788 RCAR_DMACHCR_TE | RCAR_DMACHCR_DE | 789 RCAR_DMACHCR_CAE | RCAR_DMACHCR_CAIE); 790 rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr); 791 rcar_dmac_chcr_de_barrier(chan); 792 } 793 794 static void rcar_dmac_chan_reinit(struct rcar_dmac_chan *chan) 795 { 796 struct rcar_dmac_desc *desc, *_desc; 797 unsigned long flags; 798 LIST_HEAD(descs); 799 800 spin_lock_irqsave(&chan->lock, flags); 801 802 /* Move all non-free descriptors to the local lists. */ 803 list_splice_init(&chan->desc.pending, &descs); 804 list_splice_init(&chan->desc.active, &descs); 805 list_splice_init(&chan->desc.done, &descs); 806 list_splice_init(&chan->desc.wait, &descs); 807 808 chan->desc.running = NULL; 809 810 spin_unlock_irqrestore(&chan->lock, flags); 811 812 list_for_each_entry_safe(desc, _desc, &descs, node) { 813 list_del(&desc->node); 814 rcar_dmac_desc_put(chan, desc); 815 } 816 } 817 818 static void rcar_dmac_stop_all_chan(struct rcar_dmac *dmac) 819 { 820 unsigned int i; 821 822 /* Stop all channels. */ 823 for (i = 0; i < dmac->n_channels; ++i) { 824 struct rcar_dmac_chan *chan = &dmac->channels[i]; 825 826 if (!(dmac->channels_mask & BIT(i))) 827 continue; 828 829 /* Stop and reinitialize the channel. */ 830 spin_lock_irq(&chan->lock); 831 rcar_dmac_chan_halt(chan); 832 spin_unlock_irq(&chan->lock); 833 } 834 } 835 836 static int rcar_dmac_chan_pause(struct dma_chan *chan) 837 { 838 unsigned long flags; 839 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 840 841 spin_lock_irqsave(&rchan->lock, flags); 842 rcar_dmac_clear_chcr_de(rchan); 843 spin_unlock_irqrestore(&rchan->lock, flags); 844 845 return 0; 846 } 847 848 /* ----------------------------------------------------------------------------- 849 * Descriptors preparation 850 */ 851 852 static void rcar_dmac_chan_configure_desc(struct rcar_dmac_chan *chan, 853 struct rcar_dmac_desc *desc) 854 { 855 static const u32 chcr_ts[] = { 856 RCAR_DMACHCR_TS_1B, RCAR_DMACHCR_TS_2B, 857 RCAR_DMACHCR_TS_4B, RCAR_DMACHCR_TS_8B, 858 RCAR_DMACHCR_TS_16B, RCAR_DMACHCR_TS_32B, 859 RCAR_DMACHCR_TS_64B, 860 }; 861 862 unsigned int xfer_size; 863 u32 chcr; 864 865 switch (desc->direction) { 866 case DMA_DEV_TO_MEM: 867 chcr = RCAR_DMACHCR_DM_INC | RCAR_DMACHCR_SM_FIXED 868 | RCAR_DMACHCR_RS_DMARS; 869 xfer_size = chan->src.xfer_size; 870 break; 871 872 case DMA_MEM_TO_DEV: 873 chcr = RCAR_DMACHCR_DM_FIXED | RCAR_DMACHCR_SM_INC 874 | RCAR_DMACHCR_RS_DMARS; 875 xfer_size = chan->dst.xfer_size; 876 break; 877 878 case DMA_MEM_TO_MEM: 879 default: 880 chcr = RCAR_DMACHCR_DM_INC | RCAR_DMACHCR_SM_INC 881 | RCAR_DMACHCR_RS_AUTO; 882 xfer_size = RCAR_DMAC_MEMCPY_XFER_SIZE; 883 break; 884 } 885 886 desc->xfer_shift = ilog2(xfer_size); 887 desc->chcr = chcr | chcr_ts[desc->xfer_shift]; 888 } 889 890 /* 891 * rcar_dmac_chan_prep_sg - prepare transfer descriptors from an SG list 892 * 893 * Common routine for public (MEMCPY) and slave DMA. The MEMCPY case is also 894 * converted to scatter-gather to guarantee consistent locking and a correct 895 * list manipulation. For slave DMA direction carries the usual meaning, and, 896 * logically, the SG list is RAM and the addr variable contains slave address, 897 * e.g., the FIFO I/O register. For MEMCPY direction equals DMA_MEM_TO_MEM 898 * and the SG list contains only one element and points at the source buffer. 899 */ 900 static struct dma_async_tx_descriptor * 901 rcar_dmac_chan_prep_sg(struct rcar_dmac_chan *chan, struct scatterlist *sgl, 902 unsigned int sg_len, dma_addr_t dev_addr, 903 enum dma_transfer_direction dir, unsigned long dma_flags, 904 bool cyclic) 905 { 906 struct rcar_dmac_xfer_chunk *chunk; 907 struct rcar_dmac_desc *desc; 908 struct scatterlist *sg; 909 unsigned int nchunks = 0; 910 unsigned int max_chunk_size; 911 unsigned int full_size = 0; 912 bool cross_boundary = false; 913 unsigned int i; 914 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 915 u32 high_dev_addr; 916 u32 high_mem_addr; 917 #endif 918 919 desc = rcar_dmac_desc_get(chan); 920 if (!desc) 921 return NULL; 922 923 desc->async_tx.flags = dma_flags; 924 desc->async_tx.cookie = -EBUSY; 925 926 desc->cyclic = cyclic; 927 desc->direction = dir; 928 929 rcar_dmac_chan_configure_desc(chan, desc); 930 931 max_chunk_size = RCAR_DMATCR_MASK << desc->xfer_shift; 932 933 /* 934 * Allocate and fill the transfer chunk descriptors. We own the only 935 * reference to the DMA descriptor, there's no need for locking. 936 */ 937 for_each_sg(sgl, sg, sg_len, i) { 938 dma_addr_t mem_addr = sg_dma_address(sg); 939 unsigned int len = sg_dma_len(sg); 940 941 full_size += len; 942 943 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 944 if (i == 0) { 945 high_dev_addr = dev_addr >> 32; 946 high_mem_addr = mem_addr >> 32; 947 } 948 949 if ((dev_addr >> 32 != high_dev_addr) || 950 (mem_addr >> 32 != high_mem_addr)) 951 cross_boundary = true; 952 #endif 953 while (len) { 954 unsigned int size = min(len, max_chunk_size); 955 956 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 957 /* 958 * Prevent individual transfers from crossing 4GB 959 * boundaries. 960 */ 961 if (dev_addr >> 32 != (dev_addr + size - 1) >> 32) { 962 size = ALIGN(dev_addr, 1ULL << 32) - dev_addr; 963 cross_boundary = true; 964 } 965 if (mem_addr >> 32 != (mem_addr + size - 1) >> 32) { 966 size = ALIGN(mem_addr, 1ULL << 32) - mem_addr; 967 cross_boundary = true; 968 } 969 #endif 970 971 chunk = rcar_dmac_xfer_chunk_get(chan); 972 if (!chunk) { 973 rcar_dmac_desc_put(chan, desc); 974 return NULL; 975 } 976 977 if (dir == DMA_DEV_TO_MEM) { 978 chunk->src_addr = dev_addr; 979 chunk->dst_addr = mem_addr; 980 } else { 981 chunk->src_addr = mem_addr; 982 chunk->dst_addr = dev_addr; 983 } 984 985 chunk->size = size; 986 987 dev_dbg(chan->chan.device->dev, 988 "chan%u: chunk %p/%p sgl %u@%p, %u/%u %pad -> %pad\n", 989 chan->index, chunk, desc, i, sg, size, len, 990 &chunk->src_addr, &chunk->dst_addr); 991 992 mem_addr += size; 993 if (dir == DMA_MEM_TO_MEM) 994 dev_addr += size; 995 996 len -= size; 997 998 list_add_tail(&chunk->node, &desc->chunks); 999 nchunks++; 1000 } 1001 } 1002 1003 desc->nchunks = nchunks; 1004 desc->size = full_size; 1005 1006 /* 1007 * Use hardware descriptor lists if possible when more than one chunk 1008 * needs to be transferred (otherwise they don't make much sense). 1009 * 1010 * Source/Destination address should be located in same 4GiB region 1011 * in the 40bit address space when it uses Hardware descriptor, 1012 * and cross_boundary is checking it. 1013 */ 1014 desc->hwdescs.use = !cross_boundary && nchunks > 1; 1015 if (desc->hwdescs.use) { 1016 if (rcar_dmac_fill_hwdesc(chan, desc) < 0) 1017 desc->hwdescs.use = false; 1018 } 1019 1020 return &desc->async_tx; 1021 } 1022 1023 /* ----------------------------------------------------------------------------- 1024 * DMA engine operations 1025 */ 1026 1027 static int rcar_dmac_alloc_chan_resources(struct dma_chan *chan) 1028 { 1029 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1030 int ret; 1031 1032 INIT_LIST_HEAD(&rchan->desc.chunks_free); 1033 INIT_LIST_HEAD(&rchan->desc.pages); 1034 1035 /* Preallocate descriptors. */ 1036 ret = rcar_dmac_xfer_chunk_alloc(rchan, GFP_KERNEL); 1037 if (ret < 0) 1038 return -ENOMEM; 1039 1040 ret = rcar_dmac_desc_alloc(rchan, GFP_KERNEL); 1041 if (ret < 0) 1042 return -ENOMEM; 1043 1044 return pm_runtime_get_sync(chan->device->dev); 1045 } 1046 1047 static void rcar_dmac_free_chan_resources(struct dma_chan *chan) 1048 { 1049 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1050 struct rcar_dmac *dmac = to_rcar_dmac(chan->device); 1051 struct rcar_dmac_chan_map *map = &rchan->map; 1052 struct rcar_dmac_desc_page *page, *_page; 1053 struct rcar_dmac_desc *desc; 1054 LIST_HEAD(list); 1055 1056 /* Protect against ISR */ 1057 spin_lock_irq(&rchan->lock); 1058 rcar_dmac_chan_halt(rchan); 1059 spin_unlock_irq(&rchan->lock); 1060 1061 /* 1062 * Now no new interrupts will occur, but one might already be 1063 * running. Wait for it to finish before freeing resources. 1064 */ 1065 synchronize_irq(rchan->irq); 1066 1067 if (rchan->mid_rid >= 0) { 1068 /* The caller is holding dma_list_mutex */ 1069 clear_bit(rchan->mid_rid, dmac->modules); 1070 rchan->mid_rid = -EINVAL; 1071 } 1072 1073 list_splice_init(&rchan->desc.free, &list); 1074 list_splice_init(&rchan->desc.pending, &list); 1075 list_splice_init(&rchan->desc.active, &list); 1076 list_splice_init(&rchan->desc.done, &list); 1077 list_splice_init(&rchan->desc.wait, &list); 1078 1079 rchan->desc.running = NULL; 1080 1081 list_for_each_entry(desc, &list, node) 1082 rcar_dmac_realloc_hwdesc(rchan, desc, 0); 1083 1084 list_for_each_entry_safe(page, _page, &rchan->desc.pages, node) { 1085 list_del(&page->node); 1086 free_page((unsigned long)page); 1087 } 1088 1089 /* Remove slave mapping if present. */ 1090 if (map->slave.xfer_size) { 1091 dma_unmap_resource(chan->device->dev, map->addr, 1092 map->slave.xfer_size, map->dir, 0); 1093 map->slave.xfer_size = 0; 1094 } 1095 1096 pm_runtime_put(chan->device->dev); 1097 } 1098 1099 static struct dma_async_tx_descriptor * 1100 rcar_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest, 1101 dma_addr_t dma_src, size_t len, unsigned long flags) 1102 { 1103 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1104 struct scatterlist sgl; 1105 1106 if (!len) 1107 return NULL; 1108 1109 sg_init_table(&sgl, 1); 1110 sg_set_page(&sgl, pfn_to_page(PFN_DOWN(dma_src)), len, 1111 offset_in_page(dma_src)); 1112 sg_dma_address(&sgl) = dma_src; 1113 sg_dma_len(&sgl) = len; 1114 1115 return rcar_dmac_chan_prep_sg(rchan, &sgl, 1, dma_dest, 1116 DMA_MEM_TO_MEM, flags, false); 1117 } 1118 1119 static int rcar_dmac_map_slave_addr(struct dma_chan *chan, 1120 enum dma_transfer_direction dir) 1121 { 1122 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1123 struct rcar_dmac_chan_map *map = &rchan->map; 1124 phys_addr_t dev_addr; 1125 size_t dev_size; 1126 enum dma_data_direction dev_dir; 1127 1128 if (dir == DMA_DEV_TO_MEM) { 1129 dev_addr = rchan->src.slave_addr; 1130 dev_size = rchan->src.xfer_size; 1131 dev_dir = DMA_TO_DEVICE; 1132 } else { 1133 dev_addr = rchan->dst.slave_addr; 1134 dev_size = rchan->dst.xfer_size; 1135 dev_dir = DMA_FROM_DEVICE; 1136 } 1137 1138 /* Reuse current map if possible. */ 1139 if (dev_addr == map->slave.slave_addr && 1140 dev_size == map->slave.xfer_size && 1141 dev_dir == map->dir) 1142 return 0; 1143 1144 /* Remove old mapping if present. */ 1145 if (map->slave.xfer_size) 1146 dma_unmap_resource(chan->device->dev, map->addr, 1147 map->slave.xfer_size, map->dir, 0); 1148 map->slave.xfer_size = 0; 1149 1150 /* Create new slave address map. */ 1151 map->addr = dma_map_resource(chan->device->dev, dev_addr, dev_size, 1152 dev_dir, 0); 1153 1154 if (dma_mapping_error(chan->device->dev, map->addr)) { 1155 dev_err(chan->device->dev, 1156 "chan%u: failed to map %zx@%pap", rchan->index, 1157 dev_size, &dev_addr); 1158 return -EIO; 1159 } 1160 1161 dev_dbg(chan->device->dev, "chan%u: map %zx@%pap to %pad dir: %s\n", 1162 rchan->index, dev_size, &dev_addr, &map->addr, 1163 dev_dir == DMA_TO_DEVICE ? "DMA_TO_DEVICE" : "DMA_FROM_DEVICE"); 1164 1165 map->slave.slave_addr = dev_addr; 1166 map->slave.xfer_size = dev_size; 1167 map->dir = dev_dir; 1168 1169 return 0; 1170 } 1171 1172 static struct dma_async_tx_descriptor * 1173 rcar_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 1174 unsigned int sg_len, enum dma_transfer_direction dir, 1175 unsigned long flags, void *context) 1176 { 1177 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1178 1179 /* Someone calling slave DMA on a generic channel? */ 1180 if (rchan->mid_rid < 0 || !sg_len || !sg_dma_len(sgl)) { 1181 dev_warn(chan->device->dev, 1182 "%s: bad parameter: len=%d, id=%d\n", 1183 __func__, sg_len, rchan->mid_rid); 1184 return NULL; 1185 } 1186 1187 if (rcar_dmac_map_slave_addr(chan, dir)) 1188 return NULL; 1189 1190 return rcar_dmac_chan_prep_sg(rchan, sgl, sg_len, rchan->map.addr, 1191 dir, flags, false); 1192 } 1193 1194 #define RCAR_DMAC_MAX_SG_LEN 32 1195 1196 static struct dma_async_tx_descriptor * 1197 rcar_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, 1198 size_t buf_len, size_t period_len, 1199 enum dma_transfer_direction dir, unsigned long flags) 1200 { 1201 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1202 struct dma_async_tx_descriptor *desc; 1203 struct scatterlist *sgl; 1204 unsigned int sg_len; 1205 unsigned int i; 1206 1207 /* Someone calling slave DMA on a generic channel? */ 1208 if (rchan->mid_rid < 0 || buf_len < period_len) { 1209 dev_warn(chan->device->dev, 1210 "%s: bad parameter: buf_len=%zu, period_len=%zu, id=%d\n", 1211 __func__, buf_len, period_len, rchan->mid_rid); 1212 return NULL; 1213 } 1214 1215 if (rcar_dmac_map_slave_addr(chan, dir)) 1216 return NULL; 1217 1218 sg_len = buf_len / period_len; 1219 if (sg_len > RCAR_DMAC_MAX_SG_LEN) { 1220 dev_err(chan->device->dev, 1221 "chan%u: sg length %d exceeds limit %d", 1222 rchan->index, sg_len, RCAR_DMAC_MAX_SG_LEN); 1223 return NULL; 1224 } 1225 1226 /* 1227 * Allocate the sg list dynamically as it would consume too much stack 1228 * space. 1229 */ 1230 sgl = kmalloc_array(sg_len, sizeof(*sgl), GFP_NOWAIT); 1231 if (!sgl) 1232 return NULL; 1233 1234 sg_init_table(sgl, sg_len); 1235 1236 for (i = 0; i < sg_len; ++i) { 1237 dma_addr_t src = buf_addr + (period_len * i); 1238 1239 sg_set_page(&sgl[i], pfn_to_page(PFN_DOWN(src)), period_len, 1240 offset_in_page(src)); 1241 sg_dma_address(&sgl[i]) = src; 1242 sg_dma_len(&sgl[i]) = period_len; 1243 } 1244 1245 desc = rcar_dmac_chan_prep_sg(rchan, sgl, sg_len, rchan->map.addr, 1246 dir, flags, true); 1247 1248 kfree(sgl); 1249 return desc; 1250 } 1251 1252 static int rcar_dmac_device_config(struct dma_chan *chan, 1253 struct dma_slave_config *cfg) 1254 { 1255 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1256 1257 /* 1258 * We could lock this, but you shouldn't be configuring the 1259 * channel, while using it... 1260 */ 1261 rchan->src.slave_addr = cfg->src_addr; 1262 rchan->dst.slave_addr = cfg->dst_addr; 1263 rchan->src.xfer_size = cfg->src_addr_width; 1264 rchan->dst.xfer_size = cfg->dst_addr_width; 1265 1266 return 0; 1267 } 1268 1269 static int rcar_dmac_chan_terminate_all(struct dma_chan *chan) 1270 { 1271 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1272 unsigned long flags; 1273 1274 spin_lock_irqsave(&rchan->lock, flags); 1275 rcar_dmac_chan_halt(rchan); 1276 spin_unlock_irqrestore(&rchan->lock, flags); 1277 1278 /* 1279 * FIXME: No new interrupt can occur now, but the IRQ thread might still 1280 * be running. 1281 */ 1282 1283 rcar_dmac_chan_reinit(rchan); 1284 1285 return 0; 1286 } 1287 1288 static unsigned int rcar_dmac_chan_get_residue(struct rcar_dmac_chan *chan, 1289 dma_cookie_t cookie) 1290 { 1291 struct rcar_dmac_desc *desc = chan->desc.running; 1292 struct rcar_dmac_xfer_chunk *running = NULL; 1293 struct rcar_dmac_xfer_chunk *chunk; 1294 enum dma_status status; 1295 unsigned int residue = 0; 1296 unsigned int dptr = 0; 1297 unsigned int chcrb; 1298 unsigned int tcrb; 1299 unsigned int i; 1300 1301 if (!desc) 1302 return 0; 1303 1304 /* 1305 * If the cookie corresponds to a descriptor that has been completed 1306 * there is no residue. The same check has already been performed by the 1307 * caller but without holding the channel lock, so the descriptor could 1308 * now be complete. 1309 */ 1310 status = dma_cookie_status(&chan->chan, cookie, NULL); 1311 if (status == DMA_COMPLETE) 1312 return 0; 1313 1314 /* 1315 * If the cookie doesn't correspond to the currently running transfer 1316 * then the descriptor hasn't been processed yet, and the residue is 1317 * equal to the full descriptor size. 1318 * Also, a client driver is possible to call this function before 1319 * rcar_dmac_isr_channel_thread() runs. In this case, the "desc.running" 1320 * will be the next descriptor, and the done list will appear. So, if 1321 * the argument cookie matches the done list's cookie, we can assume 1322 * the residue is zero. 1323 */ 1324 if (cookie != desc->async_tx.cookie) { 1325 list_for_each_entry(desc, &chan->desc.done, node) { 1326 if (cookie == desc->async_tx.cookie) 1327 return 0; 1328 } 1329 list_for_each_entry(desc, &chan->desc.pending, node) { 1330 if (cookie == desc->async_tx.cookie) 1331 return desc->size; 1332 } 1333 list_for_each_entry(desc, &chan->desc.active, node) { 1334 if (cookie == desc->async_tx.cookie) 1335 return desc->size; 1336 } 1337 1338 /* 1339 * No descriptor found for the cookie, there's thus no residue. 1340 * This shouldn't happen if the calling driver passes a correct 1341 * cookie value. 1342 */ 1343 WARN(1, "No descriptor for cookie!"); 1344 return 0; 1345 } 1346 1347 /* 1348 * We need to read two registers. 1349 * Make sure the control register does not skip to next chunk 1350 * while reading the counter. 1351 * Trying it 3 times should be enough: Initial read, retry, retry 1352 * for the paranoid. 1353 */ 1354 for (i = 0; i < 3; i++) { 1355 chcrb = rcar_dmac_chan_read(chan, RCAR_DMACHCRB) & 1356 RCAR_DMACHCRB_DPTR_MASK; 1357 tcrb = rcar_dmac_chan_read(chan, RCAR_DMATCRB); 1358 /* Still the same? */ 1359 if (chcrb == (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) & 1360 RCAR_DMACHCRB_DPTR_MASK)) 1361 break; 1362 } 1363 WARN_ONCE(i >= 3, "residue might be not continuous!"); 1364 1365 /* 1366 * In descriptor mode the descriptor running pointer is not maintained 1367 * by the interrupt handler, find the running descriptor from the 1368 * descriptor pointer field in the CHCRB register. In non-descriptor 1369 * mode just use the running descriptor pointer. 1370 */ 1371 if (desc->hwdescs.use) { 1372 dptr = chcrb >> RCAR_DMACHCRB_DPTR_SHIFT; 1373 if (dptr == 0) 1374 dptr = desc->nchunks; 1375 dptr--; 1376 WARN_ON(dptr >= desc->nchunks); 1377 } else { 1378 running = desc->running; 1379 } 1380 1381 /* Compute the size of all chunks still to be transferred. */ 1382 list_for_each_entry_reverse(chunk, &desc->chunks, node) { 1383 if (chunk == running || ++dptr == desc->nchunks) 1384 break; 1385 1386 residue += chunk->size; 1387 } 1388 1389 /* Add the residue for the current chunk. */ 1390 residue += tcrb << desc->xfer_shift; 1391 1392 return residue; 1393 } 1394 1395 static enum dma_status rcar_dmac_tx_status(struct dma_chan *chan, 1396 dma_cookie_t cookie, 1397 struct dma_tx_state *txstate) 1398 { 1399 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1400 enum dma_status status; 1401 unsigned long flags; 1402 unsigned int residue; 1403 bool cyclic; 1404 1405 status = dma_cookie_status(chan, cookie, txstate); 1406 if (status == DMA_COMPLETE || !txstate) 1407 return status; 1408 1409 spin_lock_irqsave(&rchan->lock, flags); 1410 residue = rcar_dmac_chan_get_residue(rchan, cookie); 1411 cyclic = rchan->desc.running ? rchan->desc.running->cyclic : false; 1412 spin_unlock_irqrestore(&rchan->lock, flags); 1413 1414 /* if there's no residue, the cookie is complete */ 1415 if (!residue && !cyclic) 1416 return DMA_COMPLETE; 1417 1418 dma_set_residue(txstate, residue); 1419 1420 return status; 1421 } 1422 1423 static void rcar_dmac_issue_pending(struct dma_chan *chan) 1424 { 1425 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1426 unsigned long flags; 1427 1428 spin_lock_irqsave(&rchan->lock, flags); 1429 1430 if (list_empty(&rchan->desc.pending)) 1431 goto done; 1432 1433 /* Append the pending list to the active list. */ 1434 list_splice_tail_init(&rchan->desc.pending, &rchan->desc.active); 1435 1436 /* 1437 * If no transfer is running pick the first descriptor from the active 1438 * list and start the transfer. 1439 */ 1440 if (!rchan->desc.running) { 1441 struct rcar_dmac_desc *desc; 1442 1443 desc = list_first_entry(&rchan->desc.active, 1444 struct rcar_dmac_desc, node); 1445 rchan->desc.running = desc; 1446 1447 rcar_dmac_chan_start_xfer(rchan); 1448 } 1449 1450 done: 1451 spin_unlock_irqrestore(&rchan->lock, flags); 1452 } 1453 1454 static void rcar_dmac_device_synchronize(struct dma_chan *chan) 1455 { 1456 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1457 1458 synchronize_irq(rchan->irq); 1459 } 1460 1461 /* ----------------------------------------------------------------------------- 1462 * IRQ handling 1463 */ 1464 1465 static irqreturn_t rcar_dmac_isr_desc_stage_end(struct rcar_dmac_chan *chan) 1466 { 1467 struct rcar_dmac_desc *desc = chan->desc.running; 1468 unsigned int stage; 1469 1470 if (WARN_ON(!desc || !desc->cyclic)) { 1471 /* 1472 * This should never happen, there should always be a running 1473 * cyclic descriptor when a descriptor stage end interrupt is 1474 * triggered. Warn and return. 1475 */ 1476 return IRQ_NONE; 1477 } 1478 1479 /* Program the interrupt pointer to the next stage. */ 1480 stage = (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) & 1481 RCAR_DMACHCRB_DPTR_MASK) >> RCAR_DMACHCRB_DPTR_SHIFT; 1482 rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(stage)); 1483 1484 return IRQ_WAKE_THREAD; 1485 } 1486 1487 static irqreturn_t rcar_dmac_isr_transfer_end(struct rcar_dmac_chan *chan) 1488 { 1489 struct rcar_dmac_desc *desc = chan->desc.running; 1490 irqreturn_t ret = IRQ_WAKE_THREAD; 1491 1492 if (WARN_ON_ONCE(!desc)) { 1493 /* 1494 * This should never happen, there should always be a running 1495 * descriptor when a transfer end interrupt is triggered. Warn 1496 * and return. 1497 */ 1498 return IRQ_NONE; 1499 } 1500 1501 /* 1502 * The transfer end interrupt isn't generated for each chunk when using 1503 * descriptor mode. Only update the running chunk pointer in 1504 * non-descriptor mode. 1505 */ 1506 if (!desc->hwdescs.use) { 1507 /* 1508 * If we haven't completed the last transfer chunk simply move 1509 * to the next one. Only wake the IRQ thread if the transfer is 1510 * cyclic. 1511 */ 1512 if (!list_is_last(&desc->running->node, &desc->chunks)) { 1513 desc->running = list_next_entry(desc->running, node); 1514 if (!desc->cyclic) 1515 ret = IRQ_HANDLED; 1516 goto done; 1517 } 1518 1519 /* 1520 * We've completed the last transfer chunk. If the transfer is 1521 * cyclic, move back to the first one. 1522 */ 1523 if (desc->cyclic) { 1524 desc->running = 1525 list_first_entry(&desc->chunks, 1526 struct rcar_dmac_xfer_chunk, 1527 node); 1528 goto done; 1529 } 1530 } 1531 1532 /* The descriptor is complete, move it to the done list. */ 1533 list_move_tail(&desc->node, &chan->desc.done); 1534 1535 /* Queue the next descriptor, if any. */ 1536 if (!list_empty(&chan->desc.active)) 1537 chan->desc.running = list_first_entry(&chan->desc.active, 1538 struct rcar_dmac_desc, 1539 node); 1540 else 1541 chan->desc.running = NULL; 1542 1543 done: 1544 if (chan->desc.running) 1545 rcar_dmac_chan_start_xfer(chan); 1546 1547 return ret; 1548 } 1549 1550 static irqreturn_t rcar_dmac_isr_channel(int irq, void *dev) 1551 { 1552 u32 mask = RCAR_DMACHCR_DSE | RCAR_DMACHCR_TE; 1553 struct rcar_dmac_chan *chan = dev; 1554 irqreturn_t ret = IRQ_NONE; 1555 bool reinit = false; 1556 u32 chcr; 1557 1558 spin_lock(&chan->lock); 1559 1560 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 1561 if (chcr & RCAR_DMACHCR_CAE) { 1562 struct rcar_dmac *dmac = to_rcar_dmac(chan->chan.device); 1563 1564 /* 1565 * We don't need to call rcar_dmac_chan_halt() 1566 * because channel is already stopped in error case. 1567 * We need to clear register and check DE bit as recovery. 1568 */ 1569 rcar_dmac_write(dmac, RCAR_DMACHCLR, 1 << chan->index); 1570 rcar_dmac_chcr_de_barrier(chan); 1571 reinit = true; 1572 goto spin_lock_end; 1573 } 1574 1575 if (chcr & RCAR_DMACHCR_TE) 1576 mask |= RCAR_DMACHCR_DE; 1577 rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr & ~mask); 1578 if (mask & RCAR_DMACHCR_DE) 1579 rcar_dmac_chcr_de_barrier(chan); 1580 1581 if (chcr & RCAR_DMACHCR_DSE) 1582 ret |= rcar_dmac_isr_desc_stage_end(chan); 1583 1584 if (chcr & RCAR_DMACHCR_TE) 1585 ret |= rcar_dmac_isr_transfer_end(chan); 1586 1587 spin_lock_end: 1588 spin_unlock(&chan->lock); 1589 1590 if (reinit) { 1591 dev_err(chan->chan.device->dev, "Channel Address Error\n"); 1592 1593 rcar_dmac_chan_reinit(chan); 1594 ret = IRQ_HANDLED; 1595 } 1596 1597 return ret; 1598 } 1599 1600 static irqreturn_t rcar_dmac_isr_channel_thread(int irq, void *dev) 1601 { 1602 struct rcar_dmac_chan *chan = dev; 1603 struct rcar_dmac_desc *desc; 1604 struct dmaengine_desc_callback cb; 1605 1606 spin_lock_irq(&chan->lock); 1607 1608 /* For cyclic transfers notify the user after every chunk. */ 1609 if (chan->desc.running && chan->desc.running->cyclic) { 1610 desc = chan->desc.running; 1611 dmaengine_desc_get_callback(&desc->async_tx, &cb); 1612 1613 if (dmaengine_desc_callback_valid(&cb)) { 1614 spin_unlock_irq(&chan->lock); 1615 dmaengine_desc_callback_invoke(&cb, NULL); 1616 spin_lock_irq(&chan->lock); 1617 } 1618 } 1619 1620 /* 1621 * Call the callback function for all descriptors on the done list and 1622 * move them to the ack wait list. 1623 */ 1624 while (!list_empty(&chan->desc.done)) { 1625 desc = list_first_entry(&chan->desc.done, struct rcar_dmac_desc, 1626 node); 1627 dma_cookie_complete(&desc->async_tx); 1628 list_del(&desc->node); 1629 1630 dmaengine_desc_get_callback(&desc->async_tx, &cb); 1631 if (dmaengine_desc_callback_valid(&cb)) { 1632 spin_unlock_irq(&chan->lock); 1633 /* 1634 * We own the only reference to this descriptor, we can 1635 * safely dereference it without holding the channel 1636 * lock. 1637 */ 1638 dmaengine_desc_callback_invoke(&cb, NULL); 1639 spin_lock_irq(&chan->lock); 1640 } 1641 1642 list_add_tail(&desc->node, &chan->desc.wait); 1643 } 1644 1645 spin_unlock_irq(&chan->lock); 1646 1647 /* Recycle all acked descriptors. */ 1648 rcar_dmac_desc_recycle_acked(chan); 1649 1650 return IRQ_HANDLED; 1651 } 1652 1653 /* ----------------------------------------------------------------------------- 1654 * OF xlate and channel filter 1655 */ 1656 1657 static bool rcar_dmac_chan_filter(struct dma_chan *chan, void *arg) 1658 { 1659 struct rcar_dmac *dmac = to_rcar_dmac(chan->device); 1660 struct of_phandle_args *dma_spec = arg; 1661 1662 /* 1663 * FIXME: Using a filter on OF platforms is a nonsense. The OF xlate 1664 * function knows from which device it wants to allocate a channel from, 1665 * and would be perfectly capable of selecting the channel it wants. 1666 * Forcing it to call dma_request_channel() and iterate through all 1667 * channels from all controllers is just pointless. 1668 */ 1669 if (chan->device->device_config != rcar_dmac_device_config) 1670 return false; 1671 1672 return !test_and_set_bit(dma_spec->args[0], dmac->modules); 1673 } 1674 1675 static struct dma_chan *rcar_dmac_of_xlate(struct of_phandle_args *dma_spec, 1676 struct of_dma *ofdma) 1677 { 1678 struct rcar_dmac_chan *rchan; 1679 struct dma_chan *chan; 1680 dma_cap_mask_t mask; 1681 1682 if (dma_spec->args_count != 1) 1683 return NULL; 1684 1685 /* Only slave DMA channels can be allocated via DT */ 1686 dma_cap_zero(mask); 1687 dma_cap_set(DMA_SLAVE, mask); 1688 1689 chan = __dma_request_channel(&mask, rcar_dmac_chan_filter, dma_spec, 1690 ofdma->of_node); 1691 if (!chan) 1692 return NULL; 1693 1694 rchan = to_rcar_dmac_chan(chan); 1695 rchan->mid_rid = dma_spec->args[0]; 1696 1697 return chan; 1698 } 1699 1700 /* ----------------------------------------------------------------------------- 1701 * Power management 1702 */ 1703 1704 #ifdef CONFIG_PM 1705 static int rcar_dmac_runtime_suspend(struct device *dev) 1706 { 1707 return 0; 1708 } 1709 1710 static int rcar_dmac_runtime_resume(struct device *dev) 1711 { 1712 struct rcar_dmac *dmac = dev_get_drvdata(dev); 1713 1714 return rcar_dmac_init(dmac); 1715 } 1716 #endif 1717 1718 static const struct dev_pm_ops rcar_dmac_pm = { 1719 /* 1720 * TODO for system sleep/resume: 1721 * - Wait for the current transfer to complete and stop the device, 1722 * - Resume transfers, if any. 1723 */ 1724 SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, 1725 pm_runtime_force_resume) 1726 SET_RUNTIME_PM_OPS(rcar_dmac_runtime_suspend, rcar_dmac_runtime_resume, 1727 NULL) 1728 }; 1729 1730 /* ----------------------------------------------------------------------------- 1731 * Probe and remove 1732 */ 1733 1734 static int rcar_dmac_chan_probe(struct rcar_dmac *dmac, 1735 struct rcar_dmac_chan *rchan, 1736 const struct rcar_dmac_of_data *data, 1737 unsigned int index) 1738 { 1739 struct platform_device *pdev = to_platform_device(dmac->dev); 1740 struct dma_chan *chan = &rchan->chan; 1741 char pdev_irqname[5]; 1742 char *irqname; 1743 int ret; 1744 1745 rchan->index = index; 1746 rchan->iomem = dmac->iomem + data->chan_offset_base + 1747 data->chan_offset_stride * index; 1748 rchan->mid_rid = -EINVAL; 1749 1750 spin_lock_init(&rchan->lock); 1751 1752 INIT_LIST_HEAD(&rchan->desc.free); 1753 INIT_LIST_HEAD(&rchan->desc.pending); 1754 INIT_LIST_HEAD(&rchan->desc.active); 1755 INIT_LIST_HEAD(&rchan->desc.done); 1756 INIT_LIST_HEAD(&rchan->desc.wait); 1757 1758 /* Request the channel interrupt. */ 1759 sprintf(pdev_irqname, "ch%u", index); 1760 rchan->irq = platform_get_irq_byname(pdev, pdev_irqname); 1761 if (rchan->irq < 0) 1762 return -ENODEV; 1763 1764 irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", 1765 dev_name(dmac->dev), index); 1766 if (!irqname) 1767 return -ENOMEM; 1768 1769 /* 1770 * Initialize the DMA engine channel and add it to the DMA engine 1771 * channels list. 1772 */ 1773 chan->device = &dmac->engine; 1774 dma_cookie_init(chan); 1775 1776 list_add_tail(&chan->device_node, &dmac->engine.channels); 1777 1778 ret = devm_request_threaded_irq(dmac->dev, rchan->irq, 1779 rcar_dmac_isr_channel, 1780 rcar_dmac_isr_channel_thread, 0, 1781 irqname, rchan); 1782 if (ret) { 1783 dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", 1784 rchan->irq, ret); 1785 return ret; 1786 } 1787 1788 return 0; 1789 } 1790 1791 #define RCAR_DMAC_MAX_CHANNELS 32 1792 1793 static int rcar_dmac_parse_of(struct device *dev, struct rcar_dmac *dmac) 1794 { 1795 struct device_node *np = dev->of_node; 1796 int ret; 1797 1798 ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels); 1799 if (ret < 0) { 1800 dev_err(dev, "unable to read dma-channels property\n"); 1801 return ret; 1802 } 1803 1804 /* The hardware and driver don't support more than 32 bits in CHCLR */ 1805 if (dmac->n_channels <= 0 || 1806 dmac->n_channels >= RCAR_DMAC_MAX_CHANNELS) { 1807 dev_err(dev, "invalid number of channels %u\n", 1808 dmac->n_channels); 1809 return -EINVAL; 1810 } 1811 1812 /* 1813 * If the driver is unable to read dma-channel-mask property, 1814 * the driver assumes that it can use all channels. 1815 */ 1816 dmac->channels_mask = GENMASK(dmac->n_channels - 1, 0); 1817 of_property_read_u32(np, "dma-channel-mask", &dmac->channels_mask); 1818 1819 /* If the property has out-of-channel mask, this driver clears it */ 1820 dmac->channels_mask &= GENMASK(dmac->n_channels - 1, 0); 1821 1822 return 0; 1823 } 1824 1825 static int rcar_dmac_probe(struct platform_device *pdev) 1826 { 1827 const enum dma_slave_buswidth widths = DMA_SLAVE_BUSWIDTH_1_BYTE | 1828 DMA_SLAVE_BUSWIDTH_2_BYTES | DMA_SLAVE_BUSWIDTH_4_BYTES | 1829 DMA_SLAVE_BUSWIDTH_8_BYTES | DMA_SLAVE_BUSWIDTH_16_BYTES | 1830 DMA_SLAVE_BUSWIDTH_32_BYTES | DMA_SLAVE_BUSWIDTH_64_BYTES; 1831 struct dma_device *engine; 1832 struct rcar_dmac *dmac; 1833 const struct rcar_dmac_of_data *data; 1834 unsigned int i; 1835 int ret; 1836 1837 data = of_device_get_match_data(&pdev->dev); 1838 if (!data) 1839 return -EINVAL; 1840 1841 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); 1842 if (!dmac) 1843 return -ENOMEM; 1844 1845 dmac->dev = &pdev->dev; 1846 platform_set_drvdata(pdev, dmac); 1847 dma_set_max_seg_size(dmac->dev, RCAR_DMATCR_MASK); 1848 dma_set_mask_and_coherent(dmac->dev, DMA_BIT_MASK(40)); 1849 1850 ret = rcar_dmac_parse_of(&pdev->dev, dmac); 1851 if (ret < 0) 1852 return ret; 1853 1854 /* 1855 * A still unconfirmed hardware bug prevents the IPMMU microTLB 0 to be 1856 * flushed correctly, resulting in memory corruption. DMAC 0 channel 0 1857 * is connected to microTLB 0 on currently supported platforms, so we 1858 * can't use it with the IPMMU. As the IOMMU API operates at the device 1859 * level we can't disable it selectively, so ignore channel 0 for now if 1860 * the device is part of an IOMMU group. 1861 */ 1862 if (device_iommu_mapped(&pdev->dev)) 1863 dmac->channels_mask &= ~BIT(0); 1864 1865 dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels, 1866 sizeof(*dmac->channels), GFP_KERNEL); 1867 if (!dmac->channels) 1868 return -ENOMEM; 1869 1870 /* Request resources. */ 1871 dmac->iomem = devm_platform_ioremap_resource(pdev, 0); 1872 if (IS_ERR(dmac->iomem)) 1873 return PTR_ERR(dmac->iomem); 1874 1875 /* Enable runtime PM and initialize the device. */ 1876 pm_runtime_enable(&pdev->dev); 1877 ret = pm_runtime_get_sync(&pdev->dev); 1878 if (ret < 0) { 1879 dev_err(&pdev->dev, "runtime PM get sync failed (%d)\n", ret); 1880 return ret; 1881 } 1882 1883 ret = rcar_dmac_init(dmac); 1884 pm_runtime_put(&pdev->dev); 1885 1886 if (ret) { 1887 dev_err(&pdev->dev, "failed to reset device\n"); 1888 goto error; 1889 } 1890 1891 /* Initialize engine */ 1892 engine = &dmac->engine; 1893 1894 dma_cap_set(DMA_MEMCPY, engine->cap_mask); 1895 dma_cap_set(DMA_SLAVE, engine->cap_mask); 1896 1897 engine->dev = &pdev->dev; 1898 engine->copy_align = ilog2(RCAR_DMAC_MEMCPY_XFER_SIZE); 1899 1900 engine->src_addr_widths = widths; 1901 engine->dst_addr_widths = widths; 1902 engine->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); 1903 engine->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1904 1905 engine->device_alloc_chan_resources = rcar_dmac_alloc_chan_resources; 1906 engine->device_free_chan_resources = rcar_dmac_free_chan_resources; 1907 engine->device_prep_dma_memcpy = rcar_dmac_prep_dma_memcpy; 1908 engine->device_prep_slave_sg = rcar_dmac_prep_slave_sg; 1909 engine->device_prep_dma_cyclic = rcar_dmac_prep_dma_cyclic; 1910 engine->device_config = rcar_dmac_device_config; 1911 engine->device_pause = rcar_dmac_chan_pause; 1912 engine->device_terminate_all = rcar_dmac_chan_terminate_all; 1913 engine->device_tx_status = rcar_dmac_tx_status; 1914 engine->device_issue_pending = rcar_dmac_issue_pending; 1915 engine->device_synchronize = rcar_dmac_device_synchronize; 1916 1917 INIT_LIST_HEAD(&engine->channels); 1918 1919 for (i = 0; i < dmac->n_channels; ++i) { 1920 if (!(dmac->channels_mask & BIT(i))) 1921 continue; 1922 1923 ret = rcar_dmac_chan_probe(dmac, &dmac->channels[i], data, i); 1924 if (ret < 0) 1925 goto error; 1926 } 1927 1928 /* Register the DMAC as a DMA provider for DT. */ 1929 ret = of_dma_controller_register(pdev->dev.of_node, rcar_dmac_of_xlate, 1930 NULL); 1931 if (ret < 0) 1932 goto error; 1933 1934 /* 1935 * Register the DMA engine device. 1936 * 1937 * Default transfer size of 32 bytes requires 32-byte alignment. 1938 */ 1939 ret = dma_async_device_register(engine); 1940 if (ret < 0) 1941 goto error; 1942 1943 return 0; 1944 1945 error: 1946 of_dma_controller_free(pdev->dev.of_node); 1947 pm_runtime_disable(&pdev->dev); 1948 return ret; 1949 } 1950 1951 static int rcar_dmac_remove(struct platform_device *pdev) 1952 { 1953 struct rcar_dmac *dmac = platform_get_drvdata(pdev); 1954 1955 of_dma_controller_free(pdev->dev.of_node); 1956 dma_async_device_unregister(&dmac->engine); 1957 1958 pm_runtime_disable(&pdev->dev); 1959 1960 return 0; 1961 } 1962 1963 static void rcar_dmac_shutdown(struct platform_device *pdev) 1964 { 1965 struct rcar_dmac *dmac = platform_get_drvdata(pdev); 1966 1967 rcar_dmac_stop_all_chan(dmac); 1968 } 1969 1970 static const struct rcar_dmac_of_data rcar_dmac_data = { 1971 .chan_offset_base = 0x8000, 1972 .chan_offset_stride = 0x80, 1973 }; 1974 1975 static const struct of_device_id rcar_dmac_of_ids[] = { 1976 { 1977 .compatible = "renesas,rcar-dmac", 1978 .data = &rcar_dmac_data, 1979 }, 1980 { /* Sentinel */ } 1981 }; 1982 MODULE_DEVICE_TABLE(of, rcar_dmac_of_ids); 1983 1984 static struct platform_driver rcar_dmac_driver = { 1985 .driver = { 1986 .pm = &rcar_dmac_pm, 1987 .name = "rcar-dmac", 1988 .of_match_table = rcar_dmac_of_ids, 1989 }, 1990 .probe = rcar_dmac_probe, 1991 .remove = rcar_dmac_remove, 1992 .shutdown = rcar_dmac_shutdown, 1993 }; 1994 1995 module_platform_driver(rcar_dmac_driver); 1996 1997 MODULE_DESCRIPTION("R-Car Gen2 DMA Controller Driver"); 1998 MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>"); 1999 MODULE_LICENSE("GPL v2"); 2000