1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2014 Emilio López 4 * Emilio López <emilio@elopez.com.ar> 5 */ 6 7 #include <linux/bitmap.h> 8 #include <linux/bitops.h> 9 #include <linux/clk.h> 10 #include <linux/dmaengine.h> 11 #include <linux/dmapool.h> 12 #include <linux/interrupt.h> 13 #include <linux/module.h> 14 #include <linux/of_dma.h> 15 #include <linux/platform_device.h> 16 #include <linux/slab.h> 17 #include <linux/spinlock.h> 18 19 #include "virt-dma.h" 20 21 /** Common macros to normal and dedicated DMA registers **/ 22 23 #define SUN4I_DMA_CFG_LOADING BIT(31) 24 #define SUN4I_DMA_CFG_DST_DATA_WIDTH(width) ((width) << 25) 25 #define SUN4I_DMA_CFG_DST_BURST_LENGTH(len) ((len) << 23) 26 #define SUN4I_DMA_CFG_DST_ADDR_MODE(mode) ((mode) << 21) 27 #define SUN4I_DMA_CFG_DST_DRQ_TYPE(type) ((type) << 16) 28 #define SUN4I_DMA_CFG_SRC_DATA_WIDTH(width) ((width) << 9) 29 #define SUN4I_DMA_CFG_SRC_BURST_LENGTH(len) ((len) << 7) 30 #define SUN4I_DMA_CFG_SRC_ADDR_MODE(mode) ((mode) << 5) 31 #define SUN4I_DMA_CFG_SRC_DRQ_TYPE(type) (type) 32 33 /** Normal DMA register values **/ 34 35 /* Normal DMA source/destination data request type values */ 36 #define SUN4I_NDMA_DRQ_TYPE_SDRAM 0x16 37 #define SUN4I_NDMA_DRQ_TYPE_LIMIT (0x1F + 1) 38 39 /** Normal DMA register layout **/ 40 41 /* Dedicated DMA source/destination address mode values */ 42 #define SUN4I_NDMA_ADDR_MODE_LINEAR 0 43 #define SUN4I_NDMA_ADDR_MODE_IO 1 44 45 /* Normal DMA configuration register layout */ 46 #define SUN4I_NDMA_CFG_CONT_MODE BIT(30) 47 #define SUN4I_NDMA_CFG_WAIT_STATE(n) ((n) << 27) 48 #define SUN4I_NDMA_CFG_DST_NON_SECURE BIT(22) 49 #define SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN BIT(15) 50 #define SUN4I_NDMA_CFG_SRC_NON_SECURE BIT(6) 51 52 /** Dedicated DMA register values **/ 53 54 /* Dedicated DMA source/destination address mode values */ 55 #define SUN4I_DDMA_ADDR_MODE_LINEAR 0 56 #define SUN4I_DDMA_ADDR_MODE_IO 1 57 #define SUN4I_DDMA_ADDR_MODE_HORIZONTAL_PAGE 2 58 #define SUN4I_DDMA_ADDR_MODE_VERTICAL_PAGE 3 59 60 /* Dedicated DMA source/destination data request type values */ 61 #define SUN4I_DDMA_DRQ_TYPE_SDRAM 0x1 62 #define SUN4I_DDMA_DRQ_TYPE_LIMIT (0x1F + 1) 63 64 /** Dedicated DMA register layout **/ 65 66 /* Dedicated DMA configuration register layout */ 67 #define SUN4I_DDMA_CFG_BUSY BIT(30) 68 #define SUN4I_DDMA_CFG_CONT_MODE BIT(29) 69 #define SUN4I_DDMA_CFG_DST_NON_SECURE BIT(28) 70 #define SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN BIT(15) 71 #define SUN4I_DDMA_CFG_SRC_NON_SECURE BIT(12) 72 73 /* Dedicated DMA parameter register layout */ 74 #define SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(n) (((n) - 1) << 24) 75 #define SUN4I_DDMA_PARA_DST_WAIT_CYCLES(n) (((n) - 1) << 16) 76 #define SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(n) (((n) - 1) << 8) 77 #define SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(n) (((n) - 1) << 0) 78 79 /** DMA register offsets **/ 80 81 /* General register offsets */ 82 #define SUN4I_DMA_IRQ_ENABLE_REG 0x0 83 #define SUN4I_DMA_IRQ_PENDING_STATUS_REG 0x4 84 85 /* Normal DMA register offsets */ 86 #define SUN4I_NDMA_CHANNEL_REG_BASE(n) (0x100 + (n) * 0x20) 87 #define SUN4I_NDMA_CFG_REG 0x0 88 #define SUN4I_NDMA_SRC_ADDR_REG 0x4 89 #define SUN4I_NDMA_DST_ADDR_REG 0x8 90 #define SUN4I_NDMA_BYTE_COUNT_REG 0xC 91 92 /* Dedicated DMA register offsets */ 93 #define SUN4I_DDMA_CHANNEL_REG_BASE(n) (0x300 + (n) * 0x20) 94 #define SUN4I_DDMA_CFG_REG 0x0 95 #define SUN4I_DDMA_SRC_ADDR_REG 0x4 96 #define SUN4I_DDMA_DST_ADDR_REG 0x8 97 #define SUN4I_DDMA_BYTE_COUNT_REG 0xC 98 #define SUN4I_DDMA_PARA_REG 0x18 99 100 /** DMA Driver **/ 101 102 /* 103 * Normal DMA has 8 channels, and Dedicated DMA has another 8, so 104 * that's 16 channels. As for endpoints, there's 29 and 21 105 * respectively. Given that the Normal DMA endpoints (other than 106 * SDRAM) can be used as tx/rx, we need 78 vchans in total 107 */ 108 #define SUN4I_NDMA_NR_MAX_CHANNELS 8 109 #define SUN4I_DDMA_NR_MAX_CHANNELS 8 110 #define SUN4I_DMA_NR_MAX_CHANNELS \ 111 (SUN4I_NDMA_NR_MAX_CHANNELS + SUN4I_DDMA_NR_MAX_CHANNELS) 112 #define SUN4I_NDMA_NR_MAX_VCHANS (29 * 2 - 1) 113 #define SUN4I_DDMA_NR_MAX_VCHANS 21 114 #define SUN4I_DMA_NR_MAX_VCHANS \ 115 (SUN4I_NDMA_NR_MAX_VCHANS + SUN4I_DDMA_NR_MAX_VCHANS) 116 117 /* This set of SUN4I_DDMA timing parameters were found experimentally while 118 * working with the SPI driver and seem to make it behave correctly */ 119 #define SUN4I_DDMA_MAGIC_SPI_PARAMETERS \ 120 (SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(1) | \ 121 SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(1) | \ 122 SUN4I_DDMA_PARA_DST_WAIT_CYCLES(2) | \ 123 SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(2)) 124 125 struct sun4i_dma_pchan { 126 /* Register base of channel */ 127 void __iomem *base; 128 /* vchan currently being serviced */ 129 struct sun4i_dma_vchan *vchan; 130 /* Is this a dedicated pchan? */ 131 int is_dedicated; 132 }; 133 134 struct sun4i_dma_vchan { 135 struct virt_dma_chan vc; 136 struct dma_slave_config cfg; 137 struct sun4i_dma_pchan *pchan; 138 struct sun4i_dma_promise *processing; 139 struct sun4i_dma_contract *contract; 140 u8 endpoint; 141 int is_dedicated; 142 }; 143 144 struct sun4i_dma_promise { 145 u32 cfg; 146 u32 para; 147 dma_addr_t src; 148 dma_addr_t dst; 149 size_t len; 150 struct list_head list; 151 }; 152 153 /* A contract is a set of promises */ 154 struct sun4i_dma_contract { 155 struct virt_dma_desc vd; 156 struct list_head demands; 157 struct list_head completed_demands; 158 int is_cyclic; 159 }; 160 161 struct sun4i_dma_dev { 162 DECLARE_BITMAP(pchans_used, SUN4I_DMA_NR_MAX_CHANNELS); 163 struct dma_device slave; 164 struct sun4i_dma_pchan *pchans; 165 struct sun4i_dma_vchan *vchans; 166 void __iomem *base; 167 struct clk *clk; 168 int irq; 169 spinlock_t lock; 170 }; 171 172 static struct sun4i_dma_dev *to_sun4i_dma_dev(struct dma_device *dev) 173 { 174 return container_of(dev, struct sun4i_dma_dev, slave); 175 } 176 177 static struct sun4i_dma_vchan *to_sun4i_dma_vchan(struct dma_chan *chan) 178 { 179 return container_of(chan, struct sun4i_dma_vchan, vc.chan); 180 } 181 182 static struct sun4i_dma_contract *to_sun4i_dma_contract(struct virt_dma_desc *vd) 183 { 184 return container_of(vd, struct sun4i_dma_contract, vd); 185 } 186 187 static struct device *chan2dev(struct dma_chan *chan) 188 { 189 return &chan->dev->device; 190 } 191 192 static int convert_burst(u32 maxburst) 193 { 194 if (maxburst > 8) 195 return -EINVAL; 196 197 /* 1 -> 0, 4 -> 1, 8 -> 2 */ 198 return (maxburst >> 2); 199 } 200 201 static int convert_buswidth(enum dma_slave_buswidth addr_width) 202 { 203 if (addr_width > DMA_SLAVE_BUSWIDTH_4_BYTES) 204 return -EINVAL; 205 206 /* 8 (1 byte) -> 0, 16 (2 bytes) -> 1, 32 (4 bytes) -> 2 */ 207 return (addr_width >> 1); 208 } 209 210 static void sun4i_dma_free_chan_resources(struct dma_chan *chan) 211 { 212 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); 213 214 vchan_free_chan_resources(&vchan->vc); 215 } 216 217 static struct sun4i_dma_pchan *find_and_use_pchan(struct sun4i_dma_dev *priv, 218 struct sun4i_dma_vchan *vchan) 219 { 220 struct sun4i_dma_pchan *pchan = NULL, *pchans = priv->pchans; 221 unsigned long flags; 222 int i, max; 223 224 /* 225 * pchans 0-SUN4I_NDMA_NR_MAX_CHANNELS are normal, and 226 * SUN4I_NDMA_NR_MAX_CHANNELS+ are dedicated ones 227 */ 228 if (vchan->is_dedicated) { 229 i = SUN4I_NDMA_NR_MAX_CHANNELS; 230 max = SUN4I_DMA_NR_MAX_CHANNELS; 231 } else { 232 i = 0; 233 max = SUN4I_NDMA_NR_MAX_CHANNELS; 234 } 235 236 spin_lock_irqsave(&priv->lock, flags); 237 for_each_clear_bit_from(i, priv->pchans_used, max) { 238 pchan = &pchans[i]; 239 pchan->vchan = vchan; 240 set_bit(i, priv->pchans_used); 241 break; 242 } 243 spin_unlock_irqrestore(&priv->lock, flags); 244 245 return pchan; 246 } 247 248 static void release_pchan(struct sun4i_dma_dev *priv, 249 struct sun4i_dma_pchan *pchan) 250 { 251 unsigned long flags; 252 int nr = pchan - priv->pchans; 253 254 spin_lock_irqsave(&priv->lock, flags); 255 256 pchan->vchan = NULL; 257 clear_bit(nr, priv->pchans_used); 258 259 spin_unlock_irqrestore(&priv->lock, flags); 260 } 261 262 static void configure_pchan(struct sun4i_dma_pchan *pchan, 263 struct sun4i_dma_promise *d) 264 { 265 /* 266 * Configure addresses and misc parameters depending on type 267 * SUN4I_DDMA has an extra field with timing parameters 268 */ 269 if (pchan->is_dedicated) { 270 writel_relaxed(d->src, pchan->base + SUN4I_DDMA_SRC_ADDR_REG); 271 writel_relaxed(d->dst, pchan->base + SUN4I_DDMA_DST_ADDR_REG); 272 writel_relaxed(d->len, pchan->base + SUN4I_DDMA_BYTE_COUNT_REG); 273 writel_relaxed(d->para, pchan->base + SUN4I_DDMA_PARA_REG); 274 writel_relaxed(d->cfg, pchan->base + SUN4I_DDMA_CFG_REG); 275 } else { 276 writel_relaxed(d->src, pchan->base + SUN4I_NDMA_SRC_ADDR_REG); 277 writel_relaxed(d->dst, pchan->base + SUN4I_NDMA_DST_ADDR_REG); 278 writel_relaxed(d->len, pchan->base + SUN4I_NDMA_BYTE_COUNT_REG); 279 writel_relaxed(d->cfg, pchan->base + SUN4I_NDMA_CFG_REG); 280 } 281 } 282 283 static void set_pchan_interrupt(struct sun4i_dma_dev *priv, 284 struct sun4i_dma_pchan *pchan, 285 int half, int end) 286 { 287 u32 reg; 288 int pchan_number = pchan - priv->pchans; 289 unsigned long flags; 290 291 spin_lock_irqsave(&priv->lock, flags); 292 293 reg = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG); 294 295 if (half) 296 reg |= BIT(pchan_number * 2); 297 else 298 reg &= ~BIT(pchan_number * 2); 299 300 if (end) 301 reg |= BIT(pchan_number * 2 + 1); 302 else 303 reg &= ~BIT(pchan_number * 2 + 1); 304 305 writel_relaxed(reg, priv->base + SUN4I_DMA_IRQ_ENABLE_REG); 306 307 spin_unlock_irqrestore(&priv->lock, flags); 308 } 309 310 /** 311 * Execute pending operations on a vchan 312 * 313 * When given a vchan, this function will try to acquire a suitable 314 * pchan and, if successful, will configure it to fulfill a promise 315 * from the next pending contract. 316 * 317 * This function must be called with &vchan->vc.lock held. 318 */ 319 static int __execute_vchan_pending(struct sun4i_dma_dev *priv, 320 struct sun4i_dma_vchan *vchan) 321 { 322 struct sun4i_dma_promise *promise = NULL; 323 struct sun4i_dma_contract *contract = NULL; 324 struct sun4i_dma_pchan *pchan; 325 struct virt_dma_desc *vd; 326 int ret; 327 328 lockdep_assert_held(&vchan->vc.lock); 329 330 /* We need a pchan to do anything, so secure one if available */ 331 pchan = find_and_use_pchan(priv, vchan); 332 if (!pchan) 333 return -EBUSY; 334 335 /* 336 * Channel endpoints must not be repeated, so if this vchan 337 * has already submitted some work, we can't do anything else 338 */ 339 if (vchan->processing) { 340 dev_dbg(chan2dev(&vchan->vc.chan), 341 "processing something to this endpoint already\n"); 342 ret = -EBUSY; 343 goto release_pchan; 344 } 345 346 do { 347 /* Figure out which contract we're working with today */ 348 vd = vchan_next_desc(&vchan->vc); 349 if (!vd) { 350 dev_dbg(chan2dev(&vchan->vc.chan), 351 "No pending contract found"); 352 ret = 0; 353 goto release_pchan; 354 } 355 356 contract = to_sun4i_dma_contract(vd); 357 if (list_empty(&contract->demands)) { 358 /* The contract has been completed so mark it as such */ 359 list_del(&contract->vd.node); 360 vchan_cookie_complete(&contract->vd); 361 dev_dbg(chan2dev(&vchan->vc.chan), 362 "Empty contract found and marked complete"); 363 } 364 } while (list_empty(&contract->demands)); 365 366 /* Now find out what we need to do */ 367 promise = list_first_entry(&contract->demands, 368 struct sun4i_dma_promise, list); 369 vchan->processing = promise; 370 371 /* ... and make it reality */ 372 if (promise) { 373 vchan->contract = contract; 374 vchan->pchan = pchan; 375 set_pchan_interrupt(priv, pchan, contract->is_cyclic, 1); 376 configure_pchan(pchan, promise); 377 } 378 379 return 0; 380 381 release_pchan: 382 release_pchan(priv, pchan); 383 return ret; 384 } 385 386 static int sanitize_config(struct dma_slave_config *sconfig, 387 enum dma_transfer_direction direction) 388 { 389 switch (direction) { 390 case DMA_MEM_TO_DEV: 391 if ((sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) || 392 !sconfig->dst_maxburst) 393 return -EINVAL; 394 395 if (sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) 396 sconfig->src_addr_width = sconfig->dst_addr_width; 397 398 if (!sconfig->src_maxburst) 399 sconfig->src_maxburst = sconfig->dst_maxburst; 400 401 break; 402 403 case DMA_DEV_TO_MEM: 404 if ((sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) || 405 !sconfig->src_maxburst) 406 return -EINVAL; 407 408 if (sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) 409 sconfig->dst_addr_width = sconfig->src_addr_width; 410 411 if (!sconfig->dst_maxburst) 412 sconfig->dst_maxburst = sconfig->src_maxburst; 413 414 break; 415 default: 416 return 0; 417 } 418 419 return 0; 420 } 421 422 /** 423 * Generate a promise, to be used in a normal DMA contract. 424 * 425 * A NDMA promise contains all the information required to program the 426 * normal part of the DMA Engine and get data copied. A non-executed 427 * promise will live in the demands list on a contract. Once it has been 428 * completed, it will be moved to the completed demands list for later freeing. 429 * All linked promises will be freed when the corresponding contract is freed 430 */ 431 static struct sun4i_dma_promise * 432 generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, 433 size_t len, struct dma_slave_config *sconfig, 434 enum dma_transfer_direction direction) 435 { 436 struct sun4i_dma_promise *promise; 437 int ret; 438 439 ret = sanitize_config(sconfig, direction); 440 if (ret) 441 return NULL; 442 443 promise = kzalloc(sizeof(*promise), GFP_NOWAIT); 444 if (!promise) 445 return NULL; 446 447 promise->src = src; 448 promise->dst = dest; 449 promise->len = len; 450 promise->cfg = SUN4I_DMA_CFG_LOADING | 451 SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN; 452 453 dev_dbg(chan2dev(chan), 454 "src burst %d, dst burst %d, src buswidth %d, dst buswidth %d", 455 sconfig->src_maxburst, sconfig->dst_maxburst, 456 sconfig->src_addr_width, sconfig->dst_addr_width); 457 458 /* Source burst */ 459 ret = convert_burst(sconfig->src_maxburst); 460 if (ret < 0) 461 goto fail; 462 promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret); 463 464 /* Destination burst */ 465 ret = convert_burst(sconfig->dst_maxburst); 466 if (ret < 0) 467 goto fail; 468 promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret); 469 470 /* Source bus width */ 471 ret = convert_buswidth(sconfig->src_addr_width); 472 if (ret < 0) 473 goto fail; 474 promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret); 475 476 /* Destination bus width */ 477 ret = convert_buswidth(sconfig->dst_addr_width); 478 if (ret < 0) 479 goto fail; 480 promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret); 481 482 return promise; 483 484 fail: 485 kfree(promise); 486 return NULL; 487 } 488 489 /** 490 * Generate a promise, to be used in a dedicated DMA contract. 491 * 492 * A DDMA promise contains all the information required to program the 493 * Dedicated part of the DMA Engine and get data copied. A non-executed 494 * promise will live in the demands list on a contract. Once it has been 495 * completed, it will be moved to the completed demands list for later freeing. 496 * All linked promises will be freed when the corresponding contract is freed 497 */ 498 static struct sun4i_dma_promise * 499 generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, 500 size_t len, struct dma_slave_config *sconfig) 501 { 502 struct sun4i_dma_promise *promise; 503 int ret; 504 505 promise = kzalloc(sizeof(*promise), GFP_NOWAIT); 506 if (!promise) 507 return NULL; 508 509 promise->src = src; 510 promise->dst = dest; 511 promise->len = len; 512 promise->cfg = SUN4I_DMA_CFG_LOADING | 513 SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN; 514 515 /* Source burst */ 516 ret = convert_burst(sconfig->src_maxburst); 517 if (ret < 0) 518 goto fail; 519 promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret); 520 521 /* Destination burst */ 522 ret = convert_burst(sconfig->dst_maxburst); 523 if (ret < 0) 524 goto fail; 525 promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret); 526 527 /* Source bus width */ 528 ret = convert_buswidth(sconfig->src_addr_width); 529 if (ret < 0) 530 goto fail; 531 promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret); 532 533 /* Destination bus width */ 534 ret = convert_buswidth(sconfig->dst_addr_width); 535 if (ret < 0) 536 goto fail; 537 promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret); 538 539 return promise; 540 541 fail: 542 kfree(promise); 543 return NULL; 544 } 545 546 /** 547 * Generate a contract 548 * 549 * Contracts function as DMA descriptors. As our hardware does not support 550 * linked lists, we need to implement SG via software. We use a contract 551 * to hold all the pieces of the request and process them serially one 552 * after another. Each piece is represented as a promise. 553 */ 554 static struct sun4i_dma_contract *generate_dma_contract(void) 555 { 556 struct sun4i_dma_contract *contract; 557 558 contract = kzalloc(sizeof(*contract), GFP_NOWAIT); 559 if (!contract) 560 return NULL; 561 562 INIT_LIST_HEAD(&contract->demands); 563 INIT_LIST_HEAD(&contract->completed_demands); 564 565 return contract; 566 } 567 568 /** 569 * Get next promise on a cyclic transfer 570 * 571 * Cyclic contracts contain a series of promises which are executed on a 572 * loop. This function returns the next promise from a cyclic contract, 573 * so it can be programmed into the hardware. 574 */ 575 static struct sun4i_dma_promise * 576 get_next_cyclic_promise(struct sun4i_dma_contract *contract) 577 { 578 struct sun4i_dma_promise *promise; 579 580 promise = list_first_entry_or_null(&contract->demands, 581 struct sun4i_dma_promise, list); 582 if (!promise) { 583 list_splice_init(&contract->completed_demands, 584 &contract->demands); 585 promise = list_first_entry(&contract->demands, 586 struct sun4i_dma_promise, list); 587 } 588 589 return promise; 590 } 591 592 /** 593 * Free a contract and all its associated promises 594 */ 595 static void sun4i_dma_free_contract(struct virt_dma_desc *vd) 596 { 597 struct sun4i_dma_contract *contract = to_sun4i_dma_contract(vd); 598 struct sun4i_dma_promise *promise, *tmp; 599 600 /* Free all the demands and completed demands */ 601 list_for_each_entry_safe(promise, tmp, &contract->demands, list) 602 kfree(promise); 603 604 list_for_each_entry_safe(promise, tmp, &contract->completed_demands, list) 605 kfree(promise); 606 607 kfree(contract); 608 } 609 610 static struct dma_async_tx_descriptor * 611 sun4i_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, 612 dma_addr_t src, size_t len, unsigned long flags) 613 { 614 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); 615 struct dma_slave_config *sconfig = &vchan->cfg; 616 struct sun4i_dma_promise *promise; 617 struct sun4i_dma_contract *contract; 618 619 contract = generate_dma_contract(); 620 if (!contract) 621 return NULL; 622 623 /* 624 * We can only do the copy to bus aligned addresses, so 625 * choose the best one so we get decent performance. We also 626 * maximize the burst size for this same reason. 627 */ 628 sconfig->src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 629 sconfig->dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 630 sconfig->src_maxburst = 8; 631 sconfig->dst_maxburst = 8; 632 633 if (vchan->is_dedicated) 634 promise = generate_ddma_promise(chan, src, dest, len, sconfig); 635 else 636 promise = generate_ndma_promise(chan, src, dest, len, sconfig, 637 DMA_MEM_TO_MEM); 638 639 if (!promise) { 640 kfree(contract); 641 return NULL; 642 } 643 644 /* Configure memcpy mode */ 645 if (vchan->is_dedicated) { 646 promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM) | 647 SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM); 648 } else { 649 promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM) | 650 SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM); 651 } 652 653 /* Fill the contract with our only promise */ 654 list_add_tail(&promise->list, &contract->demands); 655 656 /* And add it to the vchan */ 657 return vchan_tx_prep(&vchan->vc, &contract->vd, flags); 658 } 659 660 static struct dma_async_tx_descriptor * 661 sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len, 662 size_t period_len, enum dma_transfer_direction dir, 663 unsigned long flags) 664 { 665 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); 666 struct dma_slave_config *sconfig = &vchan->cfg; 667 struct sun4i_dma_promise *promise; 668 struct sun4i_dma_contract *contract; 669 dma_addr_t src, dest; 670 u32 endpoints; 671 int nr_periods, offset, plength, i; 672 u8 ram_type, io_mode, linear_mode; 673 674 if (!is_slave_direction(dir)) { 675 dev_err(chan2dev(chan), "Invalid DMA direction\n"); 676 return NULL; 677 } 678 679 contract = generate_dma_contract(); 680 if (!contract) 681 return NULL; 682 683 contract->is_cyclic = 1; 684 685 if (vchan->is_dedicated) { 686 io_mode = SUN4I_DDMA_ADDR_MODE_IO; 687 linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR; 688 ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM; 689 } else { 690 io_mode = SUN4I_NDMA_ADDR_MODE_IO; 691 linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR; 692 ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM; 693 } 694 695 if (dir == DMA_MEM_TO_DEV) { 696 src = buf; 697 dest = sconfig->dst_addr; 698 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) | 699 SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) | 700 SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) | 701 SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode); 702 } else { 703 src = sconfig->src_addr; 704 dest = buf; 705 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) | 706 SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) | 707 SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) | 708 SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode); 709 } 710 711 /* 712 * We will be using half done interrupts to make two periods 713 * out of a promise, so we need to program the DMA engine less 714 * often 715 */ 716 717 /* 718 * The engine can interrupt on half-transfer, so we can use 719 * this feature to program the engine half as often as if we 720 * didn't use it (keep in mind the hardware doesn't support 721 * linked lists). 722 * 723 * Say you have a set of periods (| marks the start/end, I for 724 * interrupt, P for programming the engine to do a new 725 * transfer), the easy but slow way would be to do 726 * 727 * |---|---|---|---| (periods / promises) 728 * P I,P I,P I,P I 729 * 730 * Using half transfer interrupts you can do 731 * 732 * |-------|-------| (promises as configured on hw) 733 * |---|---|---|---| (periods) 734 * P I I,P I I 735 * 736 * Which requires half the engine programming for the same 737 * functionality. 738 */ 739 nr_periods = DIV_ROUND_UP(len / period_len, 2); 740 for (i = 0; i < nr_periods; i++) { 741 /* Calculate the offset in the buffer and the length needed */ 742 offset = i * period_len * 2; 743 plength = min((len - offset), (period_len * 2)); 744 if (dir == DMA_MEM_TO_DEV) 745 src = buf + offset; 746 else 747 dest = buf + offset; 748 749 /* Make the promise */ 750 if (vchan->is_dedicated) 751 promise = generate_ddma_promise(chan, src, dest, 752 plength, sconfig); 753 else 754 promise = generate_ndma_promise(chan, src, dest, 755 plength, sconfig, dir); 756 757 if (!promise) { 758 /* TODO: should we free everything? */ 759 return NULL; 760 } 761 promise->cfg |= endpoints; 762 763 /* Then add it to the contract */ 764 list_add_tail(&promise->list, &contract->demands); 765 } 766 767 /* And add it to the vchan */ 768 return vchan_tx_prep(&vchan->vc, &contract->vd, flags); 769 } 770 771 static struct dma_async_tx_descriptor * 772 sun4i_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 773 unsigned int sg_len, enum dma_transfer_direction dir, 774 unsigned long flags, void *context) 775 { 776 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); 777 struct dma_slave_config *sconfig = &vchan->cfg; 778 struct sun4i_dma_promise *promise; 779 struct sun4i_dma_contract *contract; 780 u8 ram_type, io_mode, linear_mode; 781 struct scatterlist *sg; 782 dma_addr_t srcaddr, dstaddr; 783 u32 endpoints, para; 784 int i; 785 786 if (!sgl) 787 return NULL; 788 789 if (!is_slave_direction(dir)) { 790 dev_err(chan2dev(chan), "Invalid DMA direction\n"); 791 return NULL; 792 } 793 794 contract = generate_dma_contract(); 795 if (!contract) 796 return NULL; 797 798 if (vchan->is_dedicated) { 799 io_mode = SUN4I_DDMA_ADDR_MODE_IO; 800 linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR; 801 ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM; 802 } else { 803 io_mode = SUN4I_NDMA_ADDR_MODE_IO; 804 linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR; 805 ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM; 806 } 807 808 if (dir == DMA_MEM_TO_DEV) 809 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) | 810 SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) | 811 SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) | 812 SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode); 813 else 814 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) | 815 SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) | 816 SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) | 817 SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode); 818 819 for_each_sg(sgl, sg, sg_len, i) { 820 /* Figure out addresses */ 821 if (dir == DMA_MEM_TO_DEV) { 822 srcaddr = sg_dma_address(sg); 823 dstaddr = sconfig->dst_addr; 824 } else { 825 srcaddr = sconfig->src_addr; 826 dstaddr = sg_dma_address(sg); 827 } 828 829 /* 830 * These are the magic DMA engine timings that keep SPI going. 831 * I haven't seen any interface on DMAEngine to configure 832 * timings, and so far they seem to work for everything we 833 * support, so I've kept them here. I don't know if other 834 * devices need different timings because, as usual, we only 835 * have the "para" bitfield meanings, but no comment on what 836 * the values should be when doing a certain operation :| 837 */ 838 para = SUN4I_DDMA_MAGIC_SPI_PARAMETERS; 839 840 /* And make a suitable promise */ 841 if (vchan->is_dedicated) 842 promise = generate_ddma_promise(chan, srcaddr, dstaddr, 843 sg_dma_len(sg), 844 sconfig); 845 else 846 promise = generate_ndma_promise(chan, srcaddr, dstaddr, 847 sg_dma_len(sg), 848 sconfig, dir); 849 850 if (!promise) 851 return NULL; /* TODO: should we free everything? */ 852 853 promise->cfg |= endpoints; 854 promise->para = para; 855 856 /* Then add it to the contract */ 857 list_add_tail(&promise->list, &contract->demands); 858 } 859 860 /* 861 * Once we've got all the promises ready, add the contract 862 * to the pending list on the vchan 863 */ 864 return vchan_tx_prep(&vchan->vc, &contract->vd, flags); 865 } 866 867 static int sun4i_dma_terminate_all(struct dma_chan *chan) 868 { 869 struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device); 870 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); 871 struct sun4i_dma_pchan *pchan = vchan->pchan; 872 LIST_HEAD(head); 873 unsigned long flags; 874 875 spin_lock_irqsave(&vchan->vc.lock, flags); 876 vchan_get_all_descriptors(&vchan->vc, &head); 877 spin_unlock_irqrestore(&vchan->vc.lock, flags); 878 879 /* 880 * Clearing the configuration register will halt the pchan. Interrupts 881 * may still trigger, so don't forget to disable them. 882 */ 883 if (pchan) { 884 if (pchan->is_dedicated) 885 writel(0, pchan->base + SUN4I_DDMA_CFG_REG); 886 else 887 writel(0, pchan->base + SUN4I_NDMA_CFG_REG); 888 set_pchan_interrupt(priv, pchan, 0, 0); 889 release_pchan(priv, pchan); 890 } 891 892 spin_lock_irqsave(&vchan->vc.lock, flags); 893 /* Clear these so the vchan is usable again */ 894 vchan->processing = NULL; 895 vchan->pchan = NULL; 896 spin_unlock_irqrestore(&vchan->vc.lock, flags); 897 898 vchan_dma_desc_free_list(&vchan->vc, &head); 899 900 return 0; 901 } 902 903 static int sun4i_dma_config(struct dma_chan *chan, 904 struct dma_slave_config *config) 905 { 906 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); 907 908 memcpy(&vchan->cfg, config, sizeof(*config)); 909 910 return 0; 911 } 912 913 static struct dma_chan *sun4i_dma_of_xlate(struct of_phandle_args *dma_spec, 914 struct of_dma *ofdma) 915 { 916 struct sun4i_dma_dev *priv = ofdma->of_dma_data; 917 struct sun4i_dma_vchan *vchan; 918 struct dma_chan *chan; 919 u8 is_dedicated = dma_spec->args[0]; 920 u8 endpoint = dma_spec->args[1]; 921 922 /* Check if type is Normal or Dedicated */ 923 if (is_dedicated != 0 && is_dedicated != 1) 924 return NULL; 925 926 /* Make sure the endpoint looks sane */ 927 if ((is_dedicated && endpoint >= SUN4I_DDMA_DRQ_TYPE_LIMIT) || 928 (!is_dedicated && endpoint >= SUN4I_NDMA_DRQ_TYPE_LIMIT)) 929 return NULL; 930 931 chan = dma_get_any_slave_channel(&priv->slave); 932 if (!chan) 933 return NULL; 934 935 /* Assign the endpoint to the vchan */ 936 vchan = to_sun4i_dma_vchan(chan); 937 vchan->is_dedicated = is_dedicated; 938 vchan->endpoint = endpoint; 939 940 return chan; 941 } 942 943 static enum dma_status sun4i_dma_tx_status(struct dma_chan *chan, 944 dma_cookie_t cookie, 945 struct dma_tx_state *state) 946 { 947 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); 948 struct sun4i_dma_pchan *pchan = vchan->pchan; 949 struct sun4i_dma_contract *contract; 950 struct sun4i_dma_promise *promise; 951 struct virt_dma_desc *vd; 952 unsigned long flags; 953 enum dma_status ret; 954 size_t bytes = 0; 955 956 ret = dma_cookie_status(chan, cookie, state); 957 if (!state || (ret == DMA_COMPLETE)) 958 return ret; 959 960 spin_lock_irqsave(&vchan->vc.lock, flags); 961 vd = vchan_find_desc(&vchan->vc, cookie); 962 if (!vd) 963 goto exit; 964 contract = to_sun4i_dma_contract(vd); 965 966 list_for_each_entry(promise, &contract->demands, list) 967 bytes += promise->len; 968 969 /* 970 * The hardware is configured to return the remaining byte 971 * quantity. If possible, replace the first listed element's 972 * full size with the actual remaining amount 973 */ 974 promise = list_first_entry_or_null(&contract->demands, 975 struct sun4i_dma_promise, list); 976 if (promise && pchan) { 977 bytes -= promise->len; 978 if (pchan->is_dedicated) 979 bytes += readl(pchan->base + SUN4I_DDMA_BYTE_COUNT_REG); 980 else 981 bytes += readl(pchan->base + SUN4I_NDMA_BYTE_COUNT_REG); 982 } 983 984 exit: 985 986 dma_set_residue(state, bytes); 987 spin_unlock_irqrestore(&vchan->vc.lock, flags); 988 989 return ret; 990 } 991 992 static void sun4i_dma_issue_pending(struct dma_chan *chan) 993 { 994 struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device); 995 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); 996 unsigned long flags; 997 998 spin_lock_irqsave(&vchan->vc.lock, flags); 999 1000 /* 1001 * If there are pending transactions for this vchan, push one of 1002 * them into the engine to get the ball rolling. 1003 */ 1004 if (vchan_issue_pending(&vchan->vc)) 1005 __execute_vchan_pending(priv, vchan); 1006 1007 spin_unlock_irqrestore(&vchan->vc.lock, flags); 1008 } 1009 1010 static irqreturn_t sun4i_dma_interrupt(int irq, void *dev_id) 1011 { 1012 struct sun4i_dma_dev *priv = dev_id; 1013 struct sun4i_dma_pchan *pchans = priv->pchans, *pchan; 1014 struct sun4i_dma_vchan *vchan; 1015 struct sun4i_dma_contract *contract; 1016 struct sun4i_dma_promise *promise; 1017 unsigned long pendirq, irqs, disableirqs; 1018 int bit, i, free_room, allow_mitigation = 1; 1019 1020 pendirq = readl_relaxed(priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG); 1021 1022 handle_pending: 1023 1024 disableirqs = 0; 1025 free_room = 0; 1026 1027 for_each_set_bit(bit, &pendirq, 32) { 1028 pchan = &pchans[bit >> 1]; 1029 vchan = pchan->vchan; 1030 if (!vchan) /* a terminated channel may still interrupt */ 1031 continue; 1032 contract = vchan->contract; 1033 1034 /* 1035 * Disable the IRQ and free the pchan if it's an end 1036 * interrupt (odd bit) 1037 */ 1038 if (bit & 1) { 1039 spin_lock(&vchan->vc.lock); 1040 1041 /* 1042 * Move the promise into the completed list now that 1043 * we're done with it 1044 */ 1045 list_del(&vchan->processing->list); 1046 list_add_tail(&vchan->processing->list, 1047 &contract->completed_demands); 1048 1049 /* 1050 * Cyclic DMA transfers are special: 1051 * - There's always something we can dispatch 1052 * - We need to run the callback 1053 * - Latency is very important, as this is used by audio 1054 * We therefore just cycle through the list and dispatch 1055 * whatever we have here, reusing the pchan. There's 1056 * no need to run the thread after this. 1057 * 1058 * For non-cyclic transfers we need to look around, 1059 * so we can program some more work, or notify the 1060 * client that their transfers have been completed. 1061 */ 1062 if (contract->is_cyclic) { 1063 promise = get_next_cyclic_promise(contract); 1064 vchan->processing = promise; 1065 configure_pchan(pchan, promise); 1066 vchan_cyclic_callback(&contract->vd); 1067 } else { 1068 vchan->processing = NULL; 1069 vchan->pchan = NULL; 1070 1071 free_room = 1; 1072 disableirqs |= BIT(bit); 1073 release_pchan(priv, pchan); 1074 } 1075 1076 spin_unlock(&vchan->vc.lock); 1077 } else { 1078 /* Half done interrupt */ 1079 if (contract->is_cyclic) 1080 vchan_cyclic_callback(&contract->vd); 1081 else 1082 disableirqs |= BIT(bit); 1083 } 1084 } 1085 1086 /* Disable the IRQs for events we handled */ 1087 spin_lock(&priv->lock); 1088 irqs = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG); 1089 writel_relaxed(irqs & ~disableirqs, 1090 priv->base + SUN4I_DMA_IRQ_ENABLE_REG); 1091 spin_unlock(&priv->lock); 1092 1093 /* Writing 1 to the pending field will clear the pending interrupt */ 1094 writel_relaxed(pendirq, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG); 1095 1096 /* 1097 * If a pchan was freed, we may be able to schedule something else, 1098 * so have a look around 1099 */ 1100 if (free_room) { 1101 for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) { 1102 vchan = &priv->vchans[i]; 1103 spin_lock(&vchan->vc.lock); 1104 __execute_vchan_pending(priv, vchan); 1105 spin_unlock(&vchan->vc.lock); 1106 } 1107 } 1108 1109 /* 1110 * Handle newer interrupts if some showed up, but only do it once 1111 * to avoid a too long a loop 1112 */ 1113 if (allow_mitigation) { 1114 pendirq = readl_relaxed(priv->base + 1115 SUN4I_DMA_IRQ_PENDING_STATUS_REG); 1116 if (pendirq) { 1117 allow_mitigation = 0; 1118 goto handle_pending; 1119 } 1120 } 1121 1122 return IRQ_HANDLED; 1123 } 1124 1125 static int sun4i_dma_probe(struct platform_device *pdev) 1126 { 1127 struct sun4i_dma_dev *priv; 1128 struct resource *res; 1129 int i, j, ret; 1130 1131 priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); 1132 if (!priv) 1133 return -ENOMEM; 1134 1135 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1136 priv->base = devm_ioremap_resource(&pdev->dev, res); 1137 if (IS_ERR(priv->base)) 1138 return PTR_ERR(priv->base); 1139 1140 priv->irq = platform_get_irq(pdev, 0); 1141 if (priv->irq < 0) 1142 return priv->irq; 1143 1144 priv->clk = devm_clk_get(&pdev->dev, NULL); 1145 if (IS_ERR(priv->clk)) { 1146 dev_err(&pdev->dev, "No clock specified\n"); 1147 return PTR_ERR(priv->clk); 1148 } 1149 1150 platform_set_drvdata(pdev, priv); 1151 spin_lock_init(&priv->lock); 1152 1153 dma_cap_zero(priv->slave.cap_mask); 1154 dma_cap_set(DMA_PRIVATE, priv->slave.cap_mask); 1155 dma_cap_set(DMA_MEMCPY, priv->slave.cap_mask); 1156 dma_cap_set(DMA_CYCLIC, priv->slave.cap_mask); 1157 dma_cap_set(DMA_SLAVE, priv->slave.cap_mask); 1158 1159 INIT_LIST_HEAD(&priv->slave.channels); 1160 priv->slave.device_free_chan_resources = sun4i_dma_free_chan_resources; 1161 priv->slave.device_tx_status = sun4i_dma_tx_status; 1162 priv->slave.device_issue_pending = sun4i_dma_issue_pending; 1163 priv->slave.device_prep_slave_sg = sun4i_dma_prep_slave_sg; 1164 priv->slave.device_prep_dma_memcpy = sun4i_dma_prep_dma_memcpy; 1165 priv->slave.device_prep_dma_cyclic = sun4i_dma_prep_dma_cyclic; 1166 priv->slave.device_config = sun4i_dma_config; 1167 priv->slave.device_terminate_all = sun4i_dma_terminate_all; 1168 priv->slave.copy_align = 2; 1169 priv->slave.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1170 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1171 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 1172 priv->slave.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1173 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1174 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 1175 priv->slave.directions = BIT(DMA_DEV_TO_MEM) | 1176 BIT(DMA_MEM_TO_DEV); 1177 priv->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1178 1179 priv->slave.dev = &pdev->dev; 1180 1181 priv->pchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_CHANNELS, 1182 sizeof(struct sun4i_dma_pchan), GFP_KERNEL); 1183 priv->vchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_VCHANS, 1184 sizeof(struct sun4i_dma_vchan), GFP_KERNEL); 1185 if (!priv->vchans || !priv->pchans) 1186 return -ENOMEM; 1187 1188 /* 1189 * [0..SUN4I_NDMA_NR_MAX_CHANNELS) are normal pchans, and 1190 * [SUN4I_NDMA_NR_MAX_CHANNELS..SUN4I_DMA_NR_MAX_CHANNELS) are 1191 * dedicated ones 1192 */ 1193 for (i = 0; i < SUN4I_NDMA_NR_MAX_CHANNELS; i++) 1194 priv->pchans[i].base = priv->base + 1195 SUN4I_NDMA_CHANNEL_REG_BASE(i); 1196 1197 for (j = 0; i < SUN4I_DMA_NR_MAX_CHANNELS; i++, j++) { 1198 priv->pchans[i].base = priv->base + 1199 SUN4I_DDMA_CHANNEL_REG_BASE(j); 1200 priv->pchans[i].is_dedicated = 1; 1201 } 1202 1203 for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) { 1204 struct sun4i_dma_vchan *vchan = &priv->vchans[i]; 1205 1206 spin_lock_init(&vchan->vc.lock); 1207 vchan->vc.desc_free = sun4i_dma_free_contract; 1208 vchan_init(&vchan->vc, &priv->slave); 1209 } 1210 1211 ret = clk_prepare_enable(priv->clk); 1212 if (ret) { 1213 dev_err(&pdev->dev, "Couldn't enable the clock\n"); 1214 return ret; 1215 } 1216 1217 /* 1218 * Make sure the IRQs are all disabled and accounted for. The bootloader 1219 * likes to leave these dirty 1220 */ 1221 writel(0, priv->base + SUN4I_DMA_IRQ_ENABLE_REG); 1222 writel(0xFFFFFFFF, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG); 1223 1224 ret = devm_request_irq(&pdev->dev, priv->irq, sun4i_dma_interrupt, 1225 0, dev_name(&pdev->dev), priv); 1226 if (ret) { 1227 dev_err(&pdev->dev, "Cannot request IRQ\n"); 1228 goto err_clk_disable; 1229 } 1230 1231 ret = dma_async_device_register(&priv->slave); 1232 if (ret) { 1233 dev_warn(&pdev->dev, "Failed to register DMA engine device\n"); 1234 goto err_clk_disable; 1235 } 1236 1237 ret = of_dma_controller_register(pdev->dev.of_node, sun4i_dma_of_xlate, 1238 priv); 1239 if (ret) { 1240 dev_err(&pdev->dev, "of_dma_controller_register failed\n"); 1241 goto err_dma_unregister; 1242 } 1243 1244 dev_dbg(&pdev->dev, "Successfully probed SUN4I_DMA\n"); 1245 1246 return 0; 1247 1248 err_dma_unregister: 1249 dma_async_device_unregister(&priv->slave); 1250 err_clk_disable: 1251 clk_disable_unprepare(priv->clk); 1252 return ret; 1253 } 1254 1255 static int sun4i_dma_remove(struct platform_device *pdev) 1256 { 1257 struct sun4i_dma_dev *priv = platform_get_drvdata(pdev); 1258 1259 /* Disable IRQ so no more work is scheduled */ 1260 disable_irq(priv->irq); 1261 1262 of_dma_controller_free(pdev->dev.of_node); 1263 dma_async_device_unregister(&priv->slave); 1264 1265 clk_disable_unprepare(priv->clk); 1266 1267 return 0; 1268 } 1269 1270 static const struct of_device_id sun4i_dma_match[] = { 1271 { .compatible = "allwinner,sun4i-a10-dma" }, 1272 { /* sentinel */ }, 1273 }; 1274 MODULE_DEVICE_TABLE(of, sun4i_dma_match); 1275 1276 static struct platform_driver sun4i_dma_driver = { 1277 .probe = sun4i_dma_probe, 1278 .remove = sun4i_dma_remove, 1279 .driver = { 1280 .name = "sun4i-dma", 1281 .of_match_table = sun4i_dma_match, 1282 }, 1283 }; 1284 1285 module_platform_driver(sun4i_dma_driver); 1286 1287 MODULE_DESCRIPTION("Allwinner A10 Dedicated DMA Controller Driver"); 1288 MODULE_AUTHOR("Emilio López <emilio@elopez.com.ar>"); 1289 MODULE_LICENSE("GPL"); 1290