1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2013-2014 Allwinner Tech Co., Ltd 4 * Author: Sugar <shuge@allwinnertech.com> 5 * 6 * Copyright (C) 2014 Maxime Ripard 7 * Maxime Ripard <maxime.ripard@free-electrons.com> 8 */ 9 10 #include <linux/clk.h> 11 #include <linux/delay.h> 12 #include <linux/dmaengine.h> 13 #include <linux/dmapool.h> 14 #include <linux/interrupt.h> 15 #include <linux/module.h> 16 #include <linux/of_dma.h> 17 #include <linux/of_device.h> 18 #include <linux/platform_device.h> 19 #include <linux/reset.h> 20 #include <linux/slab.h> 21 #include <linux/types.h> 22 23 #include "virt-dma.h" 24 25 /* 26 * Common registers 27 */ 28 #define DMA_IRQ_EN(x) ((x) * 0x04) 29 #define DMA_IRQ_HALF BIT(0) 30 #define DMA_IRQ_PKG BIT(1) 31 #define DMA_IRQ_QUEUE BIT(2) 32 33 #define DMA_IRQ_CHAN_NR 8 34 #define DMA_IRQ_CHAN_WIDTH 4 35 36 37 #define DMA_IRQ_STAT(x) ((x) * 0x04 + 0x10) 38 39 #define DMA_STAT 0x30 40 41 /* Offset between DMA_IRQ_EN and DMA_IRQ_STAT limits number of channels */ 42 #define DMA_MAX_CHANNELS (DMA_IRQ_CHAN_NR * 0x10 / 4) 43 44 /* 45 * sun8i specific registers 46 */ 47 #define SUN8I_DMA_GATE 0x20 48 #define SUN8I_DMA_GATE_ENABLE 0x4 49 50 #define SUNXI_H3_SECURE_REG 0x20 51 #define SUNXI_H3_DMA_GATE 0x28 52 #define SUNXI_H3_DMA_GATE_ENABLE 0x4 53 /* 54 * Channels specific registers 55 */ 56 #define DMA_CHAN_ENABLE 0x00 57 #define DMA_CHAN_ENABLE_START BIT(0) 58 #define DMA_CHAN_ENABLE_STOP 0 59 60 #define DMA_CHAN_PAUSE 0x04 61 #define DMA_CHAN_PAUSE_PAUSE BIT(1) 62 #define DMA_CHAN_PAUSE_RESUME 0 63 64 #define DMA_CHAN_LLI_ADDR 0x08 65 66 #define DMA_CHAN_CUR_CFG 0x0c 67 #define DMA_CHAN_MAX_DRQ_A31 0x1f 68 #define DMA_CHAN_MAX_DRQ_H6 0x3f 69 #define DMA_CHAN_CFG_SRC_DRQ_A31(x) ((x) & DMA_CHAN_MAX_DRQ_A31) 70 #define DMA_CHAN_CFG_SRC_DRQ_H6(x) ((x) & DMA_CHAN_MAX_DRQ_H6) 71 #define DMA_CHAN_CFG_SRC_MODE_A31(x) (((x) & 0x1) << 5) 72 #define DMA_CHAN_CFG_SRC_MODE_H6(x) (((x) & 0x1) << 8) 73 #define DMA_CHAN_CFG_SRC_BURST_A31(x) (((x) & 0x3) << 7) 74 #define DMA_CHAN_CFG_SRC_BURST_H3(x) (((x) & 0x3) << 6) 75 #define DMA_CHAN_CFG_SRC_WIDTH(x) (((x) & 0x3) << 9) 76 77 #define DMA_CHAN_CFG_DST_DRQ_A31(x) (DMA_CHAN_CFG_SRC_DRQ_A31(x) << 16) 78 #define DMA_CHAN_CFG_DST_DRQ_H6(x) (DMA_CHAN_CFG_SRC_DRQ_H6(x) << 16) 79 #define DMA_CHAN_CFG_DST_MODE_A31(x) (DMA_CHAN_CFG_SRC_MODE_A31(x) << 16) 80 #define DMA_CHAN_CFG_DST_MODE_H6(x) (DMA_CHAN_CFG_SRC_MODE_H6(x) << 16) 81 #define DMA_CHAN_CFG_DST_BURST_A31(x) (DMA_CHAN_CFG_SRC_BURST_A31(x) << 16) 82 #define DMA_CHAN_CFG_DST_BURST_H3(x) (DMA_CHAN_CFG_SRC_BURST_H3(x) << 16) 83 #define DMA_CHAN_CFG_DST_WIDTH(x) (DMA_CHAN_CFG_SRC_WIDTH(x) << 16) 84 85 #define DMA_CHAN_CUR_SRC 0x10 86 87 #define DMA_CHAN_CUR_DST 0x14 88 89 #define DMA_CHAN_CUR_CNT 0x18 90 91 #define DMA_CHAN_CUR_PARA 0x1c 92 93 /* 94 * LLI address mangling 95 * 96 * The LLI link physical address is also mangled, but we avoid dealing 97 * with that by allocating LLIs from the DMA32 zone. 98 */ 99 #define SRC_HIGH_ADDR(x) (((x) & 0x3U) << 16) 100 #define DST_HIGH_ADDR(x) (((x) & 0x3U) << 18) 101 102 /* 103 * Various hardware related defines 104 */ 105 #define LLI_LAST_ITEM 0xfffff800 106 #define NORMAL_WAIT 8 107 #define DRQ_SDRAM 1 108 #define LINEAR_MODE 0 109 #define IO_MODE 1 110 111 /* forward declaration */ 112 struct sun6i_dma_dev; 113 114 /* 115 * Hardware channels / ports representation 116 * 117 * The hardware is used in several SoCs, with differing numbers 118 * of channels and endpoints. This structure ties those numbers 119 * to a certain compatible string. 120 */ 121 struct sun6i_dma_config { 122 u32 nr_max_channels; 123 u32 nr_max_requests; 124 u32 nr_max_vchans; 125 /* 126 * In the datasheets/user manuals of newer Allwinner SoCs, a special 127 * bit (bit 2 at register 0x20) is present. 128 * It's named "DMA MCLK interface circuit auto gating bit" in the 129 * documents, and the footnote of this register says that this bit 130 * should be set up when initializing the DMA controller. 131 * Allwinner A23/A33 user manuals do not have this bit documented, 132 * however these SoCs really have and need this bit, as seen in the 133 * BSP kernel source code. 134 */ 135 void (*clock_autogate_enable)(struct sun6i_dma_dev *); 136 void (*set_burst_length)(u32 *p_cfg, s8 src_burst, s8 dst_burst); 137 void (*set_drq)(u32 *p_cfg, s8 src_drq, s8 dst_drq); 138 void (*set_mode)(u32 *p_cfg, s8 src_mode, s8 dst_mode); 139 u32 src_burst_lengths; 140 u32 dst_burst_lengths; 141 u32 src_addr_widths; 142 u32 dst_addr_widths; 143 bool has_high_addr; 144 bool has_mbus_clk; 145 }; 146 147 /* 148 * Hardware representation of the LLI 149 * 150 * The hardware will be fed the physical address of this structure, 151 * and read its content in order to start the transfer. 152 */ 153 struct sun6i_dma_lli { 154 u32 cfg; 155 u32 src; 156 u32 dst; 157 u32 len; 158 u32 para; 159 u32 p_lli_next; 160 161 /* 162 * This field is not used by the DMA controller, but will be 163 * used by the CPU to go through the list (mostly for dumping 164 * or freeing it). 165 */ 166 struct sun6i_dma_lli *v_lli_next; 167 }; 168 169 170 struct sun6i_desc { 171 struct virt_dma_desc vd; 172 dma_addr_t p_lli; 173 struct sun6i_dma_lli *v_lli; 174 }; 175 176 struct sun6i_pchan { 177 u32 idx; 178 void __iomem *base; 179 struct sun6i_vchan *vchan; 180 struct sun6i_desc *desc; 181 struct sun6i_desc *done; 182 }; 183 184 struct sun6i_vchan { 185 struct virt_dma_chan vc; 186 struct list_head node; 187 struct dma_slave_config cfg; 188 struct sun6i_pchan *phy; 189 u8 port; 190 u8 irq_type; 191 bool cyclic; 192 }; 193 194 struct sun6i_dma_dev { 195 struct dma_device slave; 196 void __iomem *base; 197 struct clk *clk; 198 struct clk *clk_mbus; 199 int irq; 200 spinlock_t lock; 201 struct reset_control *rstc; 202 struct tasklet_struct task; 203 atomic_t tasklet_shutdown; 204 struct list_head pending; 205 struct dma_pool *pool; 206 struct sun6i_pchan *pchans; 207 struct sun6i_vchan *vchans; 208 const struct sun6i_dma_config *cfg; 209 u32 num_pchans; 210 u32 num_vchans; 211 u32 max_request; 212 }; 213 214 static struct device *chan2dev(struct dma_chan *chan) 215 { 216 return &chan->dev->device; 217 } 218 219 static inline struct sun6i_dma_dev *to_sun6i_dma_dev(struct dma_device *d) 220 { 221 return container_of(d, struct sun6i_dma_dev, slave); 222 } 223 224 static inline struct sun6i_vchan *to_sun6i_vchan(struct dma_chan *chan) 225 { 226 return container_of(chan, struct sun6i_vchan, vc.chan); 227 } 228 229 static inline struct sun6i_desc * 230 to_sun6i_desc(struct dma_async_tx_descriptor *tx) 231 { 232 return container_of(tx, struct sun6i_desc, vd.tx); 233 } 234 235 static inline void sun6i_dma_dump_com_regs(struct sun6i_dma_dev *sdev) 236 { 237 dev_dbg(sdev->slave.dev, "Common register:\n" 238 "\tmask0(%04x): 0x%08x\n" 239 "\tmask1(%04x): 0x%08x\n" 240 "\tpend0(%04x): 0x%08x\n" 241 "\tpend1(%04x): 0x%08x\n" 242 "\tstats(%04x): 0x%08x\n", 243 DMA_IRQ_EN(0), readl(sdev->base + DMA_IRQ_EN(0)), 244 DMA_IRQ_EN(1), readl(sdev->base + DMA_IRQ_EN(1)), 245 DMA_IRQ_STAT(0), readl(sdev->base + DMA_IRQ_STAT(0)), 246 DMA_IRQ_STAT(1), readl(sdev->base + DMA_IRQ_STAT(1)), 247 DMA_STAT, readl(sdev->base + DMA_STAT)); 248 } 249 250 static inline void sun6i_dma_dump_chan_regs(struct sun6i_dma_dev *sdev, 251 struct sun6i_pchan *pchan) 252 { 253 dev_dbg(sdev->slave.dev, "Chan %d reg:\n" 254 "\t___en(%04x): \t0x%08x\n" 255 "\tpause(%04x): \t0x%08x\n" 256 "\tstart(%04x): \t0x%08x\n" 257 "\t__cfg(%04x): \t0x%08x\n" 258 "\t__src(%04x): \t0x%08x\n" 259 "\t__dst(%04x): \t0x%08x\n" 260 "\tcount(%04x): \t0x%08x\n" 261 "\t_para(%04x): \t0x%08x\n\n", 262 pchan->idx, 263 DMA_CHAN_ENABLE, 264 readl(pchan->base + DMA_CHAN_ENABLE), 265 DMA_CHAN_PAUSE, 266 readl(pchan->base + DMA_CHAN_PAUSE), 267 DMA_CHAN_LLI_ADDR, 268 readl(pchan->base + DMA_CHAN_LLI_ADDR), 269 DMA_CHAN_CUR_CFG, 270 readl(pchan->base + DMA_CHAN_CUR_CFG), 271 DMA_CHAN_CUR_SRC, 272 readl(pchan->base + DMA_CHAN_CUR_SRC), 273 DMA_CHAN_CUR_DST, 274 readl(pchan->base + DMA_CHAN_CUR_DST), 275 DMA_CHAN_CUR_CNT, 276 readl(pchan->base + DMA_CHAN_CUR_CNT), 277 DMA_CHAN_CUR_PARA, 278 readl(pchan->base + DMA_CHAN_CUR_PARA)); 279 } 280 281 static inline s8 convert_burst(u32 maxburst) 282 { 283 switch (maxburst) { 284 case 1: 285 return 0; 286 case 4: 287 return 1; 288 case 8: 289 return 2; 290 case 16: 291 return 3; 292 default: 293 return -EINVAL; 294 } 295 } 296 297 static inline s8 convert_buswidth(enum dma_slave_buswidth addr_width) 298 { 299 return ilog2(addr_width); 300 } 301 302 static void sun6i_enable_clock_autogate_a23(struct sun6i_dma_dev *sdev) 303 { 304 writel(SUN8I_DMA_GATE_ENABLE, sdev->base + SUN8I_DMA_GATE); 305 } 306 307 static void sun6i_enable_clock_autogate_h3(struct sun6i_dma_dev *sdev) 308 { 309 writel(SUNXI_H3_DMA_GATE_ENABLE, sdev->base + SUNXI_H3_DMA_GATE); 310 } 311 312 static void sun6i_set_burst_length_a31(u32 *p_cfg, s8 src_burst, s8 dst_burst) 313 { 314 *p_cfg |= DMA_CHAN_CFG_SRC_BURST_A31(src_burst) | 315 DMA_CHAN_CFG_DST_BURST_A31(dst_burst); 316 } 317 318 static void sun6i_set_burst_length_h3(u32 *p_cfg, s8 src_burst, s8 dst_burst) 319 { 320 *p_cfg |= DMA_CHAN_CFG_SRC_BURST_H3(src_burst) | 321 DMA_CHAN_CFG_DST_BURST_H3(dst_burst); 322 } 323 324 static void sun6i_set_drq_a31(u32 *p_cfg, s8 src_drq, s8 dst_drq) 325 { 326 *p_cfg |= DMA_CHAN_CFG_SRC_DRQ_A31(src_drq) | 327 DMA_CHAN_CFG_DST_DRQ_A31(dst_drq); 328 } 329 330 static void sun6i_set_drq_h6(u32 *p_cfg, s8 src_drq, s8 dst_drq) 331 { 332 *p_cfg |= DMA_CHAN_CFG_SRC_DRQ_H6(src_drq) | 333 DMA_CHAN_CFG_DST_DRQ_H6(dst_drq); 334 } 335 336 static void sun6i_set_mode_a31(u32 *p_cfg, s8 src_mode, s8 dst_mode) 337 { 338 *p_cfg |= DMA_CHAN_CFG_SRC_MODE_A31(src_mode) | 339 DMA_CHAN_CFG_DST_MODE_A31(dst_mode); 340 } 341 342 static void sun6i_set_mode_h6(u32 *p_cfg, s8 src_mode, s8 dst_mode) 343 { 344 *p_cfg |= DMA_CHAN_CFG_SRC_MODE_H6(src_mode) | 345 DMA_CHAN_CFG_DST_MODE_H6(dst_mode); 346 } 347 348 static size_t sun6i_get_chan_size(struct sun6i_pchan *pchan) 349 { 350 struct sun6i_desc *txd = pchan->desc; 351 struct sun6i_dma_lli *lli; 352 size_t bytes; 353 dma_addr_t pos; 354 355 pos = readl(pchan->base + DMA_CHAN_LLI_ADDR); 356 bytes = readl(pchan->base + DMA_CHAN_CUR_CNT); 357 358 if (pos == LLI_LAST_ITEM) 359 return bytes; 360 361 for (lli = txd->v_lli; lli; lli = lli->v_lli_next) { 362 if (lli->p_lli_next == pos) { 363 for (lli = lli->v_lli_next; lli; lli = lli->v_lli_next) 364 bytes += lli->len; 365 break; 366 } 367 } 368 369 return bytes; 370 } 371 372 static void *sun6i_dma_lli_add(struct sun6i_dma_lli *prev, 373 struct sun6i_dma_lli *next, 374 dma_addr_t next_phy, 375 struct sun6i_desc *txd) 376 { 377 if ((!prev && !txd) || !next) 378 return NULL; 379 380 if (!prev) { 381 txd->p_lli = next_phy; 382 txd->v_lli = next; 383 } else { 384 prev->p_lli_next = next_phy; 385 prev->v_lli_next = next; 386 } 387 388 next->p_lli_next = LLI_LAST_ITEM; 389 next->v_lli_next = NULL; 390 391 return next; 392 } 393 394 static inline void sun6i_dma_dump_lli(struct sun6i_vchan *vchan, 395 struct sun6i_dma_lli *v_lli, 396 dma_addr_t p_lli) 397 { 398 dev_dbg(chan2dev(&vchan->vc.chan), 399 "\n\tdesc:\tp - %pad v - 0x%p\n" 400 "\t\tc - 0x%08x s - 0x%08x d - 0x%08x\n" 401 "\t\tl - 0x%08x p - 0x%08x n - 0x%08x\n", 402 &p_lli, v_lli, 403 v_lli->cfg, v_lli->src, v_lli->dst, 404 v_lli->len, v_lli->para, v_lli->p_lli_next); 405 } 406 407 static void sun6i_dma_free_desc(struct virt_dma_desc *vd) 408 { 409 struct sun6i_desc *txd = to_sun6i_desc(&vd->tx); 410 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(vd->tx.chan->device); 411 struct sun6i_dma_lli *v_lli, *v_next; 412 dma_addr_t p_lli, p_next; 413 414 if (unlikely(!txd)) 415 return; 416 417 p_lli = txd->p_lli; 418 v_lli = txd->v_lli; 419 420 while (v_lli) { 421 v_next = v_lli->v_lli_next; 422 p_next = v_lli->p_lli_next; 423 424 dma_pool_free(sdev->pool, v_lli, p_lli); 425 426 v_lli = v_next; 427 p_lli = p_next; 428 } 429 430 kfree(txd); 431 } 432 433 static int sun6i_dma_start_desc(struct sun6i_vchan *vchan) 434 { 435 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(vchan->vc.chan.device); 436 struct virt_dma_desc *desc = vchan_next_desc(&vchan->vc); 437 struct sun6i_pchan *pchan = vchan->phy; 438 u32 irq_val, irq_reg, irq_offset; 439 440 if (!pchan) 441 return -EAGAIN; 442 443 if (!desc) { 444 pchan->desc = NULL; 445 pchan->done = NULL; 446 return -EAGAIN; 447 } 448 449 list_del(&desc->node); 450 451 pchan->desc = to_sun6i_desc(&desc->tx); 452 pchan->done = NULL; 453 454 sun6i_dma_dump_lli(vchan, pchan->desc->v_lli, pchan->desc->p_lli); 455 456 irq_reg = pchan->idx / DMA_IRQ_CHAN_NR; 457 irq_offset = pchan->idx % DMA_IRQ_CHAN_NR; 458 459 vchan->irq_type = vchan->cyclic ? DMA_IRQ_PKG : DMA_IRQ_QUEUE; 460 461 irq_val = readl(sdev->base + DMA_IRQ_EN(irq_reg)); 462 irq_val &= ~((DMA_IRQ_HALF | DMA_IRQ_PKG | DMA_IRQ_QUEUE) << 463 (irq_offset * DMA_IRQ_CHAN_WIDTH)); 464 irq_val |= vchan->irq_type << (irq_offset * DMA_IRQ_CHAN_WIDTH); 465 writel(irq_val, sdev->base + DMA_IRQ_EN(irq_reg)); 466 467 writel(pchan->desc->p_lli, pchan->base + DMA_CHAN_LLI_ADDR); 468 writel(DMA_CHAN_ENABLE_START, pchan->base + DMA_CHAN_ENABLE); 469 470 sun6i_dma_dump_com_regs(sdev); 471 sun6i_dma_dump_chan_regs(sdev, pchan); 472 473 return 0; 474 } 475 476 static void sun6i_dma_tasklet(struct tasklet_struct *t) 477 { 478 struct sun6i_dma_dev *sdev = from_tasklet(sdev, t, task); 479 struct sun6i_vchan *vchan; 480 struct sun6i_pchan *pchan; 481 unsigned int pchan_alloc = 0; 482 unsigned int pchan_idx; 483 484 list_for_each_entry(vchan, &sdev->slave.channels, vc.chan.device_node) { 485 spin_lock_irq(&vchan->vc.lock); 486 487 pchan = vchan->phy; 488 489 if (pchan && pchan->done) { 490 if (sun6i_dma_start_desc(vchan)) { 491 /* 492 * No current txd associated with this channel 493 */ 494 dev_dbg(sdev->slave.dev, "pchan %u: free\n", 495 pchan->idx); 496 497 /* Mark this channel free */ 498 vchan->phy = NULL; 499 pchan->vchan = NULL; 500 } 501 } 502 spin_unlock_irq(&vchan->vc.lock); 503 } 504 505 spin_lock_irq(&sdev->lock); 506 for (pchan_idx = 0; pchan_idx < sdev->num_pchans; pchan_idx++) { 507 pchan = &sdev->pchans[pchan_idx]; 508 509 if (pchan->vchan || list_empty(&sdev->pending)) 510 continue; 511 512 vchan = list_first_entry(&sdev->pending, 513 struct sun6i_vchan, node); 514 515 /* Remove from pending channels */ 516 list_del_init(&vchan->node); 517 pchan_alloc |= BIT(pchan_idx); 518 519 /* Mark this channel allocated */ 520 pchan->vchan = vchan; 521 vchan->phy = pchan; 522 dev_dbg(sdev->slave.dev, "pchan %u: alloc vchan %p\n", 523 pchan->idx, &vchan->vc); 524 } 525 spin_unlock_irq(&sdev->lock); 526 527 for (pchan_idx = 0; pchan_idx < sdev->num_pchans; pchan_idx++) { 528 if (!(pchan_alloc & BIT(pchan_idx))) 529 continue; 530 531 pchan = sdev->pchans + pchan_idx; 532 vchan = pchan->vchan; 533 if (vchan) { 534 spin_lock_irq(&vchan->vc.lock); 535 sun6i_dma_start_desc(vchan); 536 spin_unlock_irq(&vchan->vc.lock); 537 } 538 } 539 } 540 541 static irqreturn_t sun6i_dma_interrupt(int irq, void *dev_id) 542 { 543 struct sun6i_dma_dev *sdev = dev_id; 544 struct sun6i_vchan *vchan; 545 struct sun6i_pchan *pchan; 546 int i, j, ret = IRQ_NONE; 547 u32 status; 548 549 for (i = 0; i < sdev->num_pchans / DMA_IRQ_CHAN_NR; i++) { 550 status = readl(sdev->base + DMA_IRQ_STAT(i)); 551 if (!status) 552 continue; 553 554 dev_dbg(sdev->slave.dev, "DMA irq status %s: 0x%x\n", 555 i ? "high" : "low", status); 556 557 writel(status, sdev->base + DMA_IRQ_STAT(i)); 558 559 for (j = 0; (j < DMA_IRQ_CHAN_NR) && status; j++) { 560 pchan = sdev->pchans + j; 561 vchan = pchan->vchan; 562 if (vchan && (status & vchan->irq_type)) { 563 if (vchan->cyclic) { 564 vchan_cyclic_callback(&pchan->desc->vd); 565 } else { 566 spin_lock(&vchan->vc.lock); 567 vchan_cookie_complete(&pchan->desc->vd); 568 pchan->done = pchan->desc; 569 spin_unlock(&vchan->vc.lock); 570 } 571 } 572 573 status = status >> DMA_IRQ_CHAN_WIDTH; 574 } 575 576 if (!atomic_read(&sdev->tasklet_shutdown)) 577 tasklet_schedule(&sdev->task); 578 ret = IRQ_HANDLED; 579 } 580 581 return ret; 582 } 583 584 static int set_config(struct sun6i_dma_dev *sdev, 585 struct dma_slave_config *sconfig, 586 enum dma_transfer_direction direction, 587 u32 *p_cfg) 588 { 589 enum dma_slave_buswidth src_addr_width, dst_addr_width; 590 u32 src_maxburst, dst_maxburst; 591 s8 src_width, dst_width, src_burst, dst_burst; 592 593 src_addr_width = sconfig->src_addr_width; 594 dst_addr_width = sconfig->dst_addr_width; 595 src_maxburst = sconfig->src_maxburst; 596 dst_maxburst = sconfig->dst_maxburst; 597 598 switch (direction) { 599 case DMA_MEM_TO_DEV: 600 if (src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) 601 src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 602 src_maxburst = src_maxburst ? src_maxburst : 8; 603 break; 604 case DMA_DEV_TO_MEM: 605 if (dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) 606 dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 607 dst_maxburst = dst_maxburst ? dst_maxburst : 8; 608 break; 609 default: 610 return -EINVAL; 611 } 612 613 if (!(BIT(src_addr_width) & sdev->slave.src_addr_widths)) 614 return -EINVAL; 615 if (!(BIT(dst_addr_width) & sdev->slave.dst_addr_widths)) 616 return -EINVAL; 617 if (!(BIT(src_maxburst) & sdev->cfg->src_burst_lengths)) 618 return -EINVAL; 619 if (!(BIT(dst_maxburst) & sdev->cfg->dst_burst_lengths)) 620 return -EINVAL; 621 622 src_width = convert_buswidth(src_addr_width); 623 dst_width = convert_buswidth(dst_addr_width); 624 dst_burst = convert_burst(dst_maxburst); 625 src_burst = convert_burst(src_maxburst); 626 627 *p_cfg = DMA_CHAN_CFG_SRC_WIDTH(src_width) | 628 DMA_CHAN_CFG_DST_WIDTH(dst_width); 629 630 sdev->cfg->set_burst_length(p_cfg, src_burst, dst_burst); 631 632 return 0; 633 } 634 635 static inline void sun6i_dma_set_addr(struct sun6i_dma_dev *sdev, 636 struct sun6i_dma_lli *v_lli, 637 dma_addr_t src, dma_addr_t dst) 638 { 639 v_lli->src = lower_32_bits(src); 640 v_lli->dst = lower_32_bits(dst); 641 642 if (sdev->cfg->has_high_addr) 643 v_lli->para |= SRC_HIGH_ADDR(upper_32_bits(src)) | 644 DST_HIGH_ADDR(upper_32_bits(dst)); 645 } 646 647 static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_memcpy( 648 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 649 size_t len, unsigned long flags) 650 { 651 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); 652 struct sun6i_vchan *vchan = to_sun6i_vchan(chan); 653 struct sun6i_dma_lli *v_lli; 654 struct sun6i_desc *txd; 655 dma_addr_t p_lli; 656 s8 burst, width; 657 658 dev_dbg(chan2dev(chan), 659 "%s; chan: %d, dest: %pad, src: %pad, len: %zu. flags: 0x%08lx\n", 660 __func__, vchan->vc.chan.chan_id, &dest, &src, len, flags); 661 662 if (!len) 663 return NULL; 664 665 txd = kzalloc(sizeof(*txd), GFP_NOWAIT); 666 if (!txd) 667 return NULL; 668 669 v_lli = dma_pool_alloc(sdev->pool, GFP_DMA32 | GFP_NOWAIT, &p_lli); 670 if (!v_lli) { 671 dev_err(sdev->slave.dev, "Failed to alloc lli memory\n"); 672 goto err_txd_free; 673 } 674 675 v_lli->len = len; 676 v_lli->para = NORMAL_WAIT; 677 sun6i_dma_set_addr(sdev, v_lli, src, dest); 678 679 burst = convert_burst(8); 680 width = convert_buswidth(DMA_SLAVE_BUSWIDTH_4_BYTES); 681 v_lli->cfg = DMA_CHAN_CFG_SRC_WIDTH(width) | 682 DMA_CHAN_CFG_DST_WIDTH(width); 683 684 sdev->cfg->set_burst_length(&v_lli->cfg, burst, burst); 685 sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, DRQ_SDRAM); 686 sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, LINEAR_MODE); 687 688 sun6i_dma_lli_add(NULL, v_lli, p_lli, txd); 689 690 sun6i_dma_dump_lli(vchan, v_lli, p_lli); 691 692 return vchan_tx_prep(&vchan->vc, &txd->vd, flags); 693 694 err_txd_free: 695 kfree(txd); 696 return NULL; 697 } 698 699 static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg( 700 struct dma_chan *chan, struct scatterlist *sgl, 701 unsigned int sg_len, enum dma_transfer_direction dir, 702 unsigned long flags, void *context) 703 { 704 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); 705 struct sun6i_vchan *vchan = to_sun6i_vchan(chan); 706 struct dma_slave_config *sconfig = &vchan->cfg; 707 struct sun6i_dma_lli *v_lli, *prev = NULL; 708 struct sun6i_desc *txd; 709 struct scatterlist *sg; 710 dma_addr_t p_lli; 711 u32 lli_cfg; 712 int i, ret; 713 714 if (!sgl) 715 return NULL; 716 717 ret = set_config(sdev, sconfig, dir, &lli_cfg); 718 if (ret) { 719 dev_err(chan2dev(chan), "Invalid DMA configuration\n"); 720 return NULL; 721 } 722 723 txd = kzalloc(sizeof(*txd), GFP_NOWAIT); 724 if (!txd) 725 return NULL; 726 727 for_each_sg(sgl, sg, sg_len, i) { 728 v_lli = dma_pool_alloc(sdev->pool, GFP_DMA32 | GFP_NOWAIT, &p_lli); 729 if (!v_lli) 730 goto err_lli_free; 731 732 v_lli->len = sg_dma_len(sg); 733 v_lli->para = NORMAL_WAIT; 734 735 if (dir == DMA_MEM_TO_DEV) { 736 sun6i_dma_set_addr(sdev, v_lli, 737 sg_dma_address(sg), 738 sconfig->dst_addr); 739 v_lli->cfg = lli_cfg; 740 sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port); 741 sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE); 742 743 dev_dbg(chan2dev(chan), 744 "%s; chan: %d, dest: %pad, src: %pad, len: %u. flags: 0x%08lx\n", 745 __func__, vchan->vc.chan.chan_id, 746 &sconfig->dst_addr, &sg_dma_address(sg), 747 sg_dma_len(sg), flags); 748 749 } else { 750 sun6i_dma_set_addr(sdev, v_lli, 751 sconfig->src_addr, 752 sg_dma_address(sg)); 753 v_lli->cfg = lli_cfg; 754 sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM); 755 sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE); 756 757 dev_dbg(chan2dev(chan), 758 "%s; chan: %d, dest: %pad, src: %pad, len: %u. flags: 0x%08lx\n", 759 __func__, vchan->vc.chan.chan_id, 760 &sg_dma_address(sg), &sconfig->src_addr, 761 sg_dma_len(sg), flags); 762 } 763 764 prev = sun6i_dma_lli_add(prev, v_lli, p_lli, txd); 765 } 766 767 dev_dbg(chan2dev(chan), "First: %pad\n", &txd->p_lli); 768 for (p_lli = txd->p_lli, v_lli = txd->v_lli; v_lli; 769 p_lli = v_lli->p_lli_next, v_lli = v_lli->v_lli_next) 770 sun6i_dma_dump_lli(vchan, v_lli, p_lli); 771 772 return vchan_tx_prep(&vchan->vc, &txd->vd, flags); 773 774 err_lli_free: 775 for (p_lli = txd->p_lli, v_lli = txd->v_lli; v_lli; 776 p_lli = v_lli->p_lli_next, v_lli = v_lli->v_lli_next) 777 dma_pool_free(sdev->pool, v_lli, p_lli); 778 kfree(txd); 779 return NULL; 780 } 781 782 static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic( 783 struct dma_chan *chan, 784 dma_addr_t buf_addr, 785 size_t buf_len, 786 size_t period_len, 787 enum dma_transfer_direction dir, 788 unsigned long flags) 789 { 790 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); 791 struct sun6i_vchan *vchan = to_sun6i_vchan(chan); 792 struct dma_slave_config *sconfig = &vchan->cfg; 793 struct sun6i_dma_lli *v_lli, *prev = NULL; 794 struct sun6i_desc *txd; 795 dma_addr_t p_lli; 796 u32 lli_cfg; 797 unsigned int i, periods = buf_len / period_len; 798 int ret; 799 800 ret = set_config(sdev, sconfig, dir, &lli_cfg); 801 if (ret) { 802 dev_err(chan2dev(chan), "Invalid DMA configuration\n"); 803 return NULL; 804 } 805 806 txd = kzalloc(sizeof(*txd), GFP_NOWAIT); 807 if (!txd) 808 return NULL; 809 810 for (i = 0; i < periods; i++) { 811 v_lli = dma_pool_alloc(sdev->pool, GFP_DMA32 | GFP_NOWAIT, &p_lli); 812 if (!v_lli) { 813 dev_err(sdev->slave.dev, "Failed to alloc lli memory\n"); 814 goto err_lli_free; 815 } 816 817 v_lli->len = period_len; 818 v_lli->para = NORMAL_WAIT; 819 820 if (dir == DMA_MEM_TO_DEV) { 821 sun6i_dma_set_addr(sdev, v_lli, 822 buf_addr + period_len * i, 823 sconfig->dst_addr); 824 v_lli->cfg = lli_cfg; 825 sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port); 826 sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE); 827 } else { 828 sun6i_dma_set_addr(sdev, v_lli, 829 sconfig->src_addr, 830 buf_addr + period_len * i); 831 v_lli->cfg = lli_cfg; 832 sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM); 833 sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE); 834 } 835 836 prev = sun6i_dma_lli_add(prev, v_lli, p_lli, txd); 837 } 838 839 prev->p_lli_next = txd->p_lli; /* cyclic list */ 840 841 vchan->cyclic = true; 842 843 return vchan_tx_prep(&vchan->vc, &txd->vd, flags); 844 845 err_lli_free: 846 for (p_lli = txd->p_lli, v_lli = txd->v_lli; v_lli; 847 p_lli = v_lli->p_lli_next, v_lli = v_lli->v_lli_next) 848 dma_pool_free(sdev->pool, v_lli, p_lli); 849 kfree(txd); 850 return NULL; 851 } 852 853 static int sun6i_dma_config(struct dma_chan *chan, 854 struct dma_slave_config *config) 855 { 856 struct sun6i_vchan *vchan = to_sun6i_vchan(chan); 857 858 memcpy(&vchan->cfg, config, sizeof(*config)); 859 860 return 0; 861 } 862 863 static int sun6i_dma_pause(struct dma_chan *chan) 864 { 865 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); 866 struct sun6i_vchan *vchan = to_sun6i_vchan(chan); 867 struct sun6i_pchan *pchan = vchan->phy; 868 869 dev_dbg(chan2dev(chan), "vchan %p: pause\n", &vchan->vc); 870 871 if (pchan) { 872 writel(DMA_CHAN_PAUSE_PAUSE, 873 pchan->base + DMA_CHAN_PAUSE); 874 } else { 875 spin_lock(&sdev->lock); 876 list_del_init(&vchan->node); 877 spin_unlock(&sdev->lock); 878 } 879 880 return 0; 881 } 882 883 static int sun6i_dma_resume(struct dma_chan *chan) 884 { 885 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); 886 struct sun6i_vchan *vchan = to_sun6i_vchan(chan); 887 struct sun6i_pchan *pchan = vchan->phy; 888 unsigned long flags; 889 890 dev_dbg(chan2dev(chan), "vchan %p: resume\n", &vchan->vc); 891 892 spin_lock_irqsave(&vchan->vc.lock, flags); 893 894 if (pchan) { 895 writel(DMA_CHAN_PAUSE_RESUME, 896 pchan->base + DMA_CHAN_PAUSE); 897 } else if (!list_empty(&vchan->vc.desc_issued)) { 898 spin_lock(&sdev->lock); 899 list_add_tail(&vchan->node, &sdev->pending); 900 spin_unlock(&sdev->lock); 901 } 902 903 spin_unlock_irqrestore(&vchan->vc.lock, flags); 904 905 return 0; 906 } 907 908 static int sun6i_dma_terminate_all(struct dma_chan *chan) 909 { 910 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); 911 struct sun6i_vchan *vchan = to_sun6i_vchan(chan); 912 struct sun6i_pchan *pchan = vchan->phy; 913 unsigned long flags; 914 LIST_HEAD(head); 915 916 spin_lock(&sdev->lock); 917 list_del_init(&vchan->node); 918 spin_unlock(&sdev->lock); 919 920 spin_lock_irqsave(&vchan->vc.lock, flags); 921 922 if (vchan->cyclic) { 923 vchan->cyclic = false; 924 if (pchan && pchan->desc) { 925 struct virt_dma_desc *vd = &pchan->desc->vd; 926 struct virt_dma_chan *vc = &vchan->vc; 927 928 list_add_tail(&vd->node, &vc->desc_completed); 929 } 930 } 931 932 vchan_get_all_descriptors(&vchan->vc, &head); 933 934 if (pchan) { 935 writel(DMA_CHAN_ENABLE_STOP, pchan->base + DMA_CHAN_ENABLE); 936 writel(DMA_CHAN_PAUSE_RESUME, pchan->base + DMA_CHAN_PAUSE); 937 938 vchan->phy = NULL; 939 pchan->vchan = NULL; 940 pchan->desc = NULL; 941 pchan->done = NULL; 942 } 943 944 spin_unlock_irqrestore(&vchan->vc.lock, flags); 945 946 vchan_dma_desc_free_list(&vchan->vc, &head); 947 948 return 0; 949 } 950 951 static enum dma_status sun6i_dma_tx_status(struct dma_chan *chan, 952 dma_cookie_t cookie, 953 struct dma_tx_state *state) 954 { 955 struct sun6i_vchan *vchan = to_sun6i_vchan(chan); 956 struct sun6i_pchan *pchan = vchan->phy; 957 struct sun6i_dma_lli *lli; 958 struct virt_dma_desc *vd; 959 struct sun6i_desc *txd; 960 enum dma_status ret; 961 unsigned long flags; 962 size_t bytes = 0; 963 964 ret = dma_cookie_status(chan, cookie, state); 965 if (ret == DMA_COMPLETE || !state) 966 return ret; 967 968 spin_lock_irqsave(&vchan->vc.lock, flags); 969 970 vd = vchan_find_desc(&vchan->vc, cookie); 971 txd = to_sun6i_desc(&vd->tx); 972 973 if (vd) { 974 for (lli = txd->v_lli; lli != NULL; lli = lli->v_lli_next) 975 bytes += lli->len; 976 } else if (!pchan || !pchan->desc) { 977 bytes = 0; 978 } else { 979 bytes = sun6i_get_chan_size(pchan); 980 } 981 982 spin_unlock_irqrestore(&vchan->vc.lock, flags); 983 984 dma_set_residue(state, bytes); 985 986 return ret; 987 } 988 989 static void sun6i_dma_issue_pending(struct dma_chan *chan) 990 { 991 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); 992 struct sun6i_vchan *vchan = to_sun6i_vchan(chan); 993 unsigned long flags; 994 995 spin_lock_irqsave(&vchan->vc.lock, flags); 996 997 if (vchan_issue_pending(&vchan->vc)) { 998 spin_lock(&sdev->lock); 999 1000 if (!vchan->phy && list_empty(&vchan->node)) { 1001 list_add_tail(&vchan->node, &sdev->pending); 1002 tasklet_schedule(&sdev->task); 1003 dev_dbg(chan2dev(chan), "vchan %p: issued\n", 1004 &vchan->vc); 1005 } 1006 1007 spin_unlock(&sdev->lock); 1008 } else { 1009 dev_dbg(chan2dev(chan), "vchan %p: nothing to issue\n", 1010 &vchan->vc); 1011 } 1012 1013 spin_unlock_irqrestore(&vchan->vc.lock, flags); 1014 } 1015 1016 static void sun6i_dma_free_chan_resources(struct dma_chan *chan) 1017 { 1018 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); 1019 struct sun6i_vchan *vchan = to_sun6i_vchan(chan); 1020 unsigned long flags; 1021 1022 spin_lock_irqsave(&sdev->lock, flags); 1023 list_del_init(&vchan->node); 1024 spin_unlock_irqrestore(&sdev->lock, flags); 1025 1026 vchan_free_chan_resources(&vchan->vc); 1027 } 1028 1029 static struct dma_chan *sun6i_dma_of_xlate(struct of_phandle_args *dma_spec, 1030 struct of_dma *ofdma) 1031 { 1032 struct sun6i_dma_dev *sdev = ofdma->of_dma_data; 1033 struct sun6i_vchan *vchan; 1034 struct dma_chan *chan; 1035 u8 port = dma_spec->args[0]; 1036 1037 if (port > sdev->max_request) 1038 return NULL; 1039 1040 chan = dma_get_any_slave_channel(&sdev->slave); 1041 if (!chan) 1042 return NULL; 1043 1044 vchan = to_sun6i_vchan(chan); 1045 vchan->port = port; 1046 1047 return chan; 1048 } 1049 1050 static inline void sun6i_kill_tasklet(struct sun6i_dma_dev *sdev) 1051 { 1052 /* Disable all interrupts from DMA */ 1053 writel(0, sdev->base + DMA_IRQ_EN(0)); 1054 writel(0, sdev->base + DMA_IRQ_EN(1)); 1055 1056 /* Prevent spurious interrupts from scheduling the tasklet */ 1057 atomic_inc(&sdev->tasklet_shutdown); 1058 1059 /* Make sure we won't have any further interrupts */ 1060 devm_free_irq(sdev->slave.dev, sdev->irq, sdev); 1061 1062 /* Actually prevent the tasklet from being scheduled */ 1063 tasklet_kill(&sdev->task); 1064 } 1065 1066 static inline void sun6i_dma_free(struct sun6i_dma_dev *sdev) 1067 { 1068 int i; 1069 1070 for (i = 0; i < sdev->num_vchans; i++) { 1071 struct sun6i_vchan *vchan = &sdev->vchans[i]; 1072 1073 list_del(&vchan->vc.chan.device_node); 1074 tasklet_kill(&vchan->vc.task); 1075 } 1076 } 1077 1078 /* 1079 * For A31: 1080 * 1081 * There's 16 physical channels that can work in parallel. 1082 * 1083 * However we have 30 different endpoints for our requests. 1084 * 1085 * Since the channels are able to handle only an unidirectional 1086 * transfer, we need to allocate more virtual channels so that 1087 * everyone can grab one channel. 1088 * 1089 * Some devices can't work in both direction (mostly because it 1090 * wouldn't make sense), so we have a bit fewer virtual channels than 1091 * 2 channels per endpoints. 1092 */ 1093 1094 static struct sun6i_dma_config sun6i_a31_dma_cfg = { 1095 .nr_max_channels = 16, 1096 .nr_max_requests = 30, 1097 .nr_max_vchans = 53, 1098 .set_burst_length = sun6i_set_burst_length_a31, 1099 .set_drq = sun6i_set_drq_a31, 1100 .set_mode = sun6i_set_mode_a31, 1101 .src_burst_lengths = BIT(1) | BIT(8), 1102 .dst_burst_lengths = BIT(1) | BIT(8), 1103 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1104 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1105 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), 1106 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1107 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1108 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), 1109 }; 1110 1111 /* 1112 * The A23 only has 8 physical channels, a maximum DRQ port id of 24, 1113 * and a total of 37 usable source and destination endpoints. 1114 */ 1115 1116 static struct sun6i_dma_config sun8i_a23_dma_cfg = { 1117 .nr_max_channels = 8, 1118 .nr_max_requests = 24, 1119 .nr_max_vchans = 37, 1120 .clock_autogate_enable = sun6i_enable_clock_autogate_a23, 1121 .set_burst_length = sun6i_set_burst_length_a31, 1122 .set_drq = sun6i_set_drq_a31, 1123 .set_mode = sun6i_set_mode_a31, 1124 .src_burst_lengths = BIT(1) | BIT(8), 1125 .dst_burst_lengths = BIT(1) | BIT(8), 1126 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1127 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1128 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), 1129 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1130 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1131 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), 1132 }; 1133 1134 static struct sun6i_dma_config sun8i_a83t_dma_cfg = { 1135 .nr_max_channels = 8, 1136 .nr_max_requests = 28, 1137 .nr_max_vchans = 39, 1138 .clock_autogate_enable = sun6i_enable_clock_autogate_a23, 1139 .set_burst_length = sun6i_set_burst_length_a31, 1140 .set_drq = sun6i_set_drq_a31, 1141 .set_mode = sun6i_set_mode_a31, 1142 .src_burst_lengths = BIT(1) | BIT(8), 1143 .dst_burst_lengths = BIT(1) | BIT(8), 1144 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1145 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1146 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), 1147 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1148 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1149 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), 1150 }; 1151 1152 /* 1153 * The H3 has 12 physical channels, a maximum DRQ port id of 27, 1154 * and a total of 34 usable source and destination endpoints. 1155 * It also supports additional burst lengths and bus widths, 1156 * and the burst length fields have different offsets. 1157 */ 1158 1159 static struct sun6i_dma_config sun8i_h3_dma_cfg = { 1160 .nr_max_channels = 12, 1161 .nr_max_requests = 27, 1162 .nr_max_vchans = 34, 1163 .clock_autogate_enable = sun6i_enable_clock_autogate_h3, 1164 .set_burst_length = sun6i_set_burst_length_h3, 1165 .set_drq = sun6i_set_drq_a31, 1166 .set_mode = sun6i_set_mode_a31, 1167 .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), 1168 .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), 1169 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1170 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1171 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1172 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), 1173 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1174 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1175 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1176 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), 1177 }; 1178 1179 /* 1180 * The A64 binding uses the number of dma channels from the 1181 * device tree node. 1182 */ 1183 static struct sun6i_dma_config sun50i_a64_dma_cfg = { 1184 .clock_autogate_enable = sun6i_enable_clock_autogate_h3, 1185 .set_burst_length = sun6i_set_burst_length_h3, 1186 .set_drq = sun6i_set_drq_a31, 1187 .set_mode = sun6i_set_mode_a31, 1188 .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), 1189 .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), 1190 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1191 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1192 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1193 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), 1194 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1195 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1196 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1197 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), 1198 }; 1199 1200 /* 1201 * The A100 binding uses the number of dma channels from the 1202 * device tree node. 1203 */ 1204 static struct sun6i_dma_config sun50i_a100_dma_cfg = { 1205 .clock_autogate_enable = sun6i_enable_clock_autogate_h3, 1206 .set_burst_length = sun6i_set_burst_length_h3, 1207 .set_drq = sun6i_set_drq_h6, 1208 .set_mode = sun6i_set_mode_h6, 1209 .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), 1210 .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), 1211 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1212 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1213 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1214 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), 1215 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1216 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1217 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1218 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), 1219 .has_high_addr = true, 1220 .has_mbus_clk = true, 1221 }; 1222 1223 /* 1224 * The H6 binding uses the number of dma channels from the 1225 * device tree node. 1226 */ 1227 static struct sun6i_dma_config sun50i_h6_dma_cfg = { 1228 .clock_autogate_enable = sun6i_enable_clock_autogate_h3, 1229 .set_burst_length = sun6i_set_burst_length_h3, 1230 .set_drq = sun6i_set_drq_h6, 1231 .set_mode = sun6i_set_mode_h6, 1232 .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), 1233 .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), 1234 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1235 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1236 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1237 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), 1238 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1239 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1240 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1241 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), 1242 .has_mbus_clk = true, 1243 }; 1244 1245 /* 1246 * The V3s have only 8 physical channels, a maximum DRQ port id of 23, 1247 * and a total of 24 usable source and destination endpoints. 1248 */ 1249 1250 static struct sun6i_dma_config sun8i_v3s_dma_cfg = { 1251 .nr_max_channels = 8, 1252 .nr_max_requests = 23, 1253 .nr_max_vchans = 24, 1254 .clock_autogate_enable = sun6i_enable_clock_autogate_a23, 1255 .set_burst_length = sun6i_set_burst_length_a31, 1256 .set_drq = sun6i_set_drq_a31, 1257 .set_mode = sun6i_set_mode_a31, 1258 .src_burst_lengths = BIT(1) | BIT(8), 1259 .dst_burst_lengths = BIT(1) | BIT(8), 1260 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1261 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1262 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), 1263 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1264 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1265 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), 1266 }; 1267 1268 static const struct of_device_id sun6i_dma_match[] = { 1269 { .compatible = "allwinner,sun6i-a31-dma", .data = &sun6i_a31_dma_cfg }, 1270 { .compatible = "allwinner,sun8i-a23-dma", .data = &sun8i_a23_dma_cfg }, 1271 { .compatible = "allwinner,sun8i-a83t-dma", .data = &sun8i_a83t_dma_cfg }, 1272 { .compatible = "allwinner,sun8i-h3-dma", .data = &sun8i_h3_dma_cfg }, 1273 { .compatible = "allwinner,sun8i-v3s-dma", .data = &sun8i_v3s_dma_cfg }, 1274 { .compatible = "allwinner,sun20i-d1-dma", .data = &sun50i_a100_dma_cfg }, 1275 { .compatible = "allwinner,sun50i-a64-dma", .data = &sun50i_a64_dma_cfg }, 1276 { .compatible = "allwinner,sun50i-a100-dma", .data = &sun50i_a100_dma_cfg }, 1277 { .compatible = "allwinner,sun50i-h6-dma", .data = &sun50i_h6_dma_cfg }, 1278 { /* sentinel */ } 1279 }; 1280 MODULE_DEVICE_TABLE(of, sun6i_dma_match); 1281 1282 static int sun6i_dma_probe(struct platform_device *pdev) 1283 { 1284 struct device_node *np = pdev->dev.of_node; 1285 struct sun6i_dma_dev *sdc; 1286 struct resource *res; 1287 int ret, i; 1288 1289 sdc = devm_kzalloc(&pdev->dev, sizeof(*sdc), GFP_KERNEL); 1290 if (!sdc) 1291 return -ENOMEM; 1292 1293 sdc->cfg = of_device_get_match_data(&pdev->dev); 1294 if (!sdc->cfg) 1295 return -ENODEV; 1296 1297 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1298 sdc->base = devm_ioremap_resource(&pdev->dev, res); 1299 if (IS_ERR(sdc->base)) 1300 return PTR_ERR(sdc->base); 1301 1302 sdc->irq = platform_get_irq(pdev, 0); 1303 if (sdc->irq < 0) 1304 return sdc->irq; 1305 1306 sdc->clk = devm_clk_get(&pdev->dev, NULL); 1307 if (IS_ERR(sdc->clk)) { 1308 dev_err(&pdev->dev, "No clock specified\n"); 1309 return PTR_ERR(sdc->clk); 1310 } 1311 1312 if (sdc->cfg->has_mbus_clk) { 1313 sdc->clk_mbus = devm_clk_get(&pdev->dev, "mbus"); 1314 if (IS_ERR(sdc->clk_mbus)) { 1315 dev_err(&pdev->dev, "No mbus clock specified\n"); 1316 return PTR_ERR(sdc->clk_mbus); 1317 } 1318 } 1319 1320 sdc->rstc = devm_reset_control_get(&pdev->dev, NULL); 1321 if (IS_ERR(sdc->rstc)) { 1322 dev_err(&pdev->dev, "No reset controller specified\n"); 1323 return PTR_ERR(sdc->rstc); 1324 } 1325 1326 sdc->pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev, 1327 sizeof(struct sun6i_dma_lli), 4, 0); 1328 if (!sdc->pool) { 1329 dev_err(&pdev->dev, "No memory for descriptors dma pool\n"); 1330 return -ENOMEM; 1331 } 1332 1333 platform_set_drvdata(pdev, sdc); 1334 INIT_LIST_HEAD(&sdc->pending); 1335 spin_lock_init(&sdc->lock); 1336 1337 dma_cap_set(DMA_PRIVATE, sdc->slave.cap_mask); 1338 dma_cap_set(DMA_MEMCPY, sdc->slave.cap_mask); 1339 dma_cap_set(DMA_SLAVE, sdc->slave.cap_mask); 1340 dma_cap_set(DMA_CYCLIC, sdc->slave.cap_mask); 1341 1342 INIT_LIST_HEAD(&sdc->slave.channels); 1343 sdc->slave.device_free_chan_resources = sun6i_dma_free_chan_resources; 1344 sdc->slave.device_tx_status = sun6i_dma_tx_status; 1345 sdc->slave.device_issue_pending = sun6i_dma_issue_pending; 1346 sdc->slave.device_prep_slave_sg = sun6i_dma_prep_slave_sg; 1347 sdc->slave.device_prep_dma_memcpy = sun6i_dma_prep_dma_memcpy; 1348 sdc->slave.device_prep_dma_cyclic = sun6i_dma_prep_dma_cyclic; 1349 sdc->slave.copy_align = DMAENGINE_ALIGN_4_BYTES; 1350 sdc->slave.device_config = sun6i_dma_config; 1351 sdc->slave.device_pause = sun6i_dma_pause; 1352 sdc->slave.device_resume = sun6i_dma_resume; 1353 sdc->slave.device_terminate_all = sun6i_dma_terminate_all; 1354 sdc->slave.src_addr_widths = sdc->cfg->src_addr_widths; 1355 sdc->slave.dst_addr_widths = sdc->cfg->dst_addr_widths; 1356 sdc->slave.directions = BIT(DMA_DEV_TO_MEM) | 1357 BIT(DMA_MEM_TO_DEV); 1358 sdc->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1359 sdc->slave.dev = &pdev->dev; 1360 1361 sdc->num_pchans = sdc->cfg->nr_max_channels; 1362 sdc->num_vchans = sdc->cfg->nr_max_vchans; 1363 sdc->max_request = sdc->cfg->nr_max_requests; 1364 1365 ret = of_property_read_u32(np, "dma-channels", &sdc->num_pchans); 1366 if (ret && !sdc->num_pchans) { 1367 dev_err(&pdev->dev, "Can't get dma-channels.\n"); 1368 return ret; 1369 } 1370 1371 ret = of_property_read_u32(np, "dma-requests", &sdc->max_request); 1372 if (ret && !sdc->max_request) { 1373 dev_info(&pdev->dev, "Missing dma-requests, using %u.\n", 1374 DMA_CHAN_MAX_DRQ_A31); 1375 sdc->max_request = DMA_CHAN_MAX_DRQ_A31; 1376 } 1377 1378 /* 1379 * If the number of vchans is not specified, derive it from the 1380 * highest port number, at most one channel per port and direction. 1381 */ 1382 if (!sdc->num_vchans) 1383 sdc->num_vchans = 2 * (sdc->max_request + 1); 1384 1385 sdc->pchans = devm_kcalloc(&pdev->dev, sdc->num_pchans, 1386 sizeof(struct sun6i_pchan), GFP_KERNEL); 1387 if (!sdc->pchans) 1388 return -ENOMEM; 1389 1390 sdc->vchans = devm_kcalloc(&pdev->dev, sdc->num_vchans, 1391 sizeof(struct sun6i_vchan), GFP_KERNEL); 1392 if (!sdc->vchans) 1393 return -ENOMEM; 1394 1395 tasklet_setup(&sdc->task, sun6i_dma_tasklet); 1396 1397 for (i = 0; i < sdc->num_pchans; i++) { 1398 struct sun6i_pchan *pchan = &sdc->pchans[i]; 1399 1400 pchan->idx = i; 1401 pchan->base = sdc->base + 0x100 + i * 0x40; 1402 } 1403 1404 for (i = 0; i < sdc->num_vchans; i++) { 1405 struct sun6i_vchan *vchan = &sdc->vchans[i]; 1406 1407 INIT_LIST_HEAD(&vchan->node); 1408 vchan->vc.desc_free = sun6i_dma_free_desc; 1409 vchan_init(&vchan->vc, &sdc->slave); 1410 } 1411 1412 ret = reset_control_deassert(sdc->rstc); 1413 if (ret) { 1414 dev_err(&pdev->dev, "Couldn't deassert the device from reset\n"); 1415 goto err_chan_free; 1416 } 1417 1418 ret = clk_prepare_enable(sdc->clk); 1419 if (ret) { 1420 dev_err(&pdev->dev, "Couldn't enable the clock\n"); 1421 goto err_reset_assert; 1422 } 1423 1424 if (sdc->cfg->has_mbus_clk) { 1425 ret = clk_prepare_enable(sdc->clk_mbus); 1426 if (ret) { 1427 dev_err(&pdev->dev, "Couldn't enable mbus clock\n"); 1428 goto err_clk_disable; 1429 } 1430 } 1431 1432 ret = devm_request_irq(&pdev->dev, sdc->irq, sun6i_dma_interrupt, 0, 1433 dev_name(&pdev->dev), sdc); 1434 if (ret) { 1435 dev_err(&pdev->dev, "Cannot request IRQ\n"); 1436 goto err_mbus_clk_disable; 1437 } 1438 1439 ret = dma_async_device_register(&sdc->slave); 1440 if (ret) { 1441 dev_warn(&pdev->dev, "Failed to register DMA engine device\n"); 1442 goto err_irq_disable; 1443 } 1444 1445 ret = of_dma_controller_register(pdev->dev.of_node, sun6i_dma_of_xlate, 1446 sdc); 1447 if (ret) { 1448 dev_err(&pdev->dev, "of_dma_controller_register failed\n"); 1449 goto err_dma_unregister; 1450 } 1451 1452 if (sdc->cfg->clock_autogate_enable) 1453 sdc->cfg->clock_autogate_enable(sdc); 1454 1455 return 0; 1456 1457 err_dma_unregister: 1458 dma_async_device_unregister(&sdc->slave); 1459 err_irq_disable: 1460 sun6i_kill_tasklet(sdc); 1461 err_mbus_clk_disable: 1462 clk_disable_unprepare(sdc->clk_mbus); 1463 err_clk_disable: 1464 clk_disable_unprepare(sdc->clk); 1465 err_reset_assert: 1466 reset_control_assert(sdc->rstc); 1467 err_chan_free: 1468 sun6i_dma_free(sdc); 1469 return ret; 1470 } 1471 1472 static int sun6i_dma_remove(struct platform_device *pdev) 1473 { 1474 struct sun6i_dma_dev *sdc = platform_get_drvdata(pdev); 1475 1476 of_dma_controller_free(pdev->dev.of_node); 1477 dma_async_device_unregister(&sdc->slave); 1478 1479 sun6i_kill_tasklet(sdc); 1480 1481 clk_disable_unprepare(sdc->clk_mbus); 1482 clk_disable_unprepare(sdc->clk); 1483 reset_control_assert(sdc->rstc); 1484 1485 sun6i_dma_free(sdc); 1486 1487 return 0; 1488 } 1489 1490 static struct platform_driver sun6i_dma_driver = { 1491 .probe = sun6i_dma_probe, 1492 .remove = sun6i_dma_remove, 1493 .driver = { 1494 .name = "sun6i-dma", 1495 .of_match_table = sun6i_dma_match, 1496 }, 1497 }; 1498 module_platform_driver(sun6i_dma_driver); 1499 1500 MODULE_DESCRIPTION("Allwinner A31 DMA Controller Driver"); 1501 MODULE_AUTHOR("Sugar <shuge@allwinnertech.com>"); 1502 MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com>"); 1503 MODULE_LICENSE("GPL"); 1504