1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2013-2014 Allwinner Tech Co., Ltd 4 * Author: Sugar <shuge@allwinnertech.com> 5 * 6 * Copyright (C) 2014 Maxime Ripard 7 * Maxime Ripard <maxime.ripard@free-electrons.com> 8 */ 9 10 #include <linux/clk.h> 11 #include <linux/delay.h> 12 #include <linux/dmaengine.h> 13 #include <linux/dmapool.h> 14 #include <linux/interrupt.h> 15 #include <linux/module.h> 16 #include <linux/of_dma.h> 17 #include <linux/of_device.h> 18 #include <linux/platform_device.h> 19 #include <linux/reset.h> 20 #include <linux/slab.h> 21 #include <linux/types.h> 22 23 #include "virt-dma.h" 24 25 /* 26 * Common registers 27 */ 28 #define DMA_IRQ_EN(x) ((x) * 0x04) 29 #define DMA_IRQ_HALF BIT(0) 30 #define DMA_IRQ_PKG BIT(1) 31 #define DMA_IRQ_QUEUE BIT(2) 32 33 #define DMA_IRQ_CHAN_NR 8 34 #define DMA_IRQ_CHAN_WIDTH 4 35 36 37 #define DMA_IRQ_STAT(x) ((x) * 0x04 + 0x10) 38 39 #define DMA_STAT 0x30 40 41 /* Offset between DMA_IRQ_EN and DMA_IRQ_STAT limits number of channels */ 42 #define DMA_MAX_CHANNELS (DMA_IRQ_CHAN_NR * 0x10 / 4) 43 44 /* 45 * sun8i specific registers 46 */ 47 #define SUN8I_DMA_GATE 0x20 48 #define SUN8I_DMA_GATE_ENABLE 0x4 49 50 #define SUNXI_H3_SECURE_REG 0x20 51 #define SUNXI_H3_DMA_GATE 0x28 52 #define SUNXI_H3_DMA_GATE_ENABLE 0x4 53 /* 54 * Channels specific registers 55 */ 56 #define DMA_CHAN_ENABLE 0x00 57 #define DMA_CHAN_ENABLE_START BIT(0) 58 #define DMA_CHAN_ENABLE_STOP 0 59 60 #define DMA_CHAN_PAUSE 0x04 61 #define DMA_CHAN_PAUSE_PAUSE BIT(1) 62 #define DMA_CHAN_PAUSE_RESUME 0 63 64 #define DMA_CHAN_LLI_ADDR 0x08 65 66 #define DMA_CHAN_CUR_CFG 0x0c 67 #define DMA_CHAN_MAX_DRQ_A31 0x1f 68 #define DMA_CHAN_MAX_DRQ_H6 0x3f 69 #define DMA_CHAN_CFG_SRC_DRQ_A31(x) ((x) & DMA_CHAN_MAX_DRQ_A31) 70 #define DMA_CHAN_CFG_SRC_DRQ_H6(x) ((x) & DMA_CHAN_MAX_DRQ_H6) 71 #define DMA_CHAN_CFG_SRC_MODE_A31(x) (((x) & 0x1) << 5) 72 #define DMA_CHAN_CFG_SRC_MODE_H6(x) (((x) & 0x1) << 8) 73 #define DMA_CHAN_CFG_SRC_BURST_A31(x) (((x) & 0x3) << 7) 74 #define DMA_CHAN_CFG_SRC_BURST_H3(x) (((x) & 0x3) << 6) 75 #define DMA_CHAN_CFG_SRC_WIDTH(x) (((x) & 0x3) << 9) 76 77 #define DMA_CHAN_CFG_DST_DRQ_A31(x) (DMA_CHAN_CFG_SRC_DRQ_A31(x) << 16) 78 #define DMA_CHAN_CFG_DST_DRQ_H6(x) (DMA_CHAN_CFG_SRC_DRQ_H6(x) << 16) 79 #define DMA_CHAN_CFG_DST_MODE_A31(x) (DMA_CHAN_CFG_SRC_MODE_A31(x) << 16) 80 #define DMA_CHAN_CFG_DST_MODE_H6(x) (DMA_CHAN_CFG_SRC_MODE_H6(x) << 16) 81 #define DMA_CHAN_CFG_DST_BURST_A31(x) (DMA_CHAN_CFG_SRC_BURST_A31(x) << 16) 82 #define DMA_CHAN_CFG_DST_BURST_H3(x) (DMA_CHAN_CFG_SRC_BURST_H3(x) << 16) 83 #define DMA_CHAN_CFG_DST_WIDTH(x) (DMA_CHAN_CFG_SRC_WIDTH(x) << 16) 84 85 #define DMA_CHAN_CUR_SRC 0x10 86 87 #define DMA_CHAN_CUR_DST 0x14 88 89 #define DMA_CHAN_CUR_CNT 0x18 90 91 #define DMA_CHAN_CUR_PARA 0x1c 92 93 94 /* 95 * Various hardware related defines 96 */ 97 #define LLI_LAST_ITEM 0xfffff800 98 #define NORMAL_WAIT 8 99 #define DRQ_SDRAM 1 100 #define LINEAR_MODE 0 101 #define IO_MODE 1 102 103 /* forward declaration */ 104 struct sun6i_dma_dev; 105 106 /* 107 * Hardware channels / ports representation 108 * 109 * The hardware is used in several SoCs, with differing numbers 110 * of channels and endpoints. This structure ties those numbers 111 * to a certain compatible string. 112 */ 113 struct sun6i_dma_config { 114 u32 nr_max_channels; 115 u32 nr_max_requests; 116 u32 nr_max_vchans; 117 /* 118 * In the datasheets/user manuals of newer Allwinner SoCs, a special 119 * bit (bit 2 at register 0x20) is present. 120 * It's named "DMA MCLK interface circuit auto gating bit" in the 121 * documents, and the footnote of this register says that this bit 122 * should be set up when initializing the DMA controller. 123 * Allwinner A23/A33 user manuals do not have this bit documented, 124 * however these SoCs really have and need this bit, as seen in the 125 * BSP kernel source code. 126 */ 127 void (*clock_autogate_enable)(struct sun6i_dma_dev *); 128 void (*set_burst_length)(u32 *p_cfg, s8 src_burst, s8 dst_burst); 129 void (*set_drq)(u32 *p_cfg, s8 src_drq, s8 dst_drq); 130 void (*set_mode)(u32 *p_cfg, s8 src_mode, s8 dst_mode); 131 u32 src_burst_lengths; 132 u32 dst_burst_lengths; 133 u32 src_addr_widths; 134 u32 dst_addr_widths; 135 bool has_mbus_clk; 136 }; 137 138 /* 139 * Hardware representation of the LLI 140 * 141 * The hardware will be fed the physical address of this structure, 142 * and read its content in order to start the transfer. 143 */ 144 struct sun6i_dma_lli { 145 u32 cfg; 146 u32 src; 147 u32 dst; 148 u32 len; 149 u32 para; 150 u32 p_lli_next; 151 152 /* 153 * This field is not used by the DMA controller, but will be 154 * used by the CPU to go through the list (mostly for dumping 155 * or freeing it). 156 */ 157 struct sun6i_dma_lli *v_lli_next; 158 }; 159 160 161 struct sun6i_desc { 162 struct virt_dma_desc vd; 163 dma_addr_t p_lli; 164 struct sun6i_dma_lli *v_lli; 165 }; 166 167 struct sun6i_pchan { 168 u32 idx; 169 void __iomem *base; 170 struct sun6i_vchan *vchan; 171 struct sun6i_desc *desc; 172 struct sun6i_desc *done; 173 }; 174 175 struct sun6i_vchan { 176 struct virt_dma_chan vc; 177 struct list_head node; 178 struct dma_slave_config cfg; 179 struct sun6i_pchan *phy; 180 u8 port; 181 u8 irq_type; 182 bool cyclic; 183 }; 184 185 struct sun6i_dma_dev { 186 struct dma_device slave; 187 void __iomem *base; 188 struct clk *clk; 189 struct clk *clk_mbus; 190 int irq; 191 spinlock_t lock; 192 struct reset_control *rstc; 193 struct tasklet_struct task; 194 atomic_t tasklet_shutdown; 195 struct list_head pending; 196 struct dma_pool *pool; 197 struct sun6i_pchan *pchans; 198 struct sun6i_vchan *vchans; 199 const struct sun6i_dma_config *cfg; 200 u32 num_pchans; 201 u32 num_vchans; 202 u32 max_request; 203 }; 204 205 static struct device *chan2dev(struct dma_chan *chan) 206 { 207 return &chan->dev->device; 208 } 209 210 static inline struct sun6i_dma_dev *to_sun6i_dma_dev(struct dma_device *d) 211 { 212 return container_of(d, struct sun6i_dma_dev, slave); 213 } 214 215 static inline struct sun6i_vchan *to_sun6i_vchan(struct dma_chan *chan) 216 { 217 return container_of(chan, struct sun6i_vchan, vc.chan); 218 } 219 220 static inline struct sun6i_desc * 221 to_sun6i_desc(struct dma_async_tx_descriptor *tx) 222 { 223 return container_of(tx, struct sun6i_desc, vd.tx); 224 } 225 226 static inline void sun6i_dma_dump_com_regs(struct sun6i_dma_dev *sdev) 227 { 228 dev_dbg(sdev->slave.dev, "Common register:\n" 229 "\tmask0(%04x): 0x%08x\n" 230 "\tmask1(%04x): 0x%08x\n" 231 "\tpend0(%04x): 0x%08x\n" 232 "\tpend1(%04x): 0x%08x\n" 233 "\tstats(%04x): 0x%08x\n", 234 DMA_IRQ_EN(0), readl(sdev->base + DMA_IRQ_EN(0)), 235 DMA_IRQ_EN(1), readl(sdev->base + DMA_IRQ_EN(1)), 236 DMA_IRQ_STAT(0), readl(sdev->base + DMA_IRQ_STAT(0)), 237 DMA_IRQ_STAT(1), readl(sdev->base + DMA_IRQ_STAT(1)), 238 DMA_STAT, readl(sdev->base + DMA_STAT)); 239 } 240 241 static inline void sun6i_dma_dump_chan_regs(struct sun6i_dma_dev *sdev, 242 struct sun6i_pchan *pchan) 243 { 244 phys_addr_t reg = virt_to_phys(pchan->base); 245 246 dev_dbg(sdev->slave.dev, "Chan %d reg: %pa\n" 247 "\t___en(%04x): \t0x%08x\n" 248 "\tpause(%04x): \t0x%08x\n" 249 "\tstart(%04x): \t0x%08x\n" 250 "\t__cfg(%04x): \t0x%08x\n" 251 "\t__src(%04x): \t0x%08x\n" 252 "\t__dst(%04x): \t0x%08x\n" 253 "\tcount(%04x): \t0x%08x\n" 254 "\t_para(%04x): \t0x%08x\n\n", 255 pchan->idx, ®, 256 DMA_CHAN_ENABLE, 257 readl(pchan->base + DMA_CHAN_ENABLE), 258 DMA_CHAN_PAUSE, 259 readl(pchan->base + DMA_CHAN_PAUSE), 260 DMA_CHAN_LLI_ADDR, 261 readl(pchan->base + DMA_CHAN_LLI_ADDR), 262 DMA_CHAN_CUR_CFG, 263 readl(pchan->base + DMA_CHAN_CUR_CFG), 264 DMA_CHAN_CUR_SRC, 265 readl(pchan->base + DMA_CHAN_CUR_SRC), 266 DMA_CHAN_CUR_DST, 267 readl(pchan->base + DMA_CHAN_CUR_DST), 268 DMA_CHAN_CUR_CNT, 269 readl(pchan->base + DMA_CHAN_CUR_CNT), 270 DMA_CHAN_CUR_PARA, 271 readl(pchan->base + DMA_CHAN_CUR_PARA)); 272 } 273 274 static inline s8 convert_burst(u32 maxburst) 275 { 276 switch (maxburst) { 277 case 1: 278 return 0; 279 case 4: 280 return 1; 281 case 8: 282 return 2; 283 case 16: 284 return 3; 285 default: 286 return -EINVAL; 287 } 288 } 289 290 static inline s8 convert_buswidth(enum dma_slave_buswidth addr_width) 291 { 292 return ilog2(addr_width); 293 } 294 295 static void sun6i_enable_clock_autogate_a23(struct sun6i_dma_dev *sdev) 296 { 297 writel(SUN8I_DMA_GATE_ENABLE, sdev->base + SUN8I_DMA_GATE); 298 } 299 300 static void sun6i_enable_clock_autogate_h3(struct sun6i_dma_dev *sdev) 301 { 302 writel(SUNXI_H3_DMA_GATE_ENABLE, sdev->base + SUNXI_H3_DMA_GATE); 303 } 304 305 static void sun6i_set_burst_length_a31(u32 *p_cfg, s8 src_burst, s8 dst_burst) 306 { 307 *p_cfg |= DMA_CHAN_CFG_SRC_BURST_A31(src_burst) | 308 DMA_CHAN_CFG_DST_BURST_A31(dst_burst); 309 } 310 311 static void sun6i_set_burst_length_h3(u32 *p_cfg, s8 src_burst, s8 dst_burst) 312 { 313 *p_cfg |= DMA_CHAN_CFG_SRC_BURST_H3(src_burst) | 314 DMA_CHAN_CFG_DST_BURST_H3(dst_burst); 315 } 316 317 static void sun6i_set_drq_a31(u32 *p_cfg, s8 src_drq, s8 dst_drq) 318 { 319 *p_cfg |= DMA_CHAN_CFG_SRC_DRQ_A31(src_drq) | 320 DMA_CHAN_CFG_DST_DRQ_A31(dst_drq); 321 } 322 323 static void sun6i_set_drq_h6(u32 *p_cfg, s8 src_drq, s8 dst_drq) 324 { 325 *p_cfg |= DMA_CHAN_CFG_SRC_DRQ_H6(src_drq) | 326 DMA_CHAN_CFG_DST_DRQ_H6(dst_drq); 327 } 328 329 static void sun6i_set_mode_a31(u32 *p_cfg, s8 src_mode, s8 dst_mode) 330 { 331 *p_cfg |= DMA_CHAN_CFG_SRC_MODE_A31(src_mode) | 332 DMA_CHAN_CFG_DST_MODE_A31(dst_mode); 333 } 334 335 static void sun6i_set_mode_h6(u32 *p_cfg, s8 src_mode, s8 dst_mode) 336 { 337 *p_cfg |= DMA_CHAN_CFG_SRC_MODE_H6(src_mode) | 338 DMA_CHAN_CFG_DST_MODE_H6(dst_mode); 339 } 340 341 static size_t sun6i_get_chan_size(struct sun6i_pchan *pchan) 342 { 343 struct sun6i_desc *txd = pchan->desc; 344 struct sun6i_dma_lli *lli; 345 size_t bytes; 346 dma_addr_t pos; 347 348 pos = readl(pchan->base + DMA_CHAN_LLI_ADDR); 349 bytes = readl(pchan->base + DMA_CHAN_CUR_CNT); 350 351 if (pos == LLI_LAST_ITEM) 352 return bytes; 353 354 for (lli = txd->v_lli; lli; lli = lli->v_lli_next) { 355 if (lli->p_lli_next == pos) { 356 for (lli = lli->v_lli_next; lli; lli = lli->v_lli_next) 357 bytes += lli->len; 358 break; 359 } 360 } 361 362 return bytes; 363 } 364 365 static void *sun6i_dma_lli_add(struct sun6i_dma_lli *prev, 366 struct sun6i_dma_lli *next, 367 dma_addr_t next_phy, 368 struct sun6i_desc *txd) 369 { 370 if ((!prev && !txd) || !next) 371 return NULL; 372 373 if (!prev) { 374 txd->p_lli = next_phy; 375 txd->v_lli = next; 376 } else { 377 prev->p_lli_next = next_phy; 378 prev->v_lli_next = next; 379 } 380 381 next->p_lli_next = LLI_LAST_ITEM; 382 next->v_lli_next = NULL; 383 384 return next; 385 } 386 387 static inline void sun6i_dma_dump_lli(struct sun6i_vchan *vchan, 388 struct sun6i_dma_lli *lli) 389 { 390 phys_addr_t p_lli = virt_to_phys(lli); 391 392 dev_dbg(chan2dev(&vchan->vc.chan), 393 "\n\tdesc: p - %pa v - 0x%p\n" 394 "\t\tc - 0x%08x s - 0x%08x d - 0x%08x\n" 395 "\t\tl - 0x%08x p - 0x%08x n - 0x%08x\n", 396 &p_lli, lli, 397 lli->cfg, lli->src, lli->dst, 398 lli->len, lli->para, lli->p_lli_next); 399 } 400 401 static void sun6i_dma_free_desc(struct virt_dma_desc *vd) 402 { 403 struct sun6i_desc *txd = to_sun6i_desc(&vd->tx); 404 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(vd->tx.chan->device); 405 struct sun6i_dma_lli *v_lli, *v_next; 406 dma_addr_t p_lli, p_next; 407 408 if (unlikely(!txd)) 409 return; 410 411 p_lli = txd->p_lli; 412 v_lli = txd->v_lli; 413 414 while (v_lli) { 415 v_next = v_lli->v_lli_next; 416 p_next = v_lli->p_lli_next; 417 418 dma_pool_free(sdev->pool, v_lli, p_lli); 419 420 v_lli = v_next; 421 p_lli = p_next; 422 } 423 424 kfree(txd); 425 } 426 427 static int sun6i_dma_start_desc(struct sun6i_vchan *vchan) 428 { 429 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(vchan->vc.chan.device); 430 struct virt_dma_desc *desc = vchan_next_desc(&vchan->vc); 431 struct sun6i_pchan *pchan = vchan->phy; 432 u32 irq_val, irq_reg, irq_offset; 433 434 if (!pchan) 435 return -EAGAIN; 436 437 if (!desc) { 438 pchan->desc = NULL; 439 pchan->done = NULL; 440 return -EAGAIN; 441 } 442 443 list_del(&desc->node); 444 445 pchan->desc = to_sun6i_desc(&desc->tx); 446 pchan->done = NULL; 447 448 sun6i_dma_dump_lli(vchan, pchan->desc->v_lli); 449 450 irq_reg = pchan->idx / DMA_IRQ_CHAN_NR; 451 irq_offset = pchan->idx % DMA_IRQ_CHAN_NR; 452 453 vchan->irq_type = vchan->cyclic ? DMA_IRQ_PKG : DMA_IRQ_QUEUE; 454 455 irq_val = readl(sdev->base + DMA_IRQ_EN(irq_reg)); 456 irq_val &= ~((DMA_IRQ_HALF | DMA_IRQ_PKG | DMA_IRQ_QUEUE) << 457 (irq_offset * DMA_IRQ_CHAN_WIDTH)); 458 irq_val |= vchan->irq_type << (irq_offset * DMA_IRQ_CHAN_WIDTH); 459 writel(irq_val, sdev->base + DMA_IRQ_EN(irq_reg)); 460 461 writel(pchan->desc->p_lli, pchan->base + DMA_CHAN_LLI_ADDR); 462 writel(DMA_CHAN_ENABLE_START, pchan->base + DMA_CHAN_ENABLE); 463 464 sun6i_dma_dump_com_regs(sdev); 465 sun6i_dma_dump_chan_regs(sdev, pchan); 466 467 return 0; 468 } 469 470 static void sun6i_dma_tasklet(struct tasklet_struct *t) 471 { 472 struct sun6i_dma_dev *sdev = from_tasklet(sdev, t, task); 473 struct sun6i_vchan *vchan; 474 struct sun6i_pchan *pchan; 475 unsigned int pchan_alloc = 0; 476 unsigned int pchan_idx; 477 478 list_for_each_entry(vchan, &sdev->slave.channels, vc.chan.device_node) { 479 spin_lock_irq(&vchan->vc.lock); 480 481 pchan = vchan->phy; 482 483 if (pchan && pchan->done) { 484 if (sun6i_dma_start_desc(vchan)) { 485 /* 486 * No current txd associated with this channel 487 */ 488 dev_dbg(sdev->slave.dev, "pchan %u: free\n", 489 pchan->idx); 490 491 /* Mark this channel free */ 492 vchan->phy = NULL; 493 pchan->vchan = NULL; 494 } 495 } 496 spin_unlock_irq(&vchan->vc.lock); 497 } 498 499 spin_lock_irq(&sdev->lock); 500 for (pchan_idx = 0; pchan_idx < sdev->num_pchans; pchan_idx++) { 501 pchan = &sdev->pchans[pchan_idx]; 502 503 if (pchan->vchan || list_empty(&sdev->pending)) 504 continue; 505 506 vchan = list_first_entry(&sdev->pending, 507 struct sun6i_vchan, node); 508 509 /* Remove from pending channels */ 510 list_del_init(&vchan->node); 511 pchan_alloc |= BIT(pchan_idx); 512 513 /* Mark this channel allocated */ 514 pchan->vchan = vchan; 515 vchan->phy = pchan; 516 dev_dbg(sdev->slave.dev, "pchan %u: alloc vchan %p\n", 517 pchan->idx, &vchan->vc); 518 } 519 spin_unlock_irq(&sdev->lock); 520 521 for (pchan_idx = 0; pchan_idx < sdev->num_pchans; pchan_idx++) { 522 if (!(pchan_alloc & BIT(pchan_idx))) 523 continue; 524 525 pchan = sdev->pchans + pchan_idx; 526 vchan = pchan->vchan; 527 if (vchan) { 528 spin_lock_irq(&vchan->vc.lock); 529 sun6i_dma_start_desc(vchan); 530 spin_unlock_irq(&vchan->vc.lock); 531 } 532 } 533 } 534 535 static irqreturn_t sun6i_dma_interrupt(int irq, void *dev_id) 536 { 537 struct sun6i_dma_dev *sdev = dev_id; 538 struct sun6i_vchan *vchan; 539 struct sun6i_pchan *pchan; 540 int i, j, ret = IRQ_NONE; 541 u32 status; 542 543 for (i = 0; i < sdev->num_pchans / DMA_IRQ_CHAN_NR; i++) { 544 status = readl(sdev->base + DMA_IRQ_STAT(i)); 545 if (!status) 546 continue; 547 548 dev_dbg(sdev->slave.dev, "DMA irq status %s: 0x%x\n", 549 i ? "high" : "low", status); 550 551 writel(status, sdev->base + DMA_IRQ_STAT(i)); 552 553 for (j = 0; (j < DMA_IRQ_CHAN_NR) && status; j++) { 554 pchan = sdev->pchans + j; 555 vchan = pchan->vchan; 556 if (vchan && (status & vchan->irq_type)) { 557 if (vchan->cyclic) { 558 vchan_cyclic_callback(&pchan->desc->vd); 559 } else { 560 spin_lock(&vchan->vc.lock); 561 vchan_cookie_complete(&pchan->desc->vd); 562 pchan->done = pchan->desc; 563 spin_unlock(&vchan->vc.lock); 564 } 565 } 566 567 status = status >> DMA_IRQ_CHAN_WIDTH; 568 } 569 570 if (!atomic_read(&sdev->tasklet_shutdown)) 571 tasklet_schedule(&sdev->task); 572 ret = IRQ_HANDLED; 573 } 574 575 return ret; 576 } 577 578 static int set_config(struct sun6i_dma_dev *sdev, 579 struct dma_slave_config *sconfig, 580 enum dma_transfer_direction direction, 581 u32 *p_cfg) 582 { 583 enum dma_slave_buswidth src_addr_width, dst_addr_width; 584 u32 src_maxburst, dst_maxburst; 585 s8 src_width, dst_width, src_burst, dst_burst; 586 587 src_addr_width = sconfig->src_addr_width; 588 dst_addr_width = sconfig->dst_addr_width; 589 src_maxburst = sconfig->src_maxburst; 590 dst_maxburst = sconfig->dst_maxburst; 591 592 switch (direction) { 593 case DMA_MEM_TO_DEV: 594 if (src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) 595 src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 596 src_maxburst = src_maxburst ? src_maxburst : 8; 597 break; 598 case DMA_DEV_TO_MEM: 599 if (dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) 600 dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 601 dst_maxburst = dst_maxburst ? dst_maxburst : 8; 602 break; 603 default: 604 return -EINVAL; 605 } 606 607 if (!(BIT(src_addr_width) & sdev->slave.src_addr_widths)) 608 return -EINVAL; 609 if (!(BIT(dst_addr_width) & sdev->slave.dst_addr_widths)) 610 return -EINVAL; 611 if (!(BIT(src_maxburst) & sdev->cfg->src_burst_lengths)) 612 return -EINVAL; 613 if (!(BIT(dst_maxburst) & sdev->cfg->dst_burst_lengths)) 614 return -EINVAL; 615 616 src_width = convert_buswidth(src_addr_width); 617 dst_width = convert_buswidth(dst_addr_width); 618 dst_burst = convert_burst(dst_maxburst); 619 src_burst = convert_burst(src_maxburst); 620 621 *p_cfg = DMA_CHAN_CFG_SRC_WIDTH(src_width) | 622 DMA_CHAN_CFG_DST_WIDTH(dst_width); 623 624 sdev->cfg->set_burst_length(p_cfg, src_burst, dst_burst); 625 626 return 0; 627 } 628 629 static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_memcpy( 630 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 631 size_t len, unsigned long flags) 632 { 633 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); 634 struct sun6i_vchan *vchan = to_sun6i_vchan(chan); 635 struct sun6i_dma_lli *v_lli; 636 struct sun6i_desc *txd; 637 dma_addr_t p_lli; 638 s8 burst, width; 639 640 dev_dbg(chan2dev(chan), 641 "%s; chan: %d, dest: %pad, src: %pad, len: %zu. flags: 0x%08lx\n", 642 __func__, vchan->vc.chan.chan_id, &dest, &src, len, flags); 643 644 if (!len) 645 return NULL; 646 647 txd = kzalloc(sizeof(*txd), GFP_NOWAIT); 648 if (!txd) 649 return NULL; 650 651 v_lli = dma_pool_alloc(sdev->pool, GFP_NOWAIT, &p_lli); 652 if (!v_lli) { 653 dev_err(sdev->slave.dev, "Failed to alloc lli memory\n"); 654 goto err_txd_free; 655 } 656 657 v_lli->src = src; 658 v_lli->dst = dest; 659 v_lli->len = len; 660 v_lli->para = NORMAL_WAIT; 661 662 burst = convert_burst(8); 663 width = convert_buswidth(DMA_SLAVE_BUSWIDTH_4_BYTES); 664 v_lli->cfg = DMA_CHAN_CFG_SRC_WIDTH(width) | 665 DMA_CHAN_CFG_DST_WIDTH(width); 666 667 sdev->cfg->set_burst_length(&v_lli->cfg, burst, burst); 668 sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, DRQ_SDRAM); 669 sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, LINEAR_MODE); 670 671 sun6i_dma_lli_add(NULL, v_lli, p_lli, txd); 672 673 sun6i_dma_dump_lli(vchan, v_lli); 674 675 return vchan_tx_prep(&vchan->vc, &txd->vd, flags); 676 677 err_txd_free: 678 kfree(txd); 679 return NULL; 680 } 681 682 static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg( 683 struct dma_chan *chan, struct scatterlist *sgl, 684 unsigned int sg_len, enum dma_transfer_direction dir, 685 unsigned long flags, void *context) 686 { 687 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); 688 struct sun6i_vchan *vchan = to_sun6i_vchan(chan); 689 struct dma_slave_config *sconfig = &vchan->cfg; 690 struct sun6i_dma_lli *v_lli, *prev = NULL; 691 struct sun6i_desc *txd; 692 struct scatterlist *sg; 693 dma_addr_t p_lli; 694 u32 lli_cfg; 695 int i, ret; 696 697 if (!sgl) 698 return NULL; 699 700 ret = set_config(sdev, sconfig, dir, &lli_cfg); 701 if (ret) { 702 dev_err(chan2dev(chan), "Invalid DMA configuration\n"); 703 return NULL; 704 } 705 706 txd = kzalloc(sizeof(*txd), GFP_NOWAIT); 707 if (!txd) 708 return NULL; 709 710 for_each_sg(sgl, sg, sg_len, i) { 711 v_lli = dma_pool_alloc(sdev->pool, GFP_NOWAIT, &p_lli); 712 if (!v_lli) 713 goto err_lli_free; 714 715 v_lli->len = sg_dma_len(sg); 716 v_lli->para = NORMAL_WAIT; 717 718 if (dir == DMA_MEM_TO_DEV) { 719 v_lli->src = sg_dma_address(sg); 720 v_lli->dst = sconfig->dst_addr; 721 v_lli->cfg = lli_cfg; 722 sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port); 723 sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE); 724 725 dev_dbg(chan2dev(chan), 726 "%s; chan: %d, dest: %pad, src: %pad, len: %u. flags: 0x%08lx\n", 727 __func__, vchan->vc.chan.chan_id, 728 &sconfig->dst_addr, &sg_dma_address(sg), 729 sg_dma_len(sg), flags); 730 731 } else { 732 v_lli->src = sconfig->src_addr; 733 v_lli->dst = sg_dma_address(sg); 734 v_lli->cfg = lli_cfg; 735 sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM); 736 sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE); 737 738 dev_dbg(chan2dev(chan), 739 "%s; chan: %d, dest: %pad, src: %pad, len: %u. flags: 0x%08lx\n", 740 __func__, vchan->vc.chan.chan_id, 741 &sg_dma_address(sg), &sconfig->src_addr, 742 sg_dma_len(sg), flags); 743 } 744 745 prev = sun6i_dma_lli_add(prev, v_lli, p_lli, txd); 746 } 747 748 dev_dbg(chan2dev(chan), "First: %pad\n", &txd->p_lli); 749 for (prev = txd->v_lli; prev; prev = prev->v_lli_next) 750 sun6i_dma_dump_lli(vchan, prev); 751 752 return vchan_tx_prep(&vchan->vc, &txd->vd, flags); 753 754 err_lli_free: 755 for (prev = txd->v_lli; prev; prev = prev->v_lli_next) 756 dma_pool_free(sdev->pool, prev, virt_to_phys(prev)); 757 kfree(txd); 758 return NULL; 759 } 760 761 static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic( 762 struct dma_chan *chan, 763 dma_addr_t buf_addr, 764 size_t buf_len, 765 size_t period_len, 766 enum dma_transfer_direction dir, 767 unsigned long flags) 768 { 769 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); 770 struct sun6i_vchan *vchan = to_sun6i_vchan(chan); 771 struct dma_slave_config *sconfig = &vchan->cfg; 772 struct sun6i_dma_lli *v_lli, *prev = NULL; 773 struct sun6i_desc *txd; 774 dma_addr_t p_lli; 775 u32 lli_cfg; 776 unsigned int i, periods = buf_len / period_len; 777 int ret; 778 779 ret = set_config(sdev, sconfig, dir, &lli_cfg); 780 if (ret) { 781 dev_err(chan2dev(chan), "Invalid DMA configuration\n"); 782 return NULL; 783 } 784 785 txd = kzalloc(sizeof(*txd), GFP_NOWAIT); 786 if (!txd) 787 return NULL; 788 789 for (i = 0; i < periods; i++) { 790 v_lli = dma_pool_alloc(sdev->pool, GFP_NOWAIT, &p_lli); 791 if (!v_lli) { 792 dev_err(sdev->slave.dev, "Failed to alloc lli memory\n"); 793 goto err_lli_free; 794 } 795 796 v_lli->len = period_len; 797 v_lli->para = NORMAL_WAIT; 798 799 if (dir == DMA_MEM_TO_DEV) { 800 v_lli->src = buf_addr + period_len * i; 801 v_lli->dst = sconfig->dst_addr; 802 v_lli->cfg = lli_cfg; 803 sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port); 804 sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE); 805 } else { 806 v_lli->src = sconfig->src_addr; 807 v_lli->dst = buf_addr + period_len * i; 808 v_lli->cfg = lli_cfg; 809 sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM); 810 sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE); 811 } 812 813 prev = sun6i_dma_lli_add(prev, v_lli, p_lli, txd); 814 } 815 816 prev->p_lli_next = txd->p_lli; /* cyclic list */ 817 818 vchan->cyclic = true; 819 820 return vchan_tx_prep(&vchan->vc, &txd->vd, flags); 821 822 err_lli_free: 823 for (prev = txd->v_lli; prev; prev = prev->v_lli_next) 824 dma_pool_free(sdev->pool, prev, virt_to_phys(prev)); 825 kfree(txd); 826 return NULL; 827 } 828 829 static int sun6i_dma_config(struct dma_chan *chan, 830 struct dma_slave_config *config) 831 { 832 struct sun6i_vchan *vchan = to_sun6i_vchan(chan); 833 834 memcpy(&vchan->cfg, config, sizeof(*config)); 835 836 return 0; 837 } 838 839 static int sun6i_dma_pause(struct dma_chan *chan) 840 { 841 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); 842 struct sun6i_vchan *vchan = to_sun6i_vchan(chan); 843 struct sun6i_pchan *pchan = vchan->phy; 844 845 dev_dbg(chan2dev(chan), "vchan %p: pause\n", &vchan->vc); 846 847 if (pchan) { 848 writel(DMA_CHAN_PAUSE_PAUSE, 849 pchan->base + DMA_CHAN_PAUSE); 850 } else { 851 spin_lock(&sdev->lock); 852 list_del_init(&vchan->node); 853 spin_unlock(&sdev->lock); 854 } 855 856 return 0; 857 } 858 859 static int sun6i_dma_resume(struct dma_chan *chan) 860 { 861 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); 862 struct sun6i_vchan *vchan = to_sun6i_vchan(chan); 863 struct sun6i_pchan *pchan = vchan->phy; 864 unsigned long flags; 865 866 dev_dbg(chan2dev(chan), "vchan %p: resume\n", &vchan->vc); 867 868 spin_lock_irqsave(&vchan->vc.lock, flags); 869 870 if (pchan) { 871 writel(DMA_CHAN_PAUSE_RESUME, 872 pchan->base + DMA_CHAN_PAUSE); 873 } else if (!list_empty(&vchan->vc.desc_issued)) { 874 spin_lock(&sdev->lock); 875 list_add_tail(&vchan->node, &sdev->pending); 876 spin_unlock(&sdev->lock); 877 } 878 879 spin_unlock_irqrestore(&vchan->vc.lock, flags); 880 881 return 0; 882 } 883 884 static int sun6i_dma_terminate_all(struct dma_chan *chan) 885 { 886 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); 887 struct sun6i_vchan *vchan = to_sun6i_vchan(chan); 888 struct sun6i_pchan *pchan = vchan->phy; 889 unsigned long flags; 890 LIST_HEAD(head); 891 892 spin_lock(&sdev->lock); 893 list_del_init(&vchan->node); 894 spin_unlock(&sdev->lock); 895 896 spin_lock_irqsave(&vchan->vc.lock, flags); 897 898 if (vchan->cyclic) { 899 vchan->cyclic = false; 900 if (pchan && pchan->desc) { 901 struct virt_dma_desc *vd = &pchan->desc->vd; 902 struct virt_dma_chan *vc = &vchan->vc; 903 904 list_add_tail(&vd->node, &vc->desc_completed); 905 } 906 } 907 908 vchan_get_all_descriptors(&vchan->vc, &head); 909 910 if (pchan) { 911 writel(DMA_CHAN_ENABLE_STOP, pchan->base + DMA_CHAN_ENABLE); 912 writel(DMA_CHAN_PAUSE_RESUME, pchan->base + DMA_CHAN_PAUSE); 913 914 vchan->phy = NULL; 915 pchan->vchan = NULL; 916 pchan->desc = NULL; 917 pchan->done = NULL; 918 } 919 920 spin_unlock_irqrestore(&vchan->vc.lock, flags); 921 922 vchan_dma_desc_free_list(&vchan->vc, &head); 923 924 return 0; 925 } 926 927 static enum dma_status sun6i_dma_tx_status(struct dma_chan *chan, 928 dma_cookie_t cookie, 929 struct dma_tx_state *state) 930 { 931 struct sun6i_vchan *vchan = to_sun6i_vchan(chan); 932 struct sun6i_pchan *pchan = vchan->phy; 933 struct sun6i_dma_lli *lli; 934 struct virt_dma_desc *vd; 935 struct sun6i_desc *txd; 936 enum dma_status ret; 937 unsigned long flags; 938 size_t bytes = 0; 939 940 ret = dma_cookie_status(chan, cookie, state); 941 if (ret == DMA_COMPLETE || !state) 942 return ret; 943 944 spin_lock_irqsave(&vchan->vc.lock, flags); 945 946 vd = vchan_find_desc(&vchan->vc, cookie); 947 txd = to_sun6i_desc(&vd->tx); 948 949 if (vd) { 950 for (lli = txd->v_lli; lli != NULL; lli = lli->v_lli_next) 951 bytes += lli->len; 952 } else if (!pchan || !pchan->desc) { 953 bytes = 0; 954 } else { 955 bytes = sun6i_get_chan_size(pchan); 956 } 957 958 spin_unlock_irqrestore(&vchan->vc.lock, flags); 959 960 dma_set_residue(state, bytes); 961 962 return ret; 963 } 964 965 static void sun6i_dma_issue_pending(struct dma_chan *chan) 966 { 967 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); 968 struct sun6i_vchan *vchan = to_sun6i_vchan(chan); 969 unsigned long flags; 970 971 spin_lock_irqsave(&vchan->vc.lock, flags); 972 973 if (vchan_issue_pending(&vchan->vc)) { 974 spin_lock(&sdev->lock); 975 976 if (!vchan->phy && list_empty(&vchan->node)) { 977 list_add_tail(&vchan->node, &sdev->pending); 978 tasklet_schedule(&sdev->task); 979 dev_dbg(chan2dev(chan), "vchan %p: issued\n", 980 &vchan->vc); 981 } 982 983 spin_unlock(&sdev->lock); 984 } else { 985 dev_dbg(chan2dev(chan), "vchan %p: nothing to issue\n", 986 &vchan->vc); 987 } 988 989 spin_unlock_irqrestore(&vchan->vc.lock, flags); 990 } 991 992 static void sun6i_dma_free_chan_resources(struct dma_chan *chan) 993 { 994 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); 995 struct sun6i_vchan *vchan = to_sun6i_vchan(chan); 996 unsigned long flags; 997 998 spin_lock_irqsave(&sdev->lock, flags); 999 list_del_init(&vchan->node); 1000 spin_unlock_irqrestore(&sdev->lock, flags); 1001 1002 vchan_free_chan_resources(&vchan->vc); 1003 } 1004 1005 static struct dma_chan *sun6i_dma_of_xlate(struct of_phandle_args *dma_spec, 1006 struct of_dma *ofdma) 1007 { 1008 struct sun6i_dma_dev *sdev = ofdma->of_dma_data; 1009 struct sun6i_vchan *vchan; 1010 struct dma_chan *chan; 1011 u8 port = dma_spec->args[0]; 1012 1013 if (port > sdev->max_request) 1014 return NULL; 1015 1016 chan = dma_get_any_slave_channel(&sdev->slave); 1017 if (!chan) 1018 return NULL; 1019 1020 vchan = to_sun6i_vchan(chan); 1021 vchan->port = port; 1022 1023 return chan; 1024 } 1025 1026 static inline void sun6i_kill_tasklet(struct sun6i_dma_dev *sdev) 1027 { 1028 /* Disable all interrupts from DMA */ 1029 writel(0, sdev->base + DMA_IRQ_EN(0)); 1030 writel(0, sdev->base + DMA_IRQ_EN(1)); 1031 1032 /* Prevent spurious interrupts from scheduling the tasklet */ 1033 atomic_inc(&sdev->tasklet_shutdown); 1034 1035 /* Make sure we won't have any further interrupts */ 1036 devm_free_irq(sdev->slave.dev, sdev->irq, sdev); 1037 1038 /* Actually prevent the tasklet from being scheduled */ 1039 tasklet_kill(&sdev->task); 1040 } 1041 1042 static inline void sun6i_dma_free(struct sun6i_dma_dev *sdev) 1043 { 1044 int i; 1045 1046 for (i = 0; i < sdev->num_vchans; i++) { 1047 struct sun6i_vchan *vchan = &sdev->vchans[i]; 1048 1049 list_del(&vchan->vc.chan.device_node); 1050 tasklet_kill(&vchan->vc.task); 1051 } 1052 } 1053 1054 /* 1055 * For A31: 1056 * 1057 * There's 16 physical channels that can work in parallel. 1058 * 1059 * However we have 30 different endpoints for our requests. 1060 * 1061 * Since the channels are able to handle only an unidirectional 1062 * transfer, we need to allocate more virtual channels so that 1063 * everyone can grab one channel. 1064 * 1065 * Some devices can't work in both direction (mostly because it 1066 * wouldn't make sense), so we have a bit fewer virtual channels than 1067 * 2 channels per endpoints. 1068 */ 1069 1070 static struct sun6i_dma_config sun6i_a31_dma_cfg = { 1071 .nr_max_channels = 16, 1072 .nr_max_requests = 30, 1073 .nr_max_vchans = 53, 1074 .set_burst_length = sun6i_set_burst_length_a31, 1075 .set_drq = sun6i_set_drq_a31, 1076 .set_mode = sun6i_set_mode_a31, 1077 .src_burst_lengths = BIT(1) | BIT(8), 1078 .dst_burst_lengths = BIT(1) | BIT(8), 1079 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1080 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1081 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), 1082 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1083 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1084 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), 1085 }; 1086 1087 /* 1088 * The A23 only has 8 physical channels, a maximum DRQ port id of 24, 1089 * and a total of 37 usable source and destination endpoints. 1090 */ 1091 1092 static struct sun6i_dma_config sun8i_a23_dma_cfg = { 1093 .nr_max_channels = 8, 1094 .nr_max_requests = 24, 1095 .nr_max_vchans = 37, 1096 .clock_autogate_enable = sun6i_enable_clock_autogate_a23, 1097 .set_burst_length = sun6i_set_burst_length_a31, 1098 .set_drq = sun6i_set_drq_a31, 1099 .set_mode = sun6i_set_mode_a31, 1100 .src_burst_lengths = BIT(1) | BIT(8), 1101 .dst_burst_lengths = BIT(1) | BIT(8), 1102 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1103 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1104 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), 1105 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1106 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1107 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), 1108 }; 1109 1110 static struct sun6i_dma_config sun8i_a83t_dma_cfg = { 1111 .nr_max_channels = 8, 1112 .nr_max_requests = 28, 1113 .nr_max_vchans = 39, 1114 .clock_autogate_enable = sun6i_enable_clock_autogate_a23, 1115 .set_burst_length = sun6i_set_burst_length_a31, 1116 .set_drq = sun6i_set_drq_a31, 1117 .set_mode = sun6i_set_mode_a31, 1118 .src_burst_lengths = BIT(1) | BIT(8), 1119 .dst_burst_lengths = BIT(1) | BIT(8), 1120 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1121 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1122 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), 1123 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1124 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1125 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), 1126 }; 1127 1128 /* 1129 * The H3 has 12 physical channels, a maximum DRQ port id of 27, 1130 * and a total of 34 usable source and destination endpoints. 1131 * It also supports additional burst lengths and bus widths, 1132 * and the burst length fields have different offsets. 1133 */ 1134 1135 static struct sun6i_dma_config sun8i_h3_dma_cfg = { 1136 .nr_max_channels = 12, 1137 .nr_max_requests = 27, 1138 .nr_max_vchans = 34, 1139 .clock_autogate_enable = sun6i_enable_clock_autogate_h3, 1140 .set_burst_length = sun6i_set_burst_length_h3, 1141 .set_drq = sun6i_set_drq_a31, 1142 .set_mode = sun6i_set_mode_a31, 1143 .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), 1144 .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), 1145 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1146 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1147 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1148 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), 1149 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1150 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1151 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1152 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), 1153 }; 1154 1155 /* 1156 * The A64 binding uses the number of dma channels from the 1157 * device tree node. 1158 */ 1159 static struct sun6i_dma_config sun50i_a64_dma_cfg = { 1160 .clock_autogate_enable = sun6i_enable_clock_autogate_h3, 1161 .set_burst_length = sun6i_set_burst_length_h3, 1162 .set_drq = sun6i_set_drq_a31, 1163 .set_mode = sun6i_set_mode_a31, 1164 .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), 1165 .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), 1166 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1167 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1168 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1169 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), 1170 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1171 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1172 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1173 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), 1174 }; 1175 1176 /* 1177 * TODO: Add support for more than 4g physical addressing. 1178 * 1179 * The A100 binding uses the number of dma channels from the 1180 * device tree node. 1181 */ 1182 static struct sun6i_dma_config sun50i_a100_dma_cfg = { 1183 .clock_autogate_enable = sun6i_enable_clock_autogate_h3, 1184 .set_burst_length = sun6i_set_burst_length_h3, 1185 .set_drq = sun6i_set_drq_h6, 1186 .set_mode = sun6i_set_mode_h6, 1187 .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), 1188 .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), 1189 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1190 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1191 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1192 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), 1193 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1194 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1195 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1196 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), 1197 .has_mbus_clk = true, 1198 }; 1199 1200 /* 1201 * The H6 binding uses the number of dma channels from the 1202 * device tree node. 1203 */ 1204 static struct sun6i_dma_config sun50i_h6_dma_cfg = { 1205 .clock_autogate_enable = sun6i_enable_clock_autogate_h3, 1206 .set_burst_length = sun6i_set_burst_length_h3, 1207 .set_drq = sun6i_set_drq_h6, 1208 .set_mode = sun6i_set_mode_h6, 1209 .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), 1210 .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), 1211 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1212 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1213 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1214 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), 1215 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1216 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1217 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1218 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), 1219 .has_mbus_clk = true, 1220 }; 1221 1222 /* 1223 * The V3s have only 8 physical channels, a maximum DRQ port id of 23, 1224 * and a total of 24 usable source and destination endpoints. 1225 */ 1226 1227 static struct sun6i_dma_config sun8i_v3s_dma_cfg = { 1228 .nr_max_channels = 8, 1229 .nr_max_requests = 23, 1230 .nr_max_vchans = 24, 1231 .clock_autogate_enable = sun6i_enable_clock_autogate_a23, 1232 .set_burst_length = sun6i_set_burst_length_a31, 1233 .set_drq = sun6i_set_drq_a31, 1234 .set_mode = sun6i_set_mode_a31, 1235 .src_burst_lengths = BIT(1) | BIT(8), 1236 .dst_burst_lengths = BIT(1) | BIT(8), 1237 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1238 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1239 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), 1240 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1241 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1242 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), 1243 }; 1244 1245 static const struct of_device_id sun6i_dma_match[] = { 1246 { .compatible = "allwinner,sun6i-a31-dma", .data = &sun6i_a31_dma_cfg }, 1247 { .compatible = "allwinner,sun8i-a23-dma", .data = &sun8i_a23_dma_cfg }, 1248 { .compatible = "allwinner,sun8i-a83t-dma", .data = &sun8i_a83t_dma_cfg }, 1249 { .compatible = "allwinner,sun8i-h3-dma", .data = &sun8i_h3_dma_cfg }, 1250 { .compatible = "allwinner,sun8i-v3s-dma", .data = &sun8i_v3s_dma_cfg }, 1251 { .compatible = "allwinner,sun50i-a64-dma", .data = &sun50i_a64_dma_cfg }, 1252 { .compatible = "allwinner,sun50i-a100-dma", .data = &sun50i_a100_dma_cfg }, 1253 { .compatible = "allwinner,sun50i-h6-dma", .data = &sun50i_h6_dma_cfg }, 1254 { /* sentinel */ } 1255 }; 1256 MODULE_DEVICE_TABLE(of, sun6i_dma_match); 1257 1258 static int sun6i_dma_probe(struct platform_device *pdev) 1259 { 1260 struct device_node *np = pdev->dev.of_node; 1261 struct sun6i_dma_dev *sdc; 1262 struct resource *res; 1263 int ret, i; 1264 1265 sdc = devm_kzalloc(&pdev->dev, sizeof(*sdc), GFP_KERNEL); 1266 if (!sdc) 1267 return -ENOMEM; 1268 1269 sdc->cfg = of_device_get_match_data(&pdev->dev); 1270 if (!sdc->cfg) 1271 return -ENODEV; 1272 1273 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1274 sdc->base = devm_ioremap_resource(&pdev->dev, res); 1275 if (IS_ERR(sdc->base)) 1276 return PTR_ERR(sdc->base); 1277 1278 sdc->irq = platform_get_irq(pdev, 0); 1279 if (sdc->irq < 0) 1280 return sdc->irq; 1281 1282 sdc->clk = devm_clk_get(&pdev->dev, NULL); 1283 if (IS_ERR(sdc->clk)) { 1284 dev_err(&pdev->dev, "No clock specified\n"); 1285 return PTR_ERR(sdc->clk); 1286 } 1287 1288 if (sdc->cfg->has_mbus_clk) { 1289 sdc->clk_mbus = devm_clk_get(&pdev->dev, "mbus"); 1290 if (IS_ERR(sdc->clk_mbus)) { 1291 dev_err(&pdev->dev, "No mbus clock specified\n"); 1292 return PTR_ERR(sdc->clk_mbus); 1293 } 1294 } 1295 1296 sdc->rstc = devm_reset_control_get(&pdev->dev, NULL); 1297 if (IS_ERR(sdc->rstc)) { 1298 dev_err(&pdev->dev, "No reset controller specified\n"); 1299 return PTR_ERR(sdc->rstc); 1300 } 1301 1302 sdc->pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev, 1303 sizeof(struct sun6i_dma_lli), 4, 0); 1304 if (!sdc->pool) { 1305 dev_err(&pdev->dev, "No memory for descriptors dma pool\n"); 1306 return -ENOMEM; 1307 } 1308 1309 platform_set_drvdata(pdev, sdc); 1310 INIT_LIST_HEAD(&sdc->pending); 1311 spin_lock_init(&sdc->lock); 1312 1313 dma_cap_set(DMA_PRIVATE, sdc->slave.cap_mask); 1314 dma_cap_set(DMA_MEMCPY, sdc->slave.cap_mask); 1315 dma_cap_set(DMA_SLAVE, sdc->slave.cap_mask); 1316 dma_cap_set(DMA_CYCLIC, sdc->slave.cap_mask); 1317 1318 INIT_LIST_HEAD(&sdc->slave.channels); 1319 sdc->slave.device_free_chan_resources = sun6i_dma_free_chan_resources; 1320 sdc->slave.device_tx_status = sun6i_dma_tx_status; 1321 sdc->slave.device_issue_pending = sun6i_dma_issue_pending; 1322 sdc->slave.device_prep_slave_sg = sun6i_dma_prep_slave_sg; 1323 sdc->slave.device_prep_dma_memcpy = sun6i_dma_prep_dma_memcpy; 1324 sdc->slave.device_prep_dma_cyclic = sun6i_dma_prep_dma_cyclic; 1325 sdc->slave.copy_align = DMAENGINE_ALIGN_4_BYTES; 1326 sdc->slave.device_config = sun6i_dma_config; 1327 sdc->slave.device_pause = sun6i_dma_pause; 1328 sdc->slave.device_resume = sun6i_dma_resume; 1329 sdc->slave.device_terminate_all = sun6i_dma_terminate_all; 1330 sdc->slave.src_addr_widths = sdc->cfg->src_addr_widths; 1331 sdc->slave.dst_addr_widths = sdc->cfg->dst_addr_widths; 1332 sdc->slave.directions = BIT(DMA_DEV_TO_MEM) | 1333 BIT(DMA_MEM_TO_DEV); 1334 sdc->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1335 sdc->slave.dev = &pdev->dev; 1336 1337 sdc->num_pchans = sdc->cfg->nr_max_channels; 1338 sdc->num_vchans = sdc->cfg->nr_max_vchans; 1339 sdc->max_request = sdc->cfg->nr_max_requests; 1340 1341 ret = of_property_read_u32(np, "dma-channels", &sdc->num_pchans); 1342 if (ret && !sdc->num_pchans) { 1343 dev_err(&pdev->dev, "Can't get dma-channels.\n"); 1344 return ret; 1345 } 1346 1347 ret = of_property_read_u32(np, "dma-requests", &sdc->max_request); 1348 if (ret && !sdc->max_request) { 1349 dev_info(&pdev->dev, "Missing dma-requests, using %u.\n", 1350 DMA_CHAN_MAX_DRQ_A31); 1351 sdc->max_request = DMA_CHAN_MAX_DRQ_A31; 1352 } 1353 1354 /* 1355 * If the number of vchans is not specified, derive it from the 1356 * highest port number, at most one channel per port and direction. 1357 */ 1358 if (!sdc->num_vchans) 1359 sdc->num_vchans = 2 * (sdc->max_request + 1); 1360 1361 sdc->pchans = devm_kcalloc(&pdev->dev, sdc->num_pchans, 1362 sizeof(struct sun6i_pchan), GFP_KERNEL); 1363 if (!sdc->pchans) 1364 return -ENOMEM; 1365 1366 sdc->vchans = devm_kcalloc(&pdev->dev, sdc->num_vchans, 1367 sizeof(struct sun6i_vchan), GFP_KERNEL); 1368 if (!sdc->vchans) 1369 return -ENOMEM; 1370 1371 tasklet_setup(&sdc->task, sun6i_dma_tasklet); 1372 1373 for (i = 0; i < sdc->num_pchans; i++) { 1374 struct sun6i_pchan *pchan = &sdc->pchans[i]; 1375 1376 pchan->idx = i; 1377 pchan->base = sdc->base + 0x100 + i * 0x40; 1378 } 1379 1380 for (i = 0; i < sdc->num_vchans; i++) { 1381 struct sun6i_vchan *vchan = &sdc->vchans[i]; 1382 1383 INIT_LIST_HEAD(&vchan->node); 1384 vchan->vc.desc_free = sun6i_dma_free_desc; 1385 vchan_init(&vchan->vc, &sdc->slave); 1386 } 1387 1388 ret = reset_control_deassert(sdc->rstc); 1389 if (ret) { 1390 dev_err(&pdev->dev, "Couldn't deassert the device from reset\n"); 1391 goto err_chan_free; 1392 } 1393 1394 ret = clk_prepare_enable(sdc->clk); 1395 if (ret) { 1396 dev_err(&pdev->dev, "Couldn't enable the clock\n"); 1397 goto err_reset_assert; 1398 } 1399 1400 if (sdc->cfg->has_mbus_clk) { 1401 ret = clk_prepare_enable(sdc->clk_mbus); 1402 if (ret) { 1403 dev_err(&pdev->dev, "Couldn't enable mbus clock\n"); 1404 goto err_clk_disable; 1405 } 1406 } 1407 1408 ret = devm_request_irq(&pdev->dev, sdc->irq, sun6i_dma_interrupt, 0, 1409 dev_name(&pdev->dev), sdc); 1410 if (ret) { 1411 dev_err(&pdev->dev, "Cannot request IRQ\n"); 1412 goto err_mbus_clk_disable; 1413 } 1414 1415 ret = dma_async_device_register(&sdc->slave); 1416 if (ret) { 1417 dev_warn(&pdev->dev, "Failed to register DMA engine device\n"); 1418 goto err_irq_disable; 1419 } 1420 1421 ret = of_dma_controller_register(pdev->dev.of_node, sun6i_dma_of_xlate, 1422 sdc); 1423 if (ret) { 1424 dev_err(&pdev->dev, "of_dma_controller_register failed\n"); 1425 goto err_dma_unregister; 1426 } 1427 1428 if (sdc->cfg->clock_autogate_enable) 1429 sdc->cfg->clock_autogate_enable(sdc); 1430 1431 return 0; 1432 1433 err_dma_unregister: 1434 dma_async_device_unregister(&sdc->slave); 1435 err_irq_disable: 1436 sun6i_kill_tasklet(sdc); 1437 err_mbus_clk_disable: 1438 clk_disable_unprepare(sdc->clk_mbus); 1439 err_clk_disable: 1440 clk_disable_unprepare(sdc->clk); 1441 err_reset_assert: 1442 reset_control_assert(sdc->rstc); 1443 err_chan_free: 1444 sun6i_dma_free(sdc); 1445 return ret; 1446 } 1447 1448 static int sun6i_dma_remove(struct platform_device *pdev) 1449 { 1450 struct sun6i_dma_dev *sdc = platform_get_drvdata(pdev); 1451 1452 of_dma_controller_free(pdev->dev.of_node); 1453 dma_async_device_unregister(&sdc->slave); 1454 1455 sun6i_kill_tasklet(sdc); 1456 1457 clk_disable_unprepare(sdc->clk_mbus); 1458 clk_disable_unprepare(sdc->clk); 1459 reset_control_assert(sdc->rstc); 1460 1461 sun6i_dma_free(sdc); 1462 1463 return 0; 1464 } 1465 1466 static struct platform_driver sun6i_dma_driver = { 1467 .probe = sun6i_dma_probe, 1468 .remove = sun6i_dma_remove, 1469 .driver = { 1470 .name = "sun6i-dma", 1471 .of_match_table = sun6i_dma_match, 1472 }, 1473 }; 1474 module_platform_driver(sun6i_dma_driver); 1475 1476 MODULE_DESCRIPTION("Allwinner A31 DMA Controller Driver"); 1477 MODULE_AUTHOR("Sugar <shuge@allwinnertech.com>"); 1478 MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com>"); 1479 MODULE_LICENSE("GPL"); 1480