1 /* 2 * SA11x0 DMAengine support 3 * 4 * Copyright (C) 2012 Russell King 5 * Derived in part from arch/arm/mach-sa1100/dma.c, 6 * Copyright (C) 2000, 2001 by Nicolas Pitre 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 */ 12 #include <linux/sched.h> 13 #include <linux/device.h> 14 #include <linux/dmaengine.h> 15 #include <linux/init.h> 16 #include <linux/interrupt.h> 17 #include <linux/kernel.h> 18 #include <linux/module.h> 19 #include <linux/platform_device.h> 20 #include <linux/sa11x0-dma.h> 21 #include <linux/slab.h> 22 #include <linux/spinlock.h> 23 24 #include "virt-dma.h" 25 26 #define NR_PHY_CHAN 6 27 #define DMA_ALIGN 3 28 #define DMA_MAX_SIZE 0x1fff 29 #define DMA_CHUNK_SIZE 0x1000 30 31 #define DMA_DDAR 0x00 32 #define DMA_DCSR_S 0x04 33 #define DMA_DCSR_C 0x08 34 #define DMA_DCSR_R 0x0c 35 #define DMA_DBSA 0x10 36 #define DMA_DBTA 0x14 37 #define DMA_DBSB 0x18 38 #define DMA_DBTB 0x1c 39 #define DMA_SIZE 0x20 40 41 #define DCSR_RUN (1 << 0) 42 #define DCSR_IE (1 << 1) 43 #define DCSR_ERROR (1 << 2) 44 #define DCSR_DONEA (1 << 3) 45 #define DCSR_STRTA (1 << 4) 46 #define DCSR_DONEB (1 << 5) 47 #define DCSR_STRTB (1 << 6) 48 #define DCSR_BIU (1 << 7) 49 50 #define DDAR_RW (1 << 0) /* 0 = W, 1 = R */ 51 #define DDAR_E (1 << 1) /* 0 = LE, 1 = BE */ 52 #define DDAR_BS (1 << 2) /* 0 = BS4, 1 = BS8 */ 53 #define DDAR_DW (1 << 3) /* 0 = 8b, 1 = 16b */ 54 #define DDAR_Ser0UDCTr (0x0 << 4) 55 #define DDAR_Ser0UDCRc (0x1 << 4) 56 #define DDAR_Ser1SDLCTr (0x2 << 4) 57 #define DDAR_Ser1SDLCRc (0x3 << 4) 58 #define DDAR_Ser1UARTTr (0x4 << 4) 59 #define DDAR_Ser1UARTRc (0x5 << 4) 60 #define DDAR_Ser2ICPTr (0x6 << 4) 61 #define DDAR_Ser2ICPRc (0x7 << 4) 62 #define DDAR_Ser3UARTTr (0x8 << 4) 63 #define DDAR_Ser3UARTRc (0x9 << 4) 64 #define DDAR_Ser4MCP0Tr (0xa << 4) 65 #define DDAR_Ser4MCP0Rc (0xb << 4) 66 #define DDAR_Ser4MCP1Tr (0xc << 4) 67 #define DDAR_Ser4MCP1Rc (0xd << 4) 68 #define DDAR_Ser4SSPTr (0xe << 4) 69 #define DDAR_Ser4SSPRc (0xf << 4) 70 71 struct sa11x0_dma_sg { 72 u32 addr; 73 u32 len; 74 }; 75 76 struct sa11x0_dma_desc { 77 struct virt_dma_desc vd; 78 79 u32 ddar; 80 size_t size; 81 unsigned period; 82 bool cyclic; 83 84 unsigned sglen; 85 struct sa11x0_dma_sg sg[0]; 86 }; 87 88 struct sa11x0_dma_phy; 89 90 struct sa11x0_dma_chan { 91 struct virt_dma_chan vc; 92 93 /* protected by c->vc.lock */ 94 struct sa11x0_dma_phy *phy; 95 enum dma_status status; 96 97 /* protected by d->lock */ 98 struct list_head node; 99 100 u32 ddar; 101 const char *name; 102 }; 103 104 struct sa11x0_dma_phy { 105 void __iomem *base; 106 struct sa11x0_dma_dev *dev; 107 unsigned num; 108 109 struct sa11x0_dma_chan *vchan; 110 111 /* Protected by c->vc.lock */ 112 unsigned sg_load; 113 struct sa11x0_dma_desc *txd_load; 114 unsigned sg_done; 115 struct sa11x0_dma_desc *txd_done; 116 #ifdef CONFIG_PM_SLEEP 117 u32 dbs[2]; 118 u32 dbt[2]; 119 u32 dcsr; 120 #endif 121 }; 122 123 struct sa11x0_dma_dev { 124 struct dma_device slave; 125 void __iomem *base; 126 spinlock_t lock; 127 struct tasklet_struct task; 128 struct list_head chan_pending; 129 struct sa11x0_dma_phy phy[NR_PHY_CHAN]; 130 }; 131 132 static struct sa11x0_dma_chan *to_sa11x0_dma_chan(struct dma_chan *chan) 133 { 134 return container_of(chan, struct sa11x0_dma_chan, vc.chan); 135 } 136 137 static struct sa11x0_dma_dev *to_sa11x0_dma(struct dma_device *dmadev) 138 { 139 return container_of(dmadev, struct sa11x0_dma_dev, slave); 140 } 141 142 static struct sa11x0_dma_desc *sa11x0_dma_next_desc(struct sa11x0_dma_chan *c) 143 { 144 struct virt_dma_desc *vd = vchan_next_desc(&c->vc); 145 146 return vd ? container_of(vd, struct sa11x0_dma_desc, vd) : NULL; 147 } 148 149 static void sa11x0_dma_free_desc(struct virt_dma_desc *vd) 150 { 151 kfree(container_of(vd, struct sa11x0_dma_desc, vd)); 152 } 153 154 static void sa11x0_dma_start_desc(struct sa11x0_dma_phy *p, struct sa11x0_dma_desc *txd) 155 { 156 list_del(&txd->vd.node); 157 p->txd_load = txd; 158 p->sg_load = 0; 159 160 dev_vdbg(p->dev->slave.dev, "pchan %u: txd %p[%x]: starting: DDAR:%x\n", 161 p->num, &txd->vd, txd->vd.tx.cookie, txd->ddar); 162 } 163 164 static void noinline sa11x0_dma_start_sg(struct sa11x0_dma_phy *p, 165 struct sa11x0_dma_chan *c) 166 { 167 struct sa11x0_dma_desc *txd = p->txd_load; 168 struct sa11x0_dma_sg *sg; 169 void __iomem *base = p->base; 170 unsigned dbsx, dbtx; 171 u32 dcsr; 172 173 if (!txd) 174 return; 175 176 dcsr = readl_relaxed(base + DMA_DCSR_R); 177 178 /* Don't try to load the next transfer if both buffers are started */ 179 if ((dcsr & (DCSR_STRTA | DCSR_STRTB)) == (DCSR_STRTA | DCSR_STRTB)) 180 return; 181 182 if (p->sg_load == txd->sglen) { 183 if (!txd->cyclic) { 184 struct sa11x0_dma_desc *txn = sa11x0_dma_next_desc(c); 185 186 /* 187 * We have reached the end of the current descriptor. 188 * Peek at the next descriptor, and if compatible with 189 * the current, start processing it. 190 */ 191 if (txn && txn->ddar == txd->ddar) { 192 txd = txn; 193 sa11x0_dma_start_desc(p, txn); 194 } else { 195 p->txd_load = NULL; 196 return; 197 } 198 } else { 199 /* Cyclic: reset back to beginning */ 200 p->sg_load = 0; 201 } 202 } 203 204 sg = &txd->sg[p->sg_load++]; 205 206 /* Select buffer to load according to channel status */ 207 if (((dcsr & (DCSR_BIU | DCSR_STRTB)) == (DCSR_BIU | DCSR_STRTB)) || 208 ((dcsr & (DCSR_BIU | DCSR_STRTA)) == 0)) { 209 dbsx = DMA_DBSA; 210 dbtx = DMA_DBTA; 211 dcsr = DCSR_STRTA | DCSR_IE | DCSR_RUN; 212 } else { 213 dbsx = DMA_DBSB; 214 dbtx = DMA_DBTB; 215 dcsr = DCSR_STRTB | DCSR_IE | DCSR_RUN; 216 } 217 218 writel_relaxed(sg->addr, base + dbsx); 219 writel_relaxed(sg->len, base + dbtx); 220 writel(dcsr, base + DMA_DCSR_S); 221 222 dev_dbg(p->dev->slave.dev, "pchan %u: load: DCSR:%02x DBS%c:%08x DBT%c:%08x\n", 223 p->num, dcsr, 224 'A' + (dbsx == DMA_DBSB), sg->addr, 225 'A' + (dbtx == DMA_DBTB), sg->len); 226 } 227 228 static void noinline sa11x0_dma_complete(struct sa11x0_dma_phy *p, 229 struct sa11x0_dma_chan *c) 230 { 231 struct sa11x0_dma_desc *txd = p->txd_done; 232 233 if (++p->sg_done == txd->sglen) { 234 if (!txd->cyclic) { 235 vchan_cookie_complete(&txd->vd); 236 237 p->sg_done = 0; 238 p->txd_done = p->txd_load; 239 240 if (!p->txd_done) 241 tasklet_schedule(&p->dev->task); 242 } else { 243 if ((p->sg_done % txd->period) == 0) 244 vchan_cyclic_callback(&txd->vd); 245 246 /* Cyclic: reset back to beginning */ 247 p->sg_done = 0; 248 } 249 } 250 251 sa11x0_dma_start_sg(p, c); 252 } 253 254 static irqreturn_t sa11x0_dma_irq(int irq, void *dev_id) 255 { 256 struct sa11x0_dma_phy *p = dev_id; 257 struct sa11x0_dma_dev *d = p->dev; 258 struct sa11x0_dma_chan *c; 259 u32 dcsr; 260 261 dcsr = readl_relaxed(p->base + DMA_DCSR_R); 262 if (!(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB))) 263 return IRQ_NONE; 264 265 /* Clear reported status bits */ 266 writel_relaxed(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB), 267 p->base + DMA_DCSR_C); 268 269 dev_dbg(d->slave.dev, "pchan %u: irq: DCSR:%02x\n", p->num, dcsr); 270 271 if (dcsr & DCSR_ERROR) { 272 dev_err(d->slave.dev, "pchan %u: error. DCSR:%02x DDAR:%08x DBSA:%08x DBTA:%08x DBSB:%08x DBTB:%08x\n", 273 p->num, dcsr, 274 readl_relaxed(p->base + DMA_DDAR), 275 readl_relaxed(p->base + DMA_DBSA), 276 readl_relaxed(p->base + DMA_DBTA), 277 readl_relaxed(p->base + DMA_DBSB), 278 readl_relaxed(p->base + DMA_DBTB)); 279 } 280 281 c = p->vchan; 282 if (c) { 283 unsigned long flags; 284 285 spin_lock_irqsave(&c->vc.lock, flags); 286 /* 287 * Now that we're holding the lock, check that the vchan 288 * really is associated with this pchan before touching the 289 * hardware. This should always succeed, because we won't 290 * change p->vchan or c->phy while the channel is actively 291 * transferring. 292 */ 293 if (c->phy == p) { 294 if (dcsr & DCSR_DONEA) 295 sa11x0_dma_complete(p, c); 296 if (dcsr & DCSR_DONEB) 297 sa11x0_dma_complete(p, c); 298 } 299 spin_unlock_irqrestore(&c->vc.lock, flags); 300 } 301 302 return IRQ_HANDLED; 303 } 304 305 static void sa11x0_dma_start_txd(struct sa11x0_dma_chan *c) 306 { 307 struct sa11x0_dma_desc *txd = sa11x0_dma_next_desc(c); 308 309 /* If the issued list is empty, we have no further txds to process */ 310 if (txd) { 311 struct sa11x0_dma_phy *p = c->phy; 312 313 sa11x0_dma_start_desc(p, txd); 314 p->txd_done = txd; 315 p->sg_done = 0; 316 317 /* The channel should not have any transfers started */ 318 WARN_ON(readl_relaxed(p->base + DMA_DCSR_R) & 319 (DCSR_STRTA | DCSR_STRTB)); 320 321 /* Clear the run and start bits before changing DDAR */ 322 writel_relaxed(DCSR_RUN | DCSR_STRTA | DCSR_STRTB, 323 p->base + DMA_DCSR_C); 324 writel_relaxed(txd->ddar, p->base + DMA_DDAR); 325 326 /* Try to start both buffers */ 327 sa11x0_dma_start_sg(p, c); 328 sa11x0_dma_start_sg(p, c); 329 } 330 } 331 332 static void sa11x0_dma_tasklet(unsigned long arg) 333 { 334 struct sa11x0_dma_dev *d = (struct sa11x0_dma_dev *)arg; 335 struct sa11x0_dma_phy *p; 336 struct sa11x0_dma_chan *c; 337 unsigned pch, pch_alloc = 0; 338 339 dev_dbg(d->slave.dev, "tasklet enter\n"); 340 341 list_for_each_entry(c, &d->slave.channels, vc.chan.device_node) { 342 spin_lock_irq(&c->vc.lock); 343 p = c->phy; 344 if (p && !p->txd_done) { 345 sa11x0_dma_start_txd(c); 346 if (!p->txd_done) { 347 /* No current txd associated with this channel */ 348 dev_dbg(d->slave.dev, "pchan %u: free\n", p->num); 349 350 /* Mark this channel free */ 351 c->phy = NULL; 352 p->vchan = NULL; 353 } 354 } 355 spin_unlock_irq(&c->vc.lock); 356 } 357 358 spin_lock_irq(&d->lock); 359 for (pch = 0; pch < NR_PHY_CHAN; pch++) { 360 p = &d->phy[pch]; 361 362 if (p->vchan == NULL && !list_empty(&d->chan_pending)) { 363 c = list_first_entry(&d->chan_pending, 364 struct sa11x0_dma_chan, node); 365 list_del_init(&c->node); 366 367 pch_alloc |= 1 << pch; 368 369 /* Mark this channel allocated */ 370 p->vchan = c; 371 372 dev_dbg(d->slave.dev, "pchan %u: alloc vchan %p\n", pch, &c->vc); 373 } 374 } 375 spin_unlock_irq(&d->lock); 376 377 for (pch = 0; pch < NR_PHY_CHAN; pch++) { 378 if (pch_alloc & (1 << pch)) { 379 p = &d->phy[pch]; 380 c = p->vchan; 381 382 spin_lock_irq(&c->vc.lock); 383 c->phy = p; 384 385 sa11x0_dma_start_txd(c); 386 spin_unlock_irq(&c->vc.lock); 387 } 388 } 389 390 dev_dbg(d->slave.dev, "tasklet exit\n"); 391 } 392 393 394 static int sa11x0_dma_alloc_chan_resources(struct dma_chan *chan) 395 { 396 return 0; 397 } 398 399 static void sa11x0_dma_free_chan_resources(struct dma_chan *chan) 400 { 401 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan); 402 struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device); 403 unsigned long flags; 404 405 spin_lock_irqsave(&d->lock, flags); 406 list_del_init(&c->node); 407 spin_unlock_irqrestore(&d->lock, flags); 408 409 vchan_free_chan_resources(&c->vc); 410 } 411 412 static dma_addr_t sa11x0_dma_pos(struct sa11x0_dma_phy *p) 413 { 414 unsigned reg; 415 u32 dcsr; 416 417 dcsr = readl_relaxed(p->base + DMA_DCSR_R); 418 419 if ((dcsr & (DCSR_BIU | DCSR_STRTA)) == DCSR_STRTA || 420 (dcsr & (DCSR_BIU | DCSR_STRTB)) == DCSR_BIU) 421 reg = DMA_DBSA; 422 else 423 reg = DMA_DBSB; 424 425 return readl_relaxed(p->base + reg); 426 } 427 428 static enum dma_status sa11x0_dma_tx_status(struct dma_chan *chan, 429 dma_cookie_t cookie, struct dma_tx_state *state) 430 { 431 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan); 432 struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device); 433 struct sa11x0_dma_phy *p; 434 struct virt_dma_desc *vd; 435 unsigned long flags; 436 enum dma_status ret; 437 438 ret = dma_cookie_status(&c->vc.chan, cookie, state); 439 if (ret == DMA_COMPLETE) 440 return ret; 441 442 if (!state) 443 return c->status; 444 445 spin_lock_irqsave(&c->vc.lock, flags); 446 p = c->phy; 447 448 /* 449 * If the cookie is on our issue queue, then the residue is 450 * its total size. 451 */ 452 vd = vchan_find_desc(&c->vc, cookie); 453 if (vd) { 454 state->residue = container_of(vd, struct sa11x0_dma_desc, vd)->size; 455 } else if (!p) { 456 state->residue = 0; 457 } else { 458 struct sa11x0_dma_desc *txd; 459 size_t bytes = 0; 460 461 if (p->txd_done && p->txd_done->vd.tx.cookie == cookie) 462 txd = p->txd_done; 463 else if (p->txd_load && p->txd_load->vd.tx.cookie == cookie) 464 txd = p->txd_load; 465 else 466 txd = NULL; 467 468 ret = c->status; 469 if (txd) { 470 dma_addr_t addr = sa11x0_dma_pos(p); 471 unsigned i; 472 473 dev_vdbg(d->slave.dev, "tx_status: addr:%x\n", addr); 474 475 for (i = 0; i < txd->sglen; i++) { 476 dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x\n", 477 i, txd->sg[i].addr, txd->sg[i].len); 478 if (addr >= txd->sg[i].addr && 479 addr < txd->sg[i].addr + txd->sg[i].len) { 480 unsigned len; 481 482 len = txd->sg[i].len - 483 (addr - txd->sg[i].addr); 484 dev_vdbg(d->slave.dev, "tx_status: [%u] +%x\n", 485 i, len); 486 bytes += len; 487 i++; 488 break; 489 } 490 } 491 for (; i < txd->sglen; i++) { 492 dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x ++\n", 493 i, txd->sg[i].addr, txd->sg[i].len); 494 bytes += txd->sg[i].len; 495 } 496 } 497 state->residue = bytes; 498 } 499 spin_unlock_irqrestore(&c->vc.lock, flags); 500 501 dev_vdbg(d->slave.dev, "tx_status: bytes 0x%zx\n", state->residue); 502 503 return ret; 504 } 505 506 /* 507 * Move pending txds to the issued list, and re-init pending list. 508 * If not already pending, add this channel to the list of pending 509 * channels and trigger the tasklet to run. 510 */ 511 static void sa11x0_dma_issue_pending(struct dma_chan *chan) 512 { 513 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan); 514 struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device); 515 unsigned long flags; 516 517 spin_lock_irqsave(&c->vc.lock, flags); 518 if (vchan_issue_pending(&c->vc)) { 519 if (!c->phy) { 520 spin_lock(&d->lock); 521 if (list_empty(&c->node)) { 522 list_add_tail(&c->node, &d->chan_pending); 523 tasklet_schedule(&d->task); 524 dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc); 525 } 526 spin_unlock(&d->lock); 527 } 528 } else 529 dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc); 530 spin_unlock_irqrestore(&c->vc.lock, flags); 531 } 532 533 static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg( 534 struct dma_chan *chan, struct scatterlist *sg, unsigned int sglen, 535 enum dma_transfer_direction dir, unsigned long flags, void *context) 536 { 537 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan); 538 struct sa11x0_dma_desc *txd; 539 struct scatterlist *sgent; 540 unsigned i, j = sglen; 541 size_t size = 0; 542 543 /* SA11x0 channels can only operate in their native direction */ 544 if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) { 545 dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n", 546 &c->vc, c->ddar, dir); 547 return NULL; 548 } 549 550 /* Do not allow zero-sized txds */ 551 if (sglen == 0) 552 return NULL; 553 554 for_each_sg(sg, sgent, sglen, i) { 555 dma_addr_t addr = sg_dma_address(sgent); 556 unsigned int len = sg_dma_len(sgent); 557 558 if (len > DMA_MAX_SIZE) 559 j += DIV_ROUND_UP(len, DMA_MAX_SIZE & ~DMA_ALIGN) - 1; 560 if (addr & DMA_ALIGN) { 561 dev_dbg(chan->device->dev, "vchan %p: bad buffer alignment: %08x\n", 562 &c->vc, addr); 563 return NULL; 564 } 565 } 566 567 txd = kzalloc(sizeof(*txd) + j * sizeof(txd->sg[0]), GFP_ATOMIC); 568 if (!txd) { 569 dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc); 570 return NULL; 571 } 572 573 j = 0; 574 for_each_sg(sg, sgent, sglen, i) { 575 dma_addr_t addr = sg_dma_address(sgent); 576 unsigned len = sg_dma_len(sgent); 577 578 size += len; 579 580 do { 581 unsigned tlen = len; 582 583 /* 584 * Check whether the transfer will fit. If not, try 585 * to split the transfer up such that we end up with 586 * equal chunks - but make sure that we preserve the 587 * alignment. This avoids small segments. 588 */ 589 if (tlen > DMA_MAX_SIZE) { 590 unsigned mult = DIV_ROUND_UP(tlen, 591 DMA_MAX_SIZE & ~DMA_ALIGN); 592 593 tlen = (tlen / mult) & ~DMA_ALIGN; 594 } 595 596 txd->sg[j].addr = addr; 597 txd->sg[j].len = tlen; 598 599 addr += tlen; 600 len -= tlen; 601 j++; 602 } while (len); 603 } 604 605 txd->ddar = c->ddar; 606 txd->size = size; 607 txd->sglen = j; 608 609 dev_dbg(chan->device->dev, "vchan %p: txd %p: size %u nr %u\n", 610 &c->vc, &txd->vd, txd->size, txd->sglen); 611 612 return vchan_tx_prep(&c->vc, &txd->vd, flags); 613 } 614 615 static struct dma_async_tx_descriptor *sa11x0_dma_prep_dma_cyclic( 616 struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period, 617 enum dma_transfer_direction dir, unsigned long flags, void *context) 618 { 619 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan); 620 struct sa11x0_dma_desc *txd; 621 unsigned i, j, k, sglen, sgperiod; 622 623 /* SA11x0 channels can only operate in their native direction */ 624 if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) { 625 dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n", 626 &c->vc, c->ddar, dir); 627 return NULL; 628 } 629 630 sgperiod = DIV_ROUND_UP(period, DMA_MAX_SIZE & ~DMA_ALIGN); 631 sglen = size * sgperiod / period; 632 633 /* Do not allow zero-sized txds */ 634 if (sglen == 0) 635 return NULL; 636 637 txd = kzalloc(sizeof(*txd) + sglen * sizeof(txd->sg[0]), GFP_ATOMIC); 638 if (!txd) { 639 dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc); 640 return NULL; 641 } 642 643 for (i = k = 0; i < size / period; i++) { 644 size_t tlen, len = period; 645 646 for (j = 0; j < sgperiod; j++, k++) { 647 tlen = len; 648 649 if (tlen > DMA_MAX_SIZE) { 650 unsigned mult = DIV_ROUND_UP(tlen, DMA_MAX_SIZE & ~DMA_ALIGN); 651 tlen = (tlen / mult) & ~DMA_ALIGN; 652 } 653 654 txd->sg[k].addr = addr; 655 txd->sg[k].len = tlen; 656 addr += tlen; 657 len -= tlen; 658 } 659 660 WARN_ON(len != 0); 661 } 662 663 WARN_ON(k != sglen); 664 665 txd->ddar = c->ddar; 666 txd->size = size; 667 txd->sglen = sglen; 668 txd->cyclic = 1; 669 txd->period = sgperiod; 670 671 return vchan_tx_prep(&c->vc, &txd->vd, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 672 } 673 674 static int sa11x0_dma_slave_config(struct sa11x0_dma_chan *c, struct dma_slave_config *cfg) 675 { 676 u32 ddar = c->ddar & ((0xf << 4) | DDAR_RW); 677 dma_addr_t addr; 678 enum dma_slave_buswidth width; 679 u32 maxburst; 680 681 if (ddar & DDAR_RW) { 682 addr = cfg->src_addr; 683 width = cfg->src_addr_width; 684 maxburst = cfg->src_maxburst; 685 } else { 686 addr = cfg->dst_addr; 687 width = cfg->dst_addr_width; 688 maxburst = cfg->dst_maxburst; 689 } 690 691 if ((width != DMA_SLAVE_BUSWIDTH_1_BYTE && 692 width != DMA_SLAVE_BUSWIDTH_2_BYTES) || 693 (maxburst != 4 && maxburst != 8)) 694 return -EINVAL; 695 696 if (width == DMA_SLAVE_BUSWIDTH_2_BYTES) 697 ddar |= DDAR_DW; 698 if (maxburst == 8) 699 ddar |= DDAR_BS; 700 701 dev_dbg(c->vc.chan.device->dev, "vchan %p: dma_slave_config addr %x width %u burst %u\n", 702 &c->vc, addr, width, maxburst); 703 704 c->ddar = ddar | (addr & 0xf0000000) | (addr & 0x003ffffc) << 6; 705 706 return 0; 707 } 708 709 static int sa11x0_dma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, 710 unsigned long arg) 711 { 712 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan); 713 struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device); 714 struct sa11x0_dma_phy *p; 715 LIST_HEAD(head); 716 unsigned long flags; 717 int ret; 718 719 switch (cmd) { 720 case DMA_SLAVE_CONFIG: 721 return sa11x0_dma_slave_config(c, (struct dma_slave_config *)arg); 722 723 case DMA_TERMINATE_ALL: 724 dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc); 725 /* Clear the tx descriptor lists */ 726 spin_lock_irqsave(&c->vc.lock, flags); 727 vchan_get_all_descriptors(&c->vc, &head); 728 729 p = c->phy; 730 if (p) { 731 dev_dbg(d->slave.dev, "pchan %u: terminating\n", p->num); 732 /* vchan is assigned to a pchan - stop the channel */ 733 writel(DCSR_RUN | DCSR_IE | 734 DCSR_STRTA | DCSR_DONEA | 735 DCSR_STRTB | DCSR_DONEB, 736 p->base + DMA_DCSR_C); 737 738 if (p->txd_load) { 739 if (p->txd_load != p->txd_done) 740 list_add_tail(&p->txd_load->vd.node, &head); 741 p->txd_load = NULL; 742 } 743 if (p->txd_done) { 744 list_add_tail(&p->txd_done->vd.node, &head); 745 p->txd_done = NULL; 746 } 747 c->phy = NULL; 748 spin_lock(&d->lock); 749 p->vchan = NULL; 750 spin_unlock(&d->lock); 751 tasklet_schedule(&d->task); 752 } 753 spin_unlock_irqrestore(&c->vc.lock, flags); 754 vchan_dma_desc_free_list(&c->vc, &head); 755 ret = 0; 756 break; 757 758 case DMA_PAUSE: 759 dev_dbg(d->slave.dev, "vchan %p: pause\n", &c->vc); 760 spin_lock_irqsave(&c->vc.lock, flags); 761 if (c->status == DMA_IN_PROGRESS) { 762 c->status = DMA_PAUSED; 763 764 p = c->phy; 765 if (p) { 766 writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C); 767 } else { 768 spin_lock(&d->lock); 769 list_del_init(&c->node); 770 spin_unlock(&d->lock); 771 } 772 } 773 spin_unlock_irqrestore(&c->vc.lock, flags); 774 ret = 0; 775 break; 776 777 case DMA_RESUME: 778 dev_dbg(d->slave.dev, "vchan %p: resume\n", &c->vc); 779 spin_lock_irqsave(&c->vc.lock, flags); 780 if (c->status == DMA_PAUSED) { 781 c->status = DMA_IN_PROGRESS; 782 783 p = c->phy; 784 if (p) { 785 writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_S); 786 } else if (!list_empty(&c->vc.desc_issued)) { 787 spin_lock(&d->lock); 788 list_add_tail(&c->node, &d->chan_pending); 789 spin_unlock(&d->lock); 790 } 791 } 792 spin_unlock_irqrestore(&c->vc.lock, flags); 793 ret = 0; 794 break; 795 796 default: 797 ret = -ENXIO; 798 break; 799 } 800 801 return ret; 802 } 803 804 struct sa11x0_dma_channel_desc { 805 u32 ddar; 806 const char *name; 807 }; 808 809 #define CD(d1, d2) { .ddar = DDAR_##d1 | d2, .name = #d1 } 810 static const struct sa11x0_dma_channel_desc chan_desc[] = { 811 CD(Ser0UDCTr, 0), 812 CD(Ser0UDCRc, DDAR_RW), 813 CD(Ser1SDLCTr, 0), 814 CD(Ser1SDLCRc, DDAR_RW), 815 CD(Ser1UARTTr, 0), 816 CD(Ser1UARTRc, DDAR_RW), 817 CD(Ser2ICPTr, 0), 818 CD(Ser2ICPRc, DDAR_RW), 819 CD(Ser3UARTTr, 0), 820 CD(Ser3UARTRc, DDAR_RW), 821 CD(Ser4MCP0Tr, 0), 822 CD(Ser4MCP0Rc, DDAR_RW), 823 CD(Ser4MCP1Tr, 0), 824 CD(Ser4MCP1Rc, DDAR_RW), 825 CD(Ser4SSPTr, 0), 826 CD(Ser4SSPRc, DDAR_RW), 827 }; 828 829 static int sa11x0_dma_init_dmadev(struct dma_device *dmadev, 830 struct device *dev) 831 { 832 unsigned i; 833 834 dmadev->chancnt = ARRAY_SIZE(chan_desc); 835 INIT_LIST_HEAD(&dmadev->channels); 836 dmadev->dev = dev; 837 dmadev->device_alloc_chan_resources = sa11x0_dma_alloc_chan_resources; 838 dmadev->device_free_chan_resources = sa11x0_dma_free_chan_resources; 839 dmadev->device_control = sa11x0_dma_control; 840 dmadev->device_tx_status = sa11x0_dma_tx_status; 841 dmadev->device_issue_pending = sa11x0_dma_issue_pending; 842 843 for (i = 0; i < dmadev->chancnt; i++) { 844 struct sa11x0_dma_chan *c; 845 846 c = kzalloc(sizeof(*c), GFP_KERNEL); 847 if (!c) { 848 dev_err(dev, "no memory for channel %u\n", i); 849 return -ENOMEM; 850 } 851 852 c->status = DMA_IN_PROGRESS; 853 c->ddar = chan_desc[i].ddar; 854 c->name = chan_desc[i].name; 855 INIT_LIST_HEAD(&c->node); 856 857 c->vc.desc_free = sa11x0_dma_free_desc; 858 vchan_init(&c->vc, dmadev); 859 } 860 861 return dma_async_device_register(dmadev); 862 } 863 864 static int sa11x0_dma_request_irq(struct platform_device *pdev, int nr, 865 void *data) 866 { 867 int irq = platform_get_irq(pdev, nr); 868 869 if (irq <= 0) 870 return -ENXIO; 871 872 return request_irq(irq, sa11x0_dma_irq, 0, dev_name(&pdev->dev), data); 873 } 874 875 static void sa11x0_dma_free_irq(struct platform_device *pdev, int nr, 876 void *data) 877 { 878 int irq = platform_get_irq(pdev, nr); 879 if (irq > 0) 880 free_irq(irq, data); 881 } 882 883 static void sa11x0_dma_free_channels(struct dma_device *dmadev) 884 { 885 struct sa11x0_dma_chan *c, *cn; 886 887 list_for_each_entry_safe(c, cn, &dmadev->channels, vc.chan.device_node) { 888 list_del(&c->vc.chan.device_node); 889 tasklet_kill(&c->vc.task); 890 kfree(c); 891 } 892 } 893 894 static int sa11x0_dma_probe(struct platform_device *pdev) 895 { 896 struct sa11x0_dma_dev *d; 897 struct resource *res; 898 unsigned i; 899 int ret; 900 901 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 902 if (!res) 903 return -ENXIO; 904 905 d = kzalloc(sizeof(*d), GFP_KERNEL); 906 if (!d) { 907 ret = -ENOMEM; 908 goto err_alloc; 909 } 910 911 spin_lock_init(&d->lock); 912 INIT_LIST_HEAD(&d->chan_pending); 913 914 d->base = ioremap(res->start, resource_size(res)); 915 if (!d->base) { 916 ret = -ENOMEM; 917 goto err_ioremap; 918 } 919 920 tasklet_init(&d->task, sa11x0_dma_tasklet, (unsigned long)d); 921 922 for (i = 0; i < NR_PHY_CHAN; i++) { 923 struct sa11x0_dma_phy *p = &d->phy[i]; 924 925 p->dev = d; 926 p->num = i; 927 p->base = d->base + i * DMA_SIZE; 928 writel_relaxed(DCSR_RUN | DCSR_IE | DCSR_ERROR | 929 DCSR_DONEA | DCSR_STRTA | DCSR_DONEB | DCSR_STRTB, 930 p->base + DMA_DCSR_C); 931 writel_relaxed(0, p->base + DMA_DDAR); 932 933 ret = sa11x0_dma_request_irq(pdev, i, p); 934 if (ret) { 935 while (i) { 936 i--; 937 sa11x0_dma_free_irq(pdev, i, &d->phy[i]); 938 } 939 goto err_irq; 940 } 941 } 942 943 dma_cap_set(DMA_SLAVE, d->slave.cap_mask); 944 dma_cap_set(DMA_CYCLIC, d->slave.cap_mask); 945 d->slave.device_prep_slave_sg = sa11x0_dma_prep_slave_sg; 946 d->slave.device_prep_dma_cyclic = sa11x0_dma_prep_dma_cyclic; 947 ret = sa11x0_dma_init_dmadev(&d->slave, &pdev->dev); 948 if (ret) { 949 dev_warn(d->slave.dev, "failed to register slave async device: %d\n", 950 ret); 951 goto err_slave_reg; 952 } 953 954 platform_set_drvdata(pdev, d); 955 return 0; 956 957 err_slave_reg: 958 sa11x0_dma_free_channels(&d->slave); 959 for (i = 0; i < NR_PHY_CHAN; i++) 960 sa11x0_dma_free_irq(pdev, i, &d->phy[i]); 961 err_irq: 962 tasklet_kill(&d->task); 963 iounmap(d->base); 964 err_ioremap: 965 kfree(d); 966 err_alloc: 967 return ret; 968 } 969 970 static int sa11x0_dma_remove(struct platform_device *pdev) 971 { 972 struct sa11x0_dma_dev *d = platform_get_drvdata(pdev); 973 unsigned pch; 974 975 dma_async_device_unregister(&d->slave); 976 977 sa11x0_dma_free_channels(&d->slave); 978 for (pch = 0; pch < NR_PHY_CHAN; pch++) 979 sa11x0_dma_free_irq(pdev, pch, &d->phy[pch]); 980 tasklet_kill(&d->task); 981 iounmap(d->base); 982 kfree(d); 983 984 return 0; 985 } 986 987 #ifdef CONFIG_PM_SLEEP 988 static int sa11x0_dma_suspend(struct device *dev) 989 { 990 struct sa11x0_dma_dev *d = dev_get_drvdata(dev); 991 unsigned pch; 992 993 for (pch = 0; pch < NR_PHY_CHAN; pch++) { 994 struct sa11x0_dma_phy *p = &d->phy[pch]; 995 u32 dcsr, saved_dcsr; 996 997 dcsr = saved_dcsr = readl_relaxed(p->base + DMA_DCSR_R); 998 if (dcsr & DCSR_RUN) { 999 writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C); 1000 dcsr = readl_relaxed(p->base + DMA_DCSR_R); 1001 } 1002 1003 saved_dcsr &= DCSR_RUN | DCSR_IE; 1004 if (dcsr & DCSR_BIU) { 1005 p->dbs[0] = readl_relaxed(p->base + DMA_DBSB); 1006 p->dbt[0] = readl_relaxed(p->base + DMA_DBTB); 1007 p->dbs[1] = readl_relaxed(p->base + DMA_DBSA); 1008 p->dbt[1] = readl_relaxed(p->base + DMA_DBTA); 1009 saved_dcsr |= (dcsr & DCSR_STRTA ? DCSR_STRTB : 0) | 1010 (dcsr & DCSR_STRTB ? DCSR_STRTA : 0); 1011 } else { 1012 p->dbs[0] = readl_relaxed(p->base + DMA_DBSA); 1013 p->dbt[0] = readl_relaxed(p->base + DMA_DBTA); 1014 p->dbs[1] = readl_relaxed(p->base + DMA_DBSB); 1015 p->dbt[1] = readl_relaxed(p->base + DMA_DBTB); 1016 saved_dcsr |= dcsr & (DCSR_STRTA | DCSR_STRTB); 1017 } 1018 p->dcsr = saved_dcsr; 1019 1020 writel(DCSR_STRTA | DCSR_STRTB, p->base + DMA_DCSR_C); 1021 } 1022 1023 return 0; 1024 } 1025 1026 static int sa11x0_dma_resume(struct device *dev) 1027 { 1028 struct sa11x0_dma_dev *d = dev_get_drvdata(dev); 1029 unsigned pch; 1030 1031 for (pch = 0; pch < NR_PHY_CHAN; pch++) { 1032 struct sa11x0_dma_phy *p = &d->phy[pch]; 1033 struct sa11x0_dma_desc *txd = NULL; 1034 u32 dcsr = readl_relaxed(p->base + DMA_DCSR_R); 1035 1036 WARN_ON(dcsr & (DCSR_BIU | DCSR_STRTA | DCSR_STRTB | DCSR_RUN)); 1037 1038 if (p->txd_done) 1039 txd = p->txd_done; 1040 else if (p->txd_load) 1041 txd = p->txd_load; 1042 1043 if (!txd) 1044 continue; 1045 1046 writel_relaxed(txd->ddar, p->base + DMA_DDAR); 1047 1048 writel_relaxed(p->dbs[0], p->base + DMA_DBSA); 1049 writel_relaxed(p->dbt[0], p->base + DMA_DBTA); 1050 writel_relaxed(p->dbs[1], p->base + DMA_DBSB); 1051 writel_relaxed(p->dbt[1], p->base + DMA_DBTB); 1052 writel_relaxed(p->dcsr, p->base + DMA_DCSR_S); 1053 } 1054 1055 return 0; 1056 } 1057 #endif 1058 1059 static const struct dev_pm_ops sa11x0_dma_pm_ops = { 1060 .suspend_noirq = sa11x0_dma_suspend, 1061 .resume_noirq = sa11x0_dma_resume, 1062 .freeze_noirq = sa11x0_dma_suspend, 1063 .thaw_noirq = sa11x0_dma_resume, 1064 .poweroff_noirq = sa11x0_dma_suspend, 1065 .restore_noirq = sa11x0_dma_resume, 1066 }; 1067 1068 static struct platform_driver sa11x0_dma_driver = { 1069 .driver = { 1070 .name = "sa11x0-dma", 1071 .owner = THIS_MODULE, 1072 .pm = &sa11x0_dma_pm_ops, 1073 }, 1074 .probe = sa11x0_dma_probe, 1075 .remove = sa11x0_dma_remove, 1076 }; 1077 1078 bool sa11x0_dma_filter_fn(struct dma_chan *chan, void *param) 1079 { 1080 if (chan->device->dev->driver == &sa11x0_dma_driver.driver) { 1081 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan); 1082 const char *p = param; 1083 1084 return !strcmp(c->name, p); 1085 } 1086 return false; 1087 } 1088 EXPORT_SYMBOL(sa11x0_dma_filter_fn); 1089 1090 static int __init sa11x0_dma_init(void) 1091 { 1092 return platform_driver_register(&sa11x0_dma_driver); 1093 } 1094 subsys_initcall(sa11x0_dma_init); 1095 1096 static void __exit sa11x0_dma_exit(void) 1097 { 1098 platform_driver_unregister(&sa11x0_dma_driver); 1099 } 1100 module_exit(sa11x0_dma_exit); 1101 1102 MODULE_AUTHOR("Russell King"); 1103 MODULE_DESCRIPTION("SA-11x0 DMA driver"); 1104 MODULE_LICENSE("GPL v2"); 1105 MODULE_ALIAS("platform:sa11x0-dma"); 1106