1 /* 2 * Copyright (c) 2006 ARM Ltd. 3 * Copyright (c) 2010 ST-Ericsson SA 4 * 5 * Author: Peter Pearse <peter.pearse@arm.com> 6 * Author: Linus Walleij <linus.walleij@stericsson.com> 7 * 8 * This program is free software; you can redistribute it and/or modify it 9 * under the terms of the GNU General Public License as published by the Free 10 * Software Foundation; either version 2 of the License, or (at your option) 11 * any later version. 12 * 13 * This program is distributed in the hope that it will be useful, but WITHOUT 14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 16 * more details. 17 * 18 * The full GNU General Public License is in this distribution in the file 19 * called COPYING. 20 * 21 * Documentation: ARM DDI 0196G == PL080 22 * Documentation: ARM DDI 0218E == PL081 23 * Documentation: S3C6410 User's Manual == PL080S 24 * 25 * PL080 & PL081 both have 16 sets of DMA signals that can be routed to any 26 * channel. 27 * 28 * The PL080 has 8 channels available for simultaneous use, and the PL081 29 * has only two channels. So on these DMA controllers the number of channels 30 * and the number of incoming DMA signals are two totally different things. 31 * It is usually not possible to theoretically handle all physical signals, 32 * so a multiplexing scheme with possible denial of use is necessary. 33 * 34 * The PL080 has a dual bus master, PL081 has a single master. 35 * 36 * PL080S is a version modified by Samsung and used in S3C64xx SoCs. 37 * It differs in following aspects: 38 * - CH_CONFIG register at different offset, 39 * - separate CH_CONTROL2 register for transfer size, 40 * - bigger maximum transfer size, 41 * - 8-word aligned LLI, instead of 4-word, due to extra CCTL2 word, 42 * - no support for peripheral flow control. 43 * 44 * Memory to peripheral transfer may be visualized as 45 * Get data from memory to DMAC 46 * Until no data left 47 * On burst request from peripheral 48 * Destination burst from DMAC to peripheral 49 * Clear burst request 50 * Raise terminal count interrupt 51 * 52 * For peripherals with a FIFO: 53 * Source burst size == half the depth of the peripheral FIFO 54 * Destination burst size == the depth of the peripheral FIFO 55 * 56 * (Bursts are irrelevant for mem to mem transfers - there are no burst 57 * signals, the DMA controller will simply facilitate its AHB master.) 58 * 59 * ASSUMES default (little) endianness for DMA transfers 60 * 61 * The PL08x has two flow control settings: 62 * - DMAC flow control: the transfer size defines the number of transfers 63 * which occur for the current LLI entry, and the DMAC raises TC at the 64 * end of every LLI entry. Observed behaviour shows the DMAC listening 65 * to both the BREQ and SREQ signals (contrary to documented), 66 * transferring data if either is active. The LBREQ and LSREQ signals 67 * are ignored. 68 * 69 * - Peripheral flow control: the transfer size is ignored (and should be 70 * zero). The data is transferred from the current LLI entry, until 71 * after the final transfer signalled by LBREQ or LSREQ. The DMAC 72 * will then move to the next LLI entry. Unsupported by PL080S. 73 */ 74 #include <linux/amba/bus.h> 75 #include <linux/amba/pl08x.h> 76 #include <linux/debugfs.h> 77 #include <linux/delay.h> 78 #include <linux/device.h> 79 #include <linux/dmaengine.h> 80 #include <linux/dmapool.h> 81 #include <linux/dma-mapping.h> 82 #include <linux/export.h> 83 #include <linux/init.h> 84 #include <linux/interrupt.h> 85 #include <linux/module.h> 86 #include <linux/pm_runtime.h> 87 #include <linux/seq_file.h> 88 #include <linux/slab.h> 89 #include <linux/amba/pl080.h> 90 91 #include "dmaengine.h" 92 #include "virt-dma.h" 93 94 #define DRIVER_NAME "pl08xdmac" 95 96 #define PL80X_DMA_BUSWIDTHS \ 97 BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \ 98 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 99 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 100 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) 101 102 static struct amba_driver pl08x_amba_driver; 103 struct pl08x_driver_data; 104 105 /** 106 * struct vendor_data - vendor-specific config parameters for PL08x derivatives 107 * @channels: the number of channels available in this variant 108 * @dualmaster: whether this version supports dual AHB masters or not. 109 * @nomadik: whether the channels have Nomadik security extension bits 110 * that need to be checked for permission before use and some registers are 111 * missing 112 * @pl080s: whether this version is a PL080S, which has separate register and 113 * LLI word for transfer size. 114 */ 115 struct vendor_data { 116 u8 config_offset; 117 u8 channels; 118 bool dualmaster; 119 bool nomadik; 120 bool pl080s; 121 u32 max_transfer_size; 122 }; 123 124 /** 125 * struct pl08x_bus_data - information of source or destination 126 * busses for a transfer 127 * @addr: current address 128 * @maxwidth: the maximum width of a transfer on this bus 129 * @buswidth: the width of this bus in bytes: 1, 2 or 4 130 */ 131 struct pl08x_bus_data { 132 dma_addr_t addr; 133 u8 maxwidth; 134 u8 buswidth; 135 }; 136 137 #define IS_BUS_ALIGNED(bus) IS_ALIGNED((bus)->addr, (bus)->buswidth) 138 139 /** 140 * struct pl08x_phy_chan - holder for the physical channels 141 * @id: physical index to this channel 142 * @lock: a lock to use when altering an instance of this struct 143 * @serving: the virtual channel currently being served by this physical 144 * channel 145 * @locked: channel unavailable for the system, e.g. dedicated to secure 146 * world 147 */ 148 struct pl08x_phy_chan { 149 unsigned int id; 150 void __iomem *base; 151 void __iomem *reg_config; 152 spinlock_t lock; 153 struct pl08x_dma_chan *serving; 154 bool locked; 155 }; 156 157 /** 158 * struct pl08x_sg - structure containing data per sg 159 * @src_addr: src address of sg 160 * @dst_addr: dst address of sg 161 * @len: transfer len in bytes 162 * @node: node for txd's dsg_list 163 */ 164 struct pl08x_sg { 165 dma_addr_t src_addr; 166 dma_addr_t dst_addr; 167 size_t len; 168 struct list_head node; 169 }; 170 171 /** 172 * struct pl08x_txd - wrapper for struct dma_async_tx_descriptor 173 * @vd: virtual DMA descriptor 174 * @dsg_list: list of children sg's 175 * @llis_bus: DMA memory address (physical) start for the LLIs 176 * @llis_va: virtual memory address start for the LLIs 177 * @cctl: control reg values for current txd 178 * @ccfg: config reg values for current txd 179 * @done: this marks completed descriptors, which should not have their 180 * mux released. 181 * @cyclic: indicate cyclic transfers 182 */ 183 struct pl08x_txd { 184 struct virt_dma_desc vd; 185 struct list_head dsg_list; 186 dma_addr_t llis_bus; 187 u32 *llis_va; 188 /* Default cctl value for LLIs */ 189 u32 cctl; 190 /* 191 * Settings to be put into the physical channel when we 192 * trigger this txd. Other registers are in llis_va[0]. 193 */ 194 u32 ccfg; 195 bool done; 196 bool cyclic; 197 }; 198 199 /** 200 * struct pl08x_dma_chan_state - holds the PL08x specific virtual channel 201 * states 202 * @PL08X_CHAN_IDLE: the channel is idle 203 * @PL08X_CHAN_RUNNING: the channel has allocated a physical transport 204 * channel and is running a transfer on it 205 * @PL08X_CHAN_PAUSED: the channel has allocated a physical transport 206 * channel, but the transfer is currently paused 207 * @PL08X_CHAN_WAITING: the channel is waiting for a physical transport 208 * channel to become available (only pertains to memcpy channels) 209 */ 210 enum pl08x_dma_chan_state { 211 PL08X_CHAN_IDLE, 212 PL08X_CHAN_RUNNING, 213 PL08X_CHAN_PAUSED, 214 PL08X_CHAN_WAITING, 215 }; 216 217 /** 218 * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel 219 * @vc: wrappped virtual channel 220 * @phychan: the physical channel utilized by this channel, if there is one 221 * @name: name of channel 222 * @cd: channel platform data 223 * @runtime_addr: address for RX/TX according to the runtime config 224 * @at: active transaction on this channel 225 * @lock: a lock for this channel data 226 * @host: a pointer to the host (internal use) 227 * @state: whether the channel is idle, paused, running etc 228 * @slave: whether this channel is a device (slave) or for memcpy 229 * @signal: the physical DMA request signal which this channel is using 230 * @mux_use: count of descriptors using this DMA request signal setting 231 */ 232 struct pl08x_dma_chan { 233 struct virt_dma_chan vc; 234 struct pl08x_phy_chan *phychan; 235 const char *name; 236 const struct pl08x_channel_data *cd; 237 struct dma_slave_config cfg; 238 struct pl08x_txd *at; 239 struct pl08x_driver_data *host; 240 enum pl08x_dma_chan_state state; 241 bool slave; 242 int signal; 243 unsigned mux_use; 244 }; 245 246 /** 247 * struct pl08x_driver_data - the local state holder for the PL08x 248 * @slave: slave engine for this instance 249 * @memcpy: memcpy engine for this instance 250 * @base: virtual memory base (remapped) for the PL08x 251 * @adev: the corresponding AMBA (PrimeCell) bus entry 252 * @vd: vendor data for this PL08x variant 253 * @pd: platform data passed in from the platform/machine 254 * @phy_chans: array of data for the physical channels 255 * @pool: a pool for the LLI descriptors 256 * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI 257 * fetches 258 * @mem_buses: set to indicate memory transfers on AHB2. 259 * @lock: a spinlock for this struct 260 */ 261 struct pl08x_driver_data { 262 struct dma_device slave; 263 struct dma_device memcpy; 264 void __iomem *base; 265 struct amba_device *adev; 266 const struct vendor_data *vd; 267 struct pl08x_platform_data *pd; 268 struct pl08x_phy_chan *phy_chans; 269 struct dma_pool *pool; 270 u8 lli_buses; 271 u8 mem_buses; 272 u8 lli_words; 273 }; 274 275 /* 276 * PL08X specific defines 277 */ 278 279 /* The order of words in an LLI. */ 280 #define PL080_LLI_SRC 0 281 #define PL080_LLI_DST 1 282 #define PL080_LLI_LLI 2 283 #define PL080_LLI_CCTL 3 284 #define PL080S_LLI_CCTL2 4 285 286 /* Total words in an LLI. */ 287 #define PL080_LLI_WORDS 4 288 #define PL080S_LLI_WORDS 8 289 290 /* 291 * Number of LLIs in each LLI buffer allocated for one transfer 292 * (maximum times we call dma_pool_alloc on this pool without freeing) 293 */ 294 #define MAX_NUM_TSFR_LLIS 512 295 #define PL08X_ALIGN 8 296 297 static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan) 298 { 299 return container_of(chan, struct pl08x_dma_chan, vc.chan); 300 } 301 302 static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx) 303 { 304 return container_of(tx, struct pl08x_txd, vd.tx); 305 } 306 307 /* 308 * Mux handling. 309 * 310 * This gives us the DMA request input to the PL08x primecell which the 311 * peripheral described by the channel data will be routed to, possibly 312 * via a board/SoC specific external MUX. One important point to note 313 * here is that this does not depend on the physical channel. 314 */ 315 static int pl08x_request_mux(struct pl08x_dma_chan *plchan) 316 { 317 const struct pl08x_platform_data *pd = plchan->host->pd; 318 int ret; 319 320 if (plchan->mux_use++ == 0 && pd->get_xfer_signal) { 321 ret = pd->get_xfer_signal(plchan->cd); 322 if (ret < 0) { 323 plchan->mux_use = 0; 324 return ret; 325 } 326 327 plchan->signal = ret; 328 } 329 return 0; 330 } 331 332 static void pl08x_release_mux(struct pl08x_dma_chan *plchan) 333 { 334 const struct pl08x_platform_data *pd = plchan->host->pd; 335 336 if (plchan->signal >= 0) { 337 WARN_ON(plchan->mux_use == 0); 338 339 if (--plchan->mux_use == 0 && pd->put_xfer_signal) { 340 pd->put_xfer_signal(plchan->cd, plchan->signal); 341 plchan->signal = -1; 342 } 343 } 344 } 345 346 /* 347 * Physical channel handling 348 */ 349 350 /* Whether a certain channel is busy or not */ 351 static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch) 352 { 353 unsigned int val; 354 355 val = readl(ch->reg_config); 356 return val & PL080_CONFIG_ACTIVE; 357 } 358 359 static void pl08x_write_lli(struct pl08x_driver_data *pl08x, 360 struct pl08x_phy_chan *phychan, const u32 *lli, u32 ccfg) 361 { 362 if (pl08x->vd->pl080s) 363 dev_vdbg(&pl08x->adev->dev, 364 "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, " 365 "clli=0x%08x, cctl=0x%08x, cctl2=0x%08x, ccfg=0x%08x\n", 366 phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST], 367 lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], 368 lli[PL080S_LLI_CCTL2], ccfg); 369 else 370 dev_vdbg(&pl08x->adev->dev, 371 "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, " 372 "clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n", 373 phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST], 374 lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], ccfg); 375 376 writel_relaxed(lli[PL080_LLI_SRC], phychan->base + PL080_CH_SRC_ADDR); 377 writel_relaxed(lli[PL080_LLI_DST], phychan->base + PL080_CH_DST_ADDR); 378 writel_relaxed(lli[PL080_LLI_LLI], phychan->base + PL080_CH_LLI); 379 writel_relaxed(lli[PL080_LLI_CCTL], phychan->base + PL080_CH_CONTROL); 380 381 if (pl08x->vd->pl080s) 382 writel_relaxed(lli[PL080S_LLI_CCTL2], 383 phychan->base + PL080S_CH_CONTROL2); 384 385 writel(ccfg, phychan->reg_config); 386 } 387 388 /* 389 * Set the initial DMA register values i.e. those for the first LLI 390 * The next LLI pointer and the configuration interrupt bit have 391 * been set when the LLIs were constructed. Poke them into the hardware 392 * and start the transfer. 393 */ 394 static void pl08x_start_next_txd(struct pl08x_dma_chan *plchan) 395 { 396 struct pl08x_driver_data *pl08x = plchan->host; 397 struct pl08x_phy_chan *phychan = plchan->phychan; 398 struct virt_dma_desc *vd = vchan_next_desc(&plchan->vc); 399 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); 400 u32 val; 401 402 list_del(&txd->vd.node); 403 404 plchan->at = txd; 405 406 /* Wait for channel inactive */ 407 while (pl08x_phy_channel_busy(phychan)) 408 cpu_relax(); 409 410 pl08x_write_lli(pl08x, phychan, &txd->llis_va[0], txd->ccfg); 411 412 /* Enable the DMA channel */ 413 /* Do not access config register until channel shows as disabled */ 414 while (readl(pl08x->base + PL080_EN_CHAN) & (1 << phychan->id)) 415 cpu_relax(); 416 417 /* Do not access config register until channel shows as inactive */ 418 val = readl(phychan->reg_config); 419 while ((val & PL080_CONFIG_ACTIVE) || (val & PL080_CONFIG_ENABLE)) 420 val = readl(phychan->reg_config); 421 422 writel(val | PL080_CONFIG_ENABLE, phychan->reg_config); 423 } 424 425 /* 426 * Pause the channel by setting the HALT bit. 427 * 428 * For M->P transfers, pause the DMAC first and then stop the peripheral - 429 * the FIFO can only drain if the peripheral is still requesting data. 430 * (note: this can still timeout if the DMAC FIFO never drains of data.) 431 * 432 * For P->M transfers, disable the peripheral first to stop it filling 433 * the DMAC FIFO, and then pause the DMAC. 434 */ 435 static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch) 436 { 437 u32 val; 438 int timeout; 439 440 /* Set the HALT bit and wait for the FIFO to drain */ 441 val = readl(ch->reg_config); 442 val |= PL080_CONFIG_HALT; 443 writel(val, ch->reg_config); 444 445 /* Wait for channel inactive */ 446 for (timeout = 1000; timeout; timeout--) { 447 if (!pl08x_phy_channel_busy(ch)) 448 break; 449 udelay(1); 450 } 451 if (pl08x_phy_channel_busy(ch)) 452 pr_err("pl08x: channel%u timeout waiting for pause\n", ch->id); 453 } 454 455 static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch) 456 { 457 u32 val; 458 459 /* Clear the HALT bit */ 460 val = readl(ch->reg_config); 461 val &= ~PL080_CONFIG_HALT; 462 writel(val, ch->reg_config); 463 } 464 465 /* 466 * pl08x_terminate_phy_chan() stops the channel, clears the FIFO and 467 * clears any pending interrupt status. This should not be used for 468 * an on-going transfer, but as a method of shutting down a channel 469 * (eg, when it's no longer used) or terminating a transfer. 470 */ 471 static void pl08x_terminate_phy_chan(struct pl08x_driver_data *pl08x, 472 struct pl08x_phy_chan *ch) 473 { 474 u32 val = readl(ch->reg_config); 475 476 val &= ~(PL080_CONFIG_ENABLE | PL080_CONFIG_ERR_IRQ_MASK | 477 PL080_CONFIG_TC_IRQ_MASK); 478 479 writel(val, ch->reg_config); 480 481 writel(1 << ch->id, pl08x->base + PL080_ERR_CLEAR); 482 writel(1 << ch->id, pl08x->base + PL080_TC_CLEAR); 483 } 484 485 static inline u32 get_bytes_in_cctl(u32 cctl) 486 { 487 /* The source width defines the number of bytes */ 488 u32 bytes = cctl & PL080_CONTROL_TRANSFER_SIZE_MASK; 489 490 cctl &= PL080_CONTROL_SWIDTH_MASK; 491 492 switch (cctl >> PL080_CONTROL_SWIDTH_SHIFT) { 493 case PL080_WIDTH_8BIT: 494 break; 495 case PL080_WIDTH_16BIT: 496 bytes *= 2; 497 break; 498 case PL080_WIDTH_32BIT: 499 bytes *= 4; 500 break; 501 } 502 return bytes; 503 } 504 505 static inline u32 get_bytes_in_cctl_pl080s(u32 cctl, u32 cctl1) 506 { 507 /* The source width defines the number of bytes */ 508 u32 bytes = cctl1 & PL080S_CONTROL_TRANSFER_SIZE_MASK; 509 510 cctl &= PL080_CONTROL_SWIDTH_MASK; 511 512 switch (cctl >> PL080_CONTROL_SWIDTH_SHIFT) { 513 case PL080_WIDTH_8BIT: 514 break; 515 case PL080_WIDTH_16BIT: 516 bytes *= 2; 517 break; 518 case PL080_WIDTH_32BIT: 519 bytes *= 4; 520 break; 521 } 522 return bytes; 523 } 524 525 /* The channel should be paused when calling this */ 526 static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan) 527 { 528 struct pl08x_driver_data *pl08x = plchan->host; 529 const u32 *llis_va, *llis_va_limit; 530 struct pl08x_phy_chan *ch; 531 dma_addr_t llis_bus; 532 struct pl08x_txd *txd; 533 u32 llis_max_words; 534 size_t bytes; 535 u32 clli; 536 537 ch = plchan->phychan; 538 txd = plchan->at; 539 540 if (!ch || !txd) 541 return 0; 542 543 /* 544 * Follow the LLIs to get the number of remaining 545 * bytes in the currently active transaction. 546 */ 547 clli = readl(ch->base + PL080_CH_LLI) & ~PL080_LLI_LM_AHB2; 548 549 /* First get the remaining bytes in the active transfer */ 550 if (pl08x->vd->pl080s) 551 bytes = get_bytes_in_cctl_pl080s( 552 readl(ch->base + PL080_CH_CONTROL), 553 readl(ch->base + PL080S_CH_CONTROL2)); 554 else 555 bytes = get_bytes_in_cctl(readl(ch->base + PL080_CH_CONTROL)); 556 557 if (!clli) 558 return bytes; 559 560 llis_va = txd->llis_va; 561 llis_bus = txd->llis_bus; 562 563 llis_max_words = pl08x->lli_words * MAX_NUM_TSFR_LLIS; 564 BUG_ON(clli < llis_bus || clli >= llis_bus + 565 sizeof(u32) * llis_max_words); 566 567 /* 568 * Locate the next LLI - as this is an array, 569 * it's simple maths to find. 570 */ 571 llis_va += (clli - llis_bus) / sizeof(u32); 572 573 llis_va_limit = llis_va + llis_max_words; 574 575 for (; llis_va < llis_va_limit; llis_va += pl08x->lli_words) { 576 if (pl08x->vd->pl080s) 577 bytes += get_bytes_in_cctl_pl080s( 578 llis_va[PL080_LLI_CCTL], 579 llis_va[PL080S_LLI_CCTL2]); 580 else 581 bytes += get_bytes_in_cctl(llis_va[PL080_LLI_CCTL]); 582 583 /* 584 * A LLI pointer going backward terminates the LLI list 585 */ 586 if (llis_va[PL080_LLI_LLI] <= clli) 587 break; 588 } 589 590 return bytes; 591 } 592 593 /* 594 * Allocate a physical channel for a virtual channel 595 * 596 * Try to locate a physical channel to be used for this transfer. If all 597 * are taken return NULL and the requester will have to cope by using 598 * some fallback PIO mode or retrying later. 599 */ 600 static struct pl08x_phy_chan * 601 pl08x_get_phy_channel(struct pl08x_driver_data *pl08x, 602 struct pl08x_dma_chan *virt_chan) 603 { 604 struct pl08x_phy_chan *ch = NULL; 605 unsigned long flags; 606 int i; 607 608 for (i = 0; i < pl08x->vd->channels; i++) { 609 ch = &pl08x->phy_chans[i]; 610 611 spin_lock_irqsave(&ch->lock, flags); 612 613 if (!ch->locked && !ch->serving) { 614 ch->serving = virt_chan; 615 spin_unlock_irqrestore(&ch->lock, flags); 616 break; 617 } 618 619 spin_unlock_irqrestore(&ch->lock, flags); 620 } 621 622 if (i == pl08x->vd->channels) { 623 /* No physical channel available, cope with it */ 624 return NULL; 625 } 626 627 return ch; 628 } 629 630 /* Mark the physical channel as free. Note, this write is atomic. */ 631 static inline void pl08x_put_phy_channel(struct pl08x_driver_data *pl08x, 632 struct pl08x_phy_chan *ch) 633 { 634 ch->serving = NULL; 635 } 636 637 /* 638 * Try to allocate a physical channel. When successful, assign it to 639 * this virtual channel, and initiate the next descriptor. The 640 * virtual channel lock must be held at this point. 641 */ 642 static void pl08x_phy_alloc_and_start(struct pl08x_dma_chan *plchan) 643 { 644 struct pl08x_driver_data *pl08x = plchan->host; 645 struct pl08x_phy_chan *ch; 646 647 ch = pl08x_get_phy_channel(pl08x, plchan); 648 if (!ch) { 649 dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name); 650 plchan->state = PL08X_CHAN_WAITING; 651 return; 652 } 653 654 dev_dbg(&pl08x->adev->dev, "allocated physical channel %d for xfer on %s\n", 655 ch->id, plchan->name); 656 657 plchan->phychan = ch; 658 plchan->state = PL08X_CHAN_RUNNING; 659 pl08x_start_next_txd(plchan); 660 } 661 662 static void pl08x_phy_reassign_start(struct pl08x_phy_chan *ch, 663 struct pl08x_dma_chan *plchan) 664 { 665 struct pl08x_driver_data *pl08x = plchan->host; 666 667 dev_dbg(&pl08x->adev->dev, "reassigned physical channel %d for xfer on %s\n", 668 ch->id, plchan->name); 669 670 /* 671 * We do this without taking the lock; we're really only concerned 672 * about whether this pointer is NULL or not, and we're guaranteed 673 * that this will only be called when it _already_ is non-NULL. 674 */ 675 ch->serving = plchan; 676 plchan->phychan = ch; 677 plchan->state = PL08X_CHAN_RUNNING; 678 pl08x_start_next_txd(plchan); 679 } 680 681 /* 682 * Free a physical DMA channel, potentially reallocating it to another 683 * virtual channel if we have any pending. 684 */ 685 static void pl08x_phy_free(struct pl08x_dma_chan *plchan) 686 { 687 struct pl08x_driver_data *pl08x = plchan->host; 688 struct pl08x_dma_chan *p, *next; 689 690 retry: 691 next = NULL; 692 693 /* Find a waiting virtual channel for the next transfer. */ 694 list_for_each_entry(p, &pl08x->memcpy.channels, vc.chan.device_node) 695 if (p->state == PL08X_CHAN_WAITING) { 696 next = p; 697 break; 698 } 699 700 if (!next) { 701 list_for_each_entry(p, &pl08x->slave.channels, vc.chan.device_node) 702 if (p->state == PL08X_CHAN_WAITING) { 703 next = p; 704 break; 705 } 706 } 707 708 /* Ensure that the physical channel is stopped */ 709 pl08x_terminate_phy_chan(pl08x, plchan->phychan); 710 711 if (next) { 712 bool success; 713 714 /* 715 * Eww. We know this isn't going to deadlock 716 * but lockdep probably doesn't. 717 */ 718 spin_lock(&next->vc.lock); 719 /* Re-check the state now that we have the lock */ 720 success = next->state == PL08X_CHAN_WAITING; 721 if (success) 722 pl08x_phy_reassign_start(plchan->phychan, next); 723 spin_unlock(&next->vc.lock); 724 725 /* If the state changed, try to find another channel */ 726 if (!success) 727 goto retry; 728 } else { 729 /* No more jobs, so free up the physical channel */ 730 pl08x_put_phy_channel(pl08x, plchan->phychan); 731 } 732 733 plchan->phychan = NULL; 734 plchan->state = PL08X_CHAN_IDLE; 735 } 736 737 /* 738 * LLI handling 739 */ 740 741 static inline unsigned int pl08x_get_bytes_for_cctl(unsigned int coded) 742 { 743 switch (coded) { 744 case PL080_WIDTH_8BIT: 745 return 1; 746 case PL080_WIDTH_16BIT: 747 return 2; 748 case PL080_WIDTH_32BIT: 749 return 4; 750 default: 751 break; 752 } 753 BUG(); 754 return 0; 755 } 756 757 static inline u32 pl08x_cctl_bits(u32 cctl, u8 srcwidth, u8 dstwidth, 758 size_t tsize) 759 { 760 u32 retbits = cctl; 761 762 /* Remove all src, dst and transfer size bits */ 763 retbits &= ~PL080_CONTROL_DWIDTH_MASK; 764 retbits &= ~PL080_CONTROL_SWIDTH_MASK; 765 retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK; 766 767 /* Then set the bits according to the parameters */ 768 switch (srcwidth) { 769 case 1: 770 retbits |= PL080_WIDTH_8BIT << PL080_CONTROL_SWIDTH_SHIFT; 771 break; 772 case 2: 773 retbits |= PL080_WIDTH_16BIT << PL080_CONTROL_SWIDTH_SHIFT; 774 break; 775 case 4: 776 retbits |= PL080_WIDTH_32BIT << PL080_CONTROL_SWIDTH_SHIFT; 777 break; 778 default: 779 BUG(); 780 break; 781 } 782 783 switch (dstwidth) { 784 case 1: 785 retbits |= PL080_WIDTH_8BIT << PL080_CONTROL_DWIDTH_SHIFT; 786 break; 787 case 2: 788 retbits |= PL080_WIDTH_16BIT << PL080_CONTROL_DWIDTH_SHIFT; 789 break; 790 case 4: 791 retbits |= PL080_WIDTH_32BIT << PL080_CONTROL_DWIDTH_SHIFT; 792 break; 793 default: 794 BUG(); 795 break; 796 } 797 798 tsize &= PL080_CONTROL_TRANSFER_SIZE_MASK; 799 retbits |= tsize << PL080_CONTROL_TRANSFER_SIZE_SHIFT; 800 return retbits; 801 } 802 803 struct pl08x_lli_build_data { 804 struct pl08x_txd *txd; 805 struct pl08x_bus_data srcbus; 806 struct pl08x_bus_data dstbus; 807 size_t remainder; 808 u32 lli_bus; 809 }; 810 811 /* 812 * Autoselect a master bus to use for the transfer. Slave will be the chosen as 813 * victim in case src & dest are not similarly aligned. i.e. If after aligning 814 * masters address with width requirements of transfer (by sending few byte by 815 * byte data), slave is still not aligned, then its width will be reduced to 816 * BYTE. 817 * - prefers the destination bus if both available 818 * - prefers bus with fixed address (i.e. peripheral) 819 */ 820 static void pl08x_choose_master_bus(struct pl08x_lli_build_data *bd, 821 struct pl08x_bus_data **mbus, struct pl08x_bus_data **sbus, u32 cctl) 822 { 823 if (!(cctl & PL080_CONTROL_DST_INCR)) { 824 *mbus = &bd->dstbus; 825 *sbus = &bd->srcbus; 826 } else if (!(cctl & PL080_CONTROL_SRC_INCR)) { 827 *mbus = &bd->srcbus; 828 *sbus = &bd->dstbus; 829 } else { 830 if (bd->dstbus.buswidth >= bd->srcbus.buswidth) { 831 *mbus = &bd->dstbus; 832 *sbus = &bd->srcbus; 833 } else { 834 *mbus = &bd->srcbus; 835 *sbus = &bd->dstbus; 836 } 837 } 838 } 839 840 /* 841 * Fills in one LLI for a certain transfer descriptor and advance the counter 842 */ 843 static void pl08x_fill_lli_for_desc(struct pl08x_driver_data *pl08x, 844 struct pl08x_lli_build_data *bd, 845 int num_llis, int len, u32 cctl, u32 cctl2) 846 { 847 u32 offset = num_llis * pl08x->lli_words; 848 u32 *llis_va = bd->txd->llis_va + offset; 849 dma_addr_t llis_bus = bd->txd->llis_bus; 850 851 BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS); 852 853 /* Advance the offset to next LLI. */ 854 offset += pl08x->lli_words; 855 856 llis_va[PL080_LLI_SRC] = bd->srcbus.addr; 857 llis_va[PL080_LLI_DST] = bd->dstbus.addr; 858 llis_va[PL080_LLI_LLI] = (llis_bus + sizeof(u32) * offset); 859 llis_va[PL080_LLI_LLI] |= bd->lli_bus; 860 llis_va[PL080_LLI_CCTL] = cctl; 861 if (pl08x->vd->pl080s) 862 llis_va[PL080S_LLI_CCTL2] = cctl2; 863 864 if (cctl & PL080_CONTROL_SRC_INCR) 865 bd->srcbus.addr += len; 866 if (cctl & PL080_CONTROL_DST_INCR) 867 bd->dstbus.addr += len; 868 869 BUG_ON(bd->remainder < len); 870 871 bd->remainder -= len; 872 } 873 874 static inline void prep_byte_width_lli(struct pl08x_driver_data *pl08x, 875 struct pl08x_lli_build_data *bd, u32 *cctl, u32 len, 876 int num_llis, size_t *total_bytes) 877 { 878 *cctl = pl08x_cctl_bits(*cctl, 1, 1, len); 879 pl08x_fill_lli_for_desc(pl08x, bd, num_llis, len, *cctl, len); 880 (*total_bytes) += len; 881 } 882 883 #ifdef VERBOSE_DEBUG 884 static void pl08x_dump_lli(struct pl08x_driver_data *pl08x, 885 const u32 *llis_va, int num_llis) 886 { 887 int i; 888 889 if (pl08x->vd->pl080s) { 890 dev_vdbg(&pl08x->adev->dev, 891 "%-3s %-9s %-10s %-10s %-10s %-10s %s\n", 892 "lli", "", "csrc", "cdst", "clli", "cctl", "cctl2"); 893 for (i = 0; i < num_llis; i++) { 894 dev_vdbg(&pl08x->adev->dev, 895 "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n", 896 i, llis_va, llis_va[PL080_LLI_SRC], 897 llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI], 898 llis_va[PL080_LLI_CCTL], 899 llis_va[PL080S_LLI_CCTL2]); 900 llis_va += pl08x->lli_words; 901 } 902 } else { 903 dev_vdbg(&pl08x->adev->dev, 904 "%-3s %-9s %-10s %-10s %-10s %s\n", 905 "lli", "", "csrc", "cdst", "clli", "cctl"); 906 for (i = 0; i < num_llis; i++) { 907 dev_vdbg(&pl08x->adev->dev, 908 "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x\n", 909 i, llis_va, llis_va[PL080_LLI_SRC], 910 llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI], 911 llis_va[PL080_LLI_CCTL]); 912 llis_va += pl08x->lli_words; 913 } 914 } 915 } 916 #else 917 static inline void pl08x_dump_lli(struct pl08x_driver_data *pl08x, 918 const u32 *llis_va, int num_llis) {} 919 #endif 920 921 /* 922 * This fills in the table of LLIs for the transfer descriptor 923 * Note that we assume we never have to change the burst sizes 924 * Return 0 for error 925 */ 926 static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, 927 struct pl08x_txd *txd) 928 { 929 struct pl08x_bus_data *mbus, *sbus; 930 struct pl08x_lli_build_data bd; 931 int num_llis = 0; 932 u32 cctl, early_bytes = 0; 933 size_t max_bytes_per_lli, total_bytes; 934 u32 *llis_va, *last_lli; 935 struct pl08x_sg *dsg; 936 937 txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus); 938 if (!txd->llis_va) { 939 dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__); 940 return 0; 941 } 942 943 bd.txd = txd; 944 bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0; 945 cctl = txd->cctl; 946 947 /* Find maximum width of the source bus */ 948 bd.srcbus.maxwidth = 949 pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_SWIDTH_MASK) >> 950 PL080_CONTROL_SWIDTH_SHIFT); 951 952 /* Find maximum width of the destination bus */ 953 bd.dstbus.maxwidth = 954 pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_DWIDTH_MASK) >> 955 PL080_CONTROL_DWIDTH_SHIFT); 956 957 list_for_each_entry(dsg, &txd->dsg_list, node) { 958 total_bytes = 0; 959 cctl = txd->cctl; 960 961 bd.srcbus.addr = dsg->src_addr; 962 bd.dstbus.addr = dsg->dst_addr; 963 bd.remainder = dsg->len; 964 bd.srcbus.buswidth = bd.srcbus.maxwidth; 965 bd.dstbus.buswidth = bd.dstbus.maxwidth; 966 967 pl08x_choose_master_bus(&bd, &mbus, &sbus, cctl); 968 969 dev_vdbg(&pl08x->adev->dev, 970 "src=0x%08llx%s/%u dst=0x%08llx%s/%u len=%zu\n", 971 (u64)bd.srcbus.addr, 972 cctl & PL080_CONTROL_SRC_INCR ? "+" : "", 973 bd.srcbus.buswidth, 974 (u64)bd.dstbus.addr, 975 cctl & PL080_CONTROL_DST_INCR ? "+" : "", 976 bd.dstbus.buswidth, 977 bd.remainder); 978 dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n", 979 mbus == &bd.srcbus ? "src" : "dst", 980 sbus == &bd.srcbus ? "src" : "dst"); 981 982 /* 983 * Zero length is only allowed if all these requirements are 984 * met: 985 * - flow controller is peripheral. 986 * - src.addr is aligned to src.width 987 * - dst.addr is aligned to dst.width 988 * 989 * sg_len == 1 should be true, as there can be two cases here: 990 * 991 * - Memory addresses are contiguous and are not scattered. 992 * Here, Only one sg will be passed by user driver, with 993 * memory address and zero length. We pass this to controller 994 * and after the transfer it will receive the last burst 995 * request from peripheral and so transfer finishes. 996 * 997 * - Memory addresses are scattered and are not contiguous. 998 * Here, Obviously as DMA controller doesn't know when a lli's 999 * transfer gets over, it can't load next lli. So in this 1000 * case, there has to be an assumption that only one lli is 1001 * supported. Thus, we can't have scattered addresses. 1002 */ 1003 if (!bd.remainder) { 1004 u32 fc = (txd->ccfg & PL080_CONFIG_FLOW_CONTROL_MASK) >> 1005 PL080_CONFIG_FLOW_CONTROL_SHIFT; 1006 if (!((fc >= PL080_FLOW_SRC2DST_DST) && 1007 (fc <= PL080_FLOW_SRC2DST_SRC))) { 1008 dev_err(&pl08x->adev->dev, "%s sg len can't be zero", 1009 __func__); 1010 return 0; 1011 } 1012 1013 if (!IS_BUS_ALIGNED(&bd.srcbus) || 1014 !IS_BUS_ALIGNED(&bd.dstbus)) { 1015 dev_err(&pl08x->adev->dev, 1016 "%s src & dst address must be aligned to src" 1017 " & dst width if peripheral is flow controller", 1018 __func__); 1019 return 0; 1020 } 1021 1022 cctl = pl08x_cctl_bits(cctl, bd.srcbus.buswidth, 1023 bd.dstbus.buswidth, 0); 1024 pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++, 1025 0, cctl, 0); 1026 break; 1027 } 1028 1029 /* 1030 * Send byte by byte for following cases 1031 * - Less than a bus width available 1032 * - until master bus is aligned 1033 */ 1034 if (bd.remainder < mbus->buswidth) 1035 early_bytes = bd.remainder; 1036 else if (!IS_BUS_ALIGNED(mbus)) { 1037 early_bytes = mbus->buswidth - 1038 (mbus->addr & (mbus->buswidth - 1)); 1039 if ((bd.remainder - early_bytes) < mbus->buswidth) 1040 early_bytes = bd.remainder; 1041 } 1042 1043 if (early_bytes) { 1044 dev_vdbg(&pl08x->adev->dev, 1045 "%s byte width LLIs (remain 0x%08zx)\n", 1046 __func__, bd.remainder); 1047 prep_byte_width_lli(pl08x, &bd, &cctl, early_bytes, 1048 num_llis++, &total_bytes); 1049 } 1050 1051 if (bd.remainder) { 1052 /* 1053 * Master now aligned 1054 * - if slave is not then we must set its width down 1055 */ 1056 if (!IS_BUS_ALIGNED(sbus)) { 1057 dev_dbg(&pl08x->adev->dev, 1058 "%s set down bus width to one byte\n", 1059 __func__); 1060 1061 sbus->buswidth = 1; 1062 } 1063 1064 /* 1065 * Bytes transferred = tsize * src width, not 1066 * MIN(buswidths) 1067 */ 1068 max_bytes_per_lli = bd.srcbus.buswidth * 1069 pl08x->vd->max_transfer_size; 1070 dev_vdbg(&pl08x->adev->dev, 1071 "%s max bytes per lli = %zu\n", 1072 __func__, max_bytes_per_lli); 1073 1074 /* 1075 * Make largest possible LLIs until less than one bus 1076 * width left 1077 */ 1078 while (bd.remainder > (mbus->buswidth - 1)) { 1079 size_t lli_len, tsize, width; 1080 1081 /* 1082 * If enough left try to send max possible, 1083 * otherwise try to send the remainder 1084 */ 1085 lli_len = min(bd.remainder, max_bytes_per_lli); 1086 1087 /* 1088 * Check against maximum bus alignment: 1089 * Calculate actual transfer size in relation to 1090 * bus width an get a maximum remainder of the 1091 * highest bus width - 1 1092 */ 1093 width = max(mbus->buswidth, sbus->buswidth); 1094 lli_len = (lli_len / width) * width; 1095 tsize = lli_len / bd.srcbus.buswidth; 1096 1097 dev_vdbg(&pl08x->adev->dev, 1098 "%s fill lli with single lli chunk of " 1099 "size 0x%08zx (remainder 0x%08zx)\n", 1100 __func__, lli_len, bd.remainder); 1101 1102 cctl = pl08x_cctl_bits(cctl, bd.srcbus.buswidth, 1103 bd.dstbus.buswidth, tsize); 1104 pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++, 1105 lli_len, cctl, tsize); 1106 total_bytes += lli_len; 1107 } 1108 1109 /* 1110 * Send any odd bytes 1111 */ 1112 if (bd.remainder) { 1113 dev_vdbg(&pl08x->adev->dev, 1114 "%s align with boundary, send odd bytes (remain %zu)\n", 1115 __func__, bd.remainder); 1116 prep_byte_width_lli(pl08x, &bd, &cctl, 1117 bd.remainder, num_llis++, &total_bytes); 1118 } 1119 } 1120 1121 if (total_bytes != dsg->len) { 1122 dev_err(&pl08x->adev->dev, 1123 "%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n", 1124 __func__, total_bytes, dsg->len); 1125 return 0; 1126 } 1127 1128 if (num_llis >= MAX_NUM_TSFR_LLIS) { 1129 dev_err(&pl08x->adev->dev, 1130 "%s need to increase MAX_NUM_TSFR_LLIS from 0x%08x\n", 1131 __func__, MAX_NUM_TSFR_LLIS); 1132 return 0; 1133 } 1134 } 1135 1136 llis_va = txd->llis_va; 1137 last_lli = llis_va + (num_llis - 1) * pl08x->lli_words; 1138 1139 if (txd->cyclic) { 1140 /* Link back to the first LLI. */ 1141 last_lli[PL080_LLI_LLI] = txd->llis_bus | bd.lli_bus; 1142 } else { 1143 /* The final LLI terminates the LLI. */ 1144 last_lli[PL080_LLI_LLI] = 0; 1145 /* The final LLI element shall also fire an interrupt. */ 1146 last_lli[PL080_LLI_CCTL] |= PL080_CONTROL_TC_IRQ_EN; 1147 } 1148 1149 pl08x_dump_lli(pl08x, llis_va, num_llis); 1150 1151 return num_llis; 1152 } 1153 1154 static void pl08x_free_txd(struct pl08x_driver_data *pl08x, 1155 struct pl08x_txd *txd) 1156 { 1157 struct pl08x_sg *dsg, *_dsg; 1158 1159 if (txd->llis_va) 1160 dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus); 1161 1162 list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) { 1163 list_del(&dsg->node); 1164 kfree(dsg); 1165 } 1166 1167 kfree(txd); 1168 } 1169 1170 static void pl08x_desc_free(struct virt_dma_desc *vd) 1171 { 1172 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); 1173 struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan); 1174 1175 dma_descriptor_unmap(&vd->tx); 1176 if (!txd->done) 1177 pl08x_release_mux(plchan); 1178 1179 pl08x_free_txd(plchan->host, txd); 1180 } 1181 1182 static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x, 1183 struct pl08x_dma_chan *plchan) 1184 { 1185 LIST_HEAD(head); 1186 1187 vchan_get_all_descriptors(&plchan->vc, &head); 1188 vchan_dma_desc_free_list(&plchan->vc, &head); 1189 } 1190 1191 /* 1192 * The DMA ENGINE API 1193 */ 1194 static void pl08x_free_chan_resources(struct dma_chan *chan) 1195 { 1196 /* Ensure all queued descriptors are freed */ 1197 vchan_free_chan_resources(to_virt_chan(chan)); 1198 } 1199 1200 static struct dma_async_tx_descriptor *pl08x_prep_dma_interrupt( 1201 struct dma_chan *chan, unsigned long flags) 1202 { 1203 struct dma_async_tx_descriptor *retval = NULL; 1204 1205 return retval; 1206 } 1207 1208 /* 1209 * Code accessing dma_async_is_complete() in a tight loop may give problems. 1210 * If slaves are relying on interrupts to signal completion this function 1211 * must not be called with interrupts disabled. 1212 */ 1213 static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan, 1214 dma_cookie_t cookie, struct dma_tx_state *txstate) 1215 { 1216 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1217 struct virt_dma_desc *vd; 1218 unsigned long flags; 1219 enum dma_status ret; 1220 size_t bytes = 0; 1221 1222 ret = dma_cookie_status(chan, cookie, txstate); 1223 if (ret == DMA_COMPLETE) 1224 return ret; 1225 1226 /* 1227 * There's no point calculating the residue if there's 1228 * no txstate to store the value. 1229 */ 1230 if (!txstate) { 1231 if (plchan->state == PL08X_CHAN_PAUSED) 1232 ret = DMA_PAUSED; 1233 return ret; 1234 } 1235 1236 spin_lock_irqsave(&plchan->vc.lock, flags); 1237 ret = dma_cookie_status(chan, cookie, txstate); 1238 if (ret != DMA_COMPLETE) { 1239 vd = vchan_find_desc(&plchan->vc, cookie); 1240 if (vd) { 1241 /* On the issued list, so hasn't been processed yet */ 1242 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); 1243 struct pl08x_sg *dsg; 1244 1245 list_for_each_entry(dsg, &txd->dsg_list, node) 1246 bytes += dsg->len; 1247 } else { 1248 bytes = pl08x_getbytes_chan(plchan); 1249 } 1250 } 1251 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1252 1253 /* 1254 * This cookie not complete yet 1255 * Get number of bytes left in the active transactions and queue 1256 */ 1257 dma_set_residue(txstate, bytes); 1258 1259 if (plchan->state == PL08X_CHAN_PAUSED && ret == DMA_IN_PROGRESS) 1260 ret = DMA_PAUSED; 1261 1262 /* Whether waiting or running, we're in progress */ 1263 return ret; 1264 } 1265 1266 /* PrimeCell DMA extension */ 1267 struct burst_table { 1268 u32 burstwords; 1269 u32 reg; 1270 }; 1271 1272 static const struct burst_table burst_sizes[] = { 1273 { 1274 .burstwords = 256, 1275 .reg = PL080_BSIZE_256, 1276 }, 1277 { 1278 .burstwords = 128, 1279 .reg = PL080_BSIZE_128, 1280 }, 1281 { 1282 .burstwords = 64, 1283 .reg = PL080_BSIZE_64, 1284 }, 1285 { 1286 .burstwords = 32, 1287 .reg = PL080_BSIZE_32, 1288 }, 1289 { 1290 .burstwords = 16, 1291 .reg = PL080_BSIZE_16, 1292 }, 1293 { 1294 .burstwords = 8, 1295 .reg = PL080_BSIZE_8, 1296 }, 1297 { 1298 .burstwords = 4, 1299 .reg = PL080_BSIZE_4, 1300 }, 1301 { 1302 .burstwords = 0, 1303 .reg = PL080_BSIZE_1, 1304 }, 1305 }; 1306 1307 /* 1308 * Given the source and destination available bus masks, select which 1309 * will be routed to each port. We try to have source and destination 1310 * on separate ports, but always respect the allowable settings. 1311 */ 1312 static u32 pl08x_select_bus(u8 src, u8 dst) 1313 { 1314 u32 cctl = 0; 1315 1316 if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1))) 1317 cctl |= PL080_CONTROL_DST_AHB2; 1318 if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2))) 1319 cctl |= PL080_CONTROL_SRC_AHB2; 1320 1321 return cctl; 1322 } 1323 1324 static u32 pl08x_cctl(u32 cctl) 1325 { 1326 cctl &= ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 | 1327 PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR | 1328 PL080_CONTROL_PROT_MASK); 1329 1330 /* Access the cell in privileged mode, non-bufferable, non-cacheable */ 1331 return cctl | PL080_CONTROL_PROT_SYS; 1332 } 1333 1334 static u32 pl08x_width(enum dma_slave_buswidth width) 1335 { 1336 switch (width) { 1337 case DMA_SLAVE_BUSWIDTH_1_BYTE: 1338 return PL080_WIDTH_8BIT; 1339 case DMA_SLAVE_BUSWIDTH_2_BYTES: 1340 return PL080_WIDTH_16BIT; 1341 case DMA_SLAVE_BUSWIDTH_4_BYTES: 1342 return PL080_WIDTH_32BIT; 1343 default: 1344 return ~0; 1345 } 1346 } 1347 1348 static u32 pl08x_burst(u32 maxburst) 1349 { 1350 int i; 1351 1352 for (i = 0; i < ARRAY_SIZE(burst_sizes); i++) 1353 if (burst_sizes[i].burstwords <= maxburst) 1354 break; 1355 1356 return burst_sizes[i].reg; 1357 } 1358 1359 static u32 pl08x_get_cctl(struct pl08x_dma_chan *plchan, 1360 enum dma_slave_buswidth addr_width, u32 maxburst) 1361 { 1362 u32 width, burst, cctl = 0; 1363 1364 width = pl08x_width(addr_width); 1365 if (width == ~0) 1366 return ~0; 1367 1368 cctl |= width << PL080_CONTROL_SWIDTH_SHIFT; 1369 cctl |= width << PL080_CONTROL_DWIDTH_SHIFT; 1370 1371 /* 1372 * If this channel will only request single transfers, set this 1373 * down to ONE element. Also select one element if no maxburst 1374 * is specified. 1375 */ 1376 if (plchan->cd->single) 1377 maxburst = 1; 1378 1379 burst = pl08x_burst(maxburst); 1380 cctl |= burst << PL080_CONTROL_SB_SIZE_SHIFT; 1381 cctl |= burst << PL080_CONTROL_DB_SIZE_SHIFT; 1382 1383 return pl08x_cctl(cctl); 1384 } 1385 1386 /* 1387 * Slave transactions callback to the slave device to allow 1388 * synchronization of slave DMA signals with the DMAC enable 1389 */ 1390 static void pl08x_issue_pending(struct dma_chan *chan) 1391 { 1392 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1393 unsigned long flags; 1394 1395 spin_lock_irqsave(&plchan->vc.lock, flags); 1396 if (vchan_issue_pending(&plchan->vc)) { 1397 if (!plchan->phychan && plchan->state != PL08X_CHAN_WAITING) 1398 pl08x_phy_alloc_and_start(plchan); 1399 } 1400 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1401 } 1402 1403 static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan) 1404 { 1405 struct pl08x_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT); 1406 1407 if (txd) { 1408 INIT_LIST_HEAD(&txd->dsg_list); 1409 1410 /* Always enable error and terminal interrupts */ 1411 txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK | 1412 PL080_CONFIG_TC_IRQ_MASK; 1413 } 1414 return txd; 1415 } 1416 1417 /* 1418 * Initialize a descriptor to be used by memcpy submit 1419 */ 1420 static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy( 1421 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 1422 size_t len, unsigned long flags) 1423 { 1424 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1425 struct pl08x_driver_data *pl08x = plchan->host; 1426 struct pl08x_txd *txd; 1427 struct pl08x_sg *dsg; 1428 int ret; 1429 1430 txd = pl08x_get_txd(plchan); 1431 if (!txd) { 1432 dev_err(&pl08x->adev->dev, 1433 "%s no memory for descriptor\n", __func__); 1434 return NULL; 1435 } 1436 1437 dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT); 1438 if (!dsg) { 1439 pl08x_free_txd(pl08x, txd); 1440 dev_err(&pl08x->adev->dev, "%s no memory for pl080 sg\n", 1441 __func__); 1442 return NULL; 1443 } 1444 list_add_tail(&dsg->node, &txd->dsg_list); 1445 1446 dsg->src_addr = src; 1447 dsg->dst_addr = dest; 1448 dsg->len = len; 1449 1450 /* Set platform data for m2m */ 1451 txd->ccfg |= PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT; 1452 txd->cctl = pl08x->pd->memcpy_channel.cctl_memcpy & 1453 ~(PL080_CONTROL_DST_AHB2 | PL080_CONTROL_SRC_AHB2); 1454 1455 /* Both to be incremented or the code will break */ 1456 txd->cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR; 1457 1458 if (pl08x->vd->dualmaster) 1459 txd->cctl |= pl08x_select_bus(pl08x->mem_buses, 1460 pl08x->mem_buses); 1461 1462 ret = pl08x_fill_llis_for_desc(plchan->host, txd); 1463 if (!ret) { 1464 pl08x_free_txd(pl08x, txd); 1465 return NULL; 1466 } 1467 1468 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); 1469 } 1470 1471 static struct pl08x_txd *pl08x_init_txd( 1472 struct dma_chan *chan, 1473 enum dma_transfer_direction direction, 1474 dma_addr_t *slave_addr) 1475 { 1476 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1477 struct pl08x_driver_data *pl08x = plchan->host; 1478 struct pl08x_txd *txd; 1479 enum dma_slave_buswidth addr_width; 1480 int ret, tmp; 1481 u8 src_buses, dst_buses; 1482 u32 maxburst, cctl; 1483 1484 txd = pl08x_get_txd(plchan); 1485 if (!txd) { 1486 dev_err(&pl08x->adev->dev, "%s no txd\n", __func__); 1487 return NULL; 1488 } 1489 1490 /* 1491 * Set up addresses, the PrimeCell configured address 1492 * will take precedence since this may configure the 1493 * channel target address dynamically at runtime. 1494 */ 1495 if (direction == DMA_MEM_TO_DEV) { 1496 cctl = PL080_CONTROL_SRC_INCR; 1497 *slave_addr = plchan->cfg.dst_addr; 1498 addr_width = plchan->cfg.dst_addr_width; 1499 maxburst = plchan->cfg.dst_maxburst; 1500 src_buses = pl08x->mem_buses; 1501 dst_buses = plchan->cd->periph_buses; 1502 } else if (direction == DMA_DEV_TO_MEM) { 1503 cctl = PL080_CONTROL_DST_INCR; 1504 *slave_addr = plchan->cfg.src_addr; 1505 addr_width = plchan->cfg.src_addr_width; 1506 maxburst = plchan->cfg.src_maxburst; 1507 src_buses = plchan->cd->periph_buses; 1508 dst_buses = pl08x->mem_buses; 1509 } else { 1510 pl08x_free_txd(pl08x, txd); 1511 dev_err(&pl08x->adev->dev, 1512 "%s direction unsupported\n", __func__); 1513 return NULL; 1514 } 1515 1516 cctl |= pl08x_get_cctl(plchan, addr_width, maxburst); 1517 if (cctl == ~0) { 1518 pl08x_free_txd(pl08x, txd); 1519 dev_err(&pl08x->adev->dev, 1520 "DMA slave configuration botched?\n"); 1521 return NULL; 1522 } 1523 1524 txd->cctl = cctl | pl08x_select_bus(src_buses, dst_buses); 1525 1526 if (plchan->cfg.device_fc) 1527 tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER_PER : 1528 PL080_FLOW_PER2MEM_PER; 1529 else 1530 tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER : 1531 PL080_FLOW_PER2MEM; 1532 1533 txd->ccfg |= tmp << PL080_CONFIG_FLOW_CONTROL_SHIFT; 1534 1535 ret = pl08x_request_mux(plchan); 1536 if (ret < 0) { 1537 pl08x_free_txd(pl08x, txd); 1538 dev_dbg(&pl08x->adev->dev, 1539 "unable to mux for transfer on %s due to platform restrictions\n", 1540 plchan->name); 1541 return NULL; 1542 } 1543 1544 dev_dbg(&pl08x->adev->dev, "allocated DMA request signal %d for xfer on %s\n", 1545 plchan->signal, plchan->name); 1546 1547 /* Assign the flow control signal to this channel */ 1548 if (direction == DMA_MEM_TO_DEV) 1549 txd->ccfg |= plchan->signal << PL080_CONFIG_DST_SEL_SHIFT; 1550 else 1551 txd->ccfg |= plchan->signal << PL080_CONFIG_SRC_SEL_SHIFT; 1552 1553 return txd; 1554 } 1555 1556 static int pl08x_tx_add_sg(struct pl08x_txd *txd, 1557 enum dma_transfer_direction direction, 1558 dma_addr_t slave_addr, 1559 dma_addr_t buf_addr, 1560 unsigned int len) 1561 { 1562 struct pl08x_sg *dsg; 1563 1564 dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT); 1565 if (!dsg) 1566 return -ENOMEM; 1567 1568 list_add_tail(&dsg->node, &txd->dsg_list); 1569 1570 dsg->len = len; 1571 if (direction == DMA_MEM_TO_DEV) { 1572 dsg->src_addr = buf_addr; 1573 dsg->dst_addr = slave_addr; 1574 } else { 1575 dsg->src_addr = slave_addr; 1576 dsg->dst_addr = buf_addr; 1577 } 1578 1579 return 0; 1580 } 1581 1582 static struct dma_async_tx_descriptor *pl08x_prep_slave_sg( 1583 struct dma_chan *chan, struct scatterlist *sgl, 1584 unsigned int sg_len, enum dma_transfer_direction direction, 1585 unsigned long flags, void *context) 1586 { 1587 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1588 struct pl08x_driver_data *pl08x = plchan->host; 1589 struct pl08x_txd *txd; 1590 struct scatterlist *sg; 1591 int ret, tmp; 1592 dma_addr_t slave_addr; 1593 1594 dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n", 1595 __func__, sg_dma_len(sgl), plchan->name); 1596 1597 txd = pl08x_init_txd(chan, direction, &slave_addr); 1598 if (!txd) 1599 return NULL; 1600 1601 for_each_sg(sgl, sg, sg_len, tmp) { 1602 ret = pl08x_tx_add_sg(txd, direction, slave_addr, 1603 sg_dma_address(sg), 1604 sg_dma_len(sg)); 1605 if (ret) { 1606 pl08x_release_mux(plchan); 1607 pl08x_free_txd(pl08x, txd); 1608 dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n", 1609 __func__); 1610 return NULL; 1611 } 1612 } 1613 1614 ret = pl08x_fill_llis_for_desc(plchan->host, txd); 1615 if (!ret) { 1616 pl08x_release_mux(plchan); 1617 pl08x_free_txd(pl08x, txd); 1618 return NULL; 1619 } 1620 1621 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); 1622 } 1623 1624 static struct dma_async_tx_descriptor *pl08x_prep_dma_cyclic( 1625 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 1626 size_t period_len, enum dma_transfer_direction direction, 1627 unsigned long flags) 1628 { 1629 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1630 struct pl08x_driver_data *pl08x = plchan->host; 1631 struct pl08x_txd *txd; 1632 int ret, tmp; 1633 dma_addr_t slave_addr; 1634 1635 dev_dbg(&pl08x->adev->dev, 1636 "%s prepare cyclic transaction of %zd/%zd bytes %s %s\n", 1637 __func__, period_len, buf_len, 1638 direction == DMA_MEM_TO_DEV ? "to" : "from", 1639 plchan->name); 1640 1641 txd = pl08x_init_txd(chan, direction, &slave_addr); 1642 if (!txd) 1643 return NULL; 1644 1645 txd->cyclic = true; 1646 txd->cctl |= PL080_CONTROL_TC_IRQ_EN; 1647 for (tmp = 0; tmp < buf_len; tmp += period_len) { 1648 ret = pl08x_tx_add_sg(txd, direction, slave_addr, 1649 buf_addr + tmp, period_len); 1650 if (ret) { 1651 pl08x_release_mux(plchan); 1652 pl08x_free_txd(pl08x, txd); 1653 return NULL; 1654 } 1655 } 1656 1657 ret = pl08x_fill_llis_for_desc(plchan->host, txd); 1658 if (!ret) { 1659 pl08x_release_mux(plchan); 1660 pl08x_free_txd(pl08x, txd); 1661 return NULL; 1662 } 1663 1664 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); 1665 } 1666 1667 static int pl08x_config(struct dma_chan *chan, 1668 struct dma_slave_config *config) 1669 { 1670 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1671 struct pl08x_driver_data *pl08x = plchan->host; 1672 1673 if (!plchan->slave) 1674 return -EINVAL; 1675 1676 /* Reject definitely invalid configurations */ 1677 if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES || 1678 config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) 1679 return -EINVAL; 1680 1681 if (config->device_fc && pl08x->vd->pl080s) { 1682 dev_err(&pl08x->adev->dev, 1683 "%s: PL080S does not support peripheral flow control\n", 1684 __func__); 1685 return -EINVAL; 1686 } 1687 1688 plchan->cfg = *config; 1689 1690 return 0; 1691 } 1692 1693 static int pl08x_terminate_all(struct dma_chan *chan) 1694 { 1695 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1696 struct pl08x_driver_data *pl08x = plchan->host; 1697 unsigned long flags; 1698 1699 spin_lock_irqsave(&plchan->vc.lock, flags); 1700 if (!plchan->phychan && !plchan->at) { 1701 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1702 return 0; 1703 } 1704 1705 plchan->state = PL08X_CHAN_IDLE; 1706 1707 if (plchan->phychan) { 1708 /* 1709 * Mark physical channel as free and free any slave 1710 * signal 1711 */ 1712 pl08x_phy_free(plchan); 1713 } 1714 /* Dequeue jobs and free LLIs */ 1715 if (plchan->at) { 1716 pl08x_desc_free(&plchan->at->vd); 1717 plchan->at = NULL; 1718 } 1719 /* Dequeue jobs not yet fired as well */ 1720 pl08x_free_txd_list(pl08x, plchan); 1721 1722 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1723 1724 return 0; 1725 } 1726 1727 static int pl08x_pause(struct dma_chan *chan) 1728 { 1729 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1730 unsigned long flags; 1731 1732 /* 1733 * Anything succeeds on channels with no physical allocation and 1734 * no queued transfers. 1735 */ 1736 spin_lock_irqsave(&plchan->vc.lock, flags); 1737 if (!plchan->phychan && !plchan->at) { 1738 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1739 return 0; 1740 } 1741 1742 pl08x_pause_phy_chan(plchan->phychan); 1743 plchan->state = PL08X_CHAN_PAUSED; 1744 1745 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1746 1747 return 0; 1748 } 1749 1750 static int pl08x_resume(struct dma_chan *chan) 1751 { 1752 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1753 unsigned long flags; 1754 1755 /* 1756 * Anything succeeds on channels with no physical allocation and 1757 * no queued transfers. 1758 */ 1759 spin_lock_irqsave(&plchan->vc.lock, flags); 1760 if (!plchan->phychan && !plchan->at) { 1761 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1762 return 0; 1763 } 1764 1765 pl08x_resume_phy_chan(plchan->phychan); 1766 plchan->state = PL08X_CHAN_RUNNING; 1767 1768 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1769 1770 return 0; 1771 } 1772 1773 bool pl08x_filter_id(struct dma_chan *chan, void *chan_id) 1774 { 1775 struct pl08x_dma_chan *plchan; 1776 char *name = chan_id; 1777 1778 /* Reject channels for devices not bound to this driver */ 1779 if (chan->device->dev->driver != &pl08x_amba_driver.drv) 1780 return false; 1781 1782 plchan = to_pl08x_chan(chan); 1783 1784 /* Check that the channel is not taken! */ 1785 if (!strcmp(plchan->name, name)) 1786 return true; 1787 1788 return false; 1789 } 1790 EXPORT_SYMBOL_GPL(pl08x_filter_id); 1791 1792 /* 1793 * Just check that the device is there and active 1794 * TODO: turn this bit on/off depending on the number of physical channels 1795 * actually used, if it is zero... well shut it off. That will save some 1796 * power. Cut the clock at the same time. 1797 */ 1798 static void pl08x_ensure_on(struct pl08x_driver_data *pl08x) 1799 { 1800 /* The Nomadik variant does not have the config register */ 1801 if (pl08x->vd->nomadik) 1802 return; 1803 writel(PL080_CONFIG_ENABLE, pl08x->base + PL080_CONFIG); 1804 } 1805 1806 static irqreturn_t pl08x_irq(int irq, void *dev) 1807 { 1808 struct pl08x_driver_data *pl08x = dev; 1809 u32 mask = 0, err, tc, i; 1810 1811 /* check & clear - ERR & TC interrupts */ 1812 err = readl(pl08x->base + PL080_ERR_STATUS); 1813 if (err) { 1814 dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n", 1815 __func__, err); 1816 writel(err, pl08x->base + PL080_ERR_CLEAR); 1817 } 1818 tc = readl(pl08x->base + PL080_TC_STATUS); 1819 if (tc) 1820 writel(tc, pl08x->base + PL080_TC_CLEAR); 1821 1822 if (!err && !tc) 1823 return IRQ_NONE; 1824 1825 for (i = 0; i < pl08x->vd->channels; i++) { 1826 if (((1 << i) & err) || ((1 << i) & tc)) { 1827 /* Locate physical channel */ 1828 struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i]; 1829 struct pl08x_dma_chan *plchan = phychan->serving; 1830 struct pl08x_txd *tx; 1831 1832 if (!plchan) { 1833 dev_err(&pl08x->adev->dev, 1834 "%s Error TC interrupt on unused channel: 0x%08x\n", 1835 __func__, i); 1836 continue; 1837 } 1838 1839 spin_lock(&plchan->vc.lock); 1840 tx = plchan->at; 1841 if (tx && tx->cyclic) { 1842 vchan_cyclic_callback(&tx->vd); 1843 } else if (tx) { 1844 plchan->at = NULL; 1845 /* 1846 * This descriptor is done, release its mux 1847 * reservation. 1848 */ 1849 pl08x_release_mux(plchan); 1850 tx->done = true; 1851 vchan_cookie_complete(&tx->vd); 1852 1853 /* 1854 * And start the next descriptor (if any), 1855 * otherwise free this channel. 1856 */ 1857 if (vchan_next_desc(&plchan->vc)) 1858 pl08x_start_next_txd(plchan); 1859 else 1860 pl08x_phy_free(plchan); 1861 } 1862 spin_unlock(&plchan->vc.lock); 1863 1864 mask |= (1 << i); 1865 } 1866 } 1867 1868 return mask ? IRQ_HANDLED : IRQ_NONE; 1869 } 1870 1871 static void pl08x_dma_slave_init(struct pl08x_dma_chan *chan) 1872 { 1873 chan->slave = true; 1874 chan->name = chan->cd->bus_id; 1875 chan->cfg.src_addr = chan->cd->addr; 1876 chan->cfg.dst_addr = chan->cd->addr; 1877 } 1878 1879 /* 1880 * Initialise the DMAC memcpy/slave channels. 1881 * Make a local wrapper to hold required data 1882 */ 1883 static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x, 1884 struct dma_device *dmadev, unsigned int channels, bool slave) 1885 { 1886 struct pl08x_dma_chan *chan; 1887 int i; 1888 1889 INIT_LIST_HEAD(&dmadev->channels); 1890 1891 /* 1892 * Register as many many memcpy as we have physical channels, 1893 * we won't always be able to use all but the code will have 1894 * to cope with that situation. 1895 */ 1896 for (i = 0; i < channels; i++) { 1897 chan = kzalloc(sizeof(*chan), GFP_KERNEL); 1898 if (!chan) { 1899 dev_err(&pl08x->adev->dev, 1900 "%s no memory for channel\n", __func__); 1901 return -ENOMEM; 1902 } 1903 1904 chan->host = pl08x; 1905 chan->state = PL08X_CHAN_IDLE; 1906 chan->signal = -1; 1907 1908 if (slave) { 1909 chan->cd = &pl08x->pd->slave_channels[i]; 1910 pl08x_dma_slave_init(chan); 1911 } else { 1912 chan->cd = &pl08x->pd->memcpy_channel; 1913 chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i); 1914 if (!chan->name) { 1915 kfree(chan); 1916 return -ENOMEM; 1917 } 1918 } 1919 dev_dbg(&pl08x->adev->dev, 1920 "initialize virtual channel \"%s\"\n", 1921 chan->name); 1922 1923 chan->vc.desc_free = pl08x_desc_free; 1924 vchan_init(&chan->vc, dmadev); 1925 } 1926 dev_info(&pl08x->adev->dev, "initialized %d virtual %s channels\n", 1927 i, slave ? "slave" : "memcpy"); 1928 return i; 1929 } 1930 1931 static void pl08x_free_virtual_channels(struct dma_device *dmadev) 1932 { 1933 struct pl08x_dma_chan *chan = NULL; 1934 struct pl08x_dma_chan *next; 1935 1936 list_for_each_entry_safe(chan, 1937 next, &dmadev->channels, vc.chan.device_node) { 1938 list_del(&chan->vc.chan.device_node); 1939 kfree(chan); 1940 } 1941 } 1942 1943 #ifdef CONFIG_DEBUG_FS 1944 static const char *pl08x_state_str(enum pl08x_dma_chan_state state) 1945 { 1946 switch (state) { 1947 case PL08X_CHAN_IDLE: 1948 return "idle"; 1949 case PL08X_CHAN_RUNNING: 1950 return "running"; 1951 case PL08X_CHAN_PAUSED: 1952 return "paused"; 1953 case PL08X_CHAN_WAITING: 1954 return "waiting"; 1955 default: 1956 break; 1957 } 1958 return "UNKNOWN STATE"; 1959 } 1960 1961 static int pl08x_debugfs_show(struct seq_file *s, void *data) 1962 { 1963 struct pl08x_driver_data *pl08x = s->private; 1964 struct pl08x_dma_chan *chan; 1965 struct pl08x_phy_chan *ch; 1966 unsigned long flags; 1967 int i; 1968 1969 seq_printf(s, "PL08x physical channels:\n"); 1970 seq_printf(s, "CHANNEL:\tUSER:\n"); 1971 seq_printf(s, "--------\t-----\n"); 1972 for (i = 0; i < pl08x->vd->channels; i++) { 1973 struct pl08x_dma_chan *virt_chan; 1974 1975 ch = &pl08x->phy_chans[i]; 1976 1977 spin_lock_irqsave(&ch->lock, flags); 1978 virt_chan = ch->serving; 1979 1980 seq_printf(s, "%d\t\t%s%s\n", 1981 ch->id, 1982 virt_chan ? virt_chan->name : "(none)", 1983 ch->locked ? " LOCKED" : ""); 1984 1985 spin_unlock_irqrestore(&ch->lock, flags); 1986 } 1987 1988 seq_printf(s, "\nPL08x virtual memcpy channels:\n"); 1989 seq_printf(s, "CHANNEL:\tSTATE:\n"); 1990 seq_printf(s, "--------\t------\n"); 1991 list_for_each_entry(chan, &pl08x->memcpy.channels, vc.chan.device_node) { 1992 seq_printf(s, "%s\t\t%s\n", chan->name, 1993 pl08x_state_str(chan->state)); 1994 } 1995 1996 seq_printf(s, "\nPL08x virtual slave channels:\n"); 1997 seq_printf(s, "CHANNEL:\tSTATE:\n"); 1998 seq_printf(s, "--------\t------\n"); 1999 list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) { 2000 seq_printf(s, "%s\t\t%s\n", chan->name, 2001 pl08x_state_str(chan->state)); 2002 } 2003 2004 return 0; 2005 } 2006 2007 static int pl08x_debugfs_open(struct inode *inode, struct file *file) 2008 { 2009 return single_open(file, pl08x_debugfs_show, inode->i_private); 2010 } 2011 2012 static const struct file_operations pl08x_debugfs_operations = { 2013 .open = pl08x_debugfs_open, 2014 .read = seq_read, 2015 .llseek = seq_lseek, 2016 .release = single_release, 2017 }; 2018 2019 static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x) 2020 { 2021 /* Expose a simple debugfs interface to view all clocks */ 2022 (void) debugfs_create_file(dev_name(&pl08x->adev->dev), 2023 S_IFREG | S_IRUGO, NULL, pl08x, 2024 &pl08x_debugfs_operations); 2025 } 2026 2027 #else 2028 static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x) 2029 { 2030 } 2031 #endif 2032 2033 static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) 2034 { 2035 struct pl08x_driver_data *pl08x; 2036 const struct vendor_data *vd = id->data; 2037 u32 tsfr_size; 2038 int ret = 0; 2039 int i; 2040 2041 ret = amba_request_regions(adev, NULL); 2042 if (ret) 2043 return ret; 2044 2045 /* Ensure that we can do DMA */ 2046 ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32)); 2047 if (ret) 2048 goto out_no_pl08x; 2049 2050 /* Create the driver state holder */ 2051 pl08x = kzalloc(sizeof(*pl08x), GFP_KERNEL); 2052 if (!pl08x) { 2053 ret = -ENOMEM; 2054 goto out_no_pl08x; 2055 } 2056 2057 /* Initialize memcpy engine */ 2058 dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask); 2059 pl08x->memcpy.dev = &adev->dev; 2060 pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources; 2061 pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy; 2062 pl08x->memcpy.device_prep_dma_interrupt = pl08x_prep_dma_interrupt; 2063 pl08x->memcpy.device_tx_status = pl08x_dma_tx_status; 2064 pl08x->memcpy.device_issue_pending = pl08x_issue_pending; 2065 pl08x->memcpy.device_config = pl08x_config; 2066 pl08x->memcpy.device_pause = pl08x_pause; 2067 pl08x->memcpy.device_resume = pl08x_resume; 2068 pl08x->memcpy.device_terminate_all = pl08x_terminate_all; 2069 pl08x->memcpy.src_addr_widths = PL80X_DMA_BUSWIDTHS; 2070 pl08x->memcpy.dst_addr_widths = PL80X_DMA_BUSWIDTHS; 2071 pl08x->memcpy.directions = BIT(DMA_MEM_TO_MEM); 2072 pl08x->memcpy.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; 2073 2074 /* Initialize slave engine */ 2075 dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask); 2076 dma_cap_set(DMA_CYCLIC, pl08x->slave.cap_mask); 2077 pl08x->slave.dev = &adev->dev; 2078 pl08x->slave.device_free_chan_resources = pl08x_free_chan_resources; 2079 pl08x->slave.device_prep_dma_interrupt = pl08x_prep_dma_interrupt; 2080 pl08x->slave.device_tx_status = pl08x_dma_tx_status; 2081 pl08x->slave.device_issue_pending = pl08x_issue_pending; 2082 pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg; 2083 pl08x->slave.device_prep_dma_cyclic = pl08x_prep_dma_cyclic; 2084 pl08x->slave.device_config = pl08x_config; 2085 pl08x->slave.device_pause = pl08x_pause; 2086 pl08x->slave.device_resume = pl08x_resume; 2087 pl08x->slave.device_terminate_all = pl08x_terminate_all; 2088 pl08x->slave.src_addr_widths = PL80X_DMA_BUSWIDTHS; 2089 pl08x->slave.dst_addr_widths = PL80X_DMA_BUSWIDTHS; 2090 pl08x->slave.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 2091 pl08x->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; 2092 2093 /* Get the platform data */ 2094 pl08x->pd = dev_get_platdata(&adev->dev); 2095 if (!pl08x->pd) { 2096 dev_err(&adev->dev, "no platform data supplied\n"); 2097 ret = -EINVAL; 2098 goto out_no_platdata; 2099 } 2100 2101 /* Assign useful pointers to the driver state */ 2102 pl08x->adev = adev; 2103 pl08x->vd = vd; 2104 2105 /* By default, AHB1 only. If dualmaster, from platform */ 2106 pl08x->lli_buses = PL08X_AHB1; 2107 pl08x->mem_buses = PL08X_AHB1; 2108 if (pl08x->vd->dualmaster) { 2109 pl08x->lli_buses = pl08x->pd->lli_buses; 2110 pl08x->mem_buses = pl08x->pd->mem_buses; 2111 } 2112 2113 if (vd->pl080s) 2114 pl08x->lli_words = PL080S_LLI_WORDS; 2115 else 2116 pl08x->lli_words = PL080_LLI_WORDS; 2117 tsfr_size = MAX_NUM_TSFR_LLIS * pl08x->lli_words * sizeof(u32); 2118 2119 /* A DMA memory pool for LLIs, align on 1-byte boundary */ 2120 pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev, 2121 tsfr_size, PL08X_ALIGN, 0); 2122 if (!pl08x->pool) { 2123 ret = -ENOMEM; 2124 goto out_no_lli_pool; 2125 } 2126 2127 pl08x->base = ioremap(adev->res.start, resource_size(&adev->res)); 2128 if (!pl08x->base) { 2129 ret = -ENOMEM; 2130 goto out_no_ioremap; 2131 } 2132 2133 /* Turn on the PL08x */ 2134 pl08x_ensure_on(pl08x); 2135 2136 /* Attach the interrupt handler */ 2137 writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR); 2138 writel(0x000000FF, pl08x->base + PL080_TC_CLEAR); 2139 2140 ret = request_irq(adev->irq[0], pl08x_irq, 0, DRIVER_NAME, pl08x); 2141 if (ret) { 2142 dev_err(&adev->dev, "%s failed to request interrupt %d\n", 2143 __func__, adev->irq[0]); 2144 goto out_no_irq; 2145 } 2146 2147 /* Initialize physical channels */ 2148 pl08x->phy_chans = kzalloc((vd->channels * sizeof(*pl08x->phy_chans)), 2149 GFP_KERNEL); 2150 if (!pl08x->phy_chans) { 2151 dev_err(&adev->dev, "%s failed to allocate " 2152 "physical channel holders\n", 2153 __func__); 2154 ret = -ENOMEM; 2155 goto out_no_phychans; 2156 } 2157 2158 for (i = 0; i < vd->channels; i++) { 2159 struct pl08x_phy_chan *ch = &pl08x->phy_chans[i]; 2160 2161 ch->id = i; 2162 ch->base = pl08x->base + PL080_Cx_BASE(i); 2163 ch->reg_config = ch->base + vd->config_offset; 2164 spin_lock_init(&ch->lock); 2165 2166 /* 2167 * Nomadik variants can have channels that are locked 2168 * down for the secure world only. Lock up these channels 2169 * by perpetually serving a dummy virtual channel. 2170 */ 2171 if (vd->nomadik) { 2172 u32 val; 2173 2174 val = readl(ch->reg_config); 2175 if (val & (PL080N_CONFIG_ITPROT | PL080N_CONFIG_SECPROT)) { 2176 dev_info(&adev->dev, "physical channel %d reserved for secure access only\n", i); 2177 ch->locked = true; 2178 } 2179 } 2180 2181 dev_dbg(&adev->dev, "physical channel %d is %s\n", 2182 i, pl08x_phy_channel_busy(ch) ? "BUSY" : "FREE"); 2183 } 2184 2185 /* Register as many memcpy channels as there are physical channels */ 2186 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->memcpy, 2187 pl08x->vd->channels, false); 2188 if (ret <= 0) { 2189 dev_warn(&pl08x->adev->dev, 2190 "%s failed to enumerate memcpy channels - %d\n", 2191 __func__, ret); 2192 goto out_no_memcpy; 2193 } 2194 2195 /* Register slave channels */ 2196 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave, 2197 pl08x->pd->num_slave_channels, true); 2198 if (ret < 0) { 2199 dev_warn(&pl08x->adev->dev, 2200 "%s failed to enumerate slave channels - %d\n", 2201 __func__, ret); 2202 goto out_no_slave; 2203 } 2204 2205 ret = dma_async_device_register(&pl08x->memcpy); 2206 if (ret) { 2207 dev_warn(&pl08x->adev->dev, 2208 "%s failed to register memcpy as an async device - %d\n", 2209 __func__, ret); 2210 goto out_no_memcpy_reg; 2211 } 2212 2213 ret = dma_async_device_register(&pl08x->slave); 2214 if (ret) { 2215 dev_warn(&pl08x->adev->dev, 2216 "%s failed to register slave as an async device - %d\n", 2217 __func__, ret); 2218 goto out_no_slave_reg; 2219 } 2220 2221 amba_set_drvdata(adev, pl08x); 2222 init_pl08x_debugfs(pl08x); 2223 dev_info(&pl08x->adev->dev, "DMA: PL%03x%s rev%u at 0x%08llx irq %d\n", 2224 amba_part(adev), pl08x->vd->pl080s ? "s" : "", amba_rev(adev), 2225 (unsigned long long)adev->res.start, adev->irq[0]); 2226 2227 return 0; 2228 2229 out_no_slave_reg: 2230 dma_async_device_unregister(&pl08x->memcpy); 2231 out_no_memcpy_reg: 2232 pl08x_free_virtual_channels(&pl08x->slave); 2233 out_no_slave: 2234 pl08x_free_virtual_channels(&pl08x->memcpy); 2235 out_no_memcpy: 2236 kfree(pl08x->phy_chans); 2237 out_no_phychans: 2238 free_irq(adev->irq[0], pl08x); 2239 out_no_irq: 2240 iounmap(pl08x->base); 2241 out_no_ioremap: 2242 dma_pool_destroy(pl08x->pool); 2243 out_no_lli_pool: 2244 out_no_platdata: 2245 kfree(pl08x); 2246 out_no_pl08x: 2247 amba_release_regions(adev); 2248 return ret; 2249 } 2250 2251 /* PL080 has 8 channels and the PL080 have just 2 */ 2252 static struct vendor_data vendor_pl080 = { 2253 .config_offset = PL080_CH_CONFIG, 2254 .channels = 8, 2255 .dualmaster = true, 2256 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, 2257 }; 2258 2259 static struct vendor_data vendor_nomadik = { 2260 .config_offset = PL080_CH_CONFIG, 2261 .channels = 8, 2262 .dualmaster = true, 2263 .nomadik = true, 2264 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, 2265 }; 2266 2267 static struct vendor_data vendor_pl080s = { 2268 .config_offset = PL080S_CH_CONFIG, 2269 .channels = 8, 2270 .pl080s = true, 2271 .max_transfer_size = PL080S_CONTROL_TRANSFER_SIZE_MASK, 2272 }; 2273 2274 static struct vendor_data vendor_pl081 = { 2275 .config_offset = PL080_CH_CONFIG, 2276 .channels = 2, 2277 .dualmaster = false, 2278 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, 2279 }; 2280 2281 static struct amba_id pl08x_ids[] = { 2282 /* Samsung PL080S variant */ 2283 { 2284 .id = 0x0a141080, 2285 .mask = 0xffffffff, 2286 .data = &vendor_pl080s, 2287 }, 2288 /* PL080 */ 2289 { 2290 .id = 0x00041080, 2291 .mask = 0x000fffff, 2292 .data = &vendor_pl080, 2293 }, 2294 /* PL081 */ 2295 { 2296 .id = 0x00041081, 2297 .mask = 0x000fffff, 2298 .data = &vendor_pl081, 2299 }, 2300 /* Nomadik 8815 PL080 variant */ 2301 { 2302 .id = 0x00280080, 2303 .mask = 0x00ffffff, 2304 .data = &vendor_nomadik, 2305 }, 2306 { 0, 0 }, 2307 }; 2308 2309 MODULE_DEVICE_TABLE(amba, pl08x_ids); 2310 2311 static struct amba_driver pl08x_amba_driver = { 2312 .drv.name = DRIVER_NAME, 2313 .id_table = pl08x_ids, 2314 .probe = pl08x_probe, 2315 }; 2316 2317 static int __init pl08x_init(void) 2318 { 2319 int retval; 2320 retval = amba_driver_register(&pl08x_amba_driver); 2321 if (retval) 2322 printk(KERN_WARNING DRIVER_NAME 2323 "failed to register as an AMBA device (%d)\n", 2324 retval); 2325 return retval; 2326 } 2327 subsys_initcall(pl08x_init); 2328