1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (c) 2006 ARM Ltd. 4 * Copyright (c) 2010 ST-Ericsson SA 5 * Copyirght (c) 2017 Linaro Ltd. 6 * 7 * Author: Peter Pearse <peter.pearse@arm.com> 8 * Author: Linus Walleij <linus.walleij@linaro.org> 9 * 10 * Documentation: ARM DDI 0196G == PL080 11 * Documentation: ARM DDI 0218E == PL081 12 * Documentation: S3C6410 User's Manual == PL080S 13 * 14 * PL080 & PL081 both have 16 sets of DMA signals that can be routed to any 15 * channel. 16 * 17 * The PL080 has 8 channels available for simultaneous use, and the PL081 18 * has only two channels. So on these DMA controllers the number of channels 19 * and the number of incoming DMA signals are two totally different things. 20 * It is usually not possible to theoretically handle all physical signals, 21 * so a multiplexing scheme with possible denial of use is necessary. 22 * 23 * The PL080 has a dual bus master, PL081 has a single master. 24 * 25 * PL080S is a version modified by Samsung and used in S3C64xx SoCs. 26 * It differs in following aspects: 27 * - CH_CONFIG register at different offset, 28 * - separate CH_CONTROL2 register for transfer size, 29 * - bigger maximum transfer size, 30 * - 8-word aligned LLI, instead of 4-word, due to extra CCTL2 word, 31 * - no support for peripheral flow control. 32 * 33 * Memory to peripheral transfer may be visualized as 34 * Get data from memory to DMAC 35 * Until no data left 36 * On burst request from peripheral 37 * Destination burst from DMAC to peripheral 38 * Clear burst request 39 * Raise terminal count interrupt 40 * 41 * For peripherals with a FIFO: 42 * Source burst size == half the depth of the peripheral FIFO 43 * Destination burst size == the depth of the peripheral FIFO 44 * 45 * (Bursts are irrelevant for mem to mem transfers - there are no burst 46 * signals, the DMA controller will simply facilitate its AHB master.) 47 * 48 * ASSUMES default (little) endianness for DMA transfers 49 * 50 * The PL08x has two flow control settings: 51 * - DMAC flow control: the transfer size defines the number of transfers 52 * which occur for the current LLI entry, and the DMAC raises TC at the 53 * end of every LLI entry. Observed behaviour shows the DMAC listening 54 * to both the BREQ and SREQ signals (contrary to documented), 55 * transferring data if either is active. The LBREQ and LSREQ signals 56 * are ignored. 57 * 58 * - Peripheral flow control: the transfer size is ignored (and should be 59 * zero). The data is transferred from the current LLI entry, until 60 * after the final transfer signalled by LBREQ or LSREQ. The DMAC 61 * will then move to the next LLI entry. Unsupported by PL080S. 62 */ 63 #include <linux/amba/bus.h> 64 #include <linux/amba/pl08x.h> 65 #include <linux/debugfs.h> 66 #include <linux/delay.h> 67 #include <linux/device.h> 68 #include <linux/dmaengine.h> 69 #include <linux/dmapool.h> 70 #include <linux/dma-mapping.h> 71 #include <linux/export.h> 72 #include <linux/init.h> 73 #include <linux/interrupt.h> 74 #include <linux/module.h> 75 #include <linux/of.h> 76 #include <linux/of_dma.h> 77 #include <linux/pm_runtime.h> 78 #include <linux/seq_file.h> 79 #include <linux/slab.h> 80 #include <linux/amba/pl080.h> 81 82 #include "dmaengine.h" 83 #include "virt-dma.h" 84 85 #define DRIVER_NAME "pl08xdmac" 86 87 #define PL80X_DMA_BUSWIDTHS \ 88 BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \ 89 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 90 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 91 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) 92 93 static struct amba_driver pl08x_amba_driver; 94 struct pl08x_driver_data; 95 96 /** 97 * struct vendor_data - vendor-specific config parameters for PL08x derivatives 98 * @config_offset: offset to the configuration register 99 * @channels: the number of channels available in this variant 100 * @signals: the number of request signals available from the hardware 101 * @dualmaster: whether this version supports dual AHB masters or not. 102 * @nomadik: whether this variant is a ST Microelectronics Nomadik, where the 103 * channels have Nomadik security extension bits that need to be checked 104 * for permission before use and some registers are missing 105 * @pl080s: whether this variant is a Samsung PL080S, which has separate 106 * register and LLI word for transfer size. 107 * @ftdmac020: whether this variant is a Faraday Technology FTDMAC020 108 * @max_transfer_size: the maximum single element transfer size for this 109 * PL08x variant. 110 */ 111 struct vendor_data { 112 u8 config_offset; 113 u8 channels; 114 u8 signals; 115 bool dualmaster; 116 bool nomadik; 117 bool pl080s; 118 bool ftdmac020; 119 u32 max_transfer_size; 120 }; 121 122 /** 123 * struct pl08x_bus_data - information of source or destination 124 * busses for a transfer 125 * @addr: current address 126 * @maxwidth: the maximum width of a transfer on this bus 127 * @buswidth: the width of this bus in bytes: 1, 2 or 4 128 */ 129 struct pl08x_bus_data { 130 dma_addr_t addr; 131 u8 maxwidth; 132 u8 buswidth; 133 }; 134 135 #define IS_BUS_ALIGNED(bus) IS_ALIGNED((bus)->addr, (bus)->buswidth) 136 137 /** 138 * struct pl08x_phy_chan - holder for the physical channels 139 * @id: physical index to this channel 140 * @base: memory base address for this physical channel 141 * @reg_config: configuration address for this physical channel 142 * @reg_control: control address for this physical channel 143 * @reg_src: transfer source address register 144 * @reg_dst: transfer destination address register 145 * @reg_lli: transfer LLI address register 146 * @reg_busy: if the variant has a special per-channel busy register, 147 * this contains a pointer to it 148 * @lock: a lock to use when altering an instance of this struct 149 * @serving: the virtual channel currently being served by this physical 150 * channel 151 * @locked: channel unavailable for the system, e.g. dedicated to secure 152 * world 153 * @ftdmac020: channel is on a FTDMAC020 154 * @pl080s: channel is on a PL08s 155 */ 156 struct pl08x_phy_chan { 157 unsigned int id; 158 void __iomem *base; 159 void __iomem *reg_config; 160 void __iomem *reg_control; 161 void __iomem *reg_src; 162 void __iomem *reg_dst; 163 void __iomem *reg_lli; 164 void __iomem *reg_busy; 165 spinlock_t lock; 166 struct pl08x_dma_chan *serving; 167 bool locked; 168 bool ftdmac020; 169 bool pl080s; 170 }; 171 172 /** 173 * struct pl08x_sg - structure containing data per sg 174 * @src_addr: src address of sg 175 * @dst_addr: dst address of sg 176 * @len: transfer len in bytes 177 * @node: node for txd's dsg_list 178 */ 179 struct pl08x_sg { 180 dma_addr_t src_addr; 181 dma_addr_t dst_addr; 182 size_t len; 183 struct list_head node; 184 }; 185 186 /** 187 * struct pl08x_txd - wrapper for struct dma_async_tx_descriptor 188 * @vd: virtual DMA descriptor 189 * @dsg_list: list of children sg's 190 * @llis_bus: DMA memory address (physical) start for the LLIs 191 * @llis_va: virtual memory address start for the LLIs 192 * @cctl: control reg values for current txd 193 * @ccfg: config reg values for current txd 194 * @done: this marks completed descriptors, which should not have their 195 * mux released. 196 * @cyclic: indicate cyclic transfers 197 */ 198 struct pl08x_txd { 199 struct virt_dma_desc vd; 200 struct list_head dsg_list; 201 dma_addr_t llis_bus; 202 u32 *llis_va; 203 /* Default cctl value for LLIs */ 204 u32 cctl; 205 /* 206 * Settings to be put into the physical channel when we 207 * trigger this txd. Other registers are in llis_va[0]. 208 */ 209 u32 ccfg; 210 bool done; 211 bool cyclic; 212 }; 213 214 /** 215 * enum pl08x_dma_chan_state - holds the PL08x specific virtual channel 216 * states 217 * @PL08X_CHAN_IDLE: the channel is idle 218 * @PL08X_CHAN_RUNNING: the channel has allocated a physical transport 219 * channel and is running a transfer on it 220 * @PL08X_CHAN_PAUSED: the channel has allocated a physical transport 221 * channel, but the transfer is currently paused 222 * @PL08X_CHAN_WAITING: the channel is waiting for a physical transport 223 * channel to become available (only pertains to memcpy channels) 224 */ 225 enum pl08x_dma_chan_state { 226 PL08X_CHAN_IDLE, 227 PL08X_CHAN_RUNNING, 228 PL08X_CHAN_PAUSED, 229 PL08X_CHAN_WAITING, 230 }; 231 232 /** 233 * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel 234 * @vc: wrappped virtual channel 235 * @phychan: the physical channel utilized by this channel, if there is one 236 * @name: name of channel 237 * @cd: channel platform data 238 * @cfg: slave configuration 239 * @at: active transaction on this channel 240 * @host: a pointer to the host (internal use) 241 * @state: whether the channel is idle, paused, running etc 242 * @slave: whether this channel is a device (slave) or for memcpy 243 * @signal: the physical DMA request signal which this channel is using 244 * @mux_use: count of descriptors using this DMA request signal setting 245 * @waiting_at: time in jiffies when this channel moved to waiting state 246 */ 247 struct pl08x_dma_chan { 248 struct virt_dma_chan vc; 249 struct pl08x_phy_chan *phychan; 250 const char *name; 251 struct pl08x_channel_data *cd; 252 struct dma_slave_config cfg; 253 struct pl08x_txd *at; 254 struct pl08x_driver_data *host; 255 enum pl08x_dma_chan_state state; 256 bool slave; 257 int signal; 258 unsigned mux_use; 259 unsigned long waiting_at; 260 }; 261 262 /** 263 * struct pl08x_driver_data - the local state holder for the PL08x 264 * @slave: optional slave engine for this instance 265 * @memcpy: memcpy engine for this instance 266 * @has_slave: the PL08x has a slave engine (routed signals) 267 * @base: virtual memory base (remapped) for the PL08x 268 * @adev: the corresponding AMBA (PrimeCell) bus entry 269 * @vd: vendor data for this PL08x variant 270 * @pd: platform data passed in from the platform/machine 271 * @phy_chans: array of data for the physical channels 272 * @pool: a pool for the LLI descriptors 273 * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI 274 * fetches 275 * @mem_buses: set to indicate memory transfers on AHB2. 276 * @lli_words: how many words are used in each LLI item for this variant 277 */ 278 struct pl08x_driver_data { 279 struct dma_device slave; 280 struct dma_device memcpy; 281 bool has_slave; 282 void __iomem *base; 283 struct amba_device *adev; 284 const struct vendor_data *vd; 285 struct pl08x_platform_data *pd; 286 struct pl08x_phy_chan *phy_chans; 287 struct dma_pool *pool; 288 u8 lli_buses; 289 u8 mem_buses; 290 u8 lli_words; 291 }; 292 293 /* 294 * PL08X specific defines 295 */ 296 297 /* The order of words in an LLI. */ 298 #define PL080_LLI_SRC 0 299 #define PL080_LLI_DST 1 300 #define PL080_LLI_LLI 2 301 #define PL080_LLI_CCTL 3 302 #define PL080S_LLI_CCTL2 4 303 304 /* Total words in an LLI. */ 305 #define PL080_LLI_WORDS 4 306 #define PL080S_LLI_WORDS 8 307 308 /* 309 * Number of LLIs in each LLI buffer allocated for one transfer 310 * (maximum times we call dma_pool_alloc on this pool without freeing) 311 */ 312 #define MAX_NUM_TSFR_LLIS 512 313 #define PL08X_ALIGN 8 314 315 static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan) 316 { 317 return container_of(chan, struct pl08x_dma_chan, vc.chan); 318 } 319 320 static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx) 321 { 322 return container_of(tx, struct pl08x_txd, vd.tx); 323 } 324 325 /* 326 * Mux handling. 327 * 328 * This gives us the DMA request input to the PL08x primecell which the 329 * peripheral described by the channel data will be routed to, possibly 330 * via a board/SoC specific external MUX. One important point to note 331 * here is that this does not depend on the physical channel. 332 */ 333 static int pl08x_request_mux(struct pl08x_dma_chan *plchan) 334 { 335 const struct pl08x_platform_data *pd = plchan->host->pd; 336 int ret; 337 338 if (plchan->mux_use++ == 0 && pd->get_xfer_signal) { 339 ret = pd->get_xfer_signal(plchan->cd); 340 if (ret < 0) { 341 plchan->mux_use = 0; 342 return ret; 343 } 344 345 plchan->signal = ret; 346 } 347 return 0; 348 } 349 350 static void pl08x_release_mux(struct pl08x_dma_chan *plchan) 351 { 352 const struct pl08x_platform_data *pd = plchan->host->pd; 353 354 if (plchan->signal >= 0) { 355 WARN_ON(plchan->mux_use == 0); 356 357 if (--plchan->mux_use == 0 && pd->put_xfer_signal) { 358 pd->put_xfer_signal(plchan->cd, plchan->signal); 359 plchan->signal = -1; 360 } 361 } 362 } 363 364 /* 365 * Physical channel handling 366 */ 367 368 /* Whether a certain channel is busy or not */ 369 static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch) 370 { 371 unsigned int val; 372 373 /* If we have a special busy register, take a shortcut */ 374 if (ch->reg_busy) { 375 val = readl(ch->reg_busy); 376 return !!(val & BIT(ch->id)); 377 } 378 val = readl(ch->reg_config); 379 return val & PL080_CONFIG_ACTIVE; 380 } 381 382 /* 383 * pl08x_write_lli() - Write an LLI into the DMA controller. 384 * 385 * The PL08x derivatives support linked lists, but the first item of the 386 * list containing the source, destination, control word and next LLI is 387 * ignored. Instead the driver has to write those values directly into the 388 * SRC, DST, LLI and control registers. On FTDMAC020 also the SIZE 389 * register need to be set up for the first transfer. 390 */ 391 static void pl08x_write_lli(struct pl08x_driver_data *pl08x, 392 struct pl08x_phy_chan *phychan, const u32 *lli, u32 ccfg) 393 { 394 if (pl08x->vd->pl080s) 395 dev_vdbg(&pl08x->adev->dev, 396 "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, " 397 "clli=0x%08x, cctl=0x%08x, cctl2=0x%08x, ccfg=0x%08x\n", 398 phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST], 399 lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], 400 lli[PL080S_LLI_CCTL2], ccfg); 401 else 402 dev_vdbg(&pl08x->adev->dev, 403 "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, " 404 "clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n", 405 phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST], 406 lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], ccfg); 407 408 writel_relaxed(lli[PL080_LLI_SRC], phychan->reg_src); 409 writel_relaxed(lli[PL080_LLI_DST], phychan->reg_dst); 410 writel_relaxed(lli[PL080_LLI_LLI], phychan->reg_lli); 411 412 /* 413 * The FTMAC020 has a different layout in the CCTL word of the LLI 414 * and the CCTL register which is split in CSR and SIZE registers. 415 * Convert the LLI item CCTL into the proper values to write into 416 * the CSR and SIZE registers. 417 */ 418 if (phychan->ftdmac020) { 419 u32 llictl = lli[PL080_LLI_CCTL]; 420 u32 val = 0; 421 422 /* Write the transfer size (12 bits) to the size register */ 423 writel_relaxed(llictl & FTDMAC020_LLI_TRANSFER_SIZE_MASK, 424 phychan->base + FTDMAC020_CH_SIZE); 425 /* 426 * Then write the control bits 28..16 to the control register 427 * by shuffleing the bits around to where they are in the 428 * main register. The mapping is as follows: 429 * Bit 28: TC_MSK - mask on all except last LLI 430 * Bit 27..25: SRC_WIDTH 431 * Bit 24..22: DST_WIDTH 432 * Bit 21..20: SRCAD_CTRL 433 * Bit 19..17: DSTAD_CTRL 434 * Bit 17: SRC_SEL 435 * Bit 16: DST_SEL 436 */ 437 if (llictl & FTDMAC020_LLI_TC_MSK) 438 val |= FTDMAC020_CH_CSR_TC_MSK; 439 val |= ((llictl & FTDMAC020_LLI_SRC_WIDTH_MSK) >> 440 (FTDMAC020_LLI_SRC_WIDTH_SHIFT - 441 FTDMAC020_CH_CSR_SRC_WIDTH_SHIFT)); 442 val |= ((llictl & FTDMAC020_LLI_DST_WIDTH_MSK) >> 443 (FTDMAC020_LLI_DST_WIDTH_SHIFT - 444 FTDMAC020_CH_CSR_DST_WIDTH_SHIFT)); 445 val |= ((llictl & FTDMAC020_LLI_SRCAD_CTL_MSK) >> 446 (FTDMAC020_LLI_SRCAD_CTL_SHIFT - 447 FTDMAC020_CH_CSR_SRCAD_CTL_SHIFT)); 448 val |= ((llictl & FTDMAC020_LLI_DSTAD_CTL_MSK) >> 449 (FTDMAC020_LLI_DSTAD_CTL_SHIFT - 450 FTDMAC020_CH_CSR_DSTAD_CTL_SHIFT)); 451 if (llictl & FTDMAC020_LLI_SRC_SEL) 452 val |= FTDMAC020_CH_CSR_SRC_SEL; 453 if (llictl & FTDMAC020_LLI_DST_SEL) 454 val |= FTDMAC020_CH_CSR_DST_SEL; 455 456 /* 457 * Set up the bits that exist in the CSR but are not 458 * part the LLI, i.e. only gets written to the control 459 * register right here. 460 * 461 * FIXME: do not just handle memcpy, also handle slave DMA. 462 */ 463 switch (pl08x->pd->memcpy_burst_size) { 464 default: 465 case PL08X_BURST_SZ_1: 466 val |= PL080_BSIZE_1 << 467 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 468 break; 469 case PL08X_BURST_SZ_4: 470 val |= PL080_BSIZE_4 << 471 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 472 break; 473 case PL08X_BURST_SZ_8: 474 val |= PL080_BSIZE_8 << 475 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 476 break; 477 case PL08X_BURST_SZ_16: 478 val |= PL080_BSIZE_16 << 479 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 480 break; 481 case PL08X_BURST_SZ_32: 482 val |= PL080_BSIZE_32 << 483 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 484 break; 485 case PL08X_BURST_SZ_64: 486 val |= PL080_BSIZE_64 << 487 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 488 break; 489 case PL08X_BURST_SZ_128: 490 val |= PL080_BSIZE_128 << 491 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 492 break; 493 case PL08X_BURST_SZ_256: 494 val |= PL080_BSIZE_256 << 495 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 496 break; 497 } 498 499 /* Protection flags */ 500 if (pl08x->pd->memcpy_prot_buff) 501 val |= FTDMAC020_CH_CSR_PROT2; 502 if (pl08x->pd->memcpy_prot_cache) 503 val |= FTDMAC020_CH_CSR_PROT3; 504 /* We are the kernel, so we are in privileged mode */ 505 val |= FTDMAC020_CH_CSR_PROT1; 506 507 writel_relaxed(val, phychan->reg_control); 508 } else { 509 /* Bits are just identical */ 510 writel_relaxed(lli[PL080_LLI_CCTL], phychan->reg_control); 511 } 512 513 /* Second control word on the PL080s */ 514 if (pl08x->vd->pl080s) 515 writel_relaxed(lli[PL080S_LLI_CCTL2], 516 phychan->base + PL080S_CH_CONTROL2); 517 518 writel(ccfg, phychan->reg_config); 519 } 520 521 /* 522 * Set the initial DMA register values i.e. those for the first LLI 523 * The next LLI pointer and the configuration interrupt bit have 524 * been set when the LLIs were constructed. Poke them into the hardware 525 * and start the transfer. 526 */ 527 static void pl08x_start_next_txd(struct pl08x_dma_chan *plchan) 528 { 529 struct pl08x_driver_data *pl08x = plchan->host; 530 struct pl08x_phy_chan *phychan = plchan->phychan; 531 struct virt_dma_desc *vd = vchan_next_desc(&plchan->vc); 532 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); 533 u32 val; 534 535 list_del(&txd->vd.node); 536 537 plchan->at = txd; 538 539 /* Wait for channel inactive */ 540 while (pl08x_phy_channel_busy(phychan)) 541 cpu_relax(); 542 543 pl08x_write_lli(pl08x, phychan, &txd->llis_va[0], txd->ccfg); 544 545 /* Enable the DMA channel */ 546 /* Do not access config register until channel shows as disabled */ 547 while (readl(pl08x->base + PL080_EN_CHAN) & BIT(phychan->id)) 548 cpu_relax(); 549 550 /* Do not access config register until channel shows as inactive */ 551 if (phychan->ftdmac020) { 552 val = readl(phychan->reg_config); 553 while (val & FTDMAC020_CH_CFG_BUSY) 554 val = readl(phychan->reg_config); 555 556 val = readl(phychan->reg_control); 557 while (val & FTDMAC020_CH_CSR_EN) 558 val = readl(phychan->reg_control); 559 560 writel(val | FTDMAC020_CH_CSR_EN, 561 phychan->reg_control); 562 } else { 563 val = readl(phychan->reg_config); 564 while ((val & PL080_CONFIG_ACTIVE) || 565 (val & PL080_CONFIG_ENABLE)) 566 val = readl(phychan->reg_config); 567 568 writel(val | PL080_CONFIG_ENABLE, phychan->reg_config); 569 } 570 } 571 572 /* 573 * Pause the channel by setting the HALT bit. 574 * 575 * For M->P transfers, pause the DMAC first and then stop the peripheral - 576 * the FIFO can only drain if the peripheral is still requesting data. 577 * (note: this can still timeout if the DMAC FIFO never drains of data.) 578 * 579 * For P->M transfers, disable the peripheral first to stop it filling 580 * the DMAC FIFO, and then pause the DMAC. 581 */ 582 static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch) 583 { 584 u32 val; 585 int timeout; 586 587 if (ch->ftdmac020) { 588 /* Use the enable bit on the FTDMAC020 */ 589 val = readl(ch->reg_control); 590 val &= ~FTDMAC020_CH_CSR_EN; 591 writel(val, ch->reg_control); 592 return; 593 } 594 595 /* Set the HALT bit and wait for the FIFO to drain */ 596 val = readl(ch->reg_config); 597 val |= PL080_CONFIG_HALT; 598 writel(val, ch->reg_config); 599 600 /* Wait for channel inactive */ 601 for (timeout = 1000; timeout; timeout--) { 602 if (!pl08x_phy_channel_busy(ch)) 603 break; 604 udelay(1); 605 } 606 if (pl08x_phy_channel_busy(ch)) 607 pr_err("pl08x: channel%u timeout waiting for pause\n", ch->id); 608 } 609 610 static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch) 611 { 612 u32 val; 613 614 /* Use the enable bit on the FTDMAC020 */ 615 if (ch->ftdmac020) { 616 val = readl(ch->reg_control); 617 val |= FTDMAC020_CH_CSR_EN; 618 writel(val, ch->reg_control); 619 return; 620 } 621 622 /* Clear the HALT bit */ 623 val = readl(ch->reg_config); 624 val &= ~PL080_CONFIG_HALT; 625 writel(val, ch->reg_config); 626 } 627 628 /* 629 * pl08x_terminate_phy_chan() stops the channel, clears the FIFO and 630 * clears any pending interrupt status. This should not be used for 631 * an on-going transfer, but as a method of shutting down a channel 632 * (eg, when it's no longer used) or terminating a transfer. 633 */ 634 static void pl08x_terminate_phy_chan(struct pl08x_driver_data *pl08x, 635 struct pl08x_phy_chan *ch) 636 { 637 u32 val; 638 639 /* The layout for the FTDMAC020 is different */ 640 if (ch->ftdmac020) { 641 /* Disable all interrupts */ 642 val = readl(ch->reg_config); 643 val |= (FTDMAC020_CH_CFG_INT_ABT_MASK | 644 FTDMAC020_CH_CFG_INT_ERR_MASK | 645 FTDMAC020_CH_CFG_INT_TC_MASK); 646 writel(val, ch->reg_config); 647 648 /* Abort and disable channel */ 649 val = readl(ch->reg_control); 650 val &= ~FTDMAC020_CH_CSR_EN; 651 val |= FTDMAC020_CH_CSR_ABT; 652 writel(val, ch->reg_control); 653 654 /* Clear ABT and ERR interrupt flags */ 655 writel(BIT(ch->id) | BIT(ch->id + 16), 656 pl08x->base + PL080_ERR_CLEAR); 657 writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR); 658 659 return; 660 } 661 662 val = readl(ch->reg_config); 663 val &= ~(PL080_CONFIG_ENABLE | PL080_CONFIG_ERR_IRQ_MASK | 664 PL080_CONFIG_TC_IRQ_MASK); 665 writel(val, ch->reg_config); 666 667 writel(BIT(ch->id), pl08x->base + PL080_ERR_CLEAR); 668 writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR); 669 } 670 671 static u32 get_bytes_in_phy_channel(struct pl08x_phy_chan *ch) 672 { 673 u32 val; 674 u32 bytes; 675 676 if (ch->ftdmac020) { 677 bytes = readl(ch->base + FTDMAC020_CH_SIZE); 678 679 val = readl(ch->reg_control); 680 val &= FTDMAC020_CH_CSR_SRC_WIDTH_MSK; 681 val >>= FTDMAC020_CH_CSR_SRC_WIDTH_SHIFT; 682 } else if (ch->pl080s) { 683 val = readl(ch->base + PL080S_CH_CONTROL2); 684 bytes = val & PL080S_CONTROL_TRANSFER_SIZE_MASK; 685 686 val = readl(ch->reg_control); 687 val &= PL080_CONTROL_SWIDTH_MASK; 688 val >>= PL080_CONTROL_SWIDTH_SHIFT; 689 } else { 690 /* Plain PL08x */ 691 val = readl(ch->reg_control); 692 bytes = val & PL080_CONTROL_TRANSFER_SIZE_MASK; 693 694 val &= PL080_CONTROL_SWIDTH_MASK; 695 val >>= PL080_CONTROL_SWIDTH_SHIFT; 696 } 697 698 switch (val) { 699 case PL080_WIDTH_8BIT: 700 break; 701 case PL080_WIDTH_16BIT: 702 bytes *= 2; 703 break; 704 case PL080_WIDTH_32BIT: 705 bytes *= 4; 706 break; 707 } 708 return bytes; 709 } 710 711 static u32 get_bytes_in_lli(struct pl08x_phy_chan *ch, const u32 *llis_va) 712 { 713 u32 val; 714 u32 bytes; 715 716 if (ch->ftdmac020) { 717 val = llis_va[PL080_LLI_CCTL]; 718 bytes = val & FTDMAC020_LLI_TRANSFER_SIZE_MASK; 719 720 val = llis_va[PL080_LLI_CCTL]; 721 val &= FTDMAC020_LLI_SRC_WIDTH_MSK; 722 val >>= FTDMAC020_LLI_SRC_WIDTH_SHIFT; 723 } else if (ch->pl080s) { 724 val = llis_va[PL080S_LLI_CCTL2]; 725 bytes = val & PL080S_CONTROL_TRANSFER_SIZE_MASK; 726 727 val = llis_va[PL080_LLI_CCTL]; 728 val &= PL080_CONTROL_SWIDTH_MASK; 729 val >>= PL080_CONTROL_SWIDTH_SHIFT; 730 } else { 731 /* Plain PL08x */ 732 val = llis_va[PL080_LLI_CCTL]; 733 bytes = val & PL080_CONTROL_TRANSFER_SIZE_MASK; 734 735 val &= PL080_CONTROL_SWIDTH_MASK; 736 val >>= PL080_CONTROL_SWIDTH_SHIFT; 737 } 738 739 switch (val) { 740 case PL080_WIDTH_8BIT: 741 break; 742 case PL080_WIDTH_16BIT: 743 bytes *= 2; 744 break; 745 case PL080_WIDTH_32BIT: 746 bytes *= 4; 747 break; 748 } 749 return bytes; 750 } 751 752 /* The channel should be paused when calling this */ 753 static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan) 754 { 755 struct pl08x_driver_data *pl08x = plchan->host; 756 const u32 *llis_va, *llis_va_limit; 757 struct pl08x_phy_chan *ch; 758 dma_addr_t llis_bus; 759 struct pl08x_txd *txd; 760 u32 llis_max_words; 761 size_t bytes; 762 u32 clli; 763 764 ch = plchan->phychan; 765 txd = plchan->at; 766 767 if (!ch || !txd) 768 return 0; 769 770 /* 771 * Follow the LLIs to get the number of remaining 772 * bytes in the currently active transaction. 773 */ 774 clli = readl(ch->reg_lli) & ~PL080_LLI_LM_AHB2; 775 776 /* First get the remaining bytes in the active transfer */ 777 bytes = get_bytes_in_phy_channel(ch); 778 779 if (!clli) 780 return bytes; 781 782 llis_va = txd->llis_va; 783 llis_bus = txd->llis_bus; 784 785 llis_max_words = pl08x->lli_words * MAX_NUM_TSFR_LLIS; 786 BUG_ON(clli < llis_bus || clli >= llis_bus + 787 sizeof(u32) * llis_max_words); 788 789 /* 790 * Locate the next LLI - as this is an array, 791 * it's simple maths to find. 792 */ 793 llis_va += (clli - llis_bus) / sizeof(u32); 794 795 llis_va_limit = llis_va + llis_max_words; 796 797 for (; llis_va < llis_va_limit; llis_va += pl08x->lli_words) { 798 bytes += get_bytes_in_lli(ch, llis_va); 799 800 /* 801 * A LLI pointer going backward terminates the LLI list 802 */ 803 if (llis_va[PL080_LLI_LLI] <= clli) 804 break; 805 } 806 807 return bytes; 808 } 809 810 /* 811 * Allocate a physical channel for a virtual channel 812 * 813 * Try to locate a physical channel to be used for this transfer. If all 814 * are taken return NULL and the requester will have to cope by using 815 * some fallback PIO mode or retrying later. 816 */ 817 static struct pl08x_phy_chan * 818 pl08x_get_phy_channel(struct pl08x_driver_data *pl08x, 819 struct pl08x_dma_chan *virt_chan) 820 { 821 struct pl08x_phy_chan *ch = NULL; 822 unsigned long flags; 823 int i; 824 825 for (i = 0; i < pl08x->vd->channels; i++) { 826 ch = &pl08x->phy_chans[i]; 827 828 spin_lock_irqsave(&ch->lock, flags); 829 830 if (!ch->locked && !ch->serving) { 831 ch->serving = virt_chan; 832 spin_unlock_irqrestore(&ch->lock, flags); 833 break; 834 } 835 836 spin_unlock_irqrestore(&ch->lock, flags); 837 } 838 839 if (i == pl08x->vd->channels) { 840 /* No physical channel available, cope with it */ 841 return NULL; 842 } 843 844 return ch; 845 } 846 847 /* Mark the physical channel as free. Note, this write is atomic. */ 848 static inline void pl08x_put_phy_channel(struct pl08x_driver_data *pl08x, 849 struct pl08x_phy_chan *ch) 850 { 851 ch->serving = NULL; 852 } 853 854 /* 855 * Try to allocate a physical channel. When successful, assign it to 856 * this virtual channel, and initiate the next descriptor. The 857 * virtual channel lock must be held at this point. 858 */ 859 static void pl08x_phy_alloc_and_start(struct pl08x_dma_chan *plchan) 860 { 861 struct pl08x_driver_data *pl08x = plchan->host; 862 struct pl08x_phy_chan *ch; 863 864 ch = pl08x_get_phy_channel(pl08x, plchan); 865 if (!ch) { 866 dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name); 867 plchan->state = PL08X_CHAN_WAITING; 868 plchan->waiting_at = jiffies; 869 return; 870 } 871 872 dev_dbg(&pl08x->adev->dev, "allocated physical channel %d for xfer on %s\n", 873 ch->id, plchan->name); 874 875 plchan->phychan = ch; 876 plchan->state = PL08X_CHAN_RUNNING; 877 pl08x_start_next_txd(plchan); 878 } 879 880 static void pl08x_phy_reassign_start(struct pl08x_phy_chan *ch, 881 struct pl08x_dma_chan *plchan) 882 { 883 struct pl08x_driver_data *pl08x = plchan->host; 884 885 dev_dbg(&pl08x->adev->dev, "reassigned physical channel %d for xfer on %s\n", 886 ch->id, plchan->name); 887 888 /* 889 * We do this without taking the lock; we're really only concerned 890 * about whether this pointer is NULL or not, and we're guaranteed 891 * that this will only be called when it _already_ is non-NULL. 892 */ 893 ch->serving = plchan; 894 plchan->phychan = ch; 895 plchan->state = PL08X_CHAN_RUNNING; 896 pl08x_start_next_txd(plchan); 897 } 898 899 /* 900 * Free a physical DMA channel, potentially reallocating it to another 901 * virtual channel if we have any pending. 902 */ 903 static void pl08x_phy_free(struct pl08x_dma_chan *plchan) 904 { 905 struct pl08x_driver_data *pl08x = plchan->host; 906 struct pl08x_dma_chan *p, *next; 907 unsigned long waiting_at; 908 retry: 909 next = NULL; 910 waiting_at = jiffies; 911 912 /* 913 * Find a waiting virtual channel for the next transfer. 914 * To be fair, time when each channel reached waiting state is compared 915 * to select channel that is waiting for the longest time. 916 */ 917 list_for_each_entry(p, &pl08x->memcpy.channels, vc.chan.device_node) 918 if (p->state == PL08X_CHAN_WAITING && 919 p->waiting_at <= waiting_at) { 920 next = p; 921 waiting_at = p->waiting_at; 922 } 923 924 if (!next && pl08x->has_slave) { 925 list_for_each_entry(p, &pl08x->slave.channels, vc.chan.device_node) 926 if (p->state == PL08X_CHAN_WAITING && 927 p->waiting_at <= waiting_at) { 928 next = p; 929 waiting_at = p->waiting_at; 930 } 931 } 932 933 /* Ensure that the physical channel is stopped */ 934 pl08x_terminate_phy_chan(pl08x, plchan->phychan); 935 936 if (next) { 937 bool success; 938 939 /* 940 * Eww. We know this isn't going to deadlock 941 * but lockdep probably doesn't. 942 */ 943 spin_lock(&next->vc.lock); 944 /* Re-check the state now that we have the lock */ 945 success = next->state == PL08X_CHAN_WAITING; 946 if (success) 947 pl08x_phy_reassign_start(plchan->phychan, next); 948 spin_unlock(&next->vc.lock); 949 950 /* If the state changed, try to find another channel */ 951 if (!success) 952 goto retry; 953 } else { 954 /* No more jobs, so free up the physical channel */ 955 pl08x_put_phy_channel(pl08x, plchan->phychan); 956 } 957 958 plchan->phychan = NULL; 959 plchan->state = PL08X_CHAN_IDLE; 960 } 961 962 /* 963 * LLI handling 964 */ 965 966 static inline unsigned int 967 pl08x_get_bytes_for_lli(struct pl08x_driver_data *pl08x, 968 u32 cctl, 969 bool source) 970 { 971 u32 val; 972 973 if (pl08x->vd->ftdmac020) { 974 if (source) 975 val = (cctl & FTDMAC020_LLI_SRC_WIDTH_MSK) >> 976 FTDMAC020_LLI_SRC_WIDTH_SHIFT; 977 else 978 val = (cctl & FTDMAC020_LLI_DST_WIDTH_MSK) >> 979 FTDMAC020_LLI_DST_WIDTH_SHIFT; 980 } else { 981 if (source) 982 val = (cctl & PL080_CONTROL_SWIDTH_MASK) >> 983 PL080_CONTROL_SWIDTH_SHIFT; 984 else 985 val = (cctl & PL080_CONTROL_DWIDTH_MASK) >> 986 PL080_CONTROL_DWIDTH_SHIFT; 987 } 988 989 switch (val) { 990 case PL080_WIDTH_8BIT: 991 return 1; 992 case PL080_WIDTH_16BIT: 993 return 2; 994 case PL080_WIDTH_32BIT: 995 return 4; 996 default: 997 break; 998 } 999 BUG(); 1000 return 0; 1001 } 1002 1003 static inline u32 pl08x_lli_control_bits(struct pl08x_driver_data *pl08x, 1004 u32 cctl, 1005 u8 srcwidth, u8 dstwidth, 1006 size_t tsize) 1007 { 1008 u32 retbits = cctl; 1009 1010 /* 1011 * Remove all src, dst and transfer size bits, then set the 1012 * width and size according to the parameters. The bit offsets 1013 * are different in the FTDMAC020 so we need to accound for this. 1014 */ 1015 if (pl08x->vd->ftdmac020) { 1016 retbits &= ~FTDMAC020_LLI_DST_WIDTH_MSK; 1017 retbits &= ~FTDMAC020_LLI_SRC_WIDTH_MSK; 1018 retbits &= ~FTDMAC020_LLI_TRANSFER_SIZE_MASK; 1019 1020 switch (srcwidth) { 1021 case 1: 1022 retbits |= PL080_WIDTH_8BIT << 1023 FTDMAC020_LLI_SRC_WIDTH_SHIFT; 1024 break; 1025 case 2: 1026 retbits |= PL080_WIDTH_16BIT << 1027 FTDMAC020_LLI_SRC_WIDTH_SHIFT; 1028 break; 1029 case 4: 1030 retbits |= PL080_WIDTH_32BIT << 1031 FTDMAC020_LLI_SRC_WIDTH_SHIFT; 1032 break; 1033 default: 1034 BUG(); 1035 break; 1036 } 1037 1038 switch (dstwidth) { 1039 case 1: 1040 retbits |= PL080_WIDTH_8BIT << 1041 FTDMAC020_LLI_DST_WIDTH_SHIFT; 1042 break; 1043 case 2: 1044 retbits |= PL080_WIDTH_16BIT << 1045 FTDMAC020_LLI_DST_WIDTH_SHIFT; 1046 break; 1047 case 4: 1048 retbits |= PL080_WIDTH_32BIT << 1049 FTDMAC020_LLI_DST_WIDTH_SHIFT; 1050 break; 1051 default: 1052 BUG(); 1053 break; 1054 } 1055 1056 tsize &= FTDMAC020_LLI_TRANSFER_SIZE_MASK; 1057 retbits |= tsize << FTDMAC020_LLI_TRANSFER_SIZE_SHIFT; 1058 } else { 1059 retbits &= ~PL080_CONTROL_DWIDTH_MASK; 1060 retbits &= ~PL080_CONTROL_SWIDTH_MASK; 1061 retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK; 1062 1063 switch (srcwidth) { 1064 case 1: 1065 retbits |= PL080_WIDTH_8BIT << 1066 PL080_CONTROL_SWIDTH_SHIFT; 1067 break; 1068 case 2: 1069 retbits |= PL080_WIDTH_16BIT << 1070 PL080_CONTROL_SWIDTH_SHIFT; 1071 break; 1072 case 4: 1073 retbits |= PL080_WIDTH_32BIT << 1074 PL080_CONTROL_SWIDTH_SHIFT; 1075 break; 1076 default: 1077 BUG(); 1078 break; 1079 } 1080 1081 switch (dstwidth) { 1082 case 1: 1083 retbits |= PL080_WIDTH_8BIT << 1084 PL080_CONTROL_DWIDTH_SHIFT; 1085 break; 1086 case 2: 1087 retbits |= PL080_WIDTH_16BIT << 1088 PL080_CONTROL_DWIDTH_SHIFT; 1089 break; 1090 case 4: 1091 retbits |= PL080_WIDTH_32BIT << 1092 PL080_CONTROL_DWIDTH_SHIFT; 1093 break; 1094 default: 1095 BUG(); 1096 break; 1097 } 1098 1099 tsize &= PL080_CONTROL_TRANSFER_SIZE_MASK; 1100 retbits |= tsize << PL080_CONTROL_TRANSFER_SIZE_SHIFT; 1101 } 1102 1103 return retbits; 1104 } 1105 1106 struct pl08x_lli_build_data { 1107 struct pl08x_txd *txd; 1108 struct pl08x_bus_data srcbus; 1109 struct pl08x_bus_data dstbus; 1110 size_t remainder; 1111 u32 lli_bus; 1112 }; 1113 1114 /* 1115 * Autoselect a master bus to use for the transfer. Slave will be the chosen as 1116 * victim in case src & dest are not similarly aligned. i.e. If after aligning 1117 * masters address with width requirements of transfer (by sending few byte by 1118 * byte data), slave is still not aligned, then its width will be reduced to 1119 * BYTE. 1120 * - prefers the destination bus if both available 1121 * - prefers bus with fixed address (i.e. peripheral) 1122 */ 1123 static void pl08x_choose_master_bus(struct pl08x_driver_data *pl08x, 1124 struct pl08x_lli_build_data *bd, 1125 struct pl08x_bus_data **mbus, 1126 struct pl08x_bus_data **sbus, 1127 u32 cctl) 1128 { 1129 bool dst_incr; 1130 bool src_incr; 1131 1132 /* 1133 * The FTDMAC020 only supports memory-to-memory transfer, so 1134 * source and destination always increase. 1135 */ 1136 if (pl08x->vd->ftdmac020) { 1137 dst_incr = true; 1138 src_incr = true; 1139 } else { 1140 dst_incr = !!(cctl & PL080_CONTROL_DST_INCR); 1141 src_incr = !!(cctl & PL080_CONTROL_SRC_INCR); 1142 } 1143 1144 /* 1145 * If either bus is not advancing, i.e. it is a peripheral, that 1146 * one becomes master 1147 */ 1148 if (!dst_incr) { 1149 *mbus = &bd->dstbus; 1150 *sbus = &bd->srcbus; 1151 } else if (!src_incr) { 1152 *mbus = &bd->srcbus; 1153 *sbus = &bd->dstbus; 1154 } else { 1155 if (bd->dstbus.buswidth >= bd->srcbus.buswidth) { 1156 *mbus = &bd->dstbus; 1157 *sbus = &bd->srcbus; 1158 } else { 1159 *mbus = &bd->srcbus; 1160 *sbus = &bd->dstbus; 1161 } 1162 } 1163 } 1164 1165 /* 1166 * Fills in one LLI for a certain transfer descriptor and advance the counter 1167 */ 1168 static void pl08x_fill_lli_for_desc(struct pl08x_driver_data *pl08x, 1169 struct pl08x_lli_build_data *bd, 1170 int num_llis, int len, u32 cctl, u32 cctl2) 1171 { 1172 u32 offset = num_llis * pl08x->lli_words; 1173 u32 *llis_va = bd->txd->llis_va + offset; 1174 dma_addr_t llis_bus = bd->txd->llis_bus; 1175 1176 BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS); 1177 1178 /* Advance the offset to next LLI. */ 1179 offset += pl08x->lli_words; 1180 1181 llis_va[PL080_LLI_SRC] = bd->srcbus.addr; 1182 llis_va[PL080_LLI_DST] = bd->dstbus.addr; 1183 llis_va[PL080_LLI_LLI] = (llis_bus + sizeof(u32) * offset); 1184 llis_va[PL080_LLI_LLI] |= bd->lli_bus; 1185 llis_va[PL080_LLI_CCTL] = cctl; 1186 if (pl08x->vd->pl080s) 1187 llis_va[PL080S_LLI_CCTL2] = cctl2; 1188 1189 if (pl08x->vd->ftdmac020) { 1190 /* FIXME: only memcpy so far so both increase */ 1191 bd->srcbus.addr += len; 1192 bd->dstbus.addr += len; 1193 } else { 1194 if (cctl & PL080_CONTROL_SRC_INCR) 1195 bd->srcbus.addr += len; 1196 if (cctl & PL080_CONTROL_DST_INCR) 1197 bd->dstbus.addr += len; 1198 } 1199 1200 BUG_ON(bd->remainder < len); 1201 1202 bd->remainder -= len; 1203 } 1204 1205 static inline void prep_byte_width_lli(struct pl08x_driver_data *pl08x, 1206 struct pl08x_lli_build_data *bd, u32 *cctl, u32 len, 1207 int num_llis, size_t *total_bytes) 1208 { 1209 *cctl = pl08x_lli_control_bits(pl08x, *cctl, 1, 1, len); 1210 pl08x_fill_lli_for_desc(pl08x, bd, num_llis, len, *cctl, len); 1211 (*total_bytes) += len; 1212 } 1213 1214 #if 1 1215 static void pl08x_dump_lli(struct pl08x_driver_data *pl08x, 1216 const u32 *llis_va, int num_llis) 1217 { 1218 int i; 1219 1220 if (pl08x->vd->pl080s) { 1221 dev_vdbg(&pl08x->adev->dev, 1222 "%-3s %-9s %-10s %-10s %-10s %-10s %s\n", 1223 "lli", "", "csrc", "cdst", "clli", "cctl", "cctl2"); 1224 for (i = 0; i < num_llis; i++) { 1225 dev_vdbg(&pl08x->adev->dev, 1226 "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n", 1227 i, llis_va, llis_va[PL080_LLI_SRC], 1228 llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI], 1229 llis_va[PL080_LLI_CCTL], 1230 llis_va[PL080S_LLI_CCTL2]); 1231 llis_va += pl08x->lli_words; 1232 } 1233 } else { 1234 dev_vdbg(&pl08x->adev->dev, 1235 "%-3s %-9s %-10s %-10s %-10s %s\n", 1236 "lli", "", "csrc", "cdst", "clli", "cctl"); 1237 for (i = 0; i < num_llis; i++) { 1238 dev_vdbg(&pl08x->adev->dev, 1239 "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x\n", 1240 i, llis_va, llis_va[PL080_LLI_SRC], 1241 llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI], 1242 llis_va[PL080_LLI_CCTL]); 1243 llis_va += pl08x->lli_words; 1244 } 1245 } 1246 } 1247 #else 1248 static inline void pl08x_dump_lli(struct pl08x_driver_data *pl08x, 1249 const u32 *llis_va, int num_llis) {} 1250 #endif 1251 1252 /* 1253 * This fills in the table of LLIs for the transfer descriptor 1254 * Note that we assume we never have to change the burst sizes 1255 * Return 0 for error 1256 */ 1257 static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, 1258 struct pl08x_txd *txd) 1259 { 1260 struct pl08x_bus_data *mbus, *sbus; 1261 struct pl08x_lli_build_data bd; 1262 int num_llis = 0; 1263 u32 cctl, early_bytes = 0; 1264 size_t max_bytes_per_lli, total_bytes; 1265 u32 *llis_va, *last_lli; 1266 struct pl08x_sg *dsg; 1267 1268 txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus); 1269 if (!txd->llis_va) { 1270 dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__); 1271 return 0; 1272 } 1273 1274 bd.txd = txd; 1275 bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0; 1276 cctl = txd->cctl; 1277 1278 /* Find maximum width of the source bus */ 1279 bd.srcbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, true); 1280 1281 /* Find maximum width of the destination bus */ 1282 bd.dstbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, false); 1283 1284 list_for_each_entry(dsg, &txd->dsg_list, node) { 1285 total_bytes = 0; 1286 cctl = txd->cctl; 1287 1288 bd.srcbus.addr = dsg->src_addr; 1289 bd.dstbus.addr = dsg->dst_addr; 1290 bd.remainder = dsg->len; 1291 bd.srcbus.buswidth = bd.srcbus.maxwidth; 1292 bd.dstbus.buswidth = bd.dstbus.maxwidth; 1293 1294 pl08x_choose_master_bus(pl08x, &bd, &mbus, &sbus, cctl); 1295 1296 dev_vdbg(&pl08x->adev->dev, 1297 "src=0x%08llx%s/%u dst=0x%08llx%s/%u len=%zu\n", 1298 (u64)bd.srcbus.addr, 1299 cctl & PL080_CONTROL_SRC_INCR ? "+" : "", 1300 bd.srcbus.buswidth, 1301 (u64)bd.dstbus.addr, 1302 cctl & PL080_CONTROL_DST_INCR ? "+" : "", 1303 bd.dstbus.buswidth, 1304 bd.remainder); 1305 dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n", 1306 mbus == &bd.srcbus ? "src" : "dst", 1307 sbus == &bd.srcbus ? "src" : "dst"); 1308 1309 /* 1310 * Zero length is only allowed if all these requirements are 1311 * met: 1312 * - flow controller is peripheral. 1313 * - src.addr is aligned to src.width 1314 * - dst.addr is aligned to dst.width 1315 * 1316 * sg_len == 1 should be true, as there can be two cases here: 1317 * 1318 * - Memory addresses are contiguous and are not scattered. 1319 * Here, Only one sg will be passed by user driver, with 1320 * memory address and zero length. We pass this to controller 1321 * and after the transfer it will receive the last burst 1322 * request from peripheral and so transfer finishes. 1323 * 1324 * - Memory addresses are scattered and are not contiguous. 1325 * Here, Obviously as DMA controller doesn't know when a lli's 1326 * transfer gets over, it can't load next lli. So in this 1327 * case, there has to be an assumption that only one lli is 1328 * supported. Thus, we can't have scattered addresses. 1329 */ 1330 if (!bd.remainder) { 1331 u32 fc; 1332 1333 /* FTDMAC020 only does memory-to-memory */ 1334 if (pl08x->vd->ftdmac020) 1335 fc = PL080_FLOW_MEM2MEM; 1336 else 1337 fc = (txd->ccfg & PL080_CONFIG_FLOW_CONTROL_MASK) >> 1338 PL080_CONFIG_FLOW_CONTROL_SHIFT; 1339 if (!((fc >= PL080_FLOW_SRC2DST_DST) && 1340 (fc <= PL080_FLOW_SRC2DST_SRC))) { 1341 dev_err(&pl08x->adev->dev, "%s sg len can't be zero", 1342 __func__); 1343 return 0; 1344 } 1345 1346 if (!IS_BUS_ALIGNED(&bd.srcbus) || 1347 !IS_BUS_ALIGNED(&bd.dstbus)) { 1348 dev_err(&pl08x->adev->dev, 1349 "%s src & dst address must be aligned to src" 1350 " & dst width if peripheral is flow controller", 1351 __func__); 1352 return 0; 1353 } 1354 1355 cctl = pl08x_lli_control_bits(pl08x, cctl, 1356 bd.srcbus.buswidth, bd.dstbus.buswidth, 1357 0); 1358 pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++, 1359 0, cctl, 0); 1360 break; 1361 } 1362 1363 /* 1364 * Send byte by byte for following cases 1365 * - Less than a bus width available 1366 * - until master bus is aligned 1367 */ 1368 if (bd.remainder < mbus->buswidth) 1369 early_bytes = bd.remainder; 1370 else if (!IS_BUS_ALIGNED(mbus)) { 1371 early_bytes = mbus->buswidth - 1372 (mbus->addr & (mbus->buswidth - 1)); 1373 if ((bd.remainder - early_bytes) < mbus->buswidth) 1374 early_bytes = bd.remainder; 1375 } 1376 1377 if (early_bytes) { 1378 dev_vdbg(&pl08x->adev->dev, 1379 "%s byte width LLIs (remain 0x%08zx)\n", 1380 __func__, bd.remainder); 1381 prep_byte_width_lli(pl08x, &bd, &cctl, early_bytes, 1382 num_llis++, &total_bytes); 1383 } 1384 1385 if (bd.remainder) { 1386 /* 1387 * Master now aligned 1388 * - if slave is not then we must set its width down 1389 */ 1390 if (!IS_BUS_ALIGNED(sbus)) { 1391 dev_dbg(&pl08x->adev->dev, 1392 "%s set down bus width to one byte\n", 1393 __func__); 1394 1395 sbus->buswidth = 1; 1396 } 1397 1398 /* 1399 * Bytes transferred = tsize * src width, not 1400 * MIN(buswidths) 1401 */ 1402 max_bytes_per_lli = bd.srcbus.buswidth * 1403 pl08x->vd->max_transfer_size; 1404 dev_vdbg(&pl08x->adev->dev, 1405 "%s max bytes per lli = %zu\n", 1406 __func__, max_bytes_per_lli); 1407 1408 /* 1409 * Make largest possible LLIs until less than one bus 1410 * width left 1411 */ 1412 while (bd.remainder > (mbus->buswidth - 1)) { 1413 size_t lli_len, tsize, width; 1414 1415 /* 1416 * If enough left try to send max possible, 1417 * otherwise try to send the remainder 1418 */ 1419 lli_len = min(bd.remainder, max_bytes_per_lli); 1420 1421 /* 1422 * Check against maximum bus alignment: 1423 * Calculate actual transfer size in relation to 1424 * bus width an get a maximum remainder of the 1425 * highest bus width - 1 1426 */ 1427 width = max(mbus->buswidth, sbus->buswidth); 1428 lli_len = (lli_len / width) * width; 1429 tsize = lli_len / bd.srcbus.buswidth; 1430 1431 dev_vdbg(&pl08x->adev->dev, 1432 "%s fill lli with single lli chunk of " 1433 "size 0x%08zx (remainder 0x%08zx)\n", 1434 __func__, lli_len, bd.remainder); 1435 1436 cctl = pl08x_lli_control_bits(pl08x, cctl, 1437 bd.srcbus.buswidth, bd.dstbus.buswidth, 1438 tsize); 1439 pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++, 1440 lli_len, cctl, tsize); 1441 total_bytes += lli_len; 1442 } 1443 1444 /* 1445 * Send any odd bytes 1446 */ 1447 if (bd.remainder) { 1448 dev_vdbg(&pl08x->adev->dev, 1449 "%s align with boundary, send odd bytes (remain %zu)\n", 1450 __func__, bd.remainder); 1451 prep_byte_width_lli(pl08x, &bd, &cctl, 1452 bd.remainder, num_llis++, &total_bytes); 1453 } 1454 } 1455 1456 if (total_bytes != dsg->len) { 1457 dev_err(&pl08x->adev->dev, 1458 "%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n", 1459 __func__, total_bytes, dsg->len); 1460 return 0; 1461 } 1462 1463 if (num_llis >= MAX_NUM_TSFR_LLIS) { 1464 dev_err(&pl08x->adev->dev, 1465 "%s need to increase MAX_NUM_TSFR_LLIS from 0x%08x\n", 1466 __func__, MAX_NUM_TSFR_LLIS); 1467 return 0; 1468 } 1469 } 1470 1471 llis_va = txd->llis_va; 1472 last_lli = llis_va + (num_llis - 1) * pl08x->lli_words; 1473 1474 if (txd->cyclic) { 1475 /* Link back to the first LLI. */ 1476 last_lli[PL080_LLI_LLI] = txd->llis_bus | bd.lli_bus; 1477 } else { 1478 /* The final LLI terminates the LLI. */ 1479 last_lli[PL080_LLI_LLI] = 0; 1480 /* The final LLI element shall also fire an interrupt. */ 1481 if (pl08x->vd->ftdmac020) 1482 last_lli[PL080_LLI_CCTL] &= ~FTDMAC020_LLI_TC_MSK; 1483 else 1484 last_lli[PL080_LLI_CCTL] |= PL080_CONTROL_TC_IRQ_EN; 1485 } 1486 1487 pl08x_dump_lli(pl08x, llis_va, num_llis); 1488 1489 return num_llis; 1490 } 1491 1492 static void pl08x_free_txd(struct pl08x_driver_data *pl08x, 1493 struct pl08x_txd *txd) 1494 { 1495 struct pl08x_sg *dsg, *_dsg; 1496 1497 if (txd->llis_va) 1498 dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus); 1499 1500 list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) { 1501 list_del(&dsg->node); 1502 kfree(dsg); 1503 } 1504 1505 kfree(txd); 1506 } 1507 1508 static void pl08x_desc_free(struct virt_dma_desc *vd) 1509 { 1510 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); 1511 struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan); 1512 1513 dma_descriptor_unmap(&vd->tx); 1514 if (!txd->done) 1515 pl08x_release_mux(plchan); 1516 1517 pl08x_free_txd(plchan->host, txd); 1518 } 1519 1520 static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x, 1521 struct pl08x_dma_chan *plchan) 1522 { 1523 LIST_HEAD(head); 1524 1525 vchan_get_all_descriptors(&plchan->vc, &head); 1526 vchan_dma_desc_free_list(&plchan->vc, &head); 1527 } 1528 1529 /* 1530 * The DMA ENGINE API 1531 */ 1532 static void pl08x_free_chan_resources(struct dma_chan *chan) 1533 { 1534 /* Ensure all queued descriptors are freed */ 1535 vchan_free_chan_resources(to_virt_chan(chan)); 1536 } 1537 1538 static struct dma_async_tx_descriptor *pl08x_prep_dma_interrupt( 1539 struct dma_chan *chan, unsigned long flags) 1540 { 1541 struct dma_async_tx_descriptor *retval = NULL; 1542 1543 return retval; 1544 } 1545 1546 /* 1547 * Code accessing dma_async_is_complete() in a tight loop may give problems. 1548 * If slaves are relying on interrupts to signal completion this function 1549 * must not be called with interrupts disabled. 1550 */ 1551 static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan, 1552 dma_cookie_t cookie, struct dma_tx_state *txstate) 1553 { 1554 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1555 struct virt_dma_desc *vd; 1556 unsigned long flags; 1557 enum dma_status ret; 1558 size_t bytes = 0; 1559 1560 ret = dma_cookie_status(chan, cookie, txstate); 1561 if (ret == DMA_COMPLETE) 1562 return ret; 1563 1564 /* 1565 * There's no point calculating the residue if there's 1566 * no txstate to store the value. 1567 */ 1568 if (!txstate) { 1569 if (plchan->state == PL08X_CHAN_PAUSED) 1570 ret = DMA_PAUSED; 1571 return ret; 1572 } 1573 1574 spin_lock_irqsave(&plchan->vc.lock, flags); 1575 ret = dma_cookie_status(chan, cookie, txstate); 1576 if (ret != DMA_COMPLETE) { 1577 vd = vchan_find_desc(&plchan->vc, cookie); 1578 if (vd) { 1579 /* On the issued list, so hasn't been processed yet */ 1580 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); 1581 struct pl08x_sg *dsg; 1582 1583 list_for_each_entry(dsg, &txd->dsg_list, node) 1584 bytes += dsg->len; 1585 } else { 1586 bytes = pl08x_getbytes_chan(plchan); 1587 } 1588 } 1589 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1590 1591 /* 1592 * This cookie not complete yet 1593 * Get number of bytes left in the active transactions and queue 1594 */ 1595 dma_set_residue(txstate, bytes); 1596 1597 if (plchan->state == PL08X_CHAN_PAUSED && ret == DMA_IN_PROGRESS) 1598 ret = DMA_PAUSED; 1599 1600 /* Whether waiting or running, we're in progress */ 1601 return ret; 1602 } 1603 1604 /* PrimeCell DMA extension */ 1605 struct burst_table { 1606 u32 burstwords; 1607 u32 reg; 1608 }; 1609 1610 static const struct burst_table burst_sizes[] = { 1611 { 1612 .burstwords = 256, 1613 .reg = PL080_BSIZE_256, 1614 }, 1615 { 1616 .burstwords = 128, 1617 .reg = PL080_BSIZE_128, 1618 }, 1619 { 1620 .burstwords = 64, 1621 .reg = PL080_BSIZE_64, 1622 }, 1623 { 1624 .burstwords = 32, 1625 .reg = PL080_BSIZE_32, 1626 }, 1627 { 1628 .burstwords = 16, 1629 .reg = PL080_BSIZE_16, 1630 }, 1631 { 1632 .burstwords = 8, 1633 .reg = PL080_BSIZE_8, 1634 }, 1635 { 1636 .burstwords = 4, 1637 .reg = PL080_BSIZE_4, 1638 }, 1639 { 1640 .burstwords = 0, 1641 .reg = PL080_BSIZE_1, 1642 }, 1643 }; 1644 1645 /* 1646 * Given the source and destination available bus masks, select which 1647 * will be routed to each port. We try to have source and destination 1648 * on separate ports, but always respect the allowable settings. 1649 */ 1650 static u32 pl08x_select_bus(bool ftdmac020, u8 src, u8 dst) 1651 { 1652 u32 cctl = 0; 1653 u32 dst_ahb2; 1654 u32 src_ahb2; 1655 1656 /* The FTDMAC020 use different bits to indicate src/dst bus */ 1657 if (ftdmac020) { 1658 dst_ahb2 = FTDMAC020_LLI_DST_SEL; 1659 src_ahb2 = FTDMAC020_LLI_SRC_SEL; 1660 } else { 1661 dst_ahb2 = PL080_CONTROL_DST_AHB2; 1662 src_ahb2 = PL080_CONTROL_SRC_AHB2; 1663 } 1664 1665 if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1))) 1666 cctl |= dst_ahb2; 1667 if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2))) 1668 cctl |= src_ahb2; 1669 1670 return cctl; 1671 } 1672 1673 static u32 pl08x_cctl(u32 cctl) 1674 { 1675 cctl &= ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 | 1676 PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR | 1677 PL080_CONTROL_PROT_MASK); 1678 1679 /* Access the cell in privileged mode, non-bufferable, non-cacheable */ 1680 return cctl | PL080_CONTROL_PROT_SYS; 1681 } 1682 1683 static u32 pl08x_width(enum dma_slave_buswidth width) 1684 { 1685 switch (width) { 1686 case DMA_SLAVE_BUSWIDTH_1_BYTE: 1687 return PL080_WIDTH_8BIT; 1688 case DMA_SLAVE_BUSWIDTH_2_BYTES: 1689 return PL080_WIDTH_16BIT; 1690 case DMA_SLAVE_BUSWIDTH_4_BYTES: 1691 return PL080_WIDTH_32BIT; 1692 default: 1693 return ~0; 1694 } 1695 } 1696 1697 static u32 pl08x_burst(u32 maxburst) 1698 { 1699 int i; 1700 1701 for (i = 0; i < ARRAY_SIZE(burst_sizes); i++) 1702 if (burst_sizes[i].burstwords <= maxburst) 1703 break; 1704 1705 return burst_sizes[i].reg; 1706 } 1707 1708 static u32 pl08x_get_cctl(struct pl08x_dma_chan *plchan, 1709 enum dma_slave_buswidth addr_width, u32 maxburst) 1710 { 1711 u32 width, burst, cctl = 0; 1712 1713 width = pl08x_width(addr_width); 1714 if (width == ~0) 1715 return ~0; 1716 1717 cctl |= width << PL080_CONTROL_SWIDTH_SHIFT; 1718 cctl |= width << PL080_CONTROL_DWIDTH_SHIFT; 1719 1720 /* 1721 * If this channel will only request single transfers, set this 1722 * down to ONE element. Also select one element if no maxburst 1723 * is specified. 1724 */ 1725 if (plchan->cd->single) 1726 maxburst = 1; 1727 1728 burst = pl08x_burst(maxburst); 1729 cctl |= burst << PL080_CONTROL_SB_SIZE_SHIFT; 1730 cctl |= burst << PL080_CONTROL_DB_SIZE_SHIFT; 1731 1732 return pl08x_cctl(cctl); 1733 } 1734 1735 /* 1736 * Slave transactions callback to the slave device to allow 1737 * synchronization of slave DMA signals with the DMAC enable 1738 */ 1739 static void pl08x_issue_pending(struct dma_chan *chan) 1740 { 1741 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1742 unsigned long flags; 1743 1744 spin_lock_irqsave(&plchan->vc.lock, flags); 1745 if (vchan_issue_pending(&plchan->vc)) { 1746 if (!plchan->phychan && plchan->state != PL08X_CHAN_WAITING) 1747 pl08x_phy_alloc_and_start(plchan); 1748 } 1749 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1750 } 1751 1752 static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan) 1753 { 1754 struct pl08x_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT); 1755 1756 if (txd) 1757 INIT_LIST_HEAD(&txd->dsg_list); 1758 return txd; 1759 } 1760 1761 static u32 pl08x_memcpy_cctl(struct pl08x_driver_data *pl08x) 1762 { 1763 u32 cctl = 0; 1764 1765 /* Conjure cctl */ 1766 switch (pl08x->pd->memcpy_burst_size) { 1767 default: 1768 dev_err(&pl08x->adev->dev, 1769 "illegal burst size for memcpy, set to 1\n"); 1770 fallthrough; 1771 case PL08X_BURST_SZ_1: 1772 cctl |= PL080_BSIZE_1 << PL080_CONTROL_SB_SIZE_SHIFT | 1773 PL080_BSIZE_1 << PL080_CONTROL_DB_SIZE_SHIFT; 1774 break; 1775 case PL08X_BURST_SZ_4: 1776 cctl |= PL080_BSIZE_4 << PL080_CONTROL_SB_SIZE_SHIFT | 1777 PL080_BSIZE_4 << PL080_CONTROL_DB_SIZE_SHIFT; 1778 break; 1779 case PL08X_BURST_SZ_8: 1780 cctl |= PL080_BSIZE_8 << PL080_CONTROL_SB_SIZE_SHIFT | 1781 PL080_BSIZE_8 << PL080_CONTROL_DB_SIZE_SHIFT; 1782 break; 1783 case PL08X_BURST_SZ_16: 1784 cctl |= PL080_BSIZE_16 << PL080_CONTROL_SB_SIZE_SHIFT | 1785 PL080_BSIZE_16 << PL080_CONTROL_DB_SIZE_SHIFT; 1786 break; 1787 case PL08X_BURST_SZ_32: 1788 cctl |= PL080_BSIZE_32 << PL080_CONTROL_SB_SIZE_SHIFT | 1789 PL080_BSIZE_32 << PL080_CONTROL_DB_SIZE_SHIFT; 1790 break; 1791 case PL08X_BURST_SZ_64: 1792 cctl |= PL080_BSIZE_64 << PL080_CONTROL_SB_SIZE_SHIFT | 1793 PL080_BSIZE_64 << PL080_CONTROL_DB_SIZE_SHIFT; 1794 break; 1795 case PL08X_BURST_SZ_128: 1796 cctl |= PL080_BSIZE_128 << PL080_CONTROL_SB_SIZE_SHIFT | 1797 PL080_BSIZE_128 << PL080_CONTROL_DB_SIZE_SHIFT; 1798 break; 1799 case PL08X_BURST_SZ_256: 1800 cctl |= PL080_BSIZE_256 << PL080_CONTROL_SB_SIZE_SHIFT | 1801 PL080_BSIZE_256 << PL080_CONTROL_DB_SIZE_SHIFT; 1802 break; 1803 } 1804 1805 switch (pl08x->pd->memcpy_bus_width) { 1806 default: 1807 dev_err(&pl08x->adev->dev, 1808 "illegal bus width for memcpy, set to 8 bits\n"); 1809 fallthrough; 1810 case PL08X_BUS_WIDTH_8_BITS: 1811 cctl |= PL080_WIDTH_8BIT << PL080_CONTROL_SWIDTH_SHIFT | 1812 PL080_WIDTH_8BIT << PL080_CONTROL_DWIDTH_SHIFT; 1813 break; 1814 case PL08X_BUS_WIDTH_16_BITS: 1815 cctl |= PL080_WIDTH_16BIT << PL080_CONTROL_SWIDTH_SHIFT | 1816 PL080_WIDTH_16BIT << PL080_CONTROL_DWIDTH_SHIFT; 1817 break; 1818 case PL08X_BUS_WIDTH_32_BITS: 1819 cctl |= PL080_WIDTH_32BIT << PL080_CONTROL_SWIDTH_SHIFT | 1820 PL080_WIDTH_32BIT << PL080_CONTROL_DWIDTH_SHIFT; 1821 break; 1822 } 1823 1824 /* Protection flags */ 1825 if (pl08x->pd->memcpy_prot_buff) 1826 cctl |= PL080_CONTROL_PROT_BUFF; 1827 if (pl08x->pd->memcpy_prot_cache) 1828 cctl |= PL080_CONTROL_PROT_CACHE; 1829 1830 /* We are the kernel, so we are in privileged mode */ 1831 cctl |= PL080_CONTROL_PROT_SYS; 1832 1833 /* Both to be incremented or the code will break */ 1834 cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR; 1835 1836 if (pl08x->vd->dualmaster) 1837 cctl |= pl08x_select_bus(false, 1838 pl08x->mem_buses, 1839 pl08x->mem_buses); 1840 1841 return cctl; 1842 } 1843 1844 static u32 pl08x_ftdmac020_memcpy_cctl(struct pl08x_driver_data *pl08x) 1845 { 1846 u32 cctl = 0; 1847 1848 /* Conjure cctl */ 1849 switch (pl08x->pd->memcpy_bus_width) { 1850 default: 1851 dev_err(&pl08x->adev->dev, 1852 "illegal bus width for memcpy, set to 8 bits\n"); 1853 fallthrough; 1854 case PL08X_BUS_WIDTH_8_BITS: 1855 cctl |= PL080_WIDTH_8BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT | 1856 PL080_WIDTH_8BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT; 1857 break; 1858 case PL08X_BUS_WIDTH_16_BITS: 1859 cctl |= PL080_WIDTH_16BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT | 1860 PL080_WIDTH_16BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT; 1861 break; 1862 case PL08X_BUS_WIDTH_32_BITS: 1863 cctl |= PL080_WIDTH_32BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT | 1864 PL080_WIDTH_32BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT; 1865 break; 1866 } 1867 1868 /* 1869 * By default mask the TC IRQ on all LLIs, it will be unmasked on 1870 * the last LLI item by other code. 1871 */ 1872 cctl |= FTDMAC020_LLI_TC_MSK; 1873 1874 /* 1875 * Both to be incremented so leave bits FTDMAC020_LLI_SRCAD_CTL 1876 * and FTDMAC020_LLI_DSTAD_CTL as zero 1877 */ 1878 if (pl08x->vd->dualmaster) 1879 cctl |= pl08x_select_bus(true, 1880 pl08x->mem_buses, 1881 pl08x->mem_buses); 1882 1883 return cctl; 1884 } 1885 1886 /* 1887 * Initialize a descriptor to be used by memcpy submit 1888 */ 1889 static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy( 1890 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 1891 size_t len, unsigned long flags) 1892 { 1893 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1894 struct pl08x_driver_data *pl08x = plchan->host; 1895 struct pl08x_txd *txd; 1896 struct pl08x_sg *dsg; 1897 int ret; 1898 1899 txd = pl08x_get_txd(plchan); 1900 if (!txd) { 1901 dev_err(&pl08x->adev->dev, 1902 "%s no memory for descriptor\n", __func__); 1903 return NULL; 1904 } 1905 1906 dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT); 1907 if (!dsg) { 1908 pl08x_free_txd(pl08x, txd); 1909 return NULL; 1910 } 1911 list_add_tail(&dsg->node, &txd->dsg_list); 1912 1913 dsg->src_addr = src; 1914 dsg->dst_addr = dest; 1915 dsg->len = len; 1916 if (pl08x->vd->ftdmac020) { 1917 /* Writing CCFG zero ENABLES all interrupts */ 1918 txd->ccfg = 0; 1919 txd->cctl = pl08x_ftdmac020_memcpy_cctl(pl08x); 1920 } else { 1921 txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK | 1922 PL080_CONFIG_TC_IRQ_MASK | 1923 PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT; 1924 txd->cctl = pl08x_memcpy_cctl(pl08x); 1925 } 1926 1927 ret = pl08x_fill_llis_for_desc(plchan->host, txd); 1928 if (!ret) { 1929 pl08x_free_txd(pl08x, txd); 1930 return NULL; 1931 } 1932 1933 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); 1934 } 1935 1936 static struct pl08x_txd *pl08x_init_txd( 1937 struct dma_chan *chan, 1938 enum dma_transfer_direction direction, 1939 dma_addr_t *slave_addr) 1940 { 1941 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1942 struct pl08x_driver_data *pl08x = plchan->host; 1943 struct pl08x_txd *txd; 1944 enum dma_slave_buswidth addr_width; 1945 int ret, tmp; 1946 u8 src_buses, dst_buses; 1947 u32 maxburst, cctl; 1948 1949 txd = pl08x_get_txd(plchan); 1950 if (!txd) { 1951 dev_err(&pl08x->adev->dev, "%s no txd\n", __func__); 1952 return NULL; 1953 } 1954 1955 /* 1956 * Set up addresses, the PrimeCell configured address 1957 * will take precedence since this may configure the 1958 * channel target address dynamically at runtime. 1959 */ 1960 if (direction == DMA_MEM_TO_DEV) { 1961 cctl = PL080_CONTROL_SRC_INCR; 1962 *slave_addr = plchan->cfg.dst_addr; 1963 addr_width = plchan->cfg.dst_addr_width; 1964 maxburst = plchan->cfg.dst_maxburst; 1965 src_buses = pl08x->mem_buses; 1966 dst_buses = plchan->cd->periph_buses; 1967 } else if (direction == DMA_DEV_TO_MEM) { 1968 cctl = PL080_CONTROL_DST_INCR; 1969 *slave_addr = plchan->cfg.src_addr; 1970 addr_width = plchan->cfg.src_addr_width; 1971 maxburst = plchan->cfg.src_maxburst; 1972 src_buses = plchan->cd->periph_buses; 1973 dst_buses = pl08x->mem_buses; 1974 } else { 1975 pl08x_free_txd(pl08x, txd); 1976 dev_err(&pl08x->adev->dev, 1977 "%s direction unsupported\n", __func__); 1978 return NULL; 1979 } 1980 1981 cctl |= pl08x_get_cctl(plchan, addr_width, maxburst); 1982 if (cctl == ~0) { 1983 pl08x_free_txd(pl08x, txd); 1984 dev_err(&pl08x->adev->dev, 1985 "DMA slave configuration botched?\n"); 1986 return NULL; 1987 } 1988 1989 txd->cctl = cctl | pl08x_select_bus(false, src_buses, dst_buses); 1990 1991 if (plchan->cfg.device_fc) 1992 tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER_PER : 1993 PL080_FLOW_PER2MEM_PER; 1994 else 1995 tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER : 1996 PL080_FLOW_PER2MEM; 1997 1998 txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK | 1999 PL080_CONFIG_TC_IRQ_MASK | 2000 tmp << PL080_CONFIG_FLOW_CONTROL_SHIFT; 2001 2002 ret = pl08x_request_mux(plchan); 2003 if (ret < 0) { 2004 pl08x_free_txd(pl08x, txd); 2005 dev_dbg(&pl08x->adev->dev, 2006 "unable to mux for transfer on %s due to platform restrictions\n", 2007 plchan->name); 2008 return NULL; 2009 } 2010 2011 dev_dbg(&pl08x->adev->dev, "allocated DMA request signal %d for xfer on %s\n", 2012 plchan->signal, plchan->name); 2013 2014 /* Assign the flow control signal to this channel */ 2015 if (direction == DMA_MEM_TO_DEV) 2016 txd->ccfg |= plchan->signal << PL080_CONFIG_DST_SEL_SHIFT; 2017 else 2018 txd->ccfg |= plchan->signal << PL080_CONFIG_SRC_SEL_SHIFT; 2019 2020 return txd; 2021 } 2022 2023 static int pl08x_tx_add_sg(struct pl08x_txd *txd, 2024 enum dma_transfer_direction direction, 2025 dma_addr_t slave_addr, 2026 dma_addr_t buf_addr, 2027 unsigned int len) 2028 { 2029 struct pl08x_sg *dsg; 2030 2031 dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT); 2032 if (!dsg) 2033 return -ENOMEM; 2034 2035 list_add_tail(&dsg->node, &txd->dsg_list); 2036 2037 dsg->len = len; 2038 if (direction == DMA_MEM_TO_DEV) { 2039 dsg->src_addr = buf_addr; 2040 dsg->dst_addr = slave_addr; 2041 } else { 2042 dsg->src_addr = slave_addr; 2043 dsg->dst_addr = buf_addr; 2044 } 2045 2046 return 0; 2047 } 2048 2049 static struct dma_async_tx_descriptor *pl08x_prep_slave_sg( 2050 struct dma_chan *chan, struct scatterlist *sgl, 2051 unsigned int sg_len, enum dma_transfer_direction direction, 2052 unsigned long flags, void *context) 2053 { 2054 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2055 struct pl08x_driver_data *pl08x = plchan->host; 2056 struct pl08x_txd *txd; 2057 struct scatterlist *sg; 2058 int ret, tmp; 2059 dma_addr_t slave_addr; 2060 2061 dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n", 2062 __func__, sg_dma_len(sgl), plchan->name); 2063 2064 txd = pl08x_init_txd(chan, direction, &slave_addr); 2065 if (!txd) 2066 return NULL; 2067 2068 for_each_sg(sgl, sg, sg_len, tmp) { 2069 ret = pl08x_tx_add_sg(txd, direction, slave_addr, 2070 sg_dma_address(sg), 2071 sg_dma_len(sg)); 2072 if (ret) { 2073 pl08x_release_mux(plchan); 2074 pl08x_free_txd(pl08x, txd); 2075 dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n", 2076 __func__); 2077 return NULL; 2078 } 2079 } 2080 2081 ret = pl08x_fill_llis_for_desc(plchan->host, txd); 2082 if (!ret) { 2083 pl08x_release_mux(plchan); 2084 pl08x_free_txd(pl08x, txd); 2085 return NULL; 2086 } 2087 2088 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); 2089 } 2090 2091 static struct dma_async_tx_descriptor *pl08x_prep_dma_cyclic( 2092 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 2093 size_t period_len, enum dma_transfer_direction direction, 2094 unsigned long flags) 2095 { 2096 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2097 struct pl08x_driver_data *pl08x = plchan->host; 2098 struct pl08x_txd *txd; 2099 int ret, tmp; 2100 dma_addr_t slave_addr; 2101 2102 dev_dbg(&pl08x->adev->dev, 2103 "%s prepare cyclic transaction of %zd/%zd bytes %s %s\n", 2104 __func__, period_len, buf_len, 2105 direction == DMA_MEM_TO_DEV ? "to" : "from", 2106 plchan->name); 2107 2108 txd = pl08x_init_txd(chan, direction, &slave_addr); 2109 if (!txd) 2110 return NULL; 2111 2112 txd->cyclic = true; 2113 txd->cctl |= PL080_CONTROL_TC_IRQ_EN; 2114 for (tmp = 0; tmp < buf_len; tmp += period_len) { 2115 ret = pl08x_tx_add_sg(txd, direction, slave_addr, 2116 buf_addr + tmp, period_len); 2117 if (ret) { 2118 pl08x_release_mux(plchan); 2119 pl08x_free_txd(pl08x, txd); 2120 return NULL; 2121 } 2122 } 2123 2124 ret = pl08x_fill_llis_for_desc(plchan->host, txd); 2125 if (!ret) { 2126 pl08x_release_mux(plchan); 2127 pl08x_free_txd(pl08x, txd); 2128 return NULL; 2129 } 2130 2131 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); 2132 } 2133 2134 static int pl08x_config(struct dma_chan *chan, 2135 struct dma_slave_config *config) 2136 { 2137 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2138 struct pl08x_driver_data *pl08x = plchan->host; 2139 2140 if (!plchan->slave) 2141 return -EINVAL; 2142 2143 /* Reject definitely invalid configurations */ 2144 if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES || 2145 config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) 2146 return -EINVAL; 2147 2148 if (config->device_fc && pl08x->vd->pl080s) { 2149 dev_err(&pl08x->adev->dev, 2150 "%s: PL080S does not support peripheral flow control\n", 2151 __func__); 2152 return -EINVAL; 2153 } 2154 2155 plchan->cfg = *config; 2156 2157 return 0; 2158 } 2159 2160 static int pl08x_terminate_all(struct dma_chan *chan) 2161 { 2162 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2163 struct pl08x_driver_data *pl08x = plchan->host; 2164 unsigned long flags; 2165 2166 spin_lock_irqsave(&plchan->vc.lock, flags); 2167 if (!plchan->phychan && !plchan->at) { 2168 spin_unlock_irqrestore(&plchan->vc.lock, flags); 2169 return 0; 2170 } 2171 2172 plchan->state = PL08X_CHAN_IDLE; 2173 2174 if (plchan->phychan) { 2175 /* 2176 * Mark physical channel as free and free any slave 2177 * signal 2178 */ 2179 pl08x_phy_free(plchan); 2180 } 2181 /* Dequeue jobs and free LLIs */ 2182 if (plchan->at) { 2183 vchan_terminate_vdesc(&plchan->at->vd); 2184 plchan->at = NULL; 2185 } 2186 /* Dequeue jobs not yet fired as well */ 2187 pl08x_free_txd_list(pl08x, plchan); 2188 2189 spin_unlock_irqrestore(&plchan->vc.lock, flags); 2190 2191 return 0; 2192 } 2193 2194 static void pl08x_synchronize(struct dma_chan *chan) 2195 { 2196 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2197 2198 vchan_synchronize(&plchan->vc); 2199 } 2200 2201 static int pl08x_pause(struct dma_chan *chan) 2202 { 2203 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2204 unsigned long flags; 2205 2206 /* 2207 * Anything succeeds on channels with no physical allocation and 2208 * no queued transfers. 2209 */ 2210 spin_lock_irqsave(&plchan->vc.lock, flags); 2211 if (!plchan->phychan && !plchan->at) { 2212 spin_unlock_irqrestore(&plchan->vc.lock, flags); 2213 return 0; 2214 } 2215 2216 pl08x_pause_phy_chan(plchan->phychan); 2217 plchan->state = PL08X_CHAN_PAUSED; 2218 2219 spin_unlock_irqrestore(&plchan->vc.lock, flags); 2220 2221 return 0; 2222 } 2223 2224 static int pl08x_resume(struct dma_chan *chan) 2225 { 2226 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2227 unsigned long flags; 2228 2229 /* 2230 * Anything succeeds on channels with no physical allocation and 2231 * no queued transfers. 2232 */ 2233 spin_lock_irqsave(&plchan->vc.lock, flags); 2234 if (!plchan->phychan && !plchan->at) { 2235 spin_unlock_irqrestore(&plchan->vc.lock, flags); 2236 return 0; 2237 } 2238 2239 pl08x_resume_phy_chan(plchan->phychan); 2240 plchan->state = PL08X_CHAN_RUNNING; 2241 2242 spin_unlock_irqrestore(&plchan->vc.lock, flags); 2243 2244 return 0; 2245 } 2246 2247 bool pl08x_filter_id(struct dma_chan *chan, void *chan_id) 2248 { 2249 struct pl08x_dma_chan *plchan; 2250 char *name = chan_id; 2251 2252 /* Reject channels for devices not bound to this driver */ 2253 if (chan->device->dev->driver != &pl08x_amba_driver.drv) 2254 return false; 2255 2256 plchan = to_pl08x_chan(chan); 2257 2258 /* Check that the channel is not taken! */ 2259 if (!strcmp(plchan->name, name)) 2260 return true; 2261 2262 return false; 2263 } 2264 EXPORT_SYMBOL_GPL(pl08x_filter_id); 2265 2266 static bool pl08x_filter_fn(struct dma_chan *chan, void *chan_id) 2267 { 2268 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2269 2270 return plchan->cd == chan_id; 2271 } 2272 2273 /* 2274 * Just check that the device is there and active 2275 * TODO: turn this bit on/off depending on the number of physical channels 2276 * actually used, if it is zero... well shut it off. That will save some 2277 * power. Cut the clock at the same time. 2278 */ 2279 static void pl08x_ensure_on(struct pl08x_driver_data *pl08x) 2280 { 2281 /* The Nomadik variant does not have the config register */ 2282 if (pl08x->vd->nomadik) 2283 return; 2284 /* The FTDMAC020 variant does this in another register */ 2285 if (pl08x->vd->ftdmac020) { 2286 writel(PL080_CONFIG_ENABLE, pl08x->base + FTDMAC020_CSR); 2287 return; 2288 } 2289 writel(PL080_CONFIG_ENABLE, pl08x->base + PL080_CONFIG); 2290 } 2291 2292 static irqreturn_t pl08x_irq(int irq, void *dev) 2293 { 2294 struct pl08x_driver_data *pl08x = dev; 2295 u32 mask = 0, err, tc, i; 2296 2297 /* check & clear - ERR & TC interrupts */ 2298 err = readl(pl08x->base + PL080_ERR_STATUS); 2299 if (err) { 2300 dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n", 2301 __func__, err); 2302 writel(err, pl08x->base + PL080_ERR_CLEAR); 2303 } 2304 tc = readl(pl08x->base + PL080_TC_STATUS); 2305 if (tc) 2306 writel(tc, pl08x->base + PL080_TC_CLEAR); 2307 2308 if (!err && !tc) 2309 return IRQ_NONE; 2310 2311 for (i = 0; i < pl08x->vd->channels; i++) { 2312 if ((BIT(i) & err) || (BIT(i) & tc)) { 2313 /* Locate physical channel */ 2314 struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i]; 2315 struct pl08x_dma_chan *plchan = phychan->serving; 2316 struct pl08x_txd *tx; 2317 2318 if (!plchan) { 2319 dev_err(&pl08x->adev->dev, 2320 "%s Error TC interrupt on unused channel: 0x%08x\n", 2321 __func__, i); 2322 continue; 2323 } 2324 2325 spin_lock(&plchan->vc.lock); 2326 tx = plchan->at; 2327 if (tx && tx->cyclic) { 2328 vchan_cyclic_callback(&tx->vd); 2329 } else if (tx) { 2330 plchan->at = NULL; 2331 /* 2332 * This descriptor is done, release its mux 2333 * reservation. 2334 */ 2335 pl08x_release_mux(plchan); 2336 tx->done = true; 2337 vchan_cookie_complete(&tx->vd); 2338 2339 /* 2340 * And start the next descriptor (if any), 2341 * otherwise free this channel. 2342 */ 2343 if (vchan_next_desc(&plchan->vc)) 2344 pl08x_start_next_txd(plchan); 2345 else 2346 pl08x_phy_free(plchan); 2347 } 2348 spin_unlock(&plchan->vc.lock); 2349 2350 mask |= BIT(i); 2351 } 2352 } 2353 2354 return mask ? IRQ_HANDLED : IRQ_NONE; 2355 } 2356 2357 static void pl08x_dma_slave_init(struct pl08x_dma_chan *chan) 2358 { 2359 chan->slave = true; 2360 chan->name = chan->cd->bus_id; 2361 chan->cfg.src_addr = chan->cd->addr; 2362 chan->cfg.dst_addr = chan->cd->addr; 2363 } 2364 2365 /* 2366 * Initialise the DMAC memcpy/slave channels. 2367 * Make a local wrapper to hold required data 2368 */ 2369 static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x, 2370 struct dma_device *dmadev, unsigned int channels, bool slave) 2371 { 2372 struct pl08x_dma_chan *chan; 2373 int i; 2374 2375 INIT_LIST_HEAD(&dmadev->channels); 2376 2377 /* 2378 * Register as many many memcpy as we have physical channels, 2379 * we won't always be able to use all but the code will have 2380 * to cope with that situation. 2381 */ 2382 for (i = 0; i < channels; i++) { 2383 chan = kzalloc(sizeof(*chan), GFP_KERNEL); 2384 if (!chan) 2385 return -ENOMEM; 2386 2387 chan->host = pl08x; 2388 chan->state = PL08X_CHAN_IDLE; 2389 chan->signal = -1; 2390 2391 if (slave) { 2392 chan->cd = &pl08x->pd->slave_channels[i]; 2393 /* 2394 * Some implementations have muxed signals, whereas some 2395 * use a mux in front of the signals and need dynamic 2396 * assignment of signals. 2397 */ 2398 chan->signal = i; 2399 pl08x_dma_slave_init(chan); 2400 } else { 2401 chan->cd = kzalloc(sizeof(*chan->cd), GFP_KERNEL); 2402 if (!chan->cd) { 2403 kfree(chan); 2404 return -ENOMEM; 2405 } 2406 chan->cd->bus_id = "memcpy"; 2407 chan->cd->periph_buses = pl08x->pd->mem_buses; 2408 chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i); 2409 if (!chan->name) { 2410 kfree(chan->cd); 2411 kfree(chan); 2412 return -ENOMEM; 2413 } 2414 } 2415 dev_dbg(&pl08x->adev->dev, 2416 "initialize virtual channel \"%s\"\n", 2417 chan->name); 2418 2419 chan->vc.desc_free = pl08x_desc_free; 2420 vchan_init(&chan->vc, dmadev); 2421 } 2422 dev_info(&pl08x->adev->dev, "initialized %d virtual %s channels\n", 2423 i, slave ? "slave" : "memcpy"); 2424 return i; 2425 } 2426 2427 static void pl08x_free_virtual_channels(struct dma_device *dmadev) 2428 { 2429 struct pl08x_dma_chan *chan = NULL; 2430 struct pl08x_dma_chan *next; 2431 2432 list_for_each_entry_safe(chan, 2433 next, &dmadev->channels, vc.chan.device_node) { 2434 list_del(&chan->vc.chan.device_node); 2435 kfree(chan); 2436 } 2437 } 2438 2439 #ifdef CONFIG_DEBUG_FS 2440 static const char *pl08x_state_str(enum pl08x_dma_chan_state state) 2441 { 2442 switch (state) { 2443 case PL08X_CHAN_IDLE: 2444 return "idle"; 2445 case PL08X_CHAN_RUNNING: 2446 return "running"; 2447 case PL08X_CHAN_PAUSED: 2448 return "paused"; 2449 case PL08X_CHAN_WAITING: 2450 return "waiting"; 2451 default: 2452 break; 2453 } 2454 return "UNKNOWN STATE"; 2455 } 2456 2457 static int pl08x_debugfs_show(struct seq_file *s, void *data) 2458 { 2459 struct pl08x_driver_data *pl08x = s->private; 2460 struct pl08x_dma_chan *chan; 2461 struct pl08x_phy_chan *ch; 2462 unsigned long flags; 2463 int i; 2464 2465 seq_printf(s, "PL08x physical channels:\n"); 2466 seq_printf(s, "CHANNEL:\tUSER:\n"); 2467 seq_printf(s, "--------\t-----\n"); 2468 for (i = 0; i < pl08x->vd->channels; i++) { 2469 struct pl08x_dma_chan *virt_chan; 2470 2471 ch = &pl08x->phy_chans[i]; 2472 2473 spin_lock_irqsave(&ch->lock, flags); 2474 virt_chan = ch->serving; 2475 2476 seq_printf(s, "%d\t\t%s%s\n", 2477 ch->id, 2478 virt_chan ? virt_chan->name : "(none)", 2479 ch->locked ? " LOCKED" : ""); 2480 2481 spin_unlock_irqrestore(&ch->lock, flags); 2482 } 2483 2484 seq_printf(s, "\nPL08x virtual memcpy channels:\n"); 2485 seq_printf(s, "CHANNEL:\tSTATE:\n"); 2486 seq_printf(s, "--------\t------\n"); 2487 list_for_each_entry(chan, &pl08x->memcpy.channels, vc.chan.device_node) { 2488 seq_printf(s, "%s\t\t%s\n", chan->name, 2489 pl08x_state_str(chan->state)); 2490 } 2491 2492 if (pl08x->has_slave) { 2493 seq_printf(s, "\nPL08x virtual slave channels:\n"); 2494 seq_printf(s, "CHANNEL:\tSTATE:\n"); 2495 seq_printf(s, "--------\t------\n"); 2496 list_for_each_entry(chan, &pl08x->slave.channels, 2497 vc.chan.device_node) { 2498 seq_printf(s, "%s\t\t%s\n", chan->name, 2499 pl08x_state_str(chan->state)); 2500 } 2501 } 2502 2503 return 0; 2504 } 2505 2506 DEFINE_SHOW_ATTRIBUTE(pl08x_debugfs); 2507 2508 static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x) 2509 { 2510 /* Expose a simple debugfs interface to view all clocks */ 2511 debugfs_create_file(dev_name(&pl08x->adev->dev), S_IFREG | S_IRUGO, 2512 NULL, pl08x, &pl08x_debugfs_fops); 2513 } 2514 2515 #else 2516 static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x) 2517 { 2518 } 2519 #endif 2520 2521 #ifdef CONFIG_OF 2522 static struct dma_chan *pl08x_find_chan_id(struct pl08x_driver_data *pl08x, 2523 u32 id) 2524 { 2525 struct pl08x_dma_chan *chan; 2526 2527 /* Trying to get a slave channel from something with no slave support */ 2528 if (!pl08x->has_slave) 2529 return NULL; 2530 2531 list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) { 2532 if (chan->signal == id) 2533 return &chan->vc.chan; 2534 } 2535 2536 return NULL; 2537 } 2538 2539 static struct dma_chan *pl08x_of_xlate(struct of_phandle_args *dma_spec, 2540 struct of_dma *ofdma) 2541 { 2542 struct pl08x_driver_data *pl08x = ofdma->of_dma_data; 2543 struct dma_chan *dma_chan; 2544 struct pl08x_dma_chan *plchan; 2545 2546 if (!pl08x) 2547 return NULL; 2548 2549 if (dma_spec->args_count != 2) { 2550 dev_err(&pl08x->adev->dev, 2551 "DMA channel translation requires two cells\n"); 2552 return NULL; 2553 } 2554 2555 dma_chan = pl08x_find_chan_id(pl08x, dma_spec->args[0]); 2556 if (!dma_chan) { 2557 dev_err(&pl08x->adev->dev, 2558 "DMA slave channel not found\n"); 2559 return NULL; 2560 } 2561 2562 plchan = to_pl08x_chan(dma_chan); 2563 dev_dbg(&pl08x->adev->dev, 2564 "translated channel for signal %d\n", 2565 dma_spec->args[0]); 2566 2567 /* Augment channel data for applicable AHB buses */ 2568 plchan->cd->periph_buses = dma_spec->args[1]; 2569 return dma_get_slave_channel(dma_chan); 2570 } 2571 2572 static int pl08x_of_probe(struct amba_device *adev, 2573 struct pl08x_driver_data *pl08x, 2574 struct device_node *np) 2575 { 2576 struct pl08x_platform_data *pd; 2577 struct pl08x_channel_data *chanp = NULL; 2578 u32 val; 2579 int ret; 2580 int i; 2581 2582 pd = devm_kzalloc(&adev->dev, sizeof(*pd), GFP_KERNEL); 2583 if (!pd) 2584 return -ENOMEM; 2585 2586 /* Eligible bus masters for fetching LLIs */ 2587 if (of_property_read_bool(np, "lli-bus-interface-ahb1")) 2588 pd->lli_buses |= PL08X_AHB1; 2589 if (of_property_read_bool(np, "lli-bus-interface-ahb2")) 2590 pd->lli_buses |= PL08X_AHB2; 2591 if (!pd->lli_buses) { 2592 dev_info(&adev->dev, "no bus masters for LLIs stated, assume all\n"); 2593 pd->lli_buses |= PL08X_AHB1 | PL08X_AHB2; 2594 } 2595 2596 /* Eligible bus masters for memory access */ 2597 if (of_property_read_bool(np, "mem-bus-interface-ahb1")) 2598 pd->mem_buses |= PL08X_AHB1; 2599 if (of_property_read_bool(np, "mem-bus-interface-ahb2")) 2600 pd->mem_buses |= PL08X_AHB2; 2601 if (!pd->mem_buses) { 2602 dev_info(&adev->dev, "no bus masters for memory stated, assume all\n"); 2603 pd->mem_buses |= PL08X_AHB1 | PL08X_AHB2; 2604 } 2605 2606 /* Parse the memcpy channel properties */ 2607 ret = of_property_read_u32(np, "memcpy-burst-size", &val); 2608 if (ret) { 2609 dev_info(&adev->dev, "no memcpy burst size specified, using 1 byte\n"); 2610 val = 1; 2611 } 2612 switch (val) { 2613 default: 2614 dev_err(&adev->dev, "illegal burst size for memcpy, set to 1\n"); 2615 fallthrough; 2616 case 1: 2617 pd->memcpy_burst_size = PL08X_BURST_SZ_1; 2618 break; 2619 case 4: 2620 pd->memcpy_burst_size = PL08X_BURST_SZ_4; 2621 break; 2622 case 8: 2623 pd->memcpy_burst_size = PL08X_BURST_SZ_8; 2624 break; 2625 case 16: 2626 pd->memcpy_burst_size = PL08X_BURST_SZ_16; 2627 break; 2628 case 32: 2629 pd->memcpy_burst_size = PL08X_BURST_SZ_32; 2630 break; 2631 case 64: 2632 pd->memcpy_burst_size = PL08X_BURST_SZ_64; 2633 break; 2634 case 128: 2635 pd->memcpy_burst_size = PL08X_BURST_SZ_128; 2636 break; 2637 case 256: 2638 pd->memcpy_burst_size = PL08X_BURST_SZ_256; 2639 break; 2640 } 2641 2642 ret = of_property_read_u32(np, "memcpy-bus-width", &val); 2643 if (ret) { 2644 dev_info(&adev->dev, "no memcpy bus width specified, using 8 bits\n"); 2645 val = 8; 2646 } 2647 switch (val) { 2648 default: 2649 dev_err(&adev->dev, "illegal bus width for memcpy, set to 8 bits\n"); 2650 fallthrough; 2651 case 8: 2652 pd->memcpy_bus_width = PL08X_BUS_WIDTH_8_BITS; 2653 break; 2654 case 16: 2655 pd->memcpy_bus_width = PL08X_BUS_WIDTH_16_BITS; 2656 break; 2657 case 32: 2658 pd->memcpy_bus_width = PL08X_BUS_WIDTH_32_BITS; 2659 break; 2660 } 2661 2662 /* 2663 * Allocate channel data for all possible slave channels (one 2664 * for each possible signal), channels will then be allocated 2665 * for a device and have it's AHB interfaces set up at 2666 * translation time. 2667 */ 2668 if (pl08x->vd->signals) { 2669 chanp = devm_kcalloc(&adev->dev, 2670 pl08x->vd->signals, 2671 sizeof(struct pl08x_channel_data), 2672 GFP_KERNEL); 2673 if (!chanp) 2674 return -ENOMEM; 2675 2676 pd->slave_channels = chanp; 2677 for (i = 0; i < pl08x->vd->signals; i++) { 2678 /* 2679 * chanp->periph_buses will be assigned at translation 2680 */ 2681 chanp->bus_id = kasprintf(GFP_KERNEL, "slave%d", i); 2682 chanp++; 2683 } 2684 pd->num_slave_channels = pl08x->vd->signals; 2685 } 2686 2687 pl08x->pd = pd; 2688 2689 return of_dma_controller_register(adev->dev.of_node, pl08x_of_xlate, 2690 pl08x); 2691 } 2692 #else 2693 static inline int pl08x_of_probe(struct amba_device *adev, 2694 struct pl08x_driver_data *pl08x, 2695 struct device_node *np) 2696 { 2697 return -EINVAL; 2698 } 2699 #endif 2700 2701 static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) 2702 { 2703 struct pl08x_driver_data *pl08x; 2704 struct vendor_data *vd = id->data; 2705 struct device_node *np = adev->dev.of_node; 2706 u32 tsfr_size; 2707 int ret = 0; 2708 int i; 2709 2710 ret = amba_request_regions(adev, NULL); 2711 if (ret) 2712 return ret; 2713 2714 /* Ensure that we can do DMA */ 2715 ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32)); 2716 if (ret) 2717 goto out_no_pl08x; 2718 2719 /* Create the driver state holder */ 2720 pl08x = kzalloc(sizeof(*pl08x), GFP_KERNEL); 2721 if (!pl08x) { 2722 ret = -ENOMEM; 2723 goto out_no_pl08x; 2724 } 2725 2726 /* Assign useful pointers to the driver state */ 2727 pl08x->adev = adev; 2728 pl08x->vd = vd; 2729 2730 pl08x->base = ioremap(adev->res.start, resource_size(&adev->res)); 2731 if (!pl08x->base) { 2732 ret = -ENOMEM; 2733 goto out_no_ioremap; 2734 } 2735 2736 if (vd->ftdmac020) { 2737 u32 val; 2738 2739 val = readl(pl08x->base + FTDMAC020_REVISION); 2740 dev_info(&pl08x->adev->dev, "FTDMAC020 %d.%d rel %d\n", 2741 (val >> 16) & 0xff, (val >> 8) & 0xff, val & 0xff); 2742 val = readl(pl08x->base + FTDMAC020_FEATURE); 2743 dev_info(&pl08x->adev->dev, "FTDMAC020 %d channels, " 2744 "%s built-in bridge, %s, %s linked lists\n", 2745 (val >> 12) & 0x0f, 2746 (val & BIT(10)) ? "no" : "has", 2747 (val & BIT(9)) ? "AHB0 and AHB1" : "AHB0", 2748 (val & BIT(8)) ? "supports" : "does not support"); 2749 2750 /* Vendor data from feature register */ 2751 if (!(val & BIT(8))) 2752 dev_warn(&pl08x->adev->dev, 2753 "linked lists not supported, required\n"); 2754 vd->channels = (val >> 12) & 0x0f; 2755 vd->dualmaster = !!(val & BIT(9)); 2756 } 2757 2758 /* Initialize memcpy engine */ 2759 dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask); 2760 pl08x->memcpy.dev = &adev->dev; 2761 pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources; 2762 pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy; 2763 pl08x->memcpy.device_prep_dma_interrupt = pl08x_prep_dma_interrupt; 2764 pl08x->memcpy.device_tx_status = pl08x_dma_tx_status; 2765 pl08x->memcpy.device_issue_pending = pl08x_issue_pending; 2766 pl08x->memcpy.device_config = pl08x_config; 2767 pl08x->memcpy.device_pause = pl08x_pause; 2768 pl08x->memcpy.device_resume = pl08x_resume; 2769 pl08x->memcpy.device_terminate_all = pl08x_terminate_all; 2770 pl08x->memcpy.device_synchronize = pl08x_synchronize; 2771 pl08x->memcpy.src_addr_widths = PL80X_DMA_BUSWIDTHS; 2772 pl08x->memcpy.dst_addr_widths = PL80X_DMA_BUSWIDTHS; 2773 pl08x->memcpy.directions = BIT(DMA_MEM_TO_MEM); 2774 pl08x->memcpy.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; 2775 if (vd->ftdmac020) 2776 pl08x->memcpy.copy_align = DMAENGINE_ALIGN_4_BYTES; 2777 2778 2779 /* 2780 * Initialize slave engine, if the block has no signals, that means 2781 * we have no slave support. 2782 */ 2783 if (vd->signals) { 2784 pl08x->has_slave = true; 2785 dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask); 2786 dma_cap_set(DMA_CYCLIC, pl08x->slave.cap_mask); 2787 pl08x->slave.dev = &adev->dev; 2788 pl08x->slave.device_free_chan_resources = 2789 pl08x_free_chan_resources; 2790 pl08x->slave.device_prep_dma_interrupt = 2791 pl08x_prep_dma_interrupt; 2792 pl08x->slave.device_tx_status = pl08x_dma_tx_status; 2793 pl08x->slave.device_issue_pending = pl08x_issue_pending; 2794 pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg; 2795 pl08x->slave.device_prep_dma_cyclic = pl08x_prep_dma_cyclic; 2796 pl08x->slave.device_config = pl08x_config; 2797 pl08x->slave.device_pause = pl08x_pause; 2798 pl08x->slave.device_resume = pl08x_resume; 2799 pl08x->slave.device_terminate_all = pl08x_terminate_all; 2800 pl08x->slave.device_synchronize = pl08x_synchronize; 2801 pl08x->slave.src_addr_widths = PL80X_DMA_BUSWIDTHS; 2802 pl08x->slave.dst_addr_widths = PL80X_DMA_BUSWIDTHS; 2803 pl08x->slave.directions = 2804 BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 2805 pl08x->slave.residue_granularity = 2806 DMA_RESIDUE_GRANULARITY_SEGMENT; 2807 } 2808 2809 /* Get the platform data */ 2810 pl08x->pd = dev_get_platdata(&adev->dev); 2811 if (!pl08x->pd) { 2812 if (np) { 2813 ret = pl08x_of_probe(adev, pl08x, np); 2814 if (ret) 2815 goto out_no_platdata; 2816 } else { 2817 dev_err(&adev->dev, "no platform data supplied\n"); 2818 ret = -EINVAL; 2819 goto out_no_platdata; 2820 } 2821 } else { 2822 pl08x->slave.filter.map = pl08x->pd->slave_map; 2823 pl08x->slave.filter.mapcnt = pl08x->pd->slave_map_len; 2824 pl08x->slave.filter.fn = pl08x_filter_fn; 2825 } 2826 2827 /* By default, AHB1 only. If dualmaster, from platform */ 2828 pl08x->lli_buses = PL08X_AHB1; 2829 pl08x->mem_buses = PL08X_AHB1; 2830 if (pl08x->vd->dualmaster) { 2831 pl08x->lli_buses = pl08x->pd->lli_buses; 2832 pl08x->mem_buses = pl08x->pd->mem_buses; 2833 } 2834 2835 if (vd->pl080s) 2836 pl08x->lli_words = PL080S_LLI_WORDS; 2837 else 2838 pl08x->lli_words = PL080_LLI_WORDS; 2839 tsfr_size = MAX_NUM_TSFR_LLIS * pl08x->lli_words * sizeof(u32); 2840 2841 /* A DMA memory pool for LLIs, align on 1-byte boundary */ 2842 pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev, 2843 tsfr_size, PL08X_ALIGN, 0); 2844 if (!pl08x->pool) { 2845 ret = -ENOMEM; 2846 goto out_no_lli_pool; 2847 } 2848 2849 /* Turn on the PL08x */ 2850 pl08x_ensure_on(pl08x); 2851 2852 /* Clear any pending interrupts */ 2853 if (vd->ftdmac020) 2854 /* This variant has error IRQs in bits 16-19 */ 2855 writel(0x0000FFFF, pl08x->base + PL080_ERR_CLEAR); 2856 else 2857 writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR); 2858 writel(0x000000FF, pl08x->base + PL080_TC_CLEAR); 2859 2860 /* Attach the interrupt handler */ 2861 ret = request_irq(adev->irq[0], pl08x_irq, 0, DRIVER_NAME, pl08x); 2862 if (ret) { 2863 dev_err(&adev->dev, "%s failed to request interrupt %d\n", 2864 __func__, adev->irq[0]); 2865 goto out_no_irq; 2866 } 2867 2868 /* Initialize physical channels */ 2869 pl08x->phy_chans = kzalloc((vd->channels * sizeof(*pl08x->phy_chans)), 2870 GFP_KERNEL); 2871 if (!pl08x->phy_chans) { 2872 ret = -ENOMEM; 2873 goto out_no_phychans; 2874 } 2875 2876 for (i = 0; i < vd->channels; i++) { 2877 struct pl08x_phy_chan *ch = &pl08x->phy_chans[i]; 2878 2879 ch->id = i; 2880 ch->base = pl08x->base + PL080_Cx_BASE(i); 2881 if (vd->ftdmac020) { 2882 /* FTDMA020 has a special channel busy register */ 2883 ch->reg_busy = ch->base + FTDMAC020_CH_BUSY; 2884 ch->reg_config = ch->base + FTDMAC020_CH_CFG; 2885 ch->reg_control = ch->base + FTDMAC020_CH_CSR; 2886 ch->reg_src = ch->base + FTDMAC020_CH_SRC_ADDR; 2887 ch->reg_dst = ch->base + FTDMAC020_CH_DST_ADDR; 2888 ch->reg_lli = ch->base + FTDMAC020_CH_LLP; 2889 ch->ftdmac020 = true; 2890 } else { 2891 ch->reg_config = ch->base + vd->config_offset; 2892 ch->reg_control = ch->base + PL080_CH_CONTROL; 2893 ch->reg_src = ch->base + PL080_CH_SRC_ADDR; 2894 ch->reg_dst = ch->base + PL080_CH_DST_ADDR; 2895 ch->reg_lli = ch->base + PL080_CH_LLI; 2896 } 2897 if (vd->pl080s) 2898 ch->pl080s = true; 2899 2900 spin_lock_init(&ch->lock); 2901 2902 /* 2903 * Nomadik variants can have channels that are locked 2904 * down for the secure world only. Lock up these channels 2905 * by perpetually serving a dummy virtual channel. 2906 */ 2907 if (vd->nomadik) { 2908 u32 val; 2909 2910 val = readl(ch->reg_config); 2911 if (val & (PL080N_CONFIG_ITPROT | PL080N_CONFIG_SECPROT)) { 2912 dev_info(&adev->dev, "physical channel %d reserved for secure access only\n", i); 2913 ch->locked = true; 2914 } 2915 } 2916 2917 dev_dbg(&adev->dev, "physical channel %d is %s\n", 2918 i, pl08x_phy_channel_busy(ch) ? "BUSY" : "FREE"); 2919 } 2920 2921 /* Register as many memcpy channels as there are physical channels */ 2922 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->memcpy, 2923 pl08x->vd->channels, false); 2924 if (ret <= 0) { 2925 dev_warn(&pl08x->adev->dev, 2926 "%s failed to enumerate memcpy channels - %d\n", 2927 __func__, ret); 2928 goto out_no_memcpy; 2929 } 2930 2931 /* Register slave channels */ 2932 if (pl08x->has_slave) { 2933 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave, 2934 pl08x->pd->num_slave_channels, true); 2935 if (ret < 0) { 2936 dev_warn(&pl08x->adev->dev, 2937 "%s failed to enumerate slave channels - %d\n", 2938 __func__, ret); 2939 goto out_no_slave; 2940 } 2941 } 2942 2943 ret = dma_async_device_register(&pl08x->memcpy); 2944 if (ret) { 2945 dev_warn(&pl08x->adev->dev, 2946 "%s failed to register memcpy as an async device - %d\n", 2947 __func__, ret); 2948 goto out_no_memcpy_reg; 2949 } 2950 2951 if (pl08x->has_slave) { 2952 ret = dma_async_device_register(&pl08x->slave); 2953 if (ret) { 2954 dev_warn(&pl08x->adev->dev, 2955 "%s failed to register slave as an async device - %d\n", 2956 __func__, ret); 2957 goto out_no_slave_reg; 2958 } 2959 } 2960 2961 amba_set_drvdata(adev, pl08x); 2962 init_pl08x_debugfs(pl08x); 2963 dev_info(&pl08x->adev->dev, "DMA: PL%03x%s rev%u at 0x%08llx irq %d\n", 2964 amba_part(adev), pl08x->vd->pl080s ? "s" : "", amba_rev(adev), 2965 (unsigned long long)adev->res.start, adev->irq[0]); 2966 2967 return 0; 2968 2969 out_no_slave_reg: 2970 dma_async_device_unregister(&pl08x->memcpy); 2971 out_no_memcpy_reg: 2972 if (pl08x->has_slave) 2973 pl08x_free_virtual_channels(&pl08x->slave); 2974 out_no_slave: 2975 pl08x_free_virtual_channels(&pl08x->memcpy); 2976 out_no_memcpy: 2977 kfree(pl08x->phy_chans); 2978 out_no_phychans: 2979 free_irq(adev->irq[0], pl08x); 2980 out_no_irq: 2981 dma_pool_destroy(pl08x->pool); 2982 out_no_lli_pool: 2983 out_no_platdata: 2984 iounmap(pl08x->base); 2985 out_no_ioremap: 2986 kfree(pl08x); 2987 out_no_pl08x: 2988 amba_release_regions(adev); 2989 return ret; 2990 } 2991 2992 /* PL080 has 8 channels and the PL080 have just 2 */ 2993 static struct vendor_data vendor_pl080 = { 2994 .config_offset = PL080_CH_CONFIG, 2995 .channels = 8, 2996 .signals = 16, 2997 .dualmaster = true, 2998 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, 2999 }; 3000 3001 static struct vendor_data vendor_nomadik = { 3002 .config_offset = PL080_CH_CONFIG, 3003 .channels = 8, 3004 .signals = 32, 3005 .dualmaster = true, 3006 .nomadik = true, 3007 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, 3008 }; 3009 3010 static struct vendor_data vendor_pl080s = { 3011 .config_offset = PL080S_CH_CONFIG, 3012 .channels = 8, 3013 .signals = 32, 3014 .pl080s = true, 3015 .max_transfer_size = PL080S_CONTROL_TRANSFER_SIZE_MASK, 3016 }; 3017 3018 static struct vendor_data vendor_pl081 = { 3019 .config_offset = PL080_CH_CONFIG, 3020 .channels = 2, 3021 .signals = 16, 3022 .dualmaster = false, 3023 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, 3024 }; 3025 3026 static struct vendor_data vendor_ftdmac020 = { 3027 .config_offset = PL080_CH_CONFIG, 3028 .ftdmac020 = true, 3029 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, 3030 }; 3031 3032 static const struct amba_id pl08x_ids[] = { 3033 /* Samsung PL080S variant */ 3034 { 3035 .id = 0x0a141080, 3036 .mask = 0xffffffff, 3037 .data = &vendor_pl080s, 3038 }, 3039 /* PL080 */ 3040 { 3041 .id = 0x00041080, 3042 .mask = 0x000fffff, 3043 .data = &vendor_pl080, 3044 }, 3045 /* PL081 */ 3046 { 3047 .id = 0x00041081, 3048 .mask = 0x000fffff, 3049 .data = &vendor_pl081, 3050 }, 3051 /* Nomadik 8815 PL080 variant */ 3052 { 3053 .id = 0x00280080, 3054 .mask = 0x00ffffff, 3055 .data = &vendor_nomadik, 3056 }, 3057 /* Faraday Technology FTDMAC020 */ 3058 { 3059 .id = 0x0003b080, 3060 .mask = 0x000fffff, 3061 .data = &vendor_ftdmac020, 3062 }, 3063 { 0, 0 }, 3064 }; 3065 3066 MODULE_DEVICE_TABLE(amba, pl08x_ids); 3067 3068 static struct amba_driver pl08x_amba_driver = { 3069 .drv.name = DRIVER_NAME, 3070 .id_table = pl08x_ids, 3071 .probe = pl08x_probe, 3072 }; 3073 3074 static int __init pl08x_init(void) 3075 { 3076 int retval; 3077 retval = amba_driver_register(&pl08x_amba_driver); 3078 if (retval) 3079 printk(KERN_WARNING DRIVER_NAME 3080 "failed to register as an AMBA device (%d)\n", 3081 retval); 3082 return retval; 3083 } 3084 subsys_initcall(pl08x_init); 3085